mirror of
https://github.com/hoshikawa2/agent_platform_oci.git
synced 2026-11-06 11:24:13 +00:00
new feature: JEV Integration. Now there are a current (native) LLM decision model and a JEV decision model
This commit is contained in:
210
tests/test_jev_transport_standalone.py
Normal file
210
tests/test_jev_transport_standalone.py
Normal file
@@ -0,0 +1,210 @@
|
||||
"""Run without project dependencies: python3 tests/test_jev_transport_standalone.py."""
|
||||
import ast
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MODULE = ROOT / 'libs/agent_framework/src/agent_framework/decisions/client.py'
|
||||
spec = importlib.util.spec_from_file_location('jev_transport', MODULE)
|
||||
client = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(client)
|
||||
|
||||
|
||||
def response(questions, confidence=.95):
|
||||
return {'model': 'typesafe-ai/jev', 'answers': [
|
||||
{'type': q['type'], 'name': q['name'], **(
|
||||
{'choice': q['choices'][0]['value'], 'confidence': confidence} if q['type'] == 'choice' else
|
||||
{'probability': confidence} if q['type'] == 'predicate' else
|
||||
{'score': 8, 'confidence': confidence})} for q in questions]}
|
||||
|
||||
|
||||
class DecisionTransportTests(unittest.IsolatedAsyncioTestCase):
|
||||
def setUp(self):
|
||||
self.env = patch.dict(os.environ, {k: v for k, v in os.environ.items() if not k.startswith(('DECISION_', 'JEV_'))}, clear=True)
|
||||
self.env.start()
|
||||
self.addCleanup(self.env.stop)
|
||||
self.llm = type('LLM', (), {'ainvoke': AsyncMock(return_value='legacy')})()
|
||||
|
||||
async def call(self, **kwargs):
|
||||
return await client.invoke(self.llm, [{'content': 'test context'}], decision_point='router',
|
||||
decision_questions=[client.choice('intent', ['billing', 'orders'])], **kwargs)
|
||||
|
||||
async def test_default_passes_original_kwargs_without_network(self):
|
||||
with patch.object(client, '_post', side_effect=AssertionError('network')):
|
||||
self.assertEqual(await self.call(profile_name='router'), 'legacy')
|
||||
self.llm.ainvoke.assert_awaited_once_with([{'content': 'test context'}], profile_name='router')
|
||||
|
||||
async def test_individual_override(self):
|
||||
os.environ.update(DECISION_PROVIDER='jev', DECISION_ROUTER_PROVIDER='current')
|
||||
self.assertFalse(client.selected('router'))
|
||||
self.assertTrue(client.selected('route_continuity'))
|
||||
os.environ.update(DECISION_PROVIDER='current', DECISION_ROUTER_PROVIDER='jev')
|
||||
self.assertTrue(client.selected('router'))
|
||||
self.assertFalse(client.selected('route_continuity'))
|
||||
os.environ['DECISION_JUDGE_ALUC_PROVIDER'] = 'jev'
|
||||
self.assertTrue(client.selected('judge.groundedness', 'judge_aluc'))
|
||||
os.environ['DECISION_JUDGE_GROUNDEDNESS_PROVIDER'] = 'current'
|
||||
self.assertFalse(client.selected('judge.groundedness', 'judge_aluc'))
|
||||
|
||||
async def test_typed_choices_predicates_scores_shared_request(self):
|
||||
os.environ['DECISION_ROUTER_PROVIDER'] = 'jev'
|
||||
with patch.object(client, '_post', side_effect=lambda b: response(b['questions'])) as post:
|
||||
result = json.loads(await client.invoke(self.llm, [], decision_point='router',
|
||||
decision_questions=[client.choice('intent', ['billing']), client.predicate('allowed'), client.score('score')]))
|
||||
self.assertEqual((result['intent'], result['allowed'], result['score']), ('billing', True, 8))
|
||||
self.assertEqual(post.call_count, 1)
|
||||
self.llm.ainvoke.assert_not_awaited()
|
||||
|
||||
async def test_invalid_response_falls_back_once(self):
|
||||
os.environ['DECISION_ROUTER_PROVIDER'] = 'jev'
|
||||
for failure in ['missing', 'duplicate', 'choice', 'nan', 'missing_confidence', 'low_confidence', 'type', 'network']:
|
||||
with self.subTest(failure=failure):
|
||||
self.llm.ainvoke.reset_mock()
|
||||
def transport(body):
|
||||
if failure == 'network': raise TimeoutError()
|
||||
r = response(body['questions']); a = r['answers'][0]
|
||||
if failure == 'missing': r['answers'] = []
|
||||
if failure == 'duplicate': r['answers'] *= 2
|
||||
if failure == 'choice': a['choice'] = 'unknown'
|
||||
if failure == 'nan': a['confidence'] = float('nan')
|
||||
if failure == 'missing_confidence': a.pop('confidence')
|
||||
if failure == 'low_confidence': a['confidence'] = .1
|
||||
if failure == 'type': a['type'] = 'predicate'
|
||||
return r
|
||||
with patch.object(client, '_post', side_effect=transport):
|
||||
self.assertEqual(await self.call(), 'legacy')
|
||||
self.llm.ainvoke.assert_awaited_once()
|
||||
|
||||
async def test_no_fallback_surfaces_error(self):
|
||||
os.environ.update(DECISION_ROUTER_PROVIDER='jev', DECISION_FALLBACK_TO_CURRENT='false')
|
||||
with patch.object(client, '_post', return_value={'answers': []}):
|
||||
with self.assertRaises(ValueError): await self.call()
|
||||
self.llm.ainvoke.assert_not_awaited()
|
||||
|
||||
async def test_generative_bypass(self):
|
||||
os.environ['DECISION_PROVIDER'] = 'jev'
|
||||
with patch.object(client, '_post', side_effect=AssertionError('network')):
|
||||
self.assertEqual(await self.call(decision_generative=True), 'legacy')
|
||||
|
||||
async def test_invalid_provider_rejected(self):
|
||||
os.environ['DECISION_ROUTER_PROVIDER'] = 'typo'
|
||||
with self.assertRaises(ValueError): await self.call()
|
||||
|
||||
async def test_custom_renderer_failure_falls_back(self):
|
||||
os.environ['DECISION_ROUTER_PROVIDER'] = 'jev'
|
||||
with patch.object(client, '_post', side_effect=lambda b: response(b['questions'])):
|
||||
self.assertEqual(await self.call(decision_render=lambda d: d['missing']), 'legacy')
|
||||
|
||||
async def test_threshold_override(self):
|
||||
os.environ.update(DECISION_ROUTER_PROVIDER='jev', DECISION_ROUTER_MIN_CONFIDENCE='.99')
|
||||
with patch.object(client, '_post', side_effect=lambda b: response(b['questions'])):
|
||||
self.assertEqual(await self.call(), 'legacy')
|
||||
|
||||
async def test_predicate_rejects_bool_and_out_of_range(self):
|
||||
for probability in [True, -1, 2, float('inf'), None]:
|
||||
with self.subTest(probability=probability):
|
||||
with patch.object(client, '_post', return_value={'answers': [{'type': 'predicate', 'name': 'allowed', 'probability': probability}]}):
|
||||
with self.assertRaises(ValueError):
|
||||
await client.evaluate('guardrail', [], [client.predicate('allowed')])
|
||||
|
||||
async def test_transport_requires_key_before_request(self):
|
||||
os.environ.pop('AI_GATEWAY_API_KEY', None)
|
||||
with patch.object(client.urllib.request, 'urlopen') as request:
|
||||
with self.assertRaises(ValueError): client._post({})
|
||||
request.assert_not_called()
|
||||
|
||||
async def test_transport_endpoint_auth_and_timeout(self):
|
||||
os.environ.update(AI_GATEWAY_API_KEY='test-key', JEV_TIMEOUT_SECONDS='3')
|
||||
import io
|
||||
with patch.object(client.urllib.request, 'urlopen', return_value=io.StringIO('{"answers": []}')) as request:
|
||||
self.assertEqual(client._post({'model': 'typesafe-ai/jev'}), {'answers': []})
|
||||
req = request.call_args.args[0]
|
||||
self.assertEqual(req.full_url, 'https://ai-gateway.vercel.sh/v1/decisions')
|
||||
self.assertEqual(req.get_header('Authorization'), 'Bearer test-key')
|
||||
self.assertEqual(request.call_args.kwargs['timeout'], 3)
|
||||
|
||||
async def test_integrated_modules_compile(self):
|
||||
for path in (ROOT / 'libs/agent_framework/src/agent_framework').rglob('*.py'):
|
||||
with self.subTest(path=path): ast.parse(path.read_text(), filename=str(path))
|
||||
|
||||
|
||||
|
||||
def load_functions(relative_path, names, namespace):
|
||||
"""Execute actual function bodies in isolation from unavailable optional dependencies."""
|
||||
path = ROOT / 'libs/agent_framework/src/agent_framework' / relative_path
|
||||
tree = ast.parse(path.read_text())
|
||||
nodes = [node for node in ast.walk(tree) if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name in names]
|
||||
module = ast.Module(body=[ast.ImportFrom(module='__future__', names=[ast.alias(name='annotations')], level=0)] + nodes, type_ignores=[])
|
||||
exec(compile(ast.fix_missing_locations(module), str(path), 'exec'), namespace)
|
||||
return namespace
|
||||
|
||||
|
||||
class DecisionCallsiteTests(unittest.IsolatedAsyncioTestCase):
|
||||
setUp = DecisionTransportTests.setUp
|
||||
|
||||
async def test_actual_continuity_classifier_jev(self):
|
||||
os.environ['DECISION_ROUTE_CONTINUITY_PROVIDER'] = 'jev'
|
||||
ns = load_functions('routing/continuity.py', {'_classify'}, {
|
||||
'json': json, 'decision_invoke': client.invoke, 'choice': client.choice,
|
||||
'ContinuityEvaluation': SimpleNamespace})
|
||||
self_obj = SimpleNamespace(llm=self.llm, profile_name='route_continuity',
|
||||
_agent_capabilities=lambda intents: {'billing': ['invoices']},
|
||||
_compact_history=lambda history: history, _parse_json=json.loads)
|
||||
with patch.object(client, '_post', side_effect=lambda b: response(b['questions'])):
|
||||
result = await ns['_classify'](self_obj, {}, text='continue', active_agent='billing', intents=[])
|
||||
self.assertEqual(result.decision, 'CONTINUE')
|
||||
self.assertGreater(result.confidence, .9)
|
||||
self.llm.ainvoke.assert_not_awaited()
|
||||
|
||||
async def test_actual_router_derives_agent_from_catalog(self):
|
||||
os.environ['DECISION_ROUTER_PROVIDER'] = 'jev'
|
||||
ns = load_functions('routing/enterprise_router.py', {'_route_by_llm'}, {
|
||||
'json': json, 'decision_invoke': client.invoke, 'choice': client.choice, 'RouteDecision': SimpleNamespace})
|
||||
intent = SimpleNamespace(enabled=True, name='invoice', agent='billing', description='Invoice', examples=[], mcp_tools=[], domain='telecom')
|
||||
self_obj = SimpleNamespace(llm=self.llm, intents=[intent], fallback_agent='fallback', _parse_json=json.loads,
|
||||
_agent_for_intent=lambda name: 'billing', _domain_for_intent=lambda name: 'telecom', _tools_for_intent=lambda name: ['read_invoice'])
|
||||
with patch.object(client, '_post', side_effect=lambda b: response(b['questions'])):
|
||||
result = await ns['_route_by_llm'](self_obj, 'invoice', {})
|
||||
self.assertEqual((result.intent, result.agent, result.mcp_tools), ('invoice', 'billing', ['read_invoice']))
|
||||
|
||||
async def test_actual_reconciliation_preserves_candidate_values(self):
|
||||
os.environ['DECISION_TRANSACTION_PARAMETER_RECONCILIATION_PROVIDER'] = 'jev'
|
||||
ns = load_functions('runtime/transaction_parameters.py', {'reconcile_transaction_parameters', '_response_text', '_coerce'}, {
|
||||
'json': json, 'logger': logging.getLogger('test'), 'parse_json_object': json.loads,
|
||||
'decision_invoke': client.invoke, 'selected': client.selected, 'choice': client.choice,
|
||||
'_EMPTY_VALUES': (None, '', {}, [])})
|
||||
self.llm.ainvoke.return_value = json.dumps({'fields': {
|
||||
'id': {'value': '42', 'decision': 'resolved', 'source': 'current'},
|
||||
'empty': {'value': None, 'decision': 'unresolved', 'source': ''}}})
|
||||
with patch.object(client, '_post', side_effect=lambda b: response(b['questions'])):
|
||||
result = await ns['reconcile_transaction_parameters'](self.llm, text='42', tool_name='read', parameter_names=['id', 'empty'])
|
||||
self.assertEqual(result['values'], {'id': '42'})
|
||||
self.assertEqual(result['decisions']['empty'], 'unresolved')
|
||||
self.llm.ainvoke.assert_awaited_once()
|
||||
|
||||
async def test_actual_calibrated_judge_and_generative_fallback(self):
|
||||
os.environ['DECISION_JUDGE_RQLT_PROVIDER'] = 'jev'
|
||||
ns = load_functions('judges/calibrated/llm_client.py', {'classify'}, {
|
||||
'decision_invoke': client.invoke, 'predicate': client.predicate, 'score': client.score,
|
||||
'choice': client.choice, '_parse_json': json.loads})
|
||||
self_obj = SimpleNamespace(llm=self.llm, default_profile='judge', _build_prompt=lambda task, payload: 'quality rubric')
|
||||
with patch.object(client, '_post', side_effect=lambda b: response(b['questions'])):
|
||||
result = await ns['classify'](self_obj, 'RQLT', {})
|
||||
self.assertTrue(result['allowed'])
|
||||
self.assertEqual(result['score'], 8)
|
||||
self.llm.ainvoke.assert_not_awaited()
|
||||
self.llm.ainvoke.return_value = '{"text":"fallback response"}'
|
||||
os.environ['DECISION_PROVIDER'] = 'jev'
|
||||
with patch.object(client, '_post', side_effect=AssertionError('no generative JEV')):
|
||||
self.assertEqual(await ns['classify'](self_obj, 'FALLBACK', {}), {'text': 'fallback response'})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
Reference in New Issue
Block a user