"""Run without project dependencies: python3 tests/test_jev_transport_standalone.py.""" import ast import asyncio import importlib.util import json import os import logging from types import SimpleNamespace from pathlib import Path import unittest from unittest.mock import AsyncMock, patch ROOT = Path(__file__).resolve().parents[1] MODULE = ROOT / 'libs/agent_framework/src/agent_framework/decisions/client.py' spec = importlib.util.spec_from_file_location('jev_transport', MODULE) client = importlib.util.module_from_spec(spec) spec.loader.exec_module(client) def response(questions, confidence=.95): return {'model': 'typesafe-ai/jev', 'answers': [ {'type': q['type'], 'name': q['name'], **( {'choice': q['choices'][0]['value'], 'confidence': confidence} if q['type'] == 'choice' else {'probability': confidence} if q['type'] == 'predicate' else {'score': 8, 'confidence': confidence})} for q in questions]} class DecisionTransportTests(unittest.IsolatedAsyncioTestCase): def setUp(self): self.env = patch.dict(os.environ, {k: v for k, v in os.environ.items() if not k.startswith(('DECISION_', 'JEV_'))}, clear=True) self.env.start() self.addCleanup(self.env.stop) self.llm = type('LLM', (), {'ainvoke': AsyncMock(return_value='legacy')})() async def call(self, **kwargs): return await client.invoke(self.llm, [{'content': 'test context'}], decision_point='router', decision_questions=[client.choice('intent', ['billing', 'orders'])], **kwargs) async def test_default_passes_original_kwargs_without_network(self): with patch.object(client, '_post', side_effect=AssertionError('network')): self.assertEqual(await self.call(profile_name='router'), 'legacy') self.llm.ainvoke.assert_awaited_once_with([{'content': 'test context'}], profile_name='router') async def test_individual_override(self): os.environ.update(DECISION_PROVIDER='jev', DECISION_ROUTER_PROVIDER='current') self.assertFalse(client.selected('router')) self.assertTrue(client.selected('route_continuity')) os.environ.update(DECISION_PROVIDER='current', DECISION_ROUTER_PROVIDER='jev') self.assertTrue(client.selected('router')) self.assertFalse(client.selected('route_continuity')) os.environ['DECISION_JUDGE_ALUC_PROVIDER'] = 'jev' self.assertTrue(client.selected('judge.groundedness', 'judge_aluc')) os.environ['DECISION_JUDGE_GROUNDEDNESS_PROVIDER'] = 'current' self.assertFalse(client.selected('judge.groundedness', 'judge_aluc')) async def test_typed_choices_predicates_scores_shared_request(self): os.environ['DECISION_ROUTER_PROVIDER'] = 'jev' with patch.object(client, '_post', side_effect=lambda b: response(b['questions'])) as post: result = json.loads(await client.invoke(self.llm, [], decision_point='router', decision_questions=[client.choice('intent', ['billing']), client.predicate('allowed'), client.score('score')])) self.assertEqual((result['intent'], result['allowed'], result['score']), ('billing', True, 8)) self.assertEqual(post.call_count, 1) self.llm.ainvoke.assert_not_awaited() async def test_invalid_response_falls_back_once(self): os.environ['DECISION_ROUTER_PROVIDER'] = 'jev' for failure in ['missing', 'duplicate', 'choice', 'nan', 'missing_confidence', 'low_confidence', 'type', 'network']: with self.subTest(failure=failure): self.llm.ainvoke.reset_mock() def transport(body): if failure == 'network': raise TimeoutError() r = response(body['questions']); a = r['answers'][0] if failure == 'missing': r['answers'] = [] if failure == 'duplicate': r['answers'] *= 2 if failure == 'choice': a['choice'] = 'unknown' if failure == 'nan': a['confidence'] = float('nan') if failure == 'missing_confidence': a.pop('confidence') if failure == 'low_confidence': a['confidence'] = .1 if failure == 'type': a['type'] = 'predicate' return r with patch.object(client, '_post', side_effect=transport): self.assertEqual(await self.call(), 'legacy') self.llm.ainvoke.assert_awaited_once() async def test_no_fallback_surfaces_error(self): os.environ.update(DECISION_ROUTER_PROVIDER='jev', DECISION_FALLBACK_TO_CURRENT='false') with patch.object(client, '_post', return_value={'answers': []}): with self.assertRaises(ValueError): await self.call() self.llm.ainvoke.assert_not_awaited() async def test_generative_bypass(self): os.environ['DECISION_PROVIDER'] = 'jev' with patch.object(client, '_post', side_effect=AssertionError('network')): self.assertEqual(await self.call(decision_generative=True), 'legacy') async def test_invalid_provider_rejected(self): os.environ['DECISION_ROUTER_PROVIDER'] = 'typo' with self.assertRaises(ValueError): await self.call() async def test_custom_renderer_failure_falls_back(self): os.environ['DECISION_ROUTER_PROVIDER'] = 'jev' with patch.object(client, '_post', side_effect=lambda b: response(b['questions'])): self.assertEqual(await self.call(decision_render=lambda d: d['missing']), 'legacy') async def test_threshold_override(self): os.environ.update(DECISION_ROUTER_PROVIDER='jev', DECISION_ROUTER_MIN_CONFIDENCE='.99') with patch.object(client, '_post', side_effect=lambda b: response(b['questions'])): self.assertEqual(await self.call(), 'legacy') async def test_predicate_rejects_bool_and_out_of_range(self): for probability in [True, -1, 2, float('inf'), None]: with self.subTest(probability=probability): with patch.object(client, '_post', return_value={'answers': [{'type': 'predicate', 'name': 'allowed', 'probability': probability}]}): with self.assertRaises(ValueError): await client.evaluate('guardrail', [], [client.predicate('allowed')]) async def test_transport_requires_key_before_request(self): os.environ.pop('AI_GATEWAY_API_KEY', None) with patch.object(client.urllib.request, 'urlopen') as request: with self.assertRaises(ValueError): client._post({}) request.assert_not_called() async def test_transport_endpoint_auth_and_timeout(self): os.environ.update(AI_GATEWAY_API_KEY='test-key', JEV_TIMEOUT_SECONDS='3') import io with patch.object(client.urllib.request, 'urlopen', return_value=io.StringIO('{"answers": []}')) as request: self.assertEqual(client._post({'model': 'typesafe-ai/jev'}), {'answers': []}) req = request.call_args.args[0] self.assertEqual(req.full_url, 'https://ai-gateway.vercel.sh/v1/decisions') self.assertEqual(req.get_header('Authorization'), 'Bearer test-key') self.assertEqual(request.call_args.kwargs['timeout'], 3) async def test_integrated_modules_compile(self): for path in (ROOT / 'libs/agent_framework/src/agent_framework').rglob('*.py'): with self.subTest(path=path): ast.parse(path.read_text(), filename=str(path)) def load_functions(relative_path, names, namespace): """Execute actual function bodies in isolation from unavailable optional dependencies.""" path = ROOT / 'libs/agent_framework/src/agent_framework' / relative_path tree = ast.parse(path.read_text()) nodes = [node for node in ast.walk(tree) if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name in names] module = ast.Module(body=[ast.ImportFrom(module='__future__', names=[ast.alias(name='annotations')], level=0)] + nodes, type_ignores=[]) exec(compile(ast.fix_missing_locations(module), str(path), 'exec'), namespace) return namespace class DecisionCallsiteTests(unittest.IsolatedAsyncioTestCase): setUp = DecisionTransportTests.setUp async def test_actual_continuity_classifier_jev(self): os.environ['DECISION_ROUTE_CONTINUITY_PROVIDER'] = 'jev' ns = load_functions('routing/continuity.py', {'_classify'}, { 'json': json, 'decision_invoke': client.invoke, 'choice': client.choice, 'ContinuityEvaluation': SimpleNamespace}) self_obj = SimpleNamespace(llm=self.llm, profile_name='route_continuity', _agent_capabilities=lambda intents: {'billing': ['invoices']}, _compact_history=lambda history: history, _parse_json=json.loads) with patch.object(client, '_post', side_effect=lambda b: response(b['questions'])): result = await ns['_classify'](self_obj, {}, text='continue', active_agent='billing', intents=[]) self.assertEqual(result.decision, 'CONTINUE') self.assertGreater(result.confidence, .9) self.llm.ainvoke.assert_not_awaited() async def test_actual_router_derives_agent_from_catalog(self): os.environ['DECISION_ROUTER_PROVIDER'] = 'jev' ns = load_functions('routing/enterprise_router.py', {'_route_by_llm'}, { 'json': json, 'decision_invoke': client.invoke, 'choice': client.choice, 'RouteDecision': SimpleNamespace}) intent = SimpleNamespace(enabled=True, name='invoice', agent='billing', description='Invoice', examples=[], mcp_tools=[], domain='telecom') self_obj = SimpleNamespace(llm=self.llm, intents=[intent], fallback_agent='fallback', _parse_json=json.loads, _agent_for_intent=lambda name: 'billing', _domain_for_intent=lambda name: 'telecom', _tools_for_intent=lambda name: ['read_invoice']) with patch.object(client, '_post', side_effect=lambda b: response(b['questions'])): result = await ns['_route_by_llm'](self_obj, 'invoice', {}) self.assertEqual((result.intent, result.agent, result.mcp_tools), ('invoice', 'billing', ['read_invoice'])) async def test_actual_reconciliation_preserves_candidate_values(self): os.environ['DECISION_TRANSACTION_PARAMETER_RECONCILIATION_PROVIDER'] = 'jev' ns = load_functions('runtime/transaction_parameters.py', {'reconcile_transaction_parameters', '_response_text', '_coerce'}, { 'json': json, 'logger': logging.getLogger('test'), 'parse_json_object': json.loads, 'decision_invoke': client.invoke, 'selected': client.selected, 'choice': client.choice, '_EMPTY_VALUES': (None, '', {}, [])}) self.llm.ainvoke.return_value = json.dumps({'fields': { 'id': {'value': '42', 'decision': 'resolved', 'source': 'current'}, 'empty': {'value': None, 'decision': 'unresolved', 'source': ''}}}) with patch.object(client, '_post', side_effect=lambda b: response(b['questions'])): result = await ns['reconcile_transaction_parameters'](self.llm, text='42', tool_name='read', parameter_names=['id', 'empty']) self.assertEqual(result['values'], {'id': '42'}) self.assertEqual(result['decisions']['empty'], 'unresolved') self.llm.ainvoke.assert_awaited_once() async def test_actual_calibrated_judge_and_generative_fallback(self): os.environ['DECISION_JUDGE_RQLT_PROVIDER'] = 'jev' ns = load_functions('judges/calibrated/llm_client.py', {'classify'}, { 'decision_invoke': client.invoke, 'predicate': client.predicate, 'score': client.score, 'choice': client.choice, '_parse_json': json.loads}) self_obj = SimpleNamespace(llm=self.llm, default_profile='judge', _build_prompt=lambda task, payload: 'quality rubric') with patch.object(client, '_post', side_effect=lambda b: response(b['questions'])): result = await ns['classify'](self_obj, 'RQLT', {}) self.assertTrue(result['allowed']) self.assertEqual(result['score'], 8) self.llm.ainvoke.assert_not_awaited() self.llm.ainvoke.return_value = '{"text":"fallback response"}' os.environ['DECISION_PROVIDER'] = 'jev' with patch.object(client, '_post', side_effect=AssertionError('no generative JEV')): self.assertEqual(await ns['classify'](self_obj, 'FALLBACK', {}), {'text': 'fallback response'}) if __name__ == '__main__': unittest.main(verbosity=2)