133 lines
4.4 KiB
Python
133 lines
4.4 KiB
Python
import json
|
|
import pytest
|
|
|
|
from agent_framework.judges.judge import JudgePipeline, JudgeResult, _compact_judge_context
|
|
|
|
|
|
def test_compact_judge_context_drops_recursive_runtime_payload_and_preserves_business_facts():
|
|
huge = 'X' * 50000
|
|
context = {
|
|
'transaction_status': 'COMPLETED',
|
|
'route': 'contestacao_agent',
|
|
'evidence': [{
|
|
'tool_name': 'cancelar_vas_avulso',
|
|
'result': {
|
|
'subject': 'TIM Fashion Mensal',
|
|
'value': '10,00',
|
|
'success': True,
|
|
'state': {'session': huge, 'nodes': huge, 'business_events': [huge]},
|
|
},
|
|
}],
|
|
'session': {'original_context': huge},
|
|
'agent_profile': huge,
|
|
}
|
|
out = _compact_judge_context(context)
|
|
rendered = json.dumps(out, ensure_ascii=False, default=str)
|
|
assert len(rendered) < 40000
|
|
assert 'TIM Fashion Mensal' in rendered
|
|
assert '10,00' in rendered
|
|
assert 'COMPLETED' in rendered
|
|
assert 'agent_profile' not in rendered
|
|
assert 'business_events' not in rendered
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pipeline_compacts_context_before_external_judge():
|
|
seen = {}
|
|
|
|
class ExternalJudge:
|
|
async def evaluate(self, question, answer, context):
|
|
seen['context'] = context
|
|
return JudgeResult(name='external', score=1.0, passed=True, reason='ok')
|
|
|
|
pipeline = JudgePipeline(judges=[ExternalJudge()], enabled=True)
|
|
context = {
|
|
'transaction_status': 'COMPLETED',
|
|
'evidence': [{'subject': 'TIM Fashion Mensal', 'state': {'session': 'Y' * 80000}}],
|
|
}
|
|
results = await pipeline.evaluate_all('sim', 'Cancelado com sucesso', context)
|
|
assert results[0].passed is True
|
|
rendered = json.dumps(seen['context'], ensure_ascii=False, default=str)
|
|
assert len(rendered) < 40000
|
|
assert 'TIM Fashion Mensal' in rendered
|
|
assert 'session' not in rendered
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_response_quality_uses_original_request_for_transaction_confirmation():
|
|
seen = {}
|
|
|
|
class ResponseQualityJudge:
|
|
name = 'tim_response_quality'
|
|
|
|
async def evaluate(self, question, answer, context):
|
|
seen['question'] = question
|
|
return JudgeResult(name=self.name, score=1.0, passed=True, reason='ok')
|
|
|
|
pipeline = JudgePipeline(judges=[ResponseQualityJudge()], enabled=True)
|
|
context = {
|
|
'transaction_status': 'COMPLETED',
|
|
'confirmation_received': True,
|
|
'transaction_evidence': [{
|
|
'arguments': {
|
|
'operator_instructions': 'cancelar o serviço adicional e enviar a segunda via',
|
|
},
|
|
}],
|
|
}
|
|
|
|
await pipeline.evaluate_all(
|
|
'autorizo que prossiga conforme solicitado',
|
|
'Cancelamento concluído com sucesso.',
|
|
context,
|
|
)
|
|
|
|
assert 'cancelar o serviço adicional e enviar a segunda via' in seen['question']
|
|
assert 'Confirmação do usuário neste turno: autorizo que prossiga conforme solicitado' in seen['question']
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_non_quality_judge_keeps_literal_confirmation_question():
|
|
seen = {}
|
|
|
|
class GroundednessJudge:
|
|
name = 'tim_groundedness'
|
|
|
|
async def evaluate(self, question, answer, context):
|
|
seen['question'] = question
|
|
return JudgeResult(name=self.name, score=1.0, passed=True, reason='ok')
|
|
|
|
pipeline = JudgePipeline(judges=[GroundednessJudge()], enabled=True)
|
|
context = {
|
|
'transaction_status': 'COMPLETED',
|
|
'confirmation_received': True,
|
|
'transaction_evidence': [{'query': 'cancelar um serviço'}],
|
|
}
|
|
|
|
await pipeline.evaluate_all('sim', 'Cancelamento concluído.', context)
|
|
|
|
assert seen['question'] == 'sim'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_quality_judge_does_not_infer_confirmation_from_text():
|
|
seen = {}
|
|
|
|
class ResponseQualityJudge:
|
|
name = 'tim_response_quality'
|
|
|
|
async def evaluate(self, question, answer, context):
|
|
seen['question'] = question
|
|
return JudgeResult(name=self.name, score=1.0, passed=True, reason='ok')
|
|
|
|
pipeline = JudgePipeline(judges=[ResponseQualityJudge()], enabled=True)
|
|
await pipeline.evaluate_all(
|
|
'sim',
|
|
'Resposta comum.',
|
|
{
|
|
'confirmation_received': False,
|
|
'transaction_evidence': [{'query': 'solicitação anterior'}],
|
|
},
|
|
)
|
|
|
|
assert seen['question'] == 'sim'
|