new feature: Multi-Intent
This commit is contained in:
@@ -51,3 +51,82 @@ async def test_pipeline_compacts_context_before_external_judge():
|
||||
assert len(rendered) < 40000
|
||||
assert 'TIM Fashion Mensal' in rendered
|
||||
assert 'session' not in rendered
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_quality_uses_original_request_for_transaction_confirmation():
|
||||
seen = {}
|
||||
|
||||
class ResponseQualityJudge:
|
||||
name = 'tim_response_quality'
|
||||
|
||||
async def evaluate(self, question, answer, context):
|
||||
seen['question'] = question
|
||||
return JudgeResult(name=self.name, score=1.0, passed=True, reason='ok')
|
||||
|
||||
pipeline = JudgePipeline(judges=[ResponseQualityJudge()], enabled=True)
|
||||
context = {
|
||||
'transaction_status': 'COMPLETED',
|
||||
'confirmation_received': True,
|
||||
'transaction_evidence': [{
|
||||
'arguments': {
|
||||
'operator_instructions': 'cancelar o serviço adicional e enviar a segunda via',
|
||||
},
|
||||
}],
|
||||
}
|
||||
|
||||
await pipeline.evaluate_all(
|
||||
'autorizo que prossiga conforme solicitado',
|
||||
'Cancelamento concluído com sucesso.',
|
||||
context,
|
||||
)
|
||||
|
||||
assert 'cancelar o serviço adicional e enviar a segunda via' in seen['question']
|
||||
assert 'Confirmação do usuário neste turno: autorizo que prossiga conforme solicitado' in seen['question']
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_quality_judge_keeps_literal_confirmation_question():
|
||||
seen = {}
|
||||
|
||||
class GroundednessJudge:
|
||||
name = 'tim_groundedness'
|
||||
|
||||
async def evaluate(self, question, answer, context):
|
||||
seen['question'] = question
|
||||
return JudgeResult(name=self.name, score=1.0, passed=True, reason='ok')
|
||||
|
||||
pipeline = JudgePipeline(judges=[GroundednessJudge()], enabled=True)
|
||||
context = {
|
||||
'transaction_status': 'COMPLETED',
|
||||
'confirmation_received': True,
|
||||
'transaction_evidence': [{'query': 'cancelar um serviço'}],
|
||||
}
|
||||
|
||||
await pipeline.evaluate_all('sim', 'Cancelamento concluído.', context)
|
||||
|
||||
assert seen['question'] == 'sim'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quality_judge_does_not_infer_confirmation_from_text():
|
||||
seen = {}
|
||||
|
||||
class ResponseQualityJudge:
|
||||
name = 'tim_response_quality'
|
||||
|
||||
async def evaluate(self, question, answer, context):
|
||||
seen['question'] = question
|
||||
return JudgeResult(name=self.name, score=1.0, passed=True, reason='ok')
|
||||
|
||||
pipeline = JudgePipeline(judges=[ResponseQualityJudge()], enabled=True)
|
||||
await pipeline.evaluate_all(
|
||||
'sim',
|
||||
'Resposta comum.',
|
||||
{
|
||||
'confirmation_received': False,
|
||||
'transaction_evidence': [{'query': 'solicitação anterior'}],
|
||||
},
|
||||
)
|
||||
|
||||
assert seen['question'] == 'sim'
|
||||
|
||||
Reference in New Issue
Block a user