51 lines
2.9 KiB
Python
51 lines
2.9 KiB
Python
from __future__ import annotations
|
|
import json, re
|
|
from typing import Any
|
|
from agent_framework.judges.judge import JudgeResult
|
|
from .tim_prompts.aluc import build_aluc_prompt
|
|
from .tim_prompts.rqlt import build_rqlt_prompt
|
|
|
|
|
|
def _parse(raw: Any) -> dict[str, Any]:
|
|
text=str(getattr(raw,'content',raw) or '').strip(); m=re.search(r'\{[\s\S]*\}',text)
|
|
if m: text=m.group(0)
|
|
try: return json.loads(text)
|
|
except Exception: return {'allowed':False,'score':0,'reason':f'Resposta inválida do judge TIM: {text[:300]}'}
|
|
|
|
class _TimJudge:
|
|
name='tim_judge'
|
|
def __init__(self, llm=None, threshold=0.6, profile_name='judge', fail_closed=True, settings=None, **kwargs):
|
|
self.llm=llm; self.threshold=float(threshold or 0); self.profile_name=profile_name or 'judge'; self.fail_closed=bool(fail_closed)
|
|
async def _invoke(self,prompt):
|
|
if not self.llm:
|
|
if self.fail_closed: return {'allowed':False,'score':0,'reason':'LLM do framework indisponível para judge TIM'}
|
|
return {'allowed':True,'score':10,'reason':'Judge TIM sem LLM; fail-open explicitamente configurado'}
|
|
raw=await self.llm.ainvoke([{'role':'system','content':'Responda apenas JSON válido, sem markdown.'},{'role':'user','content':prompt}], profile_name=self.profile_name, component_name=f'judge.external.{self.name}', generation_name=f'llm.judge.external.{self.name}')
|
|
return _parse(raw)
|
|
def _result(self,out):
|
|
raw=out.get('score', 10 if out.get('allowed',True) else 0)
|
|
try: score=float(raw); score=score/10 if score>1 else score
|
|
except Exception: score=0.0
|
|
passed=bool(out.get('allowed',True)) and score>=self.threshold
|
|
return JudgeResult(name=self.name,score=max(0,min(1,score)),passed=passed,reason=str(out.get('reason') or ''),metadata={'external':True,'domain':'TIM_CONTAS','raw_llm_answer':out,'threshold':self.threshold})
|
|
|
|
class TimGroundednessJudge(_TimJudge):
|
|
name='tim_groundedness'
|
|
async def evaluate(self, question, answer, context):
|
|
evidence=(context or {}).get('evidence') or (context or {}).get('tool_result') or (context or {}).get('sources') or context or {}
|
|
return self._result(await self._invoke(build_aluc_prompt(str(answer or ''), evidence)))
|
|
|
|
class TimResponseQualityJudge(_TimJudge):
|
|
name='tim_response_quality'
|
|
async def evaluate(self, question, answer, context):
|
|
out = await self._invoke(build_rqlt_prompt(str(question or ''), str(answer or '')))
|
|
# RQLT contract is explicitly 0..10. The base normalizer cannot infer
|
|
# that score=1 means 1/10 (it also supports judges that already emit
|
|
# 0..1), so normalize this judge at its boundary.
|
|
try:
|
|
out = dict(out or {})
|
|
out['score'] = float(out.get('score', 0)) / 10.0
|
|
except Exception:
|
|
out = {**dict(out or {}), 'score': 0.0}
|
|
return self._result(out)
|