from __future__ import annotations import json, re from typing import Any from agent_framework.judges.judge import JudgeResult from .tim_prompts.aluc import build_aluc_prompt from .tim_prompts.rqlt import build_rqlt_prompt def _parse(raw: Any) -> dict[str, Any]: text=str(getattr(raw,'content',raw) or '').strip(); m=re.search(r'\{[\s\S]*\}',text) if m: text=m.group(0) try: return json.loads(text) except Exception: return {'allowed':False,'score':0,'reason':f'Resposta inválida do judge TIM: {text[:300]}'} class _TimJudge: name='tim_judge' def __init__(self, llm=None, threshold=0.6, profile_name='judge', fail_closed=True, settings=None, **kwargs): self.llm=llm; self.threshold=float(threshold or 0); self.profile_name=profile_name or 'judge'; self.fail_closed=bool(fail_closed) async def _invoke(self,prompt): if not self.llm: if self.fail_closed: return {'allowed':False,'score':0,'reason':'LLM do framework indisponível para judge TIM'} return {'allowed':True,'score':10,'reason':'Judge TIM sem LLM; fail-open explicitamente configurado'} raw=await self.llm.ainvoke([{'role':'system','content':'Responda apenas JSON válido, sem markdown.'},{'role':'user','content':prompt}], profile_name=self.profile_name, component_name=f'judge.external.{self.name}', generation_name=f'llm.judge.external.{self.name}') return _parse(raw) def _result(self,out): raw=out.get('score', 10 if out.get('allowed',True) else 0) try: score=float(raw); score=score/10 if score>1 else score except Exception: score=0.0 passed=bool(out.get('allowed',True)) and score>=self.threshold return JudgeResult(name=self.name,score=max(0,min(1,score)),passed=passed,reason=str(out.get('reason') or ''),metadata={'external':True,'domain':'TIM_CONTAS','raw_llm_answer':out,'threshold':self.threshold}) class TimGroundednessJudge(_TimJudge): name='tim_groundedness' async def evaluate(self, question, answer, context): evidence=(context or {}).get('evidence') or (context or {}).get('tool_result') or (context or {}).get('sources') or context or {} return self._result(await self._invoke(build_aluc_prompt(str(answer or ''), evidence))) class TimResponseQualityJudge(_TimJudge): name='tim_response_quality' async def evaluate(self, question, answer, context): out = await self._invoke(build_rqlt_prompt(str(question or ''), str(answer or ''))) # RQLT contract is explicitly 0..10. The base normalizer cannot infer # that score=1 means 1/10 (it also supports judges that already emit # 0..1), so normalize this judge at its boundary. try: out = dict(out or {}) out['score'] = float(out.get('score', 0)) / 10.0 except Exception: out = {**dict(out or {}), 'score': 0.0} return self._result(out)