From b2a368d71ab6c2367161e31509b7ca889ecf49d4 Mon Sep 17 00:00:00 2001 From: Cristiano Hoshikawa Date: Wed, 9 Sep 2026 16:50:00 -0300 Subject: [PATCH] first commit --- .DS_Store | Bin 0 -> 10244 bytes .env.example | 61 + Dockerfile | 5 + Documentation/llm-as-a-judge.pdf | Bin 0 -> 146884 bytes README.en-US.md | 1536 ++++++++++++++++ README.md | 1551 +++++++++++++++++ configs/.DS_Store | Bin 0 -> 6148 bytes configs/identity.yaml | 55 + configs/judge/agents.yaml | 28 + configs/judge/session_metrics.yaml | 17 + configs/judge/storage.yaml | 5 + configs/judge/trace_metrics.yaml | 16 + .../langfuse_credentials.example.yaml | 22 + configs/llm_profiles/llm_profiles.yaml | 9 + deploy/.DS_Store | Bin 0 -> 6148 bytes deploy/README.md | 125 ++ deploy/env/evaluator.env.example | 22 + deploy/k8s/.DS_Store | Bin 0 -> 6148 bytes deploy/k8s/base/configmap.yaml | 25 + deploy/k8s/base/cronjob.yaml | 94 + deploy/k8s/base/init-db-job.yaml | 63 + deploy/k8s/base/kustomization.yaml | 12 + deploy/k8s/base/monitor-deployment.yaml | 82 + deploy/k8s/base/monitor-service.yaml | 16 + deploy/k8s/base/namespace.yaml | 6 + deploy/k8s/base/secret-template.yaml | 35 + deploy/k8s/base/serviceaccount.yaml | 6 + deploy/k8s/overlays/dev/kustomization.yaml | 24 + deploy/k8s/overlays/hml/kustomization.yaml | 24 + deploy/k8s/overlays/prod/kustomization.yaml | 24 + .../langfuse_credentials.example.yaml | 22 + deploy/scripts/build-push.sh | 18 + deploy/scripts/common.sh | 16 + deploy/scripts/create-secrets.sh | 51 + deploy/scripts/deploy.sh | 30 + deploy/scripts/run-now.sh | 11 + deploy/scripts/status.sh | 8 + deploy/scripts/undeploy.sh | 10 + evaluator/.DS_Store | Bin 0 -> 8196 bytes evaluator/__init__.py | 0 .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 172 bytes evaluator/__pycache__/cli.cpython-313.pyc | Bin 0 -> 8541 bytes evaluator/__pycache__/engine.cpython-313.pyc | Bin 0 -> 17441 bytes .../__pycache__/time_utils.cpython-313.pyc | Bin 0 -> 1875 bytes evaluator/analytics/__init__.py | 0 .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 128 bytes .../__pycache__/vloop.cpython-313.pyc | Bin 0 -> 3369 bytes evaluator/analytics/vloop.py | 76 + evaluator/cli.py | 93 + .../agent_framework.cpython-313.pyc | Bin 0 -> 4791 bytes .../__pycache__/base.cpython-313.pyc | Bin 0 -> 915 bytes .../__pycache__/langfuse.cpython-313.pyc | Bin 0 -> 27615 bytes .../langfuse_credentials.cpython-313.pyc | Bin 0 -> 7152 bytes .../__pycache__/mock.cpython-313.pyc | Bin 0 -> 1481 bytes evaluator/collectors/agent_framework.py | 42 + evaluator/collectors/base.py | 8 + evaluator/collectors/langfuse.py | 518 ++++++ evaluator/collectors/langfuse_credentials.py | 149 ++ evaluator/collectors/mock.py | 9 + .../config/__pycache__/agents.cpython-313.pyc | Bin 0 -> 2686 bytes .../__pycache__/settings.cpython-313.pyc | Bin 0 -> 9539 bytes evaluator/config/agents.py | 39 + evaluator/config/settings.py | 154 ++ .../core/__pycache__/models.cpython-313.pyc | Bin 0 -> 3518 bytes evaluator/core/models.py | 56 + evaluator/engine.py | 192 ++ .../__pycache__/resolver.cpython-313.pyc | Bin 0 -> 2312 bytes evaluator/identity/resolver.py | 41 + .../__pycache__/llm_judge.cpython-313.pyc | Bin 0 -> 8392 bytes evaluator/judges/llm_judge.py | 126 ++ .../llm/__pycache__/client.cpython-313.pyc | Bin 0 -> 12040 bytes .../profile_resolver.cpython-313.pyc | Bin 0 -> 5805 bytes evaluator/llm/client.py | 180 ++ evaluator/llm/profile_resolver.py | 82 + .../legacy_exporter.cpython-313.pyc | Bin 0 -> 11379 bytes evaluator/output/legacy_exporter.py | 177 ++ .../__pycache__/oracle_store.cpython-313.pyc | Bin 0 -> 15512 bytes .../__pycache__/repository.cpython-313.pyc | Bin 0 -> 27646 bytes evaluator/persistence/oracle_store.py | 282 +++ evaluator/persistence/repository.py | 411 +++++ .../__pycache__/loader.cpython-313.pyc | Bin 0 -> 774 bytes evaluator/prompts/loader.py | 11 + .../__pycache__/gcs_export.cpython-313.pyc | Bin 0 -> 3626 bytes .../langfuse_job_audit.cpython-313.pyc | Bin 0 -> 3393 bytes .../langfuse_scores.cpython-313.pyc | Bin 0 -> 2564 bytes evaluator/publishers/gcs_export.py | 52 + evaluator/publishers/langfuse_job_audit.py | 53 + evaluator/publishers/langfuse_scores.py | 23 + evaluator/time_utils.py | 28 + evaluator_monitor/__init__.py | 1 + .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 202 bytes .../__pycache__/app.cpython-313.pyc | Bin 0 -> 10185 bytes evaluator_monitor/app.py | 146 ++ pyproject.toml | 28 + tests/__init__.py | 0 tests/__pycache__/__init__.cpython-313.pyc | Bin 0 -> 168 bytes ...im_compliance.cpython-313-pytest-9.0.2.pyc | Bin 0 -> 24116 bytes ...or_and_run_id.cpython-313-pytest-9.0.2.pyc | Bin 0 -> 6284 bytes ...uct_hardcodes.cpython-313-pytest-9.0.2.pyc | Bin 0 -> 3863 bytes ...test_tim_spec.cpython-313-pytest-9.0.2.pyc | Bin 0 -> 7664 bytes tests/test_full_tim_compliance.py | 279 +++ tests/test_monitor_and_run_id.py | 25 + tests/test_no_product_hardcodes.py | 51 + tests/test_tim_spec.py | 101 ++ 104 files changed, 7462 insertions(+) create mode 100644 .DS_Store create mode 100644 .env.example create mode 100644 Dockerfile create mode 100644 Documentation/llm-as-a-judge.pdf create mode 100644 README.en-US.md create mode 100644 README.md create mode 100644 configs/.DS_Store create mode 100644 configs/identity.yaml create mode 100644 configs/judge/agents.yaml create mode 100644 configs/judge/session_metrics.yaml create mode 100644 configs/judge/storage.yaml create mode 100644 configs/judge/trace_metrics.yaml create mode 100644 configs/langfuse/langfuse_credentials.example.yaml create mode 100644 configs/llm_profiles/llm_profiles.yaml create mode 100644 deploy/.DS_Store create mode 100644 deploy/README.md create mode 100644 deploy/env/evaluator.env.example create mode 100644 deploy/k8s/.DS_Store create mode 100644 deploy/k8s/base/configmap.yaml create mode 100644 deploy/k8s/base/cronjob.yaml create mode 100644 deploy/k8s/base/init-db-job.yaml create mode 100644 deploy/k8s/base/kustomization.yaml create mode 100644 deploy/k8s/base/monitor-deployment.yaml create mode 100644 deploy/k8s/base/monitor-service.yaml create mode 100644 deploy/k8s/base/namespace.yaml create mode 100644 deploy/k8s/base/secret-template.yaml create mode 100644 deploy/k8s/base/serviceaccount.yaml create mode 100644 deploy/k8s/overlays/dev/kustomization.yaml create mode 100644 deploy/k8s/overlays/hml/kustomization.yaml create mode 100644 deploy/k8s/overlays/prod/kustomization.yaml create mode 100644 deploy/langfuse/langfuse_credentials.example.yaml create mode 100644 deploy/scripts/build-push.sh create mode 100644 deploy/scripts/common.sh create mode 100644 deploy/scripts/create-secrets.sh create mode 100644 deploy/scripts/deploy.sh create mode 100644 deploy/scripts/run-now.sh create mode 100644 deploy/scripts/status.sh create mode 100644 deploy/scripts/undeploy.sh create mode 100644 evaluator/.DS_Store create mode 100644 evaluator/__init__.py create mode 100644 evaluator/__pycache__/__init__.cpython-313.pyc create mode 100644 evaluator/__pycache__/cli.cpython-313.pyc create mode 100644 evaluator/__pycache__/engine.cpython-313.pyc create mode 100644 evaluator/__pycache__/time_utils.cpython-313.pyc create mode 100644 evaluator/analytics/__init__.py create mode 100644 evaluator/analytics/__pycache__/__init__.cpython-313.pyc create mode 100644 evaluator/analytics/__pycache__/vloop.cpython-313.pyc create mode 100644 evaluator/analytics/vloop.py create mode 100644 evaluator/cli.py create mode 100644 evaluator/collectors/__pycache__/agent_framework.cpython-313.pyc create mode 100644 evaluator/collectors/__pycache__/base.cpython-313.pyc create mode 100644 evaluator/collectors/__pycache__/langfuse.cpython-313.pyc create mode 100644 evaluator/collectors/__pycache__/langfuse_credentials.cpython-313.pyc create mode 100644 evaluator/collectors/__pycache__/mock.cpython-313.pyc create mode 100644 evaluator/collectors/agent_framework.py create mode 100644 evaluator/collectors/base.py create mode 100644 evaluator/collectors/langfuse.py create mode 100644 evaluator/collectors/langfuse_credentials.py create mode 100644 evaluator/collectors/mock.py create mode 100644 evaluator/config/__pycache__/agents.cpython-313.pyc create mode 100644 evaluator/config/__pycache__/settings.cpython-313.pyc create mode 100644 evaluator/config/agents.py create mode 100644 evaluator/config/settings.py create mode 100644 evaluator/core/__pycache__/models.cpython-313.pyc create mode 100644 evaluator/core/models.py create mode 100644 evaluator/engine.py create mode 100644 evaluator/identity/__pycache__/resolver.cpython-313.pyc create mode 100644 evaluator/identity/resolver.py create mode 100644 evaluator/judges/__pycache__/llm_judge.cpython-313.pyc create mode 100644 evaluator/judges/llm_judge.py create mode 100644 evaluator/llm/__pycache__/client.cpython-313.pyc create mode 100644 evaluator/llm/__pycache__/profile_resolver.cpython-313.pyc create mode 100644 evaluator/llm/client.py create mode 100644 evaluator/llm/profile_resolver.py create mode 100644 evaluator/output/__pycache__/legacy_exporter.cpython-313.pyc create mode 100644 evaluator/output/legacy_exporter.py create mode 100644 evaluator/persistence/__pycache__/oracle_store.cpython-313.pyc create mode 100644 evaluator/persistence/__pycache__/repository.cpython-313.pyc create mode 100644 evaluator/persistence/oracle_store.py create mode 100644 evaluator/persistence/repository.py create mode 100644 evaluator/prompts/__pycache__/loader.cpython-313.pyc create mode 100644 evaluator/prompts/loader.py create mode 100644 evaluator/publishers/__pycache__/gcs_export.cpython-313.pyc create mode 100644 evaluator/publishers/__pycache__/langfuse_job_audit.cpython-313.pyc create mode 100644 evaluator/publishers/__pycache__/langfuse_scores.cpython-313.pyc create mode 100644 evaluator/publishers/gcs_export.py create mode 100644 evaluator/publishers/langfuse_job_audit.py create mode 100644 evaluator/publishers/langfuse_scores.py create mode 100644 evaluator/time_utils.py create mode 100644 evaluator_monitor/__init__.py create mode 100644 evaluator_monitor/__pycache__/__init__.cpython-313.pyc create mode 100644 evaluator_monitor/__pycache__/app.cpython-313.pyc create mode 100644 evaluator_monitor/app.py create mode 100644 pyproject.toml create mode 100644 tests/__init__.py create mode 100644 tests/__pycache__/__init__.cpython-313.pyc create mode 100644 tests/__pycache__/test_full_tim_compliance.cpython-313-pytest-9.0.2.pyc create mode 100644 tests/__pycache__/test_monitor_and_run_id.cpython-313-pytest-9.0.2.pyc create mode 100644 tests/__pycache__/test_no_product_hardcodes.cpython-313-pytest-9.0.2.pyc create mode 100644 tests/__pycache__/test_tim_spec.cpython-313-pytest-9.0.2.pyc create mode 100644 tests/test_full_tim_compliance.py create mode 100644 tests/test_monitor_and_run_id.py create mode 100644 tests/test_no_product_hardcodes.py create mode 100644 tests/test_tim_spec.py diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5cf0f444884a20a595d005db3a209ca3f4b2f412 GIT binary patch literal 10244 zcmeHMYitx%6uxJ=ZD%Y)r#z%viYyedN|91Zd5YV1QQCr3wr`Mjc4ufO%+A!E+1)}V zsS(s@LWu9j4*|O8eQu$%T%uQy_ zz2|Z7x!;^~=iV7Y2-GE{rGx|sAuL{$3RCblPN8yojVX-72_OZuCu;A~a8!@=kJ0oP z9s(W$9s(W$9s(W${}lxAolOf{U{mkzA>bk4AuxgfT^|B?Q5tmUkWH$50I3hz zR>8VJD@X$l4LWqlrYOMzwkcBE6r3Xl*yhL&IC?>c4%yT;C*T}Dz!@2w6AD-3svbcedo&lm|E>IZty?Z-mG}2$ zd>epfG$r{YN_3);9XNI`G2Ejs${VYZ=>s|V$Jw>TmJg%)iRqU-nOMu^4>>^n$s-bx zNk381t6gqy_U<1o&-7&h8S%x{N2Vte>A3uW=cyk&384p3k{~hkMzzx1^IV{`}9=U6wQQDC&stZR*SByk#t>*v?+3`TbNO= zYo@4Za@;adBpFqbu|&wjBcSQDyq?hDw=Yw zs#0a6OB`sC2h6l#G_I>^vu-G+qHCQU=0u{7E<=x-^;W;#f{-xeEi!hiB;^oxjw4N@ z18?r9-570h+(23uthAxx4i^^BnZIyR}3=2imrq7tk^IiBjG*j83$nn;c zBAI=iq7jufNzr=SlZv54RJlbHV{#E};JD%GMbiX-iNBN;%Gi!XM7cYmn7b@4Ki5Bx z6W(CAN$AWbU7|fow*FlI^(?!U^6TR=-fcEt?!So>M7G61OzbWe^Gi6v;97QK%-E(? zd^IabT&pTZWYuP^;8(Fi6W5-!XJ|9m^6S_lsNg#ATVV|6W7zCke?2Q4XSdU_H;RV2 zxnI^Ud38I>c2Y^LBq391Xl&ZJ$?u;Cg)XB{jLVx-7~jH4mbLWnM1AYoqlEUqUQ$dw z?g(2gp`d6<1(OFKhMhw!%asMo?Wa;^6GF<4q|R7%3u{iGdzZPSoUA5wqz!Y`Aoq~{ zDFAm(ID4542+b$O97vvXD_hZl>fJfjl zcpRR9=imi+(F**-R^Y!5Z^JwAE*ycQZ~{JoPvJB898SV%_zup)&!YnS4tHR~FWCb- z^9BFXh58@Am}PA?vZO-h>wa<;u9rU(qz-23KN*f|s;>2-7EM+THwQ^vZdBYT&<%o4 z;;xsc@5IOBYh}*tHu2B&Hfo&yGG~-s(>0U49vqkyjrEUo2GLo-Z>|dTpE|QtmA9A z#s_DgK)vvahk%EGhrksekYn2f==cBi6aW9ef}TAS4*?H>s|5k@g~Fja^frA29PK#$ z)*isC2QONv-jGc#6TDfUp?0=cO&2i>1MNOknZl5?%1@nbTB$z3xb4zq)2x+(s{Rde9!Sa z=Y79>$Nk3tj=y8bv-W;g%(-Sg&-~4`sg%SdnAn)P(2%L>Apg*i*#N8nu(357GCx0y znumiKi>Q%{kuBJQMajs*%ozZL+*V#HYvo3)B$9#~{KQCNAF8YAK&hn$f z-)4cZ1zFeS5582)oWZV65Go*qOMvZNeq7i94^y&800BHdCS{Re2e5&Dwm<+j&Yvv~ z02}wu7AFAsun7=u#ob*bRb7l+%pmhfs&WAyh=R1Z0S|p4J(QflCaPvGx-1Yeu&9~2 zyRb;xLH1eX=SB4AMfwLHkgYSdG7q3@c1#s|iFtY;z+#oh)9-uw|vKRI)5I&p% zKPi)R0=qguR(+7*!v_{s7Bwd$duN9q{F->Mh^n%PnYme+n5jsLKxS7nGIoY^`6t zoD(N|RP%u!?ab#3IbA7pe*V`2OxQ}56xu8l5A=3XGL??4^lMu(eh;UstDAHSVV5iV zbOVGp*Qb3`=PP}b++J>E-tITF$WL!lC==k6F^e2~E#n-p3qe}zu}^NV2|8|$-qb|) z#3~rX`H47^>9zRWAj(QYBPDQC9wwUd=Tp3YALzug(7WhaP#Tj+874x1lt|TatTU&; z-_ZW@@?s=g!t~}zM|VT)69I&@`?CvW4E|17wuPmU!L-;ntnR1Vt_ONp$szdysx-Ik zUE+k{^(63GPrh>Bj?wC#e#!8Q`I?J{pp=rYKh3g}oeLLn*QAz2Y1BXMe6pt%#c9Hs zroXX*!;8$zCPj*>k{krGA6!ffYh6ritCKLn>POevDWs;6q(C;A%6en5Vw$hxAP^%t zwbzU~eZrhwMbPTppO2_LoxUHiQS~Lz@C7B6KvX^>po=R3n+#Z~jS)8{#CCRlgoLoa zX!0r+=`m9ejqSU&%AIIf}fA6Fb_U+IvFDbp9)UL1%;G^ z7CaiFBKz7n+||^Lz(>GV?d*8`_3aq9dM;nIBA%Lp@x?OIjpEY12X41%=YS$sAUI~B zX}qQ{{A!)A;36{IT?IFY&&i2mq5`Rd!V$!yyx<=j|gs zo7cE3Q<{#vukT?A#kQb(Hmj#L=>SX|6sw=ptwM|r;hYcDnnu*&8-5#L@| z@k2o&M9XZM)cC6=1@d-t%RBfMWva@BxJKB|KYA=JusVcUQB|lt(x`c9?Fu_5(ZK?b z=b~mI`ax66#kZm()X&1%wFs=J0Iy}c4gSDB-K?62&HGkU8Frwk7jA_JqqU@y>r&1H zD8w6f474TQ^W4WX*3%Jsoy(w73~u`r$M|(@RNF5sCh8gb*DAnQc~Yr6I){iv1QzwK zi;r6L?I$F626s0nYu^_-FbJ|)+CR``L(gJQ%5SY#`0L7qy2?_ZO+km)ZYE1T;ek{4 z?dzL$@a>Zyim{8FAclE7@r9$ap(b*Q$>r^=0^8kKWzVLs;K#~WRo|(Snizbu?oHt9 zDk6)m(oPQ=2|;XCmMhTw1RqleF5H=V!cZGeMs(LUgUtzR?@&NVvySo$ix~6~YQ-{q zJU6Q1?=FGair+uKg)!p1BQ4+QrobJ5tpr(|i{~^7Nwx?}-0wW;9q%FWdIBY>J@jgs z-y+5bgtDJI)Aw-`^xACgHL_@hNWwR)Q-_)Mr=}#22(}Vy`+zo2cFOUx;tGtWr*Y2M ze00~4G6&E$kmgn$*1dD}=6F?Cnzj{71(`M6M$C4c(9G)EGuAm1MsT>wgjV?KV-Mc( zl> zdSpCPc+_V_w(6FvUNNzZ^VzuDo(5`n;h0ChGrxpB^DDjj zNA0%s*~WgdKxW|Fq6NOt0?wQEs+5ZYXXB$yt;0NlQC31OYtEmPy!j=)3#xTZQlScAd;4KYZg=P2$i(0S^GfzUgoi z$qe$F8KsdGH^SvyUP(ls`RkmJ8|ID4ci?XCn$W#ksvj0tg5#TUhbkkq4iBPc&6CZR zk{|H;h9t~P1qqFy1Ahy+xdF7vgW1*R+d}1XpZb!Ya9RrK&dD)7d42=!+wx&yJbSxQ zaGRnv_A;u}e$A$p{FC?4$KJAh=uh62z)AiSM2>muQW1Z&G>`f#ppciwGYVbvnpUEY zc4L$~{Hop2Pa2y_%*M9C=+oEqvF`2FZ$<4hjh56OZN~+KDs{B&PRSs>M>bEt*7c(! zf};$FPW#5Paq#I8^3=rP<$N)@M7!`1i|Qg-A65op*aBftSCs5K(-j{tH)?Hk} ztheQu<0#_ekfc*qkh1QuC_+}O3p-barc2^AG;hB^+B}77qHc|NPKesh0>t?!B#yK91C{a;ncs9QQ_WxaiCk>Xb(+r%?hXc$#6!n5 zQRdnf`&hKon#UYp_F;gp4r-!;j-b0Nr!A~7TGUUqX`G{LKZPvIo!fJnY~ra;WAZAF z(ShGl6pS%ak{b5q>Eg=<*jk7?6t&Omp4=<)6_loIhyUjO0m39P2HaOgrM>m zhhFpBn})BW+{=@$lWrM9p92Mm|I#Ed3l_jdq(hNX9gV60SfTbvBXNm zEiJk>ae<^#QewRcm-)5{b2A7kC4})55xTb1(Nea>x3=A3tETqVxcpVjwAr>rUER7< z78|_RjNM(0QWh(%O42UdmEXQ(^U;mY8FRj!a#j2+TQas1QF;+1M$MvFc$G}M6Q1~v zED>JPE`pSL5DR;1moF{trYYfxueq%5*M2oVF1%#`Y&rH>EoB{(THZ0Ht3|@v+ zAul91%IYW#B>Vt<7z%JIkOb~moy|4*9 z{&~2EF-$fD7f>WQ5}n5|-&4JA%Ge>TfMJr#nJH#aUg1B-Y3N>@iiLyV$!(8$beQ{z zcE}yxLY}cWw);2&trGPEo=JNlT4e@LnXMJj$rmps+c-ATQM#sD6vUKklhS{AX z|E2Q}&!TzE85_(OCWsWWnJ2lxHCw#gf$e#5N)fSQFwb~`hP9r6Of&qZ{hneWSd}J8 z<+jsRF8CUiUMVH#Z<`{hCP1Cd;IneyR_=Oto^U~hF}_ozrkM2@9{qOE-1cfwBI?4Z zVJ##$o|8U6LcHGad=5`xZQ*qt%kztpTh5nIPxpI+LiB9eRgC8LIjMRB7vkPSh@bJ$B~{51t#O%5J}Y%+jcuBWHgWA$=&O_) z6PTE$fNw69GU;ZkB&8N>&3O+xqN}tN)>o?oK!5Up)m9d~VoTz9#UZb%A_VOgj@vm6 zl=(IX^m>+6=wt(e`i{w`VEjU}2CPyit?w(#{yZ%&;5_BY3z%^2yOE98x(U>9ChA;P zRq-ip-AYV8vagQ4Yueqsi7t22(7eg5jmkSmjlj+RoJAYpFyLdz@~J z*tYH7_zzEGo`guc1)@iaJ#|?h(t7h6K4;4w;D~tqS;mE=TK$U#zs#6^&;eAwxIie> z++k$plv*`{78;8#e9MOh#xbZ1nHaCgG-!i`{g2eH6R-w!n3a4fvVtUesrDcwsD^K{ zHDpKDrGuDN;FafzP1{&ix&e>E8c3&Lj+*+e1XN|MA00K7a7eT;g+K1{qEv=Ha$@1g z#&gLJr|ImLOjOFhvZCSeFG7>YC?eq-m773{=t;noXznfh!dpBOLoK4ebqebv+bf0b zRQ2o$va9Yf0C7K`Ke-FaRkcc=)4z#niy9_2jXODs?t^x$Lb}LYeG-n|5nn>a>w=z^ zucK(&^{a?El@08IaNDC|6hbE=`eMOBcEbAb=;upj+Tj@6X`V90&8mvf!gd0_#SAKZ zg9jl^X3&D;b!NmubFUY*hEcb#6hM|LngZkS>)lKB#n)@OWxEowVn}T*pG2$3)*;Lr zR)UvWrS(r)Cz*s{U(dwYLOaiy-aap(;(&FY8z=rao7{yqAdpaOZairLjUuRHDXpY( zd*avyE2tpQKRfuyM0u9oE(Dwt7E48U@ixxmREMl>si&D~OC@6FdkhuT1Wt^-b+bDG zmq#CR?Mwn}?$Jp6>MPdEqKB!QN~p;QRjb zMDLLR!6&kc8vf77MJ+;}Z+z#- zy7~4}fp*NM+m34UHI9r*fCp<(k+maKMj97gQa%Nxcwb;|u^fHWn!QZ1M@-3=Wj64_ zDQWwdU--0(QUZ1H`&(}j!j%SRp+S2?MJ=5#e(Z%Nl&dc3{smiOoAzm<8W!HW z0*dbGD-dk6^h9V3r?hH9~YjyE=I5kI}(3NU3WNMxY>&ST5^Ahb>JQrE45^ z*AC~}?wB0p4a^F_Shx)kb8Po73HSiCvMg=Oh=TZnbZa?%mO1UMm1_(y4X(VP;=Mbv zywkIi63!aMg;FVqGuRtr>ZtT#{&co@<7hp^d^%&z=j)I}z6Hy%U9T=Y^5V)4uZfGo zh)e7(=WE;4dy`Xq+OTghXalbzl!hzj+ZtGG8dfgxQ;5|y zz||n}MW;BQ3WY4S4QTmEE^EWXX?xj6h^iRluEO3qEpww0S>>isTK-ix z2Vjf>s@CzIh_tk>BwmTLWS0)@3AGjMXdyr;__rIJ~_H`hCz4BLs6aOU8^ z9lemMR}RQb&(L|<|Mk-Dz!bXNsVeDoNIKDOGs9a|A41U!1+s}fZW?d#>+6qru`wjo z#SwuBmsAwCraLFH+uurv8&=CcdK#z<-CwjMRD9fdH3|m^$DzlChpIhxGVZFIR#t_h z;uJ3xKl`qNTMRHAH;z!-DWF_6*r2siuW%kuavI`4b2N17&ecbf61^0@@9vXy1x+5O zz>{I|Pi`=*cUu*A%bC`9jTH*ojx)k%RW}i?*hrZ5e7_*+c}wV9pk&|`*{sAhDcgJ@ z(H6OpBDj*<9)q zxvjLqmHjbNP%nt(=qmO$y8tb(tBWblt8e8VI@i#@@F8*iD*^f|hy5do{40I>kd^(B zUuIEtHFo)tWLJZv#~;2`FtRg)hlv z#O)zjS1Wr97ELRAVS8t*e|`QjNX*RH#K{UW!U^z4dfLj#*+taS$O*s>ge33(Xam`~ zAj3^vEFnpE(BG2oKSutI;6G=6$d5lz`)??OB!?}Wf2=1B!4IrVgzYVC%>b+{!plpAO|ZXdu?W?0U7epN8}+_&cwmW4*0PEfQ^ll=f|>;WciOhf~2s`fDpd_$kqP| zHvCA^|Hk*fviK|_@+=BqCp#ls784@?+h3F8Kl1ti?I2O%-v|AG6Mh@?yGlUxg7r5& zkoZq}AZ-fSRx1||CaFL6)zk>WB!p3C0NYPcMh&cPZ}lTr4*;@1AQ^vU;Gxe0F}jeQ zf)EA)PyXDKhi`t7`d8KbCz5{Yw5YV$kM)13HSi}8@xUMCYc;T>w3xh+1IsU>S;RCU zR6?W)xvA>m3=!iG^a#NA6K(o2>`%o5ust9>ESeDEJrMdE8-In06rD`XoFFzr`)e5p zUZrAYVFfWw4_aXev}E=Jz;bnPur;&$p_9MC9S^JgV)O7u(r|7jck|Em8F z*w)_||975>{WlGj{Gp*vRz|k+YX93FitP`0?Vnu~#H#=0q8@zNZ_@rpr}}SX^jF&c z8`-dO{KYr1LA3H;zKILS&GIkb1Ol=B>6?I@oE#52@o#*SoSD%Ne+^(~|HDmzesfch z(GWKUu|gok3;>X2{)?M>fc1Xk{U81FzY`PaH!(>;!~_8hox!$7&j0&z`iWjj{3$2U zpP_^e$nmF~9y)(9p?7|#d`KX1@w3C|Bs>Q-x$*02>j1Nf&hkp5z>FKCN7>og#_e; z7}}3L`@@>J{!66)yESn=bpJzMkkS9inmGOz^dBJm|KRlg7K>$mN$IZw`W0vXAqE3~ zmOK0<27?~L*k8@$q0a*mx&Yuq==*C7e)#4WiGPj3e6pWww{g7%IhLEx-~9-6Lw8FV1bw$?Gt# zU#IYs5*xsYgM{K$hDxH#bBQ+}5pUX`AP@?8)s%6KY-9_1-5#g2YCn&0e{#@PeQ(+_1D*UP3ozUtYkGLI36+Y3TKpJeGb#o{dJbGIkNX_b71MATnEabpzb) zd$Y-c^6n;fbFW>%_k^nFGPEalNXSIo<}&6$6W{r>vnbWCoxiNZ$DH@OsU4f%G4bI7}(V*t6YmILMzo3;Fw#9_$s08Bc6MmQw zLzs}^*QN<4h_umgMlbc=b(5CAL$IzF7TL~Csb8K( z<86j))IQ_fD^@=@7g^-Y9Vb2^)Meq3!EQ@I=^=?Qab|1YC`b?g|sP87g^UfkaEB9IdmrEUx}a( zxENui6<40r$f5Ik_1#yP^0_A~+3Y9GOM&n*hUagNHsEi*A8+rLH540n&bDM#A%|@C zgP2j^=0YF0^>bCH3gVa-#0d>1&@PR4W?4?`18m%)e%eOtO=hZ;LD z_?g0d&F=<5h#`1ls+&?e2r0_&t^yPH@GTbh@$3q<6_)1a`!9lDf{rD5LaO3n;#mT- z^Xj2*OIL3oBxGMzBS$h?k?lzv{n>mWiQaYjd%hqk~ZL>rgK&2UE_yml$h`kLQ z;^Auxa6fD;&$0M&e$I8UGvRX*aQ~`{Qoh4sS#Rx;W=D{EcLNT?<-33tQUeq1jI~+! zXj$RWYlG6PXKsETFrSr#6F|(i+ixt|xg-eG-S7oZhVyL@VPz{q9`(czCX;i{(DJS_ z0(%&Vtu5sW3WN;t!!x=kRYLZDfC*Gr?Ub}Y_SEH&+T!_)SHGT_*ow#c+awSDEgCo znK6Fkd)JY-v}RTV#p%+-O4y|UA6xqI+0ia@RhcsG5{bhSu?tWHdWLEUF-_6@2mi<1 zu2n+_Q^u{Nhr4jtvyy%k>j|#tlL_w^H5yu6ixAUV%IBsSfdREYrQ9Z^c)7tBRA5328idoC8juH6 z*C+xS=C7+Cbqyl|1n5ul;sDmlQ8{C^IN#LyxN@V8L`|%)bkruW+R?(O)%>DNXa|Lw z*HyO6s%z&=H#w+;#TWE$w8a)t`@-x$5vhb|41^;ZQuD#lWrv$^qHE0?LHVA=ZSoE4KYsh7DF5kLb`d%h6p&RZwL4b`GUAuG9 zI#42Fr#K0T(5x7MVrVo3yP}rxrben-YA~#lKSF)Wnd^4;nZ&6(X6EPM1eHpD+JKns zRfD(=Y2W!nOQICi$7HGO{0*rI{zHNu9F`-r@zyJp)nwI}gT+JqGJHoBVShlM&#=!_p-Q0 zw9fsOwUp-ePi3zEUo zO1(`g}SR|JcsS~tXuB9oUKBcaLQWrf)4dOs|Ab$)6B( zZPqo(B2*%g2w~Yk)B0F;(2DR9R$wr_Hu33AX9XUUNNWj^d-9=JizTNFd*@&qv5;v{ zJkn%gvQZbS5Z&l60mTQve8x)aY*Nl6b*)6b2=~#4uX`%et|;(m#N*`6<4gl#g~wUr zalMD$^iIUQB~KNfqv?E=$$$FgmAH^BWM&4`b3!4L)=54a@Tkgm`+XqzT_J~z!`Ni_ zo)N;!gT^!5>vzusjmu{&C;jebBt+D)E9eizr-xchR%VYxZ;ZHo*;Zi@Q~5G|DHR7< zpQMLMJYK8FNyzqXZE~d#4?diHN|}|XuibHuz!AarIc9=^p#<4}c^dDa>sz{7$c)3D z^=WRcO)_bj=2Wynv)irZ-t?11LKltWZ2cl>SfwuY36DEF{uQCFap|}Zv$?f|P$cnL z&ykk17tp^P`q>(y(v!+@@0WbUHw+Zs;I?^87(EmOmm%Pho53@pmA@}1J;FHHQGXls zE{{glv_QP*e!F(FPt1Uk#SKPIKc9$elAE1|`$kK{6v*(YgS7%oSpYNqu`=mo%vg=RSGiZx0p)vXNI0T- zH=xDStLAZ`MwIvlKe}mnSmu`{Gy_sn@Mw$D7Ay0H?uwJ>f`Ekrq0QK3c~wdnFeUep@6Kx^3`@!6P)JO0honU z#7>4ocH7Fjl<%t$` zrlK#eM-Q;AippA?Lp<&zT-S*L)mnQAKiq%yW6-~9F3)RzRqPmEr|AG&*#?VSBXvGG zEcyZ_lbHs%W}Ih?n7JsX5UQ~1!49h~LB-2n^C(Q6NF6U^D26^FKr)`YXf{6X3F9|% z*tpZSM~bDWww(F_LXUkckQLF->rU#kJaD55mEfJHlN~(iY9hMmTxS%os#Ecy6(3KB zyWT8izDFZ?<1P0jkTA!ruch{o&_VsD&cuQX38 z#b;@0JDRohko$==>#cOAduM0g1w&}%(7rB%588C9TQsUJC}U?iqnf3`$J%Y4-R%a8 zz|g1&{p&3`gETx$>@2yF^U%|};%Pm>c=$cNBF^TS$IpC5Q=9R()ozAo!|T$}UdD65 zj^|-b4lGZ3`u4HVqgfKr^IO$cfCzG&wD;K@ObkBg%UhDeZK?70%$oX7aAlN~^FHlV zNOOH9K2y7grk4ju&}b^(&Saabf5E3Ec79ystO;L(T!|wrF}l6Oz?jOw&>+kFLVr>x zY=)(OsSf{U<@j7@wX+}~N_-7}HF9BV^&BByh*AYNpGbcP)TXh{QA?dFg$Fh$eI_VN z$jzKtf-pEFqo5ZqUK{=XbzfZvOUgurRNxh$Ez&Zn@G47?tBBJ&`sC><&S&pW%r>Is zpS;O2NH$nL7v2mHq*JV968Y!Nj>`#GtK~M?a?cgWiF)q5OZ0`4X^{Kw7(9hr22Ug| z&H70EQaSQ*`2u)}czZ9`<=vQ}vzi5fST8Xry& zG8^ik6h`7wxFfC3?CvqXMXdNvo2$RD;~5Ea2?=I*0oKG+#3mg5fx+??=D(AtfvzLj}uf-R>j~-Ep<5_tJc~X35lve{41sSWoqiv z;p9j`wAaSaNy?e6XVFGZ4`Ek; zm|?~uK52F8UyhCxEWa2#7^r^|Ih1OL;TcG=8E1Q~{c**n)%4q8#?)EO@LDH`8}cLr zsj`pxTwmlrV{nrRUo#lbM!OS0HdpdZVB2YFkCYC1vYr~564p&RB^_gdIE-xaoC{+n z>y5Bb_(!&##mA(>iDZb*t<=Ja2OA2KpO&4tN%?Zr5eeC!V9dOb1-1JE@uK{8pmLX! zdhvqjWdkHDEy>;FEJTs^9Bh{jGhdJSZFYU*YJFB4vwiiUbW)8gb8Tcta))gUgSas- zsmkmXtW(lx<)=UxOJhE$+O+0CBu(cr(8#(JcTeD>O5vjy_!`9}#ePnFFQzMw_eu7_ zi#x1oY>9|w@#JDbc*SCT)Nh}PRtY1$z|+XFzq;r1fNAKHnu1&2d8r)Cj>eZVip1x0 zcJe4uYNW>DQ&psj) zIBhO-6|-hZIeelkC{k$BCH&s4>M@E)%T%sPM;-N6;iNwy^CH-B5`jL-j0@Vx_NpV8 z4XcOB=ZYtfz)~0*^p+ILlS&-{r!9e=>2e(RsUny5qCMJ63t^)S4Lr~bvv0%QqBUR} z92>Twmo?(V$~NV1V8U{;WB^M^(D!q2kKpS>_}23i;q@}wDgy$Db6>XPWQFM?ZzozP z+C2+)ne79h%xWwhFwz#sv+<|g8QD|Ja>a+Ux}Dbux=0%y3()Uo(q!Z?Ll+gZp1VUbRd)JrP+p1PmFb-Ry8Mxs$mDBPv_DE z^mV<$?U%t?R!|Xt;A46>a&ND={*Ug&#f&USW_Ab z(1ntE?YMk490iN@P9xx=2)(Eq<^!fxE~RkGn9Y(fianEB_?orAazMbbbFOG8!K~Y6 ztt7!tAM6@(tEtxJn%oov9fKjMuSlb9BHcTusdx8Pdi4#@g>`HI<3!`v?1dl`E|| z-_de?^fob~2oP^XtnwCb6n4qVo7HZKmtK$HGKp})D_F|YOFoHoa{U-D-iZ30I6C*K zR#Ig3CW-@2-R%DJy>2&^^Cj{%%BNBe8$zZ^C+wkiJ-HholTR3XAYE;kCD^JyI^m|{ zaI!c9LfKuDey8go^v_N{LE9A+!n4$64UC%PPn45w;=^C9=_EBVK5rGHRMTu_3(=WS z8)YqIYf+#x`;sn6mydXi;PM+BcQPSS;Lp0T{#Bx}Ar-uTY20PLb z)K$~ehwJrx6ISb{$g_L~G(9f_hR_z>sgc4v zcT9rE!aDD_>+D_z;p^5ucUi_HB6*{*8b9_i*-4ghFk0uyycL2|n2Ma$E7V=^|rfvW#~y zCg8(f1|-fGvw+=a8GyosGgK#gF`S?_ekxbsELH&c=w$sxZ4v%Jl{Q31p~d*p{v4w+qVBA6 zW^*N%v!g-5DAM5Gqin&K={YTfi60ZLcZHb7C+Gn4n-0&%(%>Ry3f%7JK%afh6n6&B zwSIrKeMgD88F7s3D>EOjYqS;wPoKN(cL5Pz@fl+LYpokfR{eeMPIpJ2uSGDQp$LEP z7#Fm6xio!>LLQyLdmUwJ1;1~R#Kc&_3?f&WM43!!q!Gg>`ff#tn;fy#xEPeA9mO~% z+d{z2AiIPssYyw<_Q`T+2#U#raI*X&q{U`B`3V7YEjiL)m;X26z|*`-md2r58_mM2 z@5J)}MYV!b8EcIOFxPor{&OTr<*iZ{8s1W;L4xMnL46ZuCIVpXXHPfHi^`@+t?(Dl zVPeS2RH%~q2RYVmrv~5f7HTGCsORZ-Gs@WLQ6tMTLrC&ROh&|&#Z5Rv z^Ik(6m~)OL{vD;2 zANEC7dNN9jXI8d>vdbZ-$14VL6)FO+w0|u2gUK{0kAf$SPyCF9z3{``mFTHU*8(xgb@O^RgyZ~IO(^&` z3htxR?)4*kQ|>MLSr~&vJ7n0`ZegSrwoC6H2Vm^53!uFKKCeXa%TIg@PgjgrnfXbak`8OPv3$FF@@byKja8XgIG<>b?8{IFQMRk!x&vO(uss*p>z2VGEs#4ipl+J8$zWFjp-EHV z!j2f-c~h}}7#=~4^7VoLDsv>i7ER&NDT&J1W{ljMOJTnqVCY9nV`AJrKNJ;Kf?QUI z*FhsNAMHcP<+xrG6;3IX&P7{!*Lc=fFK2T$B(&FqyT6C71>cfNJds}^mi6Y0zRzEs zRC406Xk%3Fe@zRc#Dp-}AmF#I&qkve{mBR`b~EEd8`CgQ%fR!4K%XAt5-XOeDfNbU zLVru^(}8qRH9ni&FLO4#s2i6kdsus9b`if~I4(?xhpqY3ARO5iF<&~AKuo|>&9HQG1PnQC%ZerWu%_Ud?i%5iHeQR9qP zcAdejwb!xeX@W=?qUEP%8AJ^im9M^vJ8qTbiXVLRolOdwwHWx+3uJq&K@B6Vw&R6P zd{?Ra^l4+$ha~vP`B=Ro#FCyXSuYo#sus4pOACHIXQkUzT^$iTf)P9lmFlFkw^}O< ze>z*+RDZIY>E{Tps_Ga(F9+ce$?Zyn>0oK3eOA>Xa0_g2J{3V`M1Fq5;+o1WhR6~N zj<&V(=_lA^?nCR%B)^l3)Myi(q)wr@3&huTvT;)jjdH(>{t{a1wD|`gda5-kY5O3V}~^V-O+g< z4-ePx&}5yiJeY(FyA`4L<)Y&`XmFVe6YlU65=j>us_X}{0FqB<&<(Ms>fz1%nm$oQJ0S@4x`|g~W%PN+LV(X!L$YT1f=AzFHCp9jr z4vq=m&Yu{VY{;$fsR^lFU^;A}LrXCcnsEomSCrZPCZtWNrGtSJMlC?^9U1rVGP%+e zYBSau@< zSRcgK+BI1^?S>g^_jXL9WDHod~WGR>MtV%GK6Y<@$^V9Uy-&^9(a z2TB{E61dYirNEAPr>ze}A$_qk>~waCd*(L~o6`st5PW^eY~XT`BlM?R7PN4DGCb zWI4m?MF2Z)o?aUAF?x3=g=B}Bh+%|5KjOQ|Blvlwj7_OiZ08Q36a2vg`W~2E4bTwu4I-NDj(r^1+w=$87-v+ zGaM#xOC&Sp<{aa-$T*8H@haF?A_?(aPdcFwB#B8Z3dB%bB5M!HeTI+b2Jc3V4+aNI z7ByZ`GAG~Ytj9;#z>Q^2VDpU)a=!k&9rJRdZKH0(Tw^B4{W7~_Pbkh|n6IuKo}}e=g`YCj2<-R-9$9`2;B*@)gq}9{ zos)JQsg$mcBd9G1Q&ei8mZX)4RfO>A9wZvz0H$se= zOxzq51>BSpND-mKlTy)jlIW#9ic{che)2>fcA$_o7RFxgHEXO>3+;z8gsgTPB3f$r zHtn60OXY3!fk5d5aPoV(G&Z?M-ZfuW(=bxLqz8IT25Rq-P^_n?0^dziDO7ef+hUt< zwdGj%)1ZMZsik9hzuDXMi*tM(DEEImn)x`Izh+f>AH77}Ou-0Ay8r)YUj5#xe=Z*xhsFR=j^&$X~CcZ*)+A0=Wgw=$=0pII>xkq z6L~KheY{$2=#z%{>lH%QlVaMZI0R`1glC+57kBI@-RqRuwOPX|*r%v)SkSDG@L1F= zzi z8HJ6noGq>zZT;eXyslx6B3)(C-O)G{I_N zq!-qvvXZ@M9}v0?$5E#_t$OeijoHfZXQH0yGVZ{x79{m1Gy!aPkm=iPY0f+I-(VoO z2Bw35}5k2E}sE~28SgDs`0rdxd5MvFV^R+2?SZIs_r+iY;s>B(9%y;0zh3Y>CK zvDQ}E&>k;&BpTWOI%(Vr-?g-(*I2_``wb#h>M>2~6=j7d8H4Q0@0F!n7&@`NcZAu= zx8p;19jndklApKvKg;*rzD5Wz!lX7UeKo<$ao}AJD($K%cD0_j)V8Rlb=`iiKSG|* zs2U+j->ZNyj+C-PdFYth8u~od0k2M~Q2_>niC$dB;L0u25p1bUr}737k7z40DQ0wo zYY10;X}PXa}d5VzU+&+fbCF*!%f!CGUOr1csQK(CYm#8=G55GdQeg%lR2NJck%kpH2#h* z!6;bH_Vc*MyNOYc=uf=uT5D3UJtf2P!&@7xQ z?DM5k@M>2@*S&I-iJ{cuVWbqDy^(~IWk4eLyv(^9?GKVTNh~`+S}}vp-qr}cSjj&~ zX=t8HzrCyWsA-?SZM(I_pGcpa(RB~cPU{ZyhLMIL&2F2%^|`P+ALLfPMRA$i#b_R6 zxLf$bb#UOTw>;5s-OfeO{33KPa_;PQ->uO+)$?{|@N{Bue|aSt>p0qiYq(lKZx6hB zzPGe3sH->7e%IQ5yR+weKdG;GIg}FHJm)V=#@P;DnA=zMy+6Ua7reZ^vG!`ccW_!+ zzS?emowcCPd^8<8vsX08qWPq4K94~YK`orX?fcjrj;@c}_x+VSH=ddvpIe}8xa@PE z+Y|UylkNV+n+9}et@U@D*x5s7M&cA<;A@f{`&;UIwXt1GgoHgRyfU5dsVrf>x?k4#jiC=jL4o(JI(h?S2SxJ;*PKgCsr+8MeDSa>~Lw_ z0!tSdXv8_Q8=QoeD+|0THmT^|soWrldm`?@$ltRDRVa zDzeByPM|#eQE;_0emJ25IaXv3`FXTI3&?+-(D^mwKfE&lIe&j;(2tY1?qWp^K7SUR zJl4Hv%KI6UmL1h})GM-!l&1#vxWu!Fmk(FcSAgDG;aw8cY@ zt%8F|!gfaq8KshX_|P|~o}5SB1BgwAJwxI9V{W+XUyk022dQrDEU*c-e4WU+( zrB+rif^UG6Vqi5 zGrAQOcIx@<&#xP9t77Z#;s$2xzVIZ%i7~WJx8I*To=t7+ww_JfRJESz*i`wv_G1;H zg(5=*zEY#4#=Bd1y8sA+z{mfKx_6F_to!=Ja3X684u-gnL4zg6p2?R(DF*`KOgt8SgWw}yO6ZLQWzo!oD1FNAx`2Pe%%%dSdS z?40OHprV+QTRKea2lv{ci$9@rwt^dF+Ro#W=>I&m;0i)J zwprRIl`CpPof~%_uHf-B&RtqUc)Yp;<#Ab@M;=DK?>POI#2`j>a8@;Oj(QY`2RlE6 z@=BVWo@d>I6Qn$F{9Ozrd>|w5obh_!+EtFA_JWZgPh|IgvAr?6_wHq{u$wR%cevkB zt5+a&L!3T5fhpg=cw*=6o$?T!{#8-}B zEXY%+a*vbrB?une#6>rE`qTOIbmu4NUr!#aUkv;@*RLrt#2-_`ye?l1B^^H3TD<4` zIX)d-o>e=J__9l_)96wBovy2+iu3`aQ(oHmN0V}B!BiG-3GC_06hqylrYmbP@n;4BeEe3=?svt2VgftpJ*6R}=-Cenm(SVLPc*)#2L@m7 zkN#wMJy3SZtQYqB9@WGJ&-X^_m)s9OC+{|rwRS5VcDwDx{zpw+UzTUIcWv*W4|n#F z=}!iXrw@uPkxrvnhR~j?zPGdTmw0&%N({8#;YQSkY~Hyje37Cl zz!A8^H_u>RLGRu$Q|a0I_{e>~79Fc2=>2=*n|CH)IzgKLjk4?}xb}JDBTwXv_t*`d zHA_{-O`K`2-|d$!d}7qa(F}i}%Bua~A&w-O!F_N#2(|!E>SqH8+HRK$`vqjk>|=&h z#P*CKirOQG$_D3`PvS3&cb6%IFNQDL*(i~}e0Oh;K<~1SCXWE+zpwpmC?oqt+o347 z$(o_7dW^M#@S4#OrWkgMqyruaFxUCo-AxxK@S5PPx*YXZ7wF#8*zfNj9Jtw?9G&W$ zEKM!VtbRGLES*{|+@CHi%rB^onU|JP(~L+;g{oFjQ_~Jjj%E}o((GNkXq5~x8*!q3 zyEfIz`JKy5gx^M{ZbgOQkc7$ER5Di1K+S$=WPa}!wvg?dwkGWWKimkBw7#eZj64PN z0uqu7EhMiVkuqwl{w(UFq@#?oGF!CL7n?Rx%F<>ZA(tjUxqGTO|7M;F<1r2~5h1Bv z8E7jk2D+AyH|XIz-ntF~sKH#j7xmxJa{{_(hAp{ESfHgW@gN)92Nt;q$DS8ztYS*< zF(?^0J1CDx^yn5#l^KeCtIleAmnnHG0j;RMAU*1WTAwm$mu>bvo>&fq z6fIIo4k<8!Q)1Y~<6M1eNT4%e8`xtVzHUpOnE;mL6-&~mB*53;$yZvh;sFpoWpJYp z0E+uoEdcDE8BFJo&T8fgA+NXUk!Rx@ulG|LdVv%${DXQV%CCViI}XSd!e$>cLqVUf z91k*e|N9dxK5kN4y@{*jKJfve9`e!_Dl7+uDFCX+rMIuF&_qtzkhegPKg%0Ll)MFb z$9|y1AWvVTVsxz9jdQno$rOB^T$zh22(-Z|jd~1hu@pQ5@_a%jm|~?r1rENVhZ!ID z>|_K>wX7~wOFeri+q~T`VwIFhB~MC@GuIU29$8M2tF_x|r~xDtcnsnK1ZcU8v#f37 z*KgvY3CDfpifSYaR4$q6>D}azfzfHZ=rrMd6-?X|s{$hX(x2cOMD^n0VRO;P137+k z_OP65<>*Q4atPR_)U51gICN~;=WJLORs+!`cqVe>OQW=~iX&KJ{i4a5NJn}hHTCG> z@lgKuEwwGSC@6+?tKvMG2u_#CX|T)7iC8>#97(H2b(+8=gS{mFWFU-#db`;CBaGV7 zS^>bJtgQL6KO>e4elfCO^7oR*jY7;W*g1tQ{H(b}4*KZEmJBv2>WDi^r!=^oeh_s@ z-Z$2rI zeg_TlXWb8Jn`?UdrB*HgZk5z_eDdp7i8aZHM%>ajo5qSf&@@y1(rUhSm3Dv%HO;*` z)86*iA7oJ1tBhOm##VJo*cHn@PKn?P;A}*o^cH*vET{twZQclfE9tR0@eBA}WzwF4 z9ZAg8Q1V$@ghV?P7?TDN)N-+^{WaS57Ev5|`W+{!faiF>%b=ujWnfPvCc+9$Xm0+z zas4U(CMT3Uw0v0?BOh)heJ7HN(VsRgOx2hfFhl&j19Tx@_oJKJ!XeT?(PP2~mvKEd zwe*sb;ByMcnU3gu1VwIZ!Fqg~iYXAdi*}P`PgUzCsrk7~llx_Mf69n@v#xp#+^3BuF(;O$LhBr+=-M)&oSB^X&4dLi5i2B3w2g`ZN0ED~^y^FGVQbM@AH}h!VgNYeO@v-7Tw6 zuGX`ukZCeQE*;XLag}>F;WkMmVCQyC%q3*ZNvly{>I`4ewrpU5=q4eIFySi(K55CY zn-`%`4dJf??FCzipwiz$6J8y1WsifNuR-YonVZ0|=wl}k)CU4BVr4WmxaZql5BqEd zU8+d-M*>Ii#U%eh(KpX|xdN_N6iz|(c%^`pCH4x2?r^a&s<}`S zQ1+9yeWb6M9ZQ4`J=1fb%A)h1(>71`>gP@#{Xzc? zL_kZ*#dQFvOwO4OYj6e=$(IaD=?*zb>iG7v?>&o543D>27=5t;=n<$Kvb)EV;WGqD)hGgk^316it7IZ0Qh9 ze(DD|>g8DwX`+c$|72>9vlr-|HK@Dx3g#Oa^HVEL7ap_$VJzlsN2No97lOD>2X+TS zvD7IYukLQy34hFk3fn5BT-qa2ZRL=u@FmBYK*aHsaeO)3;!J;53OmKWnU z%AUUJwLTr3xUC_uxOK2hAS(i)??En#JEweD>U*2B!4m{adjqWe2?Zky%)!aeAwYil z5}>o*Dx$N#B3x8SZ!4^tb~Havdv!t!GjTe0t+SpAR#nNh-Cj9zG_O3tFsD5Bcrvov z-dfu{=umLWsXg3|t9{)*=umd(>R4vb(VB}{RmnI|OcVTTW?w90abLvMv1M9ETjtKN z)xuvj3GI0P=gH)$%1H-T+y~P-^hLWB=ixlvuBd}li|{&~eMT#peMc*7hH6qJ&RoH` z?TNynizAi{O1`XZT_i=zZm)G<1p!wGsH!z@`Jx1;Ut;&HBid9ls;8JAGL1cpS= zWaVrS!>p}TPIlUAej2y8is^bfV--E|T-j{ln2rKB4XcBMWphb<{2y^nqX)X2aaI8= zU*%L-N9Al0%j;1&3#o_*m5m9i&jquk zsCl4oMo>qHnB;HV5I2?AFGM^x=g|zIM8a7O_OBo6d7bW@HB!YS>N|G6aXky^g^h-tDt5Fo!=UvN2~>jywNHF)%WfwSaapf-)iGiHK$KH%7MZ zI-G{wrf9f%LW)wbaPoSzYUe!TqiM*=h~PK+zECdT8I^kMmo!&swr&gm1>S7lZbjm! zeK~XAg$({WXj3+zxsZ`5?0R%YHo+tQPU(&@bYg7qY{GaSPw`btU)u_k049y=mudd!J*5_DCSTAPk{jd($lbu;&{ znl8_GZ6WgVA*2MxI_Xd5u2KdbIwy2w2Q$7rt8DRXv~1`Fgu*ep_{@gYAPV{X8gW^E zF%qOuE^=Ez9yzWf~%Itc#^%?7U!9ThOQn5DR zxpshv3`@mg-pnIQC3|OqBHf!whA?etMTCb9tB%Wet3&2NWG@f-93;Q}+}^IsfdVb- z_l+AG@@2GZ5@i_UjsH#-XzCeDi0vt9hfLLn37fa{zCz)7?;PIH^&Wh4k3_aWzL0yhPgS-|XjJc_z@ z2Y(vJkfDFfzjsBi)B{lP=9}nxMLvJ@p`~_wy)nKOUDZJm*45&! z_Y}K^1m!zEejfHY1c^N1v&r7P)nv1OUtZ^yUHI`4!o^$dISHSwyZ;VY7w8H*$aNLB zp7~9a&g@SFRN_}4qYb7I`|WBaK1U+(_)vHNQ87$1jg7v#vPCI)0Ffr<>pQlvK%A7Q zC#Si{NbH&j_d31KZGB7rbvL7hJ2eSEJ{f;pr_1|#(@Pz{*+X-^k_V=w-M8W4p);mD zxHl(pc!L|J43h4-YWj5bwyYZ+Y-0&c59ZKSlf&7!Dd4#WlK{!WSFjD}(4-HN7*Ea5 ztx=lFQ8GGCnZw_d;FYkLH1ko7Dp*HI4ZRO7jnwU12{}uAsTrO&rTdo4PLIkV32Vck z7@H$^8_vT(A1q{Ixl>I_T-F{JTc@+dS+4yh%thR!BZg~W*s0!oS^H-YuW%PAG8n=Z z#sx;?9CHak8gnffH900GnS+ee>xfYZmZ4*7N^YvJ-pD&t0lsr){`zJjl0qX{EdTvp z$UV`jk>xC#ppY$n#-W3o?V+>oiubU!F=>qlLDkvf$TK@RDar&A3v2H-tKGxP4wvYY z;+k0Rr2J92@9STg?Yvp!8&!_?Xn@wY?+_(L=$OA9(4zDSDK=w&d93X$rVpMj@N6u) z$@V_NQ==fDat9jbT5ERwl(OC$aK`7gEon1mCF^=y6W2fAIPhpM7`yI`yC4X~#RbNL z5p&zAsL!DU3J+l1LP0*V)f2r@#zo93$dU>H>2=HeM*Y-H)95(4`QE#loLawoSMdOs zb|hxHx{FWIDUIOtcma$1qU*fwY6WhK=*P|x9Us`lAtY%xva_#bX9pGY z1FD_nB2UaDjctKtAq9m&C;v8ZHq32w*JCr^8v2Sn@Nfe~ns38C#@TZ4p7$YUPr(4($f%H>J0R=NNzhVQ= z&UE&!Y;aUFm|dD2eKUZ=C3ORZId)oVDmpS%&tz04ecPc=ks9X8Oqzz?2Dig8vL{6JAR2{P|t%%ICQB+8E?ae=9Qg7x88aRR^=%(n3!^9QI%={gET};-2Z&yHBSs@n|!WX!>SJ)bz8=UKR z4^_%X3m?aUpD#fs-7$vvFxxXEe@NwTHR~(NU*&F{H?NxXZaJ`o8G_xev!Z`)ExbBM z1soc&ZXiJpyeyJM)c{s)QyIz?6O(+D=7rXvCyk|J-rP;Uh7eH*#jQ1)4jnb35P$|%C6Wb7wdB=K z1u5ql3{Amffk)aQNz;qPmwO^F0luv zsbux7+Bx1N#|9~kaY@Er@tzCH;|+|oRGl^V!Lo{qs*UZX_}JRvnobJN7O{@!oo$k& zMvE4Qy{rE5wsJ!pHr%G=$fkx!^=G_sfaE7^d%;naKg);Yp@pr+P*!p3Rt*My>D9Q@ z(yPAr>{bf*N+naeL-jJJaS8W@1^$l#0nhc0^~jv$H>w(}kZn&pX&O*NIH>%q$m z`z#Ny-CvWLf;cJcS2pn<5m7<>-W1I}S3qdZ7GqisCgpQiB(mo?X2ZXOj(Pl3wOpEA zcWby#QFtnx>9PrKjM(7^2Vj@nt{GR!B zS>ymZWR&Dq*eSV&Vcxxu7RpZ_x}E5{|Rpp3kV zmTxtC+`Ni$S>+@zlr4d~G|^1%llRwx4AZ}>C+0fAR$l}oS{LE`~ouA$JAqai)&{H2k$m_OXxDJJ7bF?k!`MU z)Pd(F)ic7O7_F77SJOp6W{8+a+LqmmA$)5EUj$BONVTx5mX3=GLAF}jm2vYu7+HmBmwrXp@>#>|u6n2`J!gru`TiWj;xJntC_BMud;vE2qg_4YEq;rFml5|+Rwt8cojtQTzN z2J^w0yTkd4*ee9?^?gDI9GsHh`LJ(KQF&ugBf@vHQ}5y~kLs%&%dcG#RW|5>?lBoK zH9(yDQl!ucBOz$vn2tA{qPwyZiiwKVg-X#2olUuKm60t$DQ`#VU*!Tm1-p9ktv&8HMVi^9jG*fo$-+S+n=5QBK9dMcAEn zoyD`7l_P(owLm4}l{*XN|3@`9@Do|^`&zsSJ7(V`LHCa(SgXECQq;#=8p9y1)k_{X zciZRgiww*6KXYoMb)ecNTuxR?FLH}zzSixv`wRP0rX@wi?&Fs} zO5@;kgfJ$3?u|bV5i-?-nh82OK1RzHg!Ec@TzXG~yll5uTrXUjhCL}6&cewJ#GR82Smm>uJf{koL%6(M+D%-1Gu_)rI_nNWPuz)WhwwdwgzF70kwuYMJVIa# zP;ormw^S$K_~6gDA4P`R$+*lLhH$= zpCDMF8J0a9y4FkR9Ed1IhAy@DW1&pX<-{-5975Oes;y?oJ-n0E?@oBIUe=WG6J)p{bkH3p&6OK0bKN%#}i4#dGv5G z28Cakv!{90g|{*V%Vq}QOeA}Wo@cLsp$fJo=w@~ zB|U^Mx+QwD?GKA7=$J_4pWb-jI!zgxQS6pt<<>4>g8&ay>}ql zF?joCU`jY^NeOrY4cq1FO5(Z_>m}l61YrIZYu`_PXUG|8r#~1bZbF1E$KExGAkp?u zCk*?V?cGA_&5oq`^G<#4%uYhf&!>eo%Ypz+7%6)d&d5?H3sy7}V`IA2YFJaFf+{P? zkgDN{lG&$KDRwJhx^SNqf{c#oTLCp)mXDT0ejC&0&^Nk(GN%#+O%;Y=YC0;aslaWqHiqA0+Gpiu>-F)b z;(Ta647JnQM0tCkB0%EMX#8;gR#L~H#H0ziR2T>?y1#{iCiEA8qWy_U;cNoCLGtg2 zkY+B2`!H(Mv(JV}<9Sl%7=Ge@na!4gkm4TMi5_bih#fS@B*L8rN?IT z@9dz{YXEjvgD5(x8F?DA`Q@Wy6O)n>75E?OfR7pyKJf|s{G{#jRcMmY!H?A1D1O(h9y?DK{A;qGiy)QAZ+t<%)Q1kN-4hoT9FfF^Sh(Lb_32>V2SE9fHw}h4dcm(4?ub;m>QA)pZZ)s;7ZN}5@wHA5TNi+YSyb@W)ywfL6}niZdcnxbW47IIW3Uw z;r(TSvToIJV%yAtNd`taXo}xJj@gLzq%fV(Yh)z2M@P#C9BIY;>?Ax3gHyk4?J`&n^+uyha0tb|%jxm+W! zc7y+yJg2gza+}htasImv{P#+1RUUiL;~%;=hFvTHH&mk1W*#GLJ#&r#Pm*JJaRluI zFHXR-uDi&GBFgjYsOuBxhqdUi_3X?u1I}BPDKTI1x1Z1w$GA1pBxoIyv;csOT7J&B zfjTp&SSa%mdd*pow=@CT=*Q3yTj)<#gCZLi#s#~aa4b^&OQca+gJt;|+N=Y}Q32UDVpNv!;pEs{a^3EzzJzBm>yw50zk} zjxwVTeTdVJ%Ktt=sVNsu@cJ-N4?usJ5BtgI7PZ-7uuPzA-rHY`O;u?6%cqGW2i8u` zthJ9h3^mo#K%IA-Z%7~e6P-n>PLq})pi@l7Mgk@d-M&ooY+~YKVgR4MY<7A6xc@9IqQdDG-GcbJa zn$C(k-kSA=^<5kCO6s8zs>&htsSQSN;IWFbO^HI#IGuQ=xFB$?l0VBEGwX1(2YS@w z@I|1_7IrIKA8mDIa51OUe-6`Ih|q1R1SG*L7W)a*GdkLym-L(r87=a_6bx&Lq^RRJ zS{^6H#y1!)!~(yWzLxO`3*SYB%3Ws_(+#9Hmh}5I`uW`jX`BZkqbfyf2vU7SS5CfW z+8=JDtmjALJ){5*+YjHpHtjDN+Y}+Z7B3Eg%l6^cJrEtvnH$!PEKCG+^n_Mu))}YK z-J+SUpTzXhTF#CPkR4V7eI`H=Lys+l=ZKa- zjobccD0BlTL#u5}n=g3XqwHWsX_HX4EF6U{0>JJoDpRMXq>dgSmjXRr!%&!j*JcQ9 zQueV0U_j7#CRH>f%lAubg zWeU5rTENo)B(MhahR(^cCZkDy=wY28z=XE&t%~)b z`vXy7gs|`-F(~?^D34T9+k3^m(n&cwj+06lxQtmyAg4ke+SeFUc)4C|nr0e-g%kXD zEQ4&2KEv+`ne|S&#{iT?{5>|6$U0O@a-7Jd=yBy1EID%PKrhbkaYR33e&cK>Y$+WR z^X@?QqC@sGdm$L!`9K9w*u{i`ba~1fSmEVq8HLju3x*?a>+SyzzN+r$9E0maGzag3 zzQXl+VV*J>xmAO^`p(ekLe?_`(Lv+3#ZN;TVH~(6qV+->uouNI0^mIgHvF9s&bgZX z!v~AfuCRh%!X7kS`wcz0f}eAsdC&!c!-LbUO(dQ-UE48&% zt1a9{Difv{JSB1MS_AA*eG&%#@XYmU23yL-#We%jH?8^N`{3!zIfxt7E4<1kk9DJ= z=fSxZF0O{5>%jS|e#IrMS-}D#`LxzsF;cqs6>y+}`OQds7@b0q=`I|P+dh9{8fg>? z)+8$nM(3XY82g>l-?Y%r64-p*2w;Y9*XM{kD$!{*Hk2IXj3Y%!pFqR1}r*d|XRa~z=}y3(324%)0}DpwLM zVsrMT(ODlNmo%>}4HcYIN0hXC;v;agVmR2@bU*?I?O1<|XS^u)v2vvfj~ixqEC5(u zDjHjf9J}J(HBb_q8pxArB4k&2i*LKGh1wK~oYAy=S5A2h9piM*=}$I@*`E6z{BG&= zFSt(cOQzHwb!r@@oS7eRr#B{3O}6%W?YP(>-Y^t}SEqg`YBNq(R`5V&&do0LVcCk~ zBGD)!*Jo)h1iPWB;{rY_@LDgN9^jrF!6Sc&<&}>tA*_hOm(z|f3u|fdf`Y`w+=paO z`Xlk-MlRQvA2?9zlRYatFT&yuBv(xS=}iZ+#Nu{Fd^G5_XCH-bq~1-PSC^B@25b>) zTGr)X=ewVI@t$&BadckflRPQPn=jf7a8^FL1xo4Zv;kbTW!##0ZBNvGIAgD-2inpY z7MZ*#;$3y@W?q|TP4>?yb#>-W+qI!^)}bG1V5eZhT5s3Aq#8G(JKyr0hq%-e+`+gM z;$HmVnwPoI2;#oC{L{hd?Zj8%RG00EEOXkhFalgFysA*!{&K>6sOVzhb{R}3+hS#% za1W4ci<1FV6ElwHi>VCn+zQ(bJgdW}5_T6iaQ9vUnX zm}MUGl{mWT@1@mTiVXuzZuVYE<_zBbK&5e*wCDXnvL-{|1W(|YxOi5Yg@2Y*_CRz9 z)PXC!YoV47?_TzR1R2H3O>e3y3>ck|KU6e#$otcD>L0xk*Ni(8aar22;|iJBfp%H1 zNv`egnJ%XNYVV3@;dnl}ott?Mhkh>@lP1v6@6Wx`habJ;`Lf1Y>I{oS^&8>#<)}9OK1a|=8_A@E&Ekz>)JNHpYRbEvA7h~{w)(yi~7gZO6m-=XZ z%sP|+Bnxp8Bn!nFWD}X)*IG!I>QOK<|5FY!Jr1T@b_CE}HvI%1Ovgf#Ud*iwVbHje;3j55xHG1>d%{QXzV?D&~!@dW%wRx1G>#8W&>Y>>Q z`c}1uuHZPQv!#9V-p%=rmEQ`O zjuHY0fy!0P%{-Z2xq)6G^OiUdw1+eg#9QhM#MSDFd-r7-0dSTQA24r0CJ1juCWMY^ zwbZ*{2atzE5B+`Z3+NTvJ;hepiSpLziQ|^;oUSiBs)drb>LBFZ;+5^G?N(GpS+3#x+zZygVmY7TTk1Xbz-RTxHxK(s{E2<|E=c;L zI@4me7y6X4g|G#wg{l}`yt6j?-}m;XNd{5(npaOi``P#D_hp};>S|ir|IMcMza_)} z+0_2k@>`RTg`G*8@N3h$CLt@wR{?rL0E;Fe8!PJ<#m4+K#`x9PgplKF7Qn>*MR78H zNpP_IrPwtI+1dW`F#-NEva)@VU)j@t=l{wuGk-DsO9H;^urmEK`j?l5^RFRhz+c{f zNsj-;!186Eg_Gs)@t2A1OA9jxfRN>HhZRC*_AeP$#(zw)f607xHuyWr@umM?lJl=p zmcOok7QjE`*Gm5?1+aYSXJPw? ziS^3~0OLQJ{u7kH4gfg*#scsU1IJ(9ubwaeIpz2(%KBC5{Qo+>gg80AJYo80(VTx{ z$;|Rs)mM*_FJA#Hf0xL_$n+H=PUbHk{*N;4|8&y%7q%}2Ul{(^k1s8Mwf-~vf7$=X zpZ{9wD>Pq0X8eo%*Yg+Z|7!n2{fqiS{4du3jC}$9@7#Z<=`TO$-=O|Q_up5R{|Vw> z2>xRAS1SwqUwr;2Fn8~?zW@I_Ck4RH`G02c1Sk$$`77mK%i1!8j2SB=ro2Pqe{S*BIG%s%hNxzuG2Oy&3v|Ql;^xFFAr~=Vl^rA!P{2C zlBG?lRN^D{C~Gz4*$larS`^AislAT<{d`%ubpOz6Y`%`WJ$$QM0il_KlICe;A9QQK z>LmnMXtE}ySeWA=Iw|t1FxTx7KMpuQQT9&Z9*?-THS?9|^GlVF2&|+-UFuEmmTKmv zl7hd;$H(L;i7GWL>4XkD>#pHU!d>2dq&YQi6~Nv^cIT`u!I$|FK)AvyB8&t~fr=FS z+v)eh=iq2GtOQP|?)5c`4t+4X!EY_3P$Y08BzX`1vEbw!QU^lel$vJiA0KAp&$+=0 zJ3tLc5j45Bbok-s0xO_qNkX!*QYSX7cQik7k#cP-2*S;UZa2jp>~6cFl4hUY;Yq+C zpdV>)k!f5Bm}qCJFBj>QP0ZK*b_%-3ah+h?pMj@y9wh&ptMFfZ|KBjO zuyZi7{jby_CPsDu>pw+g{&zW;GtX~&O7r)REsq`S?yBCS|dF1X2?>0<|>U zOPyj9ux{S7PCrouOD8Q>Y2{ojdEc?+KAQ-;>FmB02ocC!BPkmrFE>0(sf=^CTx`q> zCovFl)fxy0t*=$-ws%0Tyr5#%0%3gv$HJNRlFfIgS8Lrl;f72qBz!l|t+}LESJ72P zY%*o635&{Ub~9U_k0{CNmlk>E7%G#(QJIs_OyDxmY=nNNuhLwx$*$iO6xo+qEde)rFKuZtv&F!i~#}n+6 zKg9F!lwxwiIrb-Xtctb{VzaB#ASw2FepS}n3pRygf-vQWdQ99_p0S+*okU;Hy8}r~ zY^WEfB$*j#TLx+^Uq`9y7+a|v15qFc}z?9byuqPJGmPO-3T^RfKLu~}@*vF8FSpjaK#I|M6Dv}tuj6_|JO@rj@ z0bd0;KuqmHY)D>;!3aQ=vL}VF6!du9BiX7Cdl7PR)2;YFPepj;(6@J|Cb;4;@^obL zt@*!$x+Qn-`dDlce}KX5O4K8Kzx3cCY}%@FoOMjK@zOX9j6{oR4;@N>djq*2?aV-T z(>9H48-EngaO-(Hci6j}k2}mvyM$TbTlXCMgbjJVIH{uY7`Q>0(+6+`zL&j!(BVx` zjP=x2%X8_SjI7FBiaqUkH|W+FbFGHn#l2TGK7Vrc@R55jatTvO%7-e%=iY6KSZKOT zI#>MRlMFrO6I-Md`I>p550Q6?0=H3c*bM(-LM`}b^e_K^LC$y+40q+bVvJyH`oCu_ zkbQ~E5;=XT6!?ez{ZnY?(!TZ30rySi1#1g!i~VrF@s>yxGo=zI+4aK{d=G& zF%RN(%#9;Uk(4o-a!6N3?2K8TtG8}K)u-ot>g`@-!SM*7UYt|2kewkUul*l6FO|JBkqE4ifb?*3$BIPTh z2bNyWxFr5ektyKyH=ERtIn5#Q*SoI46h`5D(W+wJF~}O$8p@mREqebEZvx)wAq}88Y5IA8 zQ}Y_d`nK3;sG7m?!(nGOZ#A;raW94hbc5jeT=&;ZN%(+cKUl&CXyWGs$^q32$-1QV zti<<8hJ4Xh;s=bHSHnXC8RVq4on5q~v5p`Go~2KOctd z%9c6j0E?@qxf`Ayp0D?lIE;*4dQ)3O0YX7=1oA8ji_Q+}&&f5r1G2@ zWvRTou_xVQQYQ0SY;7vRV`x3SBEnK;Sn+vi#2wJhRzHgH2VPa1L=~JxpFGzS0_xu5 ziwhb69giS|oJQ4)I-hvcY585?IQPAm3ov-O(as+lN|{zPd5udv^dNl*B?1$OO+k4e`#v=kpdbZqE~M>-S&IF#Xgu*(q%Suo8OZl z``D2u2@{1SV5_&e%JzHn1-*HBJUu;~ea+R`Sj)^zx0uhECz(x%$xKX*O^;7TybKSE zuwAlL30uw|LN=LeS_2fS%(pkBp52GzEXwBdI1UwaGvui##;_K)g%>WI82oaU;3GZs zl$x;?e`zC4QKZYw_41sziz>yVy)7nUvzKJuX-aejDRmNzlAYUaI=#m1tE-b2pp(^< zEBsZiLbo_ju7yZjMJHrW=d@}%c?6-^)j4NSNAgQdTI!yqv3gEkI_hffW}7aZmPt-> z^eR5V{5X?2$CrFRadCW}x=1N0sin3K^!??Rg+qr)q-drB-9%B?lxYXfIEx10biEj+ z-}K11q2SVYi;jq5yyTwq!0ljRymTSI)#r$0BgkvCe$G6yC_^gu`uFtw=_G@rM9)oq zK&d!{|EaR&Y5d8pRGkPWR5sk)ev9ERf5WMJI!1$j7PO9QN8eu&6+%g+m?4-6Okb72 zTZT%s^M9$xmY4|G$kP>)lN3+SDKA%GVSTJ`8^+65LX?&AENj$^q7>7NT1ez>%_&kU(8cGluV~%=gqX8X>kxx9w^z1R z3vNHdcKmSm!lTZ_aTq?{Eek>azf z&-T%j0saD-P0Y=y)7wu_{)39jj*VOzdBj zLLm8jxnJ+`{3GZwad+hh!OZ;~b8K#LqfYL?oNslnb-{5)&AIU?P&e%Bq;Tw|fEqQr z1HV2iNMt=*Yv{3d(-E8Vy?5f|=F>)e))R@w6~#FNdP)96ZI@B;PV-m~UuBb?S&b<%L)Lkc%jn zg|xntg$VjHGntU|y4c_(?xb72Y+5muqGd*F8h%WIS`Pj1neI#A-_?n~7aebR2b zPb?HAO(UYZNo~8yZb9U(BDyny^>}GPlFm|JXA7O`#23s}ie56y>3cR8-%S3UrMpw+ zbfwqN;*w2S59hc%L8Brf65(3EoLN@ae^if^sk}UNXG-m44%INVG{(7f4S-2$B>ekj+*F-aw77WJSHhaRbgj2t`R6snZ z+xkK$G-P-e)mA@kW}{+5{>G$vO2id;}|DU4sa{r9}54nF7TCZr=(%J^GVP$Bmk=#+uW>ukm|F zIi$z5>wA17Y5qVBp26zuW_if|;gU}H`QHg7JZ4F+t*jwDzWQN;LIxZf_5~U^a^^Kb z)l`g)QuC-bj`)$XnOrs2O4z$4eFq3C80mqnq=n-JDK%N+W#(7GR%*IOltSF8 zDitv_n0MVl*#;MwPk14dVy93| ze59@`$g#J(k}nd}ja%P&l#u8_Gr=A}na2}rqc4SBc&1>#e>;QJ0hV^z8=jF;*2Ag> z=OCBN$5${eJ8??9i=B~Y#}f*UQ#d9?gdjai3Dr?FV`juzg0cXXZcQIjhtNVegOoWY z^P+0Fj|4f&Tyd+LSV5;7UW{`TwNr`6v}hyMy9!Z+)nVH}#IXv7!!9C6K2{gH?gGVJnP#twI~3p} zazUtySe*&i7%Cb}bGZ99%@sW_D#$>e%JPAC@M=}3Z_DyQ68Pz+ppp^Y@oE-Ju$u9f zpiP0=VRKOAo?*JBx=;TH0B=B$zcc{w1C@YX*{&VQ$S>QRC^chn4FWY)1xWgs!d_QdGMwkZ_ z1CxQ-z(&Bt_w$_Ax$XosvX@@45F}gaW1T-j4_e5-$nE!zh6Ac1@D3k|7=kV87BBaE zH{Fe-2)wu5ftU!q-@gSh5qPgzi5^Eb2efk@J_Bm3_*zB|FfRG z+=Y`DGqog+Uj-{&1uI0~Cf4j_&AF_( zCu>$&p+Cg4TgMmZtXyYpC1TNmj~s>FCyr;sqS^5IZ0rg95gr1x} zmP>A{p~5quD3^2fpRTQg^%T3oBQuX_)R6}5J`|B`v!{c&c9Bl=P1 zUia_Rkjz@$uOa90UG7&iZ*o7M7qTE{pEtz7_Ncn(?#!X?9WUsjYf*4dt9y-LyWH1h zj&(22)C(Gng3Fqa^2fQSdS|-Fp}n%qIc|Rw+T7(X&b-WBWJnDbb#}Q2Kn0%>?E__d zXX>4LvXIm69z1<|VTdj8_cd=aSDPoBhnVxtea+csx4DPeWlpjrSe%wvi^F2GSS%)s zY~dENB!$Aq{66spdy>g1nE0iBMzS7vauJI6;>Fg-ESx5gmMB&5ifN^+LOW1T6?0VW z^fXV1+3?_}cuE=WtW;4|stxs3gv{ZoTA{B(Ggn?(-NM*y0px0Y10G$fszWR+vevo8 zTP(YYvG{ejx&*&y-K~KDrKPMaPAg6r6+diD*#&~yNa#C9Y2D&IO0`WD)2dqsZ{Oad zG@#|{@o*UN3hjsDt(e{HPwZ5A*>3hn!2;F0rBUopT_iITB3YTb2+N>MT_lrb=pr-DiR48@Zt4=b zN$*Kw=VUSsnV4gpGBL*>Jv66p!$fwa~2cs_K?m{)V#F5&jY7p1QI?+t|uM zh28qPsjII+l^1BNQZzP5^fk8d0>y=*c&zBFQ1n$O`WovWtM^5^t5;UHSg15GdX~Z4 zxZMV~t;IRMG{xCCO84Or*=cKBdvJ8wMs{C7b9hR%7(f*DE9qAvim6=wu5-San@~8x7=K_X99;I~IOKC*m4}p&a?^qcANR$iCKRNHS8=`HrgDs>T zw4Z*(PNCM5w42&VoT|#`URq5*qV>2_&P2{lG!>VE{2#IOa69GE1GrZ{KnEdv39X?$ zl)}=&AJg@;PCATQ>nMhDFqleOMz^wy!&lNQIx62lg>*4pPK~TNTphkGd`I{pdYE=g zzYd=zJ7rKku7lyvl;4Nnh5=^Lopd)HWp`M2k{`V^L-Iblg6@%OWEP$u{u&x&(^b$x zripZr9pFB+-asF+G`3nAjaGgV*1}IiDwAqx0o_A;*kCr6XDhS96T=571$|tNR_~@( z+J!4bKc_dDLpc?GD13_2sV|Mkc-rY@c0lSlyS77oo~FP`y=fRqmeDWhIeLY8*f04q z#i8UYe&w3*tCU0oX*#rh6g59!U-C7$u9u#b$An8M7Iyi8ur@tI@30J($0oBGyfy*H!vBXGuD+`$h?Kan4ozcKabI2Mk@jCkqCbRYea#bBf=YhvGH zue0~~Xg-_&n7=FiNPbNIt+@`PzKoXAt@Jp3$r9L5Hiccv7O>T9J^KN>n;m4Yu#b2N zui}gONoj$!T>80OiffwOB;TOisN8J&sH3{$>5kuYd>PIU-$+y74{M?QopitcneQQb z1J_Y{mnqE7VsWV~n@wle;#$LQWe>7#>@n7kUS47EvX60N`+|MLaogr5?!sA0oTxnf z3LIL0#P8*YaJ|Am<$sYPs*xI`71AnclcY&6Nq>;ul}<}%aD`=??3Qz6uk4e@ z%CqGw<@@Ci+ zl$qrR=q=gCQrQXqs8k6r__;hvsith{UizuDoLxsdc{!2o8_O;5<4J5g_Jt~z&%Tnv zcp#nxFDjJYqZ?>3|2>_+I^IBcvAObmx{V6hYWk2K!K&!3TyE-PN@maVg>sN5vUcL~ zW8(k$u%1j(lIZ)aM!LszlD|P$(jnPKN2Dj9{UQESX`*~enaUPm1zblr((>?HTBTIW zzh(1DVlyaLK8D?GwUjSsgI|wbWfu0dU0B_Fun&|-6Ooe!&$$?WF&#U^J-9YwpOE1} z3$ey8!7lbPwVSGVh~_J?tU-KQEx*_?m1c$?p}WKL>GJR$)DQc?`tWMBxQ*VYjkJxe z>$sL0@pSqI*7n897=B0@6Yj@@{0%;hZ|Xk3uy!s>qfc;&Pt8XudnqXYj;2v@_?GZ% z@a~JS!`@AEaQZ!tQGSLV#z_aLpkoqm36GH)G2WvzCH!dE&1|$Fyo4sxK6==!P@UO_ zIo8;3G2Uybflm#ukQzD`!VVi@1@XVmuEhRxle}EMLH-Nff;GPh`^{FYo9$RR?fPf^ z;!SvLhKl(1fSF3$IXiALhxpz8L{jAA64}i1aYpGDlX9F(``~0&Y&rcY&F4H_bhc=c z^YfyKXN#y9apxHX0|#WsXUFFvz#*T`sM3KmeucgvRX!lTb)&%+8*vX)$V!{Eb-wBr zll~4rCndz6^xG{(CYyD*Tx1%~Sl;oo$Eo=22gR;n6`qImDJoR#x7z_so|aKRS32W4KIe z<5%fpu+tC47+}WmS8X}DgOpJCD}Ro+_aM8;hUvjWprV-UpII#yN#bO-6xrge%~o#3 zNyncY6F10ugh_G{Xa1P@K`h;|{LwU_x=&ctdDd4`q#IT!d$tGxi%%FfOkm&u=JUDy z4knvzq?quSWemXibm}mD0MK13(8JsF5BB@Rz=H#%HkNwoRL93gD4w@gcTC~6%3*TS zBryiRtydf#$P&yJr!&L~+GtCx1>7HR-V%EmNlrjsJaOi}wX1rK^!S*sW6D1E6^s4s&9mRU z5(sYE`|}QWhuU4EmAXcbi}*!OZnZfXC0K<{wk;9^Z>KHNWwD{~sdlH6PeW7V*QooEY8}gd7_8n2leveDo9C9Np|wHYhmb| zk-e^2yKm;iLmgAtG4{^B-J60lfBVhZH$Ut6Q-?+9?U(BLYtTE5p4auBU}tuli@9vF z&FYAabH*w|cMvFmW5pXmfiOj3N-E4?G7l+CHJMXV z&E8%nbFU$VUVr~B;}Y2qq@?-l`d@#|$Yoa#pFCmc$`$!*xt>$qIuF~(-moKxiBC>G_2{H{e51)*? z-!kmwTd~Hr+QpK_g8uxt^8{7_6^jsS`|Js^Ah3vcnoUf6u9$`uoOck)7r^tAgt|it zcO9&}B!Act>EOZTH+v_h*If#2N?3?5;!CmLjnwBWy^%Lcd?K3&MLfhaltu`p%Z;}R zV;`?^en5E>Pr#$tAou3+>#A3VYxR>oG=&3iVdgxENBAF_0%o z-U^wmvObp6H8t=+GzstPR6L^go`cw8#JeHk5BV_olw??JH%Z|mtx3apD15}PCJnnw zVqDrHJt=W%C9x#&rA8*cZ7O|4{3EQ9$9Cc#)^-j0Dsp~)!fE*XdZoXw<~rxo!tb$v zv|%Uj^!#L2z}REAcT}erO=c>0;qo!)v}6)3eSj6LM?UkX+wCUN zq*LUO)8P=I$m?qIT$}z$vS~fvVBZk;e5_(M+tYY?;>F1m(nq_h5@#jPN}uXlY+h`y zPh66`IK9@jieF_~X}>0Jy=k*~lk@qsH~H(P*X?h`Wpt@j;tYRI4su|6W>9MQPuS`0 zk@#159*SXpfBcNpCaXW&GiU(z533WqdWj8{It8tGWYYzIHp6P9PM~n&o;lVhUEX zoN6>aTJu*{x0}>-XC^kvR<7EA5k5vKKmrg43>CzHKp^01iAf4cdF@MLV&n`&T9?Rp zJSz4T`-Gc26NVJzr=%ocLo|7EdU+F_DFyjM;+|du+|>2{rsMliA28X18^O-|XnR&%=FDpHD~gpltismC3B$0fU_*n& zvpw-9lX)<)aX2Z;f}xq$MVr zrpJf{6(6rhKJ#0hPQocTWYH6cE;mC<%TMG zuL_0o4pN?QFki6^JQ-KWBM9(>&f0a8$)EcFc{vCgGaIn-caV zyr1#D>r{qgzx`*4Jj>>^m`pEZW+Y{1W>_*auv1zxGNqU-XNW)4HaVWfhgjN9p(hE& z+8B4(x-ECxdCT3_wcKN7*qTxgW1|&Ip6%spNhK#6>UYHNEXLFNGJZXm`5xYr+-zfu zzDR4Z@t*dH{a0U|IKvg6I9qc(zVl|iKDNIv7Td9L7mF-ezt!c;boOv&IZeL^pCU6B zss-E%bgtK-0jgm&S76nOsf}@&%`rSHBo(wT;f|yjef1_qR&PA+<}fK-DmOXXTL_05 zGGy=|tXuu&h((RtB+h6ivur+7$WwEFa?ij z>JO}W`u%x}e|T>&@z5Ke?5N(p@1YHK1L27?!XL;f@I)UwecoNdo}T6x^P84QOe`6n z0}0mKV~dG%#AVrRz5f?!UjipZb**3b*1mRi^;X?I)6+dY-Lr`UGsDs_d9)GnkARE| z1B|qw0xqDksBw+s1};QVBT?gqE5;Ck8HRB%iiEg^=!?cB(dhFbQD4X-%0mN8&pY>4 zRZoxT@0s5p=&IAz)!kL+e&;*ey-g@l!6oV|_fCYuS^i7n?8)IJ=6S&d;fwXl zf-Ci#gEz$=(-l?I4NgSxeTV>oOlaDMNC`+sqkA+h#M%A$Q6`Gd$;)tcJY1e`4a0?1 zh6}6A4OlDYB`-|kq)qH7*(DBplUVj9vFuIaiY)aVS;S<`EQY(@zm>c{yJm#lhsJG= zzKQzLI0yr_=hc#DSaVb#y2Axm<6a6cm;CoP7D_%~U^g)%h>r-?9H^DxwV6+{V4VP? zIfRbTG6Et947s$X`xL;=mL)!4MLh@7)+P0N)V3=^6j2ovQO!*)1XN9pEzLRDxq%|_ z1{77|*plc0j37)TTP9td?dYkxYw^`jKYUfow4kM~=v#T-xvPUcsn36P>8pzuEx2|~ z@zb|o7)00E_pI7@?Un0-58z9$I_ui&uTO4y>Fmu5=G}L6;@7usFaFOSl1IjX0Zquk z6h_At1bNRg+TcM)?L8g3;5rcA^CW^N$R*$63B2c91W%Al-o_I|$vs)}1kv-(l2kg! zbxo2d@SY%eg6s*dAIp#Jux4xLX!mMQYOiSgG9od#D*9~zt2 z#e8+TRqRTQ7T5SNu~AhIwldhnNz7vMX7U~ocK4F^;4MZUT1}z%HxhdgFSnoggf?@J zNu8XzzwJPqKbNCP>Z+p~Ibb%Q&j)UgmhE*2U#%SYiMR4PRnO=T6G2NN93HcIQo{Z5Yrg%)i_yTb==1`ZzZ7%qj8=qZER_2c6_Nn zH5#?}TS93+>qmI)!1eh4J70XUr#KqTdyMTpe8OYJb->(r4J-yiAW?6Me;oX(NF$Pf zhY`yY0-8J8mL~)>cStP|o*{Il^(R7Io7;x}6%ktn%i{wSgxBztP^-{v4O&fNeCy*nkqU z0)bZzU(g5y4WAA?mk*Gy&KnOw9&GsXA>>}hRv(AnBpe9NFz+W9kv-G`RYVaR}zTHm?Y4mxt5uTm;^cz>Ng`a*NV7_(XO|4 znZNZQRM>0oED#8dSR6%@#C&K{bW*a@IyJeNT_7%y&a)OIFOn{RyzO-yCv}*huoAeQApuxVZ8Duq^?B$bA3%KPocI=m zctx+Hn9}ulu$p2jsK=t}hD3+s0r{-E%SIeib@{q)mrJ|86+1Ji&zhjNCkY86;-#t*i8 zfTrNYi0Qc7BoV43I<)3g)&iK2&b@+FE`T}>4xx? zXPk0X`xrE4+xfkRk@(_md#||s9}oZXPx!URF1mE{lUH7~4xM6Ne!;Y>-(9NN9gC6l z?uW>{xA>pMFN%LHcK_;mw)N+|JMUW!@CJN81DV-ME=zkDPCYvbv49}TSZL$eHY9Kg zZUeWM4dy}mjZG`m>(o2dN7Q{P4?~(&#wwGQ4rQUTMfq3}RmBj=m56PEz#E+UltSvC zX}*o47k3TA@PgRJDdW`fd=od3!$}Tt>wMmYw(Z{yffgxzl3;6KV6W*^LFuJ4<|`0r zEn}81i*HqxVK>lJeha0d=q|S`>fMi590lDhWj^L6aAfw~l};7@Hnxk$EHAJU=C|Qa zc!9;8n{i&~N9_=Sw&z34Q|PHAhhq}gMoTOgi?cc@4BPgeXdYmqO|j{F;g212uh*+k ztE3l$c|l5dcL}$FCGvTQu%eDzjOiJ>IcXhU>02*tXzLzNN+jYh=fFBH&ndiT$GlWKy zf82z_qgix#kbMnbQ#_*^O+;k%C zJHx=8+NlN}V@xt|JSvGCW7hipj9wo^35K-c(y8iHTa{APR$ZNJOQjOb>4^)J(pb3m;&6B<~L29VUqbO$tz$WN-gB> z+&mLyN0X)!p_ZZak|T&Vut>0t-bIP$0-Y;gT$u9p}qr&Xda$MS&}2jGFeG`<8=_8-UXURejEV?|$zf*1Usg=%Ao@d1lxv zih3!C>ZQAi;^nlSARFFgct=F%$kq;I_)MDOebM6zUsKPINsKv%ID3*S3p07TZ8Ari zXG`bE^UWLCHRdb)i^6tupQ%dx9Mpl^&2!X^=9k)+`j>{xX`Id(tg6U7$7#AD38JWh zED4&37>2Y+eYD`36ty6{gjtpx3z1`NlGB3lE;+&TQbJ&bKD;z9Gm`dM9wUtVkqV5V z<}EGBEEL(3+qu`d4>@)XFl8S?d9_{JE`F%7Yc!;h=cZ45UBp+5T_P6m^u7I#!=zDo zz&{)8Di$^O?ggZ^#r96zO>w%HG)WpE$yjx?P4`aI6;fJr=T2ki&Q-j#2l!6isGc%) zW8$Pv$CGx>?%{l_B=!&PBkf=(6Is5j@GYrp=}&x|%bY44#X>P8DjyGHcg1 zB*K+4MDU-M;r-Ji8yk7MLzV+pqM@O_p8Q4z3{X`f;b$Zp-dila*Di~VY>0m}`d+2O z&I%eSPzZg4d-53)_6ZgV`2>LgL6FFm3KV)mkib$nH8bpUhi4Vu=b9avtX4rjS5wHR zX#~s~`7AAV6kOvG8HM5{42dN5T}ih$aDYalRB-zO2YV=L9YJt91q$sB6dEVYwM&)% zLfZf-jGRcHPj{+w&Yz(|Fg##b%)BIIL!eGeKU9v+G}5vFPj@ZkyHh!s_Ck=m*z~ zJ7MbNYab~78J&N6_Snwxr{1-?xPkAV^UT82A8)CBw(IOo^GC5K`@@T-Pr0!E&_kj& zZt>)kFC%r|MT39kF9ANv2sp3;g3nm0isBm1~%L z`KQ>&^k>)}{U!YkW_RV6m43sj^jB804MLs2p(0szyxtL<73zqd!!NG9!n(z}m%Ych zw_-hd6tDOH(Fia>CT0fB7)M&6o9l8k^+?v`Odl9fJdn`Xc!HD7tnUOSOJ;?{sv_Q? zN4!CgxPz{AM6yW<0WZ_1dWS@jB1IEt&2<{>jRiV@FqR;=X`o8CB6D zzD!L*R%ox+rbnnzQ4bNTsx&>dnu=QqGo6$LjLQ7pM)j3vR9|~WmG~B;a-SB4S<aDc)Jp>Qx92(v<*jiiuez{Zv;QYftWQ%oc2$Th&9 zYfvG+i7~(3)Lbizgd<_d*)iByCe`dVQy`&FJ%IlCROgT9T(siE%WvOxLva(z-Tv69 zWQJe7sk;E|Ho1FOZome+R2+6Harj zz#W3=9 zo?nYjmjcLQYigL36~P%Mfhhro$kiewL=py@N(eGS+1gCaFiWo~v-FyhrRyCvkcEH_ zYvfIm^B(4f#D--4HS)uP%;PuZ28vMWoZTA58MNGXx3LJLPqG<()* z!$@_ySvcTo(xj>+A&*zJYy@?Z$w;(;l_T|Du<=P;I$ja0h*q$ImNhe>Y*kjuaM^Um z)+V+ z{%yrK9(uRZ~yVWf5Q`hi3UDezTz2l`nzwV z6+LJ7{dDBguIVRTKjX#+cNV|yI%6#I14bSNwy!4KfhIcCni zI8YrNZ0%5!?tF{wc?B-&dl;YO8ef?&2)k62)KOAz*XQd?b#B5OyRopWG|{5u8yq+_ zwmE2`Q=W2#Ch8y%gxahEqng*3uq=Zh+?IGu4RYd8tWC=PDR`t|`{)jQaL0}Tf$tx9 z9Csd^jJpS>!`(a&#B>eloJH-lLxjEiXFUOnt|C|vj?vvT?w=)vGoB2m@|xo?SrAI6 z9&*Nmp6bfI9fa#JnSRne?gw-lN_R`EvmY_i+1J%O`*g4 zRHMo5t6pd;*!LiGBCJnCzB+z2cQgMB?sJJ(I5e7jnd}k)}14)iXf50`^_75TL6Jl<$k7;AlN5Co~SlqXUIydr3u^IF>d-KW%N~0o+1r3S_1a>}}c@7bUYz zYC}^u+NVt27$2%2kR<)xadVvB40*DzTk{ju?#o9&5KP10Dw&4aDicXD!KB^wY;zii z42v|={c<(v_^4=(BaiBM4vhLQyLlOh0GgvZ5X8x{1g3aci?xCIB*ZXc90o3un&l|9=|j2 ztHXEWJ)ajj2a8+-ok#zuQ#C~xJQvSMr1IT|pV%Ve605S_;IXBcj{V*-?tvq63*p;x zi#-J=q$S}S!+PPXV{)E%2*F+b0XXVQ8lm3JFoF;6z+@e7AT_8Cp<@t+%>@8zah#lF z=zbbb0h5BjlNow-MA9tsGVjw^nL${RRfCaatf&IzAk!tFgT0i1OomMBaPRzU@6HeR z44ZLFrpHd)zTJG|jqRj<+~`7t@n+ywiIltrx@GB>qg$SCC4%-ekuZ&lA?PCEpD|P! zROnXp3WpL=a21`!#3L=KSgk&~;UNn~2IMiIOgamNd>R$#hitWvl zcZq(fu}YUz%u6|aj4=aGWhaSwX}bPHL%AE@!>$$AO6%DtM8U#7!x+h9AR;Vjx;~PZ zK$NtTeJ7(l!dQ}IMFl5m7$zB~`Bs;Ot$w^7+{dWRd{XK|qqbQOeqb4E04C38a{$Z>fpQKHRZ2$ zzR+e5P~So7neZ%D_H=iF;J1ygBJuvl%)PYEx#}uf=7b&ino>D)_OBTY0uO2MZ3Yj% zP0N^5H)`-&T{%(dgI{kl6!M}wzvr#qRL(dmMd$eRj?Ed(W2v}h1RQne@ifj^zN`Rj zLYj*V884miIZu8v0Q=m+C;96{1CXId#*9TN2**$w`R_(G=(LgH=x8(#@!N_WPZwwN z{fEA|{r|Q9oIQMSGWXh{(cDLe2qWAFKCX)7jTlp_WNokTE_pT0%^jBN1*Vd%h4D<5 z`fEqnpHKV!Kb$FPbc-Qg5+!g$5*Arjk~xfJQQ}w#BMz0q2(~PYV7=p8fTIL~_rfQ> z6jJk!twW@nk5PjwBvq7D+tvB%QngFvRjE9Ob+g)}6HDuG$=@%R<&F%eOSx=WwANT? zY@>K7EZhIh_}L;8x^lTy90iN#m)OCNwt_p7lCWXuRGU#FNlpbw=#lc1bD+ZQy_0iN zzS$9*b0T;nQn2fdg4pbcSzlhAwBZn_Y#ZyUpia_)Lg?|4T$MF)J1-=2{P>u__r8?i6bW_ni}(= zpcxFtBk?%LnOsnfsB!MeNU!mt!A2r>949OD{uzN8k$h}6KUGL4uD`n=sEJ03bU4fgT2Ray`jCZ3IXAM<4y!vOM1eWRF zTx?J%EQ@baEn4_gWj0E)WR@*kKft7#IZ`*H+3{GDX=!HstvH*mX3j!4qA{=Z7@7e zm&53gXeU83U8lG|ZxKC7v1AOvznHCi8rMCI>!lz?|5po7N0#BD<*~|8cVU?`B<^r{ zbvT;rl2XPdL!$*K!ypAH``Zgp&5;C#aY?28AeV}nxc-jmOYWHS_u?zX8_^ZdK2Vr8 z>iXhMe7|8W>^*;5abVz=ELwf_-0MO*sX5Ocyp{hu*iVS5!z>+vyXIvd%;Km$HiXp* zu8I_Gf>pt^(7=y~G-fC8ZISWWY5cUvlx%^YnVy}!kiUYxoL|kZ=I>(eWglgpV*kke zG5iVhN#qkdR>3zi4g3V2EAV&NYqNjMa+z>Lwl$o~PO+y{99MN*dTO>qn(dz%>a6Ij z?5LWRoK<};zbLdgdqwuvid(aPw%^Z2RU3uC7j2H`7%~HOWIV^&LA!w;&vTd!*Rf(< z)(-QGkYWQd9+M{wUz14qSS-~fL^(DLG_f+!#L604v24I5qZ{yyHsGaz0b;#md;^}* z21+mroPcA=hOP$Okn&)Z@?exI8Ez^I4ni&2t7_BiliI5qtsk=2A#I3zikai0Q-8F|N(~2?31F)^h)|YI*K~`ycu3OT}lO-iVHSh48=y z1AEq={}h=2d&Q4Y{QYz0p0@D*LgT936{l@SbN}=nTG0PO@v%Q`DSr5qrow$Fw;3sS z7T+nt$;IE-PKXjbt^@DC0T{t%s`1qnk10zvkTs^FvucraeibLvB|Q?|iga5;iV0+( z=)xHyXr7>Yg4H+pakmv~h5f$n>e^O6d0tuDYPx%$yNB1`?XJu^ufzGKyC<*br+|<# zPN+B`d5Su>;{1x`@} z+!v*VKY?~%3(1p|`fPjXFkWsS42hA*viEc-(O20GPTk@CKr&m8Z!#*+{NEu%540g*^7n)5@fV!^}6 zSL94ckXK-bK|Y8&P&G9SmtUQF;x}5ek#HoCW{<|euxZM+ zbe?Bw9lm+z?=FA!&FOVBrw#7kG4q01BT`fU1FgGZ?TJ5oq&SlApYif#_q|=2sX6iD z;xaVq`qkr9ao}RMW$b0gpF`Hr%^mzJ_h0;*%t-baI%mbt)RxUF%$CWenYS=rraOXB zPl$nVOcgy(FYEPdWwKf(SCq+dPlnvwb^(Ds0XagxuH!8BEN%sR5yxd}N3*$#W7#R< zw94bECe=)?J%ybk&aIqPcT>Pflg0x9PmL#JJR$1|wVsfsxN=T*gp4O-J)xFVDkl@E zPS4ih8n!kw#@CvjlsT@cGue@znOUNqr!O`Z1sB?vsh8`Q`>rxCu33>;$=xNM)^fHYR85QI=uqV;0vuD$6Vc>d{AB7QZQu->%9M`UUYUFB01m5I0{3LJ|Cl&PtP zV*;OukBH@o4c!K@$KEhAf;NC?LSdpLYDe?YQnUsMs1I$-YaL) zTngO53G%RM+QUnxE?9ayPgsX7&iATuWBkTU7A zAsa{KPpHMmx1u;25f6`WruL0c!wDJZgti$xjp(& zFcRh>)SiiZ%g)`Z&wKf*3x9D+``ih|B`2MG_K&~#=_CJK$@lv>~>=d||=4zq`h`<<@IXo6*v;xNgFhOD=wW z#YObD8Ad`>+fU}Ap$N@j1&?w?PY}E=gZO`T83eD(ApRd+2HlnKBI}#Y&6Zw~xNx7_A8X|%NRzn}q)U8{^Pl=ejlq7hL{*T3!yIO?%pih8t$|<` zdl@}0W%Sr|yQCjE89go}!zdYU599garQxnH8~)BLp|?Y)4O6ckp(NdPTaNcsJlj!9 z-c#Y+3Yz0Rm6G>V3L%o&4|z|gGCkcaxJ`yq@ZHzevTKXq z&DUsmfi@ca-25F2iih6%eevMZ9mj9D>g`^>|L~>{iiaP$73rU`GY)Tle#@CV=&unl zGDKmM$@)BeEJZMXl()#va}68_uFH{z^L0%&QiW)7;sdsK)E=flyZ1&0D0uPcut};bn~^jL8vyl5gknNPa%QhTqQb<9WUh-`K6rU+mytE>NnRm`Y2Of1 z0PzwM{o}8kRiQo)TQGKDfi6UhGL-oS2TEtK!_M&V<{O%Y%L%1EL>cfLKzrVh^#u4n zFRC)&BRov{%JAtcbHLi*TktiyJ4M))^DQ0qOx9m-W$hd@27J{RdkS;Be~NXyJ)4>3 zpKZ;u&AX+$eb{f4HkwJdZluy;qFA&s)XKMNllV#6)X=H?soH6w1^fbSap)rcBJGNh zj}MXRyahq54^y2_bc$yY8k7+9C0LH_?L)V0a*~Ha zEINH4W)BR+q67Ac$1R++r<5^xMR1ZVxWA|5P4S&xUJ@U^16Zyy%$+-7)3(!-$~ORW zAHdvSW)zk}mQ^_>kb&14%IoLU65OJ@G4w*Wmgn>2H#rb(*DNnErE+JPPt5C@1zZ3_ zhHyx7XAG0lxnnB<$5sLs>;f$)jmXSlgdQkf@zRGiv2hBDeEx?S>5371ep9?)Tk*A8 zF%m4k0$e`vXLtQq4f~gYSn+RP-qOSV>fmIqusXT$_(PAlj6MY{E5OdCnTE=kW$g-| zC--#tOS9LFA-l0k2c@CO@4Qel9q(0rD4Ph_-a+|@6eHiz0Rd(dg;iW{)d$9*vFtc$ zoIFk+Z;ZCa1{5nmdeRoz7;Yn4clYHD>i;F|O8}#&(sk=po!a-_s<)&&Np}*`kwDTR z1fnTa7D0CH5hNNFR2G#WnqB1!H!r>eRu=zWi+`|qaGT~z(&U%vnUzhke!73#EC!(Ip0$#gXw#n;Mp zT6JI?oFGh)@h46bE`c4wB6*>9N#HuL1TGVn%S*K-fi>{^0!|y&1=a;O!F%|7qz7T2 zus`rJd`0*-`0v6y+S`Fo;Bn!&_Hm$|BZMQ0iQ@DKev}3Lhz#aeJy>wptgKMskP($k z4(H&&_PiO2Gf*?T(rp0s+v}qkfJciy)X<-2US&3D4`(?!f2(X6P5eF9j~x9;h_J?Sz!kk5clBVo5eLjGItr*5Mr$WXYJ8MYXG+fCt8Ef z6GWGBU!qm0OtfO+wLM8(m+H4u$<`ps@EP=1)go3~Fc7iM6+|R45J@twON+HlNv0oK%<&bvk6&Nl6B|qn_ zT9IxUQa%CF{Hn1RfSNZ32kG3v(EX{3vEiYu^cniMLtECg&-os>YjEb7uW9+5<~ij< z0JChn@s~n(lK4wz-S9?1f-ebBeMx{D5*4RO6t3tE!HB4=$g==ZAUY}N#%E`I@!4r# zCRvKrbgu!!OJ0=!Px7+AAczwfI1$~qPkI6tUbCOr8WLa(Juvt|#CThjEa!i>Cn5pX zqK5!l#X_Cbpuy|G_1yLHdmO~;gA;g><3x@TB?b31rzJTgNixT9B7=c~2!5AI(*Oln zfKy}+pp*m1eRRwgB?&DV6z6Gubkr6Vah@%8OEg*(du>&gl{Cf7n@!(O7RO#2Gi!xB z5X_cIf~2@f(O=y(iH`15HFv>~;2h^dC-9ITe)<^aLTyMU;hPLlxvzeLg+g8rg~@!swj#>?!wG#LmrCMnKQCI?{S~-~q_yKEA4|;}J zi-P!F^doFJ&(X3}j%=dzJir`p5Z&4Y#g7qe6ktzR+GQZeH9)c4Tq>sGycUHXGfyJp zhP#5>W}fV7GmaT&ijC53W-GbAgov#RTRqKiqlZ;EDDLVYy*naZSQ>G zSCX5QBlZv74oV{JsVGJ!(~5gmaBg0lYa}dkU|b7Mjsag09z}mSXWE(n4dZ8CS-?Em z%RIe!`m@iT;jhEpE5m;oT8dKV&nbo)<@VpSWA-~L1}~W88o< zXz34sh~8TOp8~J5zeAtUMLt1h##^yGZR=(p*Q$Ha3tRxUj{uxFs>6?xMO5wWDsCy0ory%suL+ zQ9qCTBK~VNTnrcjDT_sf0;Px4g^ZGam_fC zn2n{Bm5S!->+-Fzwf-VFt-c^E$XzQe&8<~7E3YVD07O8$zpG#8%$B?cpwW=YTa6W= z=#`^x8byyvHfZhI{n|EdSYx+oJGC!0Mtj~fqWcS);__R5<}qHLv_qKHOCyu{8iyzM zwQASGYf)mq_ZBoQ$yj~#Q$0~9`NsHsy(FD~Y4pKRGRafL{|hy_Mruql@~EqftEfrF zh%S_`e0KkgTN0HE6i19%wjzU}5Eroh!hu3E5Zkv5?j^@;Um09M!Phf``yXX|+8J`f z84n-I^wF2wT8)jFkkZv-s&`hit+*nCk;Ce~;kO;(c`t+NDO8VbC7lrERrxWk{jK!& zR?uo;_7VKymf&^TSlMW0gGad2;3Z`Zj$yIQLPO>y?l4EExOR@_LLNR5DsuKXpEJ6^ zX{0nl>^B!BH4z2>&p1-E`j~TYnKxM@F^xoK0KOq+t{zoGs*cbH2uzt38hp z-VwPXzD!u5Ua#L6xiQ|KdPg~G9gY1p_=WXF?8CATQp2fOnr+YJPaBF6)*v#bi6jcVq;OJ`@hD@dgc9{7>&3A~yRs0?9Fc(`+j6JeEkkE&N+xSo zj=Ft&r@U(7fY&(eP+rQIXN8n;39yb;NnVZIsH=pgqlEhkW)l_g+BTr-wQWGB&xc^E zO-0y&ak-^p1tv?={QztQ+rcg{0ALEVgV}%q7?8z_771tUGQ7M1(QZIcOc21@4TyH* z^rROrD{^QApE8J&VG}Zx31a0_TSfqmSaDZ56CLEeV@UZ2M=l7b9NkWa9-XkHYZX%H zULYA-*dd#cA(tRz6CCeQ8|PX*;I|5C@VKVNayo2KRh2bN$SRShfODScUA6n_on7|O z-(P%TIi0`sCu^Vimo;mjVGj)cGx%!^fEu z$Zy2ytDI=W8XoZCTge;Z7Ck}t1j7^XE%GCRdF^%`=-7PDp;jW#2m?u(k0zlEv@kE= zYt55u%@bx5-oV$IC#(L@n=g~%ka4i15&tE;C87dS$qRxPSo4GPtFypQy)+RD5=+VnJS%Q5}i7sF)e{(o|@|gJ);T4-&7_j(ye&X=NtZ3JdDc7xfF_j z0RXm^bIO3WtIxo8uNoNVSm88!i|Cn6lx!MNtvFGwi;`i_c!xdJ4!g#?q7cp;=0S>Y z=V@Le{ZJDA0G@bd-lOE=6?ggYV-xv<#j=B=hQ0xkRqp1VgtKe>0`#w|PU6mqRo*xf zGMz@pG4zL2n~=gTC=z+%+0Ov-5CIq^8jDh*05QEXUz~XJ&3+QTpFCkyEtZwA0yAy9 z%f9^0(V_ob_37Qu{zqzO?DmCQo_hSwW%q%*to?^U8IYa<^v0c!B$luE{Tpxp?uW=y zPC=UczBA=b2Z^QJAkk3Gs(JN7m2D0+C%;EuBFziUPhLkaW|xT9ggTS`sW;iTg73yY z3Vsy&()v915wVDoR4Nz8W^_gz+f;ruol!?eCeY3540^IUC3I2pds0EYPW_1cByt*@ z)C>@2G}+LRrIdM-LZ%Y=oJrCMQ(0s}h&BQ0`BH>WWlepARn@;!-7*C|YZymNV3@Yq zX?B~)uHvQQSXVQE&6i0c7;I-v4lgW|SX+}6Ww1_|8rBKZ>ohjKj$-q9@9dCaT^$gL z1a2PNo13V{soRmgFt2-dubg_rol{QLYc53%;m<{7@6EMXED@a4|ZCK^eCGd@l< zGFG0S=I^!S6UbY=`%QX0J$x!0%*~D%J8%dOoG03E7lq5TrQs#BX zQnoh*T1gG3q8Yn*#!tAHmf4B?oVRYt6DZ@96xehMvLnGi%oV%agaRe`M1? z-}}vD+W>RV+)1={dI}wQ?T`QS<)g11#J+h3^3CPQR)&$rMTo{(DJmI8IdccwAzmsk zVV1KuiA!W5>~OJ@Ykth0ho!P4;S~$K!=4VEio>yi39+%sNr9R1Ny)i^MX`Cws{%K~ zuS%}x)`w5gr=kWG0lI2gb0WBa#zd0(R%5$C8wN}yC7wD!KZVz-=f(PMa;ptwa~=#L zJBEjx2map#&OHUoR#8YwYHO-Tj;MOmIs_vUYwGg5RG`LFIGf2<=kfD?9CoHaD)PJ+ z5brBQnu`9QG!89ze%i?JcBU@x>5Hdku9>oP1>A^VRG}nMA;+jCiSm#kIIKb=U2R8> zw8k7m?;&+MMc^f`z=wyfT&}0K!LBx!-^isGAaflFyNn_(svCI&c8n|ZQ%5j%1dRyR zF?EGO8Dk7O$QTj~y5>zv5c4MS-e~)kS9CPw%%+BpE}w~Wx;#UC1t}eZ3POCwF;^U@ z|J!q)4t)ti|M3>kz?tLHp1ZEOckn1ZR~cWpd&93m!FsG0q)?2afZCz=hQ2n^I}cnB z9^7=n^*bzYw%xi5|c+2$nm?E~8WeQVjh7AT8KEW5<*9 z*3KnU`K=U)+1LcdY!#Ux7xk$nh)*RpN>sgz`~2N)eG0j)PoI&viVSiPzUjW<(}ezx zdMp}y0UV$zs8c|qJaDLZ&oNAdl44>gPUKE>I5-g|7j8A3G1-t|aw5+OD628VK!P$k zJppncSGRE^$RQK5DzTT6B0*eKKc{yE!$~^nUXp;@*4OCt-&Zv^Vagq3JCLKoekzPf zH1}-V7L4Dq_Toi}@s0B?Jbak>`Mq7s^HaVT_?0xJ^Xhxg5T2p)hvqV$Av=ND<{WIH zQAiR950S1JjcFuLkoI9N+;N6^Vioveuc4=705(One# zv~3Y<2E9pb-+;+!Co!aGV!mjSj?M~CofGxP*j+hqOuH+0a->#y#5hh;h{0}R$23mk zvl^!)fGQvhMqzMzBbCD>`iXtAL`e*bEZ3Oc6)fhJa*{-Ekl5j#HmgjcE?n5$-1~$6 zwZECsyJq>EAGe`&@b3?FJpRkUE9plz-#Y)k+XkOUR%;7Vh&DXiP4TqkOm|CXSNlsW zTu7J{b4wRze1KU=W_>dB#cpw%`_>2IuqdTOl<@J8 zW<-v8ez@QLgt`Lr13+;s2PKZrvJ~6~x5HhqAASJ2emDSW3Z~%^^a?@;XP_^IMOA}D z)u4n=0TQPPJ*Nr1jxvZ*2??tmoF$yCaI47vNm#9N{;iWbR{0BYBY!3KLy6@udm`DC z#n}}IlP4mT+S1$G3qL=6_?s}S{^lq}4?j9I7fc}f5&)YVeJM_#u)(>(8`{ z9)OF!*?w^5Vr1^nx@W$^Gx1EonHSWGRZa1`y3WWVDdw}qaK1<}?~=~j$pYu3fLwuF zC^EYGf7w$UBYLc7&Rl`y38E+9Nkp5>C0xp^ljs1K4wCZbKu@3smpcb~(Q|+$Z;*C| z0sBt$4hJC%xfXFMQU~rF=`v{@vqn0~e8}-TI8en^^I4&l8!xu2v(*J~0e2a{K)em! z!v0Krnfnubn>)sR%Ks<#wGa+S63Z}<#sm4HfL@4#kafoJ83?n^XugD&ImA6`kj0Hb zvP?;^59qeYLQ)o~6!5EXn&2~rGn}>+B~qmsd{2#K#ib*Zvs7{U6xE(CGD*={08p&m zPR&M^8q+R}CGLpmC*@F45KDn!QsjOJ4nz^Sii#=fUn-_v>(@-8m~IG-8;-k9;fCXr zxf8_}1(bGMt+-GJy}^Av!OtU271|i`=$0VW86rrDcQUjXRq;4vN7n*}^xl@l`m$D0 zC@X8j6Zd<{@ZkNMd(z}-cZExBPv&8}D7TS;8}8pzL53#xMDWvldkiwJkDkaYg*@$+ zy?I)ULgB9pybFO4ilA=`h1$pip5)jQ#XtPT?u2tV=vd&C5K2wbWO!1Skky716p%}W zoa0KJ7BFw3pR^V^!ux97f&6C+;YQi` zZSPqu^M8l0Jj*8*5B&}gc$TO0ed@RT+WrDd+!tX%X&LXZq2!yL=P_fQXJ^yCemjd| zKAlam+t~M6I2%0-uuO_w$#%2DEJT+?q8TSh!kj;zdbb{+f<9wrjF^Hf$)-;(^IhSjPfqycxKA$gv85(`a>6IaeX`b`Asxwrt=a8aI*XjFmfdOs6B68#%n6#LVoGq! z5Ft4y+cRM`F=>Kc5+7@lTB+#`hL$kQKG4w<^+2c6D3)6sQVY@S)&xk9Z%P!uDM7v| zfk`J!{7nfG!Y2rJp1@X>Bozq-e`mt0&nD1U?W5?XD(_3GJo8-Tn&(iVDhrNK0GAc$ z6orj6LyUCUw~RCiBB+R)Ih^q@kdto81)NG!755=lr1nBq@O9%Q*SDcXn> zR`acVt(d|l3tA>hGS(=`US*QE$|P@^y?i7+U%pD5+C5zZ7PLEg3kveY&*y(KYXOJ~GFUdh3=? zyZH%N{m{Bbu-J@DG#3&=@bID8z|AZ<~p7#PO-vM(HUhmnsHo1$Y2 zv;<2r@(%%CiYZgMX+nWpAY8{S6@OD-_8}Lf%9cGBg5v3;8NQp(6HX6hfH< z!Pyc--T2~$eLSi2xTRCCq)r5v5Mh8IsxD~)5&CT{lg*Fe0mU19nrHZ8I|(ihzK<*= znV-PWK9V(0$%@IE<`y0&p*27$8b)^sTQNndxKxrT6~XLX<3)@&MYzEz8+|pliZtaF zCQF+z zUj`3-{zFg>)Yo2p?e#yr`bYF%+B~!rRv@hkQ03q)Vue;K#yQ5h#tZ|tr+204RC<(B zRn{18EIYq!WqNB`m|#svOt+>d76_Lsi>yV7Wx{f0sd0m~Jkg(iBlK?c-S`{j$3n-- zKS&R!BULbGDpOp;qk zv}1)pmZHA91H)^)Fno;{#)!s?Q)f3`kY+RFt4QNTYHCXqjOe^5HeTfLUJ<2tU4{t4 z0nj?K<$}^oEkfW~O-3e%OZGA66ulS$)Ud92J}eRrk$pUBOva}$n;)O>!1Y^>EL-#b ztqboTZSGjR{^=)H-@JQhDf{9*bLZYW{P1H#-~8y}34`A-j~_bt>RYe=@!ycvOdDFt ze1NpZppxLXL~B;abdIizPNZkhw}1}{v^56(|q6TBwT zS>9d#Cihn0WA0Psv#2qOu2gd2R=Qcah@PS>q?gj~D1VNA82L2zapDZE1E_}LNtxHU zP!b|lv9u;?gtky48W~E^KZOg>Ixuuw@6@|>SWck-a-uIff&TR(^sf`>UnkJNP68glUZPifBH8rHT7X69Qp=Ce8)?AijKA?A;O`#b^n`G}KfJJrIn z^0Q%m7uEMc`W~QvvhCaY;iw0IA)+2o);Lv;6ZMcsss}LLriO7YTtA}9TwnLlr7sSB zdD9y|_(RvDgB8!Lzj?>bwQC+5T1pEOXMxdx-#&E5j{8nuz&v~C(C`2C&A0y*$2E6R z6#X*N4ik`l*zAc7L0~{o1@rI%I3HdMS3^!T1yK;yped>pBLJBg14 zldF7krBAN#$?0McxuF0nLzNT6>EeZ%LgkXm4dQ*`otYiMr|W;msG=2*T4QF^ziqJz z`cj%U8i5pDBrFmaNsHt~$|7}{uuNPgEt8ii%hcZL-WnYbYG+1`%Pf=@$cw8N*RHNw zo$1c}RQjdzK{&EvH_)qkjt)OyKWr6*K*LdFwn9T?6%1`Ac5knw~v zJR1@yZ(S(VWEBa<)74=pk1mVj(qm<;p0ugN+GDe0SH^b64#zk>mWtgJdp`zKvHN3m z>_wzTVWi=tgTW5r!wt+SXMiJsb~>Xm&=88`NoND5dIVtfqOuibbXhXYL#LsZK;<8M zaQMe|5bGjLj+Rq#5YNQyU^L%|f4q?tr=!jzHWM+jJ9jLNe=?TFKNTYgU5vCD;D4PY zI-Zlhyfk*(gA7g``n#?K+xKcc5yn|m6k zkVqn2SZ;~a8D+7}f?oN)2Oh@8x_uPL9{P7?KygOy#Hv|dt8OmWg}W(yDdh>=4wl0^ z6L*n5U5vtDP6GT?{)mNKFzz7-)SE;5Glvv9r0DBI4C=a2R6&RaCEr7Bq#lrl||HY{4W?R(E4Yn2)PJ6*^AjIzLu1ffcgK!oHG1L7r_ z_sN`3mM{-;HO9vK?FHzMntBbA+JHWokP~)&Y-s5J;D&gVeZ!%iVUG=rm4DaIlF> z18#oeBw+jjdKnc3<91(r7Wjt(x)W~s^cZ&5C-684cGhN76WROA%MKE|nFwKPVr0x> z-e`*}0E#$FO%jRkmN={eO^uBWZX%GioF*~c?qRl&eNxG^KW)S>Zo7I#{hfFA?A;s8 z)s{c9%{Xt#qx3cR0)EBNkMAA)>CE~#_A_@NKl1^sMyI{ssi4Pk<5bwn)9GLY(;f}j zu|O!F3xbRgj3^)&k&&A=(e_ zqpc#|V1?YK3%5>g5lG9zc!P>B`LG50Env-x6HW*WHN*$v^vd}5_^$YH9L9@fwqgO# zEzb4eI({*55CJ8o#UtVeA{4#AK`aIiZV6u^g?lnzMvCDXgFMFrmFuVC?`hmfoACH|ncmDlb z=+#22M9esJQRU>MiLD>O<)p>ZDYU{yS2X zib%B--ac?PWlJJ!2O@bEsd3(|IEk2Zg2TZsR(G8ce^A8V;cG#dgrMT^7>LROSQq9{=eO=q2;s9tHrvh8ERt+7P1~SS{_Ia!olnGLgB^n%@#u+T9q!hO#mbu5I+A-9k0!cf zEu~+6atO=^LGHDm4}#3N2b%!+&CrV99?Pbpxi6j@x)D|n-g(_km#n3?5Us>hu=|l# z2G}oNK$=z5NQ~zE*4Ln;y0u1k`x>vp#1?xg=%RiyBT7w04K}=?S*|oREBe~Iw9?S5 zUOFP+J3=TOfxUuvdLvemb9&y?JO_Q0Cuc?gw(ig9O7Vz*k4r@;9l@85kbD#s(34T3 zROvh7xUT|UoXI!SUUF55{%4wd=zG34m7?s?VFz&DOq`=LNXEIC%bUH5N&?&CD1pYk z+-b%6hyGs8X%6YP8?a>Bj1>ff05s$;;`X_;t+-Wf>6AC|tJQ|!cGU$ZI1m$(jN@*$SxHFTe|suogJO)zymCkY|+A65#TGE{)K7wJ|Rq|Hy*QhK%oZjx@tGv4%nG4>_!O;!2+=XdVS zzTNCeo2E(ABuzs}o3ylPOMBV6P@qMj&}vx(YFp3>Wd~VAsDQ|#BDg#1sEiK<3e+;r zxOChQ#5dzG&J*I%@j3)4Cb-nVU8%j=Z?b_^OaT>R5+(zZfS9lpn$xo5^Qn&IP^$LD;nAHUqBvF z=!J6n-5>S3d~rspbuY)Rg(guBes)CRk-jv2e~Uv3l6dX}3AxAviKRJrRFO{~a}<@}KI!qp zPaIBezc#sX@UvrT5AQ+r`_x1Un(1#&nBr)ZiQ%&zF^{ZCmqxm&gfiIh??*fT>_yE3 zvkqoE!$0AI9MmOG^}SHZ$6k2liK4M5UP|_ydLeY4n2Sd)8Q;UVCk~uGgImA*JzjtO z>Ic-TH4uO1zlr4#a9HwZbR(Kr(2oFIvywC0i)Zd6Uz*uL#ej z;a{;HJJM=HGj@m9!3_<54qX`dAy6~Zh{@IueXAR~&<_88ctGn1a}8;LGnCaZTWhal z>kPHUz-c&_(_oToC&k{*NI306n&V8GQ=li--N{UB`Z0qG43zF&u6T~6 z(eRR*6LlitAx}p-=?@w;Ii?(QjjhSlWNxyyo7&Co z);02#x;4hNmbKQcx*f)QP4}9&TMnq7*Zf>KW$d&3N&N@Q-*gG#8_S@}Z8AIb`e8UM zn_1&Z=M57LcN;i^bGQ;xr>toh@2c2f&0rn>6a#K>V@$q43LE;rK{;zjAo)o zo3E|cat+!+2D3B%Sj0dmge!Z}J?n`nAtX%DZE6Yk(`_6a!sSa&;Zn=efi#COz^(}A z4UC9_^4e)?R2EVII(0AWi4Jy*=-^Wls_WpFRKc@tW0(rI3|>4@99L%*$Mr-_9Jj`e zDahL<8i&#?U{d^ljgKBqKjNhOq=@PzPJ=@7{T7 zeAA6oN^*#1{tb~5J$%Ka1b(=?gQ#yb_2PGx*aT4tHRi0SA{S8MW-}?@5GRw15YACW zggC_2wNQntl(njQ;bxeDXDFLh4Z;#wfEOq`RO?{5a-C`?Y*lVoeFK;2j@SVRDZ{F` z@>%71P*AOLQm__cV!TuoT-;}h11?pmu~Mz}11A0l2D%9{o-YfNx~u0CA*C^zin&u^ zy&Cs|p@(=QGWjWd6GBKq2Nu|fc9!l*Js`cPpQqoX|5`6&zz^BcVsauL$Fy#~Fg0co=G$Ts&C*i>ABi^s%b zR@&Jr5Efzk_|k3|`cZH+#(_XEK6Sna%Txt5C4t70%(jUd)2x=;7x2n)b2CeB870ne zV$b}kcp_Rw2l?jkJa6k{C3jf<$0rDJ^!YE;+V$-`>l^d+l~f`zJh+9oDNNd zJ{+1L_1bZ;L3EIy|0#^0;9lY|w-O4X~GBrYM z{P|59Alv4Q3uA=F>SsQ+{-_rb4@5xR$ZrE`EV$1y^EC7p;CCl~e{I|>2lJbtfH z@7I~V9*3kk_1E9D>w_WR7;D{kEEJ_zD=ciBL3Gnj=R(}ReU&A%QdME z4W~MfE|OFwq8rzWG<>WT)oOZlrWMxGPO6QyyK8Z+iv$NfG`s4N>}X5S(l;bKDdGaj zu}Z2rdJ|bt@ey#bd0!Tbd}gLA;=+Op{^}Z}=X}>j7v~~=+JO_47OoU6XmG&F$krvZ zn$@J}@xQ5>edhhzi@j<_5ASs~yFS%#(WfL;P;_9ASXSf_^P+_Y&l4VuJU+F09ER-S z^du%>IlM|j%h4={k4O%3w2NdAJqZdiH6e*1AW5^2viyY+1t_4IRzUGRqEMQ!xmhfy z8&Wm6JXC{>uQPB4&uWlyT@9pEhlW#~N1u^Y`Jx?1!j)mXF+3Q?_2J!N={C$$upXLE zXe6H&iKAFAK70P`)bsy9a`K91aEW@MFw<2}JRD;=qy7k65h~EVYT+Di<4zNoF@H#x z+$ic*^7rW~OpI1&XFbO><^YON#c`AqRYM?7=0Y|t(n33+Z+oz<{lN#@+aAPc+aGwK zom`W=FY(OxrHeC0inD+g^3hr3!c+OAzQD<&aA9F zY_nQTCMVVJF=H%3{$dffJ0Y#_HE>7d&35Kh1lg1dl1gEbppMvHy^`08v)680dbg)C z>J#s;aX?yiS^YHCcI|;(&rF;Ep$9C+qMgm@u89@3OH2xG=Hhkp@@q!h!ANpYB9;MyWPI># z&c{58x)homLX{l0a2!TlFWf1rG0=j6lRx_kyoelOrvx$Qs4Sm45Klo&Gsj|RO&58F z3}Pf}e(Gc29gEAg-&{sR^izx)rr$1j8#++-5uC}!siiTD)cTCKde}lj)=-D=KBDF1sGZTyB9jwZ2^iyU zw2; zGH@GNLy&fxJEtY>eb!e23ZFS<_QC9>XMgwFf4(o9At^XIYQn#XpOGCxcJ%KO#z(gv zM5Rek$asw%l^Q%I)A*T&N(!LcLob!RxK~t}*~jKez8Ck1PAziMg@r3Mn}w;y$LuT7 zMu+n=FJ-0hmBf`xmlG*a=BZ!&C44oU2wxhRIx)&(Q|W=T`f9xDTsb<;fBZM5zEC8=fD6BGN{V9E&p+i4efAlp^5W!8*hOe;MGVQJN2Ah~ z5{q&Cxc*Vv3)YL9MFR%4wp8=1e5GkB;76esDvsG^t)Wjy2$u&gU!us8ONp{%k0KGJ zUQ2B6CY7E_&-?LIjqSxT7E#c;(xs6U>n#xYUEwzG%*H)?Z>Z3S?m2-Bd&R>?7OQ6# z)^%xD@M)F|lZ)Z_FT)%8QZ7j98j++L`w?P7JgZs4h`chSSxBVF#f~hY0F7)10cu%F znL*k3JgqOj0H4U-M=I1M?dVLAbvg2{AUcHSRHzp+k14q?3I9AT8Waj8C{;y3NXo5H z&yrvvg)8M(*cvDjZ&9FBX~vB-f|LFVDQ>2yQiwdq5f5wF2oPf_O;S`85#=8T@Yd%^77SVxY2%Q*$gOGe%mNnlFXd>Xe5n z!ISQ=ULM3^o}A^J&aTswpEi4-qX`N}fG7Zvb{cWWF&I~X- zX%2aU%241_oj#I}6Q4x1vt83T;I1hzr`5G|ZTB^X%2@OUmz9o-=CR z!t5zETIEEMM@sVN@?@8f%FZ5D&dqie=34~8;hHvi<+5eBv{sGZuy#!J_70QnhDIze zt!N~FS#z4*Y0YapnwvY;a;;fveeUSdfvi>(xk#Pkb7$CQOp%Ci?u>ATNGiEHLocuW zXTO7pbQaD^zrXIsKlnYNfW?wOj#nDP_pu%r-)R!5jV0BFAu?cz*7!mAi;2yf8Khc@??F|2?g`)G_oX>9!d z+KE|XGN;TWCJ0pXTe}Z>F+;&8vWVq!c-1ka?t2B#(6PW#WfYPCG!{yiQD4g z3MFsF%0MrR$3U#M{*mMva)8*QmaO^nvn{fNhqk2Arv`tp+6CA=_eL9 z(jRL%6K`or^b>|IF7g%uot*Boaz<`qax*-*wykZ?7waEh2ft6+mplw|P$a)0@xeJ# z=K`(?xlxmt8L*DAVnMKn6h^Hnq}S<)QBfe!X@UH4YVz0~lf}XrDNXcf(l+@7>GRgu`NimIwz_3+!| zj;#8oJD#|ud1L3)88Ft=?hj3@m@YbD&6>xH_H?eFQWYy96#I~rJVGdDBTP0;G}w(H znTz&TYC>y#xWD={>uGLV6(;Ol+&SV8v zDO69WUQ%IDk>*XOWHvI}ZcUN-`^Ei)mY9@QGPFgEm_>{uNqNaNkfK^inUkM9A6d}^6k!PBJA9>W%FGSP^8&1NQoz`Sy??Xd4F*#wKV(5!`pN|;IBcoD|3u03|J+r1fG9s?LGZnZAWje0q0F&so% z7~UNzXp8g$iV|zllTe?|Ypx)XL&`BAF+Ps3KDN57a?P=>RmWD9SFSnQb^pFS5AEHz zmmgbq-`Tsjd~jb~-F+Wyx%=#Wbyq9Sz5UL)58i$I99vgGihDC(OytLZQvIGv2AD*T zboVg1x|hp60$LL75o#HUpq!17vUq()dCK?1Hy@qY)Ai^_#Vfd)F)e|YM&DgcbF4v# z`y;7)4tiE>t@XoN+YTGvVTCnTSY(7oBW%~hYCY5{p@xU7U`|d(*A6+9$=AtoNKWKS zqscuTgolHNf_*`5X>fB8Hw5Pe@%SL*1WSTA2&OKxxgG$_@~oiA6cpT!K0>n~K6M0X z8R>{vnc{KM)Gd@V&q&lW(GSU)h!%CtT83Xu)P2ljOrSN z8%II)C}`Y zx{Fxrt_=pLH&CEl&Jd{_4WmbgQMuRc$uJ1{0%Qmof;sFiBusY9>{jbjS<<_-}X`%O7u!8|I7 z1<7@M86PA3tw+^jx=x|t0jb_wq2H>aW`(Fc6lTRn+8@XHUpmE0yv`jhKm>rc5|SG(}- ziG8fiTX`k#Cz04aV%=)zJx-h>lnA(358Ktyre3YawQ88GYF2fqxGDuq;-DEIJzdsi zGHKjSheJccYP+^egRt9S*J!PFt=4Yk46Z(W3Z-$m#|&;|svViz#@aDT)EqG-EWlum z$n42Tn@NjhkuNJdIJ8`1*v3G(d(R0Fl7IQ^MDnurRonjCcN{vr3x*gNGhIy32z9=P;r4Ekx&qWUmv?*M|;>w-)*)`tJ5|0UvZ{!OAS? z_CT!@vaKMqTCKR<4BY~(6kxdl?$tr5Zn6$fQo#(xLIqyP!Hpc002m1gh>lDr)bM+e zy^C%dZ8W*v#2iSBoI?K<_+?!6yS&YO{3$ZUWEWurY}%NS3U2TyiG+zs0^AZ&v)8>nos z+ybo@Xfnc0!cGC-rH3j#6zXBJ8akA46M6HCdF;}5p*;?8IOG8{p-EPk++=pT5yxd( z-3EQ8TQy`fh6ynwy)7|nmw$u~atV`=!J1O!@c?rcO-9)@gy7~C_1@N#1bq4ogpz-~ z`s_u=i=orc9!b8x<@A=20Z>W@AKTk_51nq*gU_3>W)A0Prs->Kbu-~8bI zdtavhm&De$54)5*-zad7`!06P4;V&Pc>R6PTe^P)=q-< ziLgk7MiE*|VO9uk&4HOYQ0<0^88F!iApy`RkBtXUHB?vYYdrz4cWglK^~BsZ+xT>M zfE(}TRVos#5d$fa{yNIX`Uj%d(cbGL;k2F`vfy-Jmgz@gYlal0(PD9PKrD}*=vBId zE775{+b$X$%AA{5A6&HN)w?Ff@BZBrubj_yPv6x!_ugB|46j+5?*H)K$3NOUMy4{Q z52Eb$jk{|XY^ZSsrf-;i)6cd}&uS?NOsfo*-S)`*<~^PD&h&`Ta(`Q~uWrQ+j?P2N zN~`WWzc2Y-+y2h_@ni8PTBp;gF~p}d=f~$v^cODLH-}J=_EL8e3c5*D`-FJgwg4>j zgTin3P$6OBe#bTIo;nb@|qV@)QLWMbR4?M!Ujwr$(CZCf|?**o>xHp+zWDrx)*;5t?3n`SA;qlc@yi^QTq`h=+dia!+nO>$yE1R2X0oh_Bst3BL@H(x<8dv( zhIA^Iy{v`zLy;L56QbpbMlrY@S346p6P2%_s${3#Qt6Xas5o z%W_i^m?ZSkhM($0RIH0Th{AXC2K->?%$E*_{P%cBOT9p*HM5dXPd!)RcuHJA$&blI z(0T0013p=nTzwx(EEBg%QX~#Wf^;a(waG32db`9kuU{;cswHKOb3kXkO4pTP0u?N- z{@g^7;n=hZRHyb|P91E4oWUvvMx5Kgw!DOZ!GGq1P=Q#`By8Cfa6*M}m_Z`pAcd%( zq9INl_IaSXRTv~2W9<1` z1a(WgncmJ-Qc|*7{QkHBX?JB^a!caPXK2Q(iVu;n-b1WRMk`n0yBG|bZ zDG!my2=YQioqE}L^VLu?X~ zZW6JB(2dcvGDGV;kG#b1=R_Vd<%e<o+?&Iu(jUfGI%sW^y1-+1q)p)0K@ zQo2%GDMRSjlbmtUkH9QexE9fORkv_jMzmI2*-f3g!%2IX-T2rt(7`W|v|M~wSIt~B zjO8tgJ&2+FP&Au*2V>=)R#t9(oSOH0L7VFOOpQjUMlnK!$svl&Y?7zhuL}o1qA$X} zbNkLD2go=g%gR8$N12K`Pn?5DiD-X+p3;2%?dn8E9`mvIOPVK|)hZ)~jV?4l(0Qdg zv!`<2D+N}Ik-(FDcX&pxWG?ArVZFht!Cnq(V%7Do)nzIAY%%JLO6ZzM#%w_1b$H~O zO5wHCY`_7LERJSLflxG@c))%-dYbuH43BV@_zJOK>}Mq>GYY9@Q~;^%9C^z4#)k`q?_cVPqMrg7P%6}$u$>AMV%^* zvPzr0Ht`eOuNBrveSG~5@w5F%Nw@0WHhUYpJo)@m`BPIAB!sgSHzB>Ap8bQ>F7dwA zXIBSs%Oe^mfsX8sGIvGUccP^pgi&VBR#sS=x7afCjmNH-|VFhD3-Zo6`0+OOkvB1|{EyMJsyHmzm2%c*!dW=l+Qu<2`MX9O|_cN9G6& z-&k7zNCJn^@UwgQJOg@IUMkC+Vl(O)m$Pzj%By%&f7ki8SW*8@4DaRp{6%kX+8lkQm_wRtWN_y6$Lmgd-nh2(PU$*(8+Jj(I-?AcoH*`o79Im5_r`(D`eq)yAPLk;{!G4&w#1;g~{ zyejP+e2dqDwE;^Z4E9#)%^|0&v`()wHK{~;9j*&D@JCiHc`PnixwKXs2o>_aG!rf3 zLkb2{&b*PlQV5Q)J}|sdMLd^Hg?@0YQqGt^aj16LF>H;3>A079stlOwjSzBbtwgF1 z#!s$%Z!jpsgm0QZMC_GZ4Ag;tft#638^r?zM6V*DWZ-+6YVE{4&R6O~>#9bq@{(xu zL-0A}t5(kq%SAtrx^0tn%&{9+CfbQebe)X((_8#X-E)}e*JjW>Q%l#iFIQ4ed?cW@ zKeU+KfO=Mqgp12AyW1dApU*oAg8>pMaj z4MqWDDL991d4&O!(4ev$^>@f13&GP1Il85`4t!@Ld9t@TvYL!HAbOyI4JC5sX@m_X zp6yqtSgEJ2@R$JI34(sTkmOC1pJ#kIxhbg!prHU@6*M}JaB@zn6IjI5?r?R=S?0`^ z)s{Wg9SIsQBk#bwe?|esD&+T@nJ`bo{OONtC9XKsGLs(gga|k^pZP7A3yXP)H@RwMu zBSeXDIe+0Q@)s^Ip(~u)A|#503*_6Hqd8_RPnk~Ig$JPT>+A-p>qwaMm%#s zyZ)H{VKofzH=Ga6)lDxQ;+-MgX{&BaO}FR4ptFVJ_FeA;Vg;$k>(ua7gkQI#B^|*$ zX*4zHFQ-{)&0K{WY{iM4I4;3fD4oDP=Tb!zY)F_$jp9^HI0ZHgM8VJRRsK`X?EZli zR9&@|Al*E(@dR(yBeW7#+KUp{>1o>Odkz^AZaa3{NHQ0vF_SIe_wJQ}P}uuy>C$Y2 z6)E(T{X^Cwr^IBf8M4Dnm9xquzqE$h+?K(3`GUHv<_rLq6=-(0E-~RHwNo=Pl1wYH!dM`NQEX0Z zB@b+xlHeD;3t{`Y3dZW0$Dl!no}Tg)z~l08tx_{YLH5roaYRzE#eOTx8;SZOHklgl zjK30IJ#QsQA~;F5+rZ*&yiz!BKX|(eQknKgT(G$kTJz>`UL~cwYBXN?qe>?9J|F8bRZ%{pc0j0ZLN znlU%k<;mNC8XE9=-c!#w`%jIxgI39eWFR)_(pecwOSJj>(Lcib0*hMGfAiGB4BXw| z?_IAkENT;7clB{bskD<*t0laRYfm z$zWT?b(F$f9`RH4tnFOwJ{f}CD*-W)=K>-~wI19gK)QZ$(;C`{08cREW{mfF$%aD% z*o4xh7nm#cVs!fXrV`9RzHPu$i<1FD#DG(4(!iJ&W}Ai@+FfuCG#6+{PSxC?nD-O< zxSpI9<#i-cOr9M3V8tLk<~+a0v-#)YE+v{WkGO;0A1=uo(Tay#ys|bu%(1PXb?s4S zEEd1dxYF)UD)1i`w54|yRJR06(%?_xZdX1)spBA+fz6Q zH|4Sh47e}Zq0b5K$@WVJkWMv$RV_|fAjz0_ZaCFp>h=^rk2o7k!C!z#eriXwL=LgSJ)y2uP~_W z4zCtp;%7-ogGME-n%JPZ1kfp7fDzX@;~JusW6Z@{Ww?W<{rpc@!Yv_ba+^Q zTeSymcvN7TBNB3bvA`))=eD78%)Horu;jXYLu&IO>tJ$Uk(6+vvoR$q0L{URC@l)+ zLf>$3)Jv=7SnAvdj}5~-4k;gMr0`ivabOOUUXEJ&(iFQ?DQ9Sh@1vy!D@ZnRQnxfj zVzO?-2cTck>(Xv!m`PFC=C`+aX4;ZRX*rJ|^RT`@?aL7gQ5jhC+;ph$au;6-dvESF zFqoMvUlt!{HaU{C@&pp4^J{&tdqxVNyy}8>%SFbc=r6HNwoN=f4O<#b4y}jHANy@e z(-_n@M{SV>-azmj-*C=vk0Tuc-I_Ns)n`{z8_x%B0*HMw|6`v?RY^(SVq&6OIpEX@ zv$&MS&~%;evzUQW*-@)(LCa8}Pd^3`lU}d6pqgPXNvBB@=8Z|9E+CRP4e}_{X;l0| zf8xE-A#gPiM+CPWD76CpH&E%*3OYSQB8&ylRkbT=NJ zia#@T>JBEC;TqIQguY9fsv1h4v~L5sDwD6?>Lf_TJK&cTsa1~o30M?eX}O4he_Vm- z?VI_Erd-*AS*WQ!MBL$I%sM5McDjqA*EOtCw)~sKB!F$nK#5v3l`>Ay>e`NAdeSej zJW=PFB;XfV(w)gno>)_?x}YK45D-R^@*?T_dbPFW$Eou`GhB*-$1An&T}HavizLGN z3ZMn2g5Ab!BdWKjvDqYtiAHXq*O4ZMwR115{WeRmTWylJQR11>twO-im0Y#gvLhR z08xn_KRZGqR)QTbVpy|G`Wo4JKcXNQXrW`4j1-Q!-E49M#6Utsmh_H?Ehe#>+6{z6 zqw4f8!p{l{wD8$4s#ZOv#4__Nh39y*I4vSKDVY`_eW3K>x8a3TvqBwM7je~rpp7gVN+RJl z_#|+63-K(=XaW+EMzx=r`WX{b>d@;y&%t>2Ky&4FLa?_6^gO{2^|Qn<6YV7p{%-Cq z&UPqQvtsc}^#QH1-Zq9MBf(WG=&$!UC>m^kah%huiJ%w_hg-qyH<&Nr4%gJc2??!j zqHOINlpwp=TKZ*ZnbJaSxEFu2oTQ6ihuPmhhhO*YdKB=6Zt0H$S)ph{oukJF;e6q( ztt@CMpcJ8f+>7elas)#MK_9MjI)Kh32w!EFO0Uxm-ZE@|Yk;w)G>&sH6H0tn9iRR^ z(i0~S(ZOO{WJEnp>8Y-_x!}sAPNGDC-X!E3t98rABY{$Dd%Uz(jdDByrP{uC+ZmE# z!Rr)WP5whT3MvKGCGD4?)uBCV+xch4m4%b=!^=i^w4=&!-W0!(8Y>CZ&mx>HdyHmw)}%Gj}W z!Las}XxJ5K50LAI(W?exV0n5rInlZLH@|}`GBqZ4jVg`Xjc;MsQySXB$LtI;97?kc zDL8?j?6y6#PuUGopu>_cON}Z=H~btlz?p7BZb2N5H6x9a(X&lrqEAd$!p%R_7a?gf zlrkn=8h43*y$;!htFaN^E|MgB*J7-U%*H@O!xMKqsFGB!j>WH?%5?htczVbcGo#LVu5yfgef!c|kk1-8{r-&z9w z34Y=4ztVNoX3rNYr$27lK0ftcK6vU6?aYl--52!&(WmsI-S}WA&-0Pt#HWMBY_JmssQu700kjFE;}Zuw zN3(5Wk7tl`q!hky`K?|vx7A##S3c{uVX~wryflEfN-)K9yg1JckP@G#4 zrwre@^t4h!aOZGp1DnpQWp?S9F7?li_gQ0Hx;ey%iNVH8uPDU#j0VT7?~V3AO(<@k+}lxh@@lR1@dRb7SMuNA>F&ZE`}Y zPC&B~=&jy2B6S_;ZEX5wS0z{AH3wZ@kqX&@y1hzF5~mQkLF8gwTN*mlPSD?8w=`uy zK37t7+7qFsP&Ah{SK6hA>v?4`W-|8UY#VC|L3EWJC2xO!M6B9r_>J~gGHn#6PjDKf zcX(gI43$T8t|u^S7G;pf9LTNR&z$L1s;i0EI+S>foA0>VxNq8-nDVWOvF{Lg7tOzK z!7kqXn}VESN^Sv-7i~X;Z@}<<^DF^%x3rs=MEl|je(M584j3}-xKT#A_Ph06;PB(i1=)GfZOht&!V$Ihd+K7iYLIYXH&OsN<( z$yq2v+c!+cpIk-GPAA6JA(}kQsIZR^GXF2ee`PIvq zt5nwmnOJ7mX%Q7Eq?G1KVniM6-^p&HcMt3x>~oj-`E5#0tlHkdOl|~E4BwWS$F|!8 zyNE`=`?0^5wZTlz7&?%Da3@^TIben{x0efiw6VXBC`vu=-5egxpvHO_@OXyWfKj1G zwr*wcOJ4knX}G>y_+>kqprw-enkEr5B(8~0>*V>YI`yFUyhla;b`5sOo0jY8-HJms zrAWh<0k+;Me$|NXDb|2~C01O4FsTyAdkN^x@Q&KP&<|YYrFc=yQ4p-KWM*Z{$6fd z{JXww%hO3bo<8=(YV6Z^be-t=7=~u^Ml&}nuO;=zKz#vBe>_b?1Zr}K23-+F6M4_| z>K;*f7}b5kdm)$Z-hOcOFhXgw^<7c%iRmf8a;~j*zq|SM&K`>k%VTbGGI2w0l55Ik z*0#K?!`<+u1&0JDLiADxk=72AVixF;qXMiEZ8M0tm6#uGhC~fKjlycd?NFlS%lK=; zC8KvqVoAI@IH)I~SV5;^tI5S0GVTM`*u(?T7E;_x%e=kvE#Y=VbO`fhVl6M@7t-;smsHF zfyXKuT}>R;a9%7mjMpOxi-YjAkLnWva&9~M!(Ea~ze!dgjl4^V!!)QK0Rlh!Ux%Py zKR6!^yNf>hE+dd4#JK=hue}qb>+rhJ`Bol{%&oMmXz4&~P8h3L@7pMwNgrgFDo!k1 zSVNDNUfeQOkRg?Su8)zdb5BHk-{V9&qU(NIXLI%d9%BKk(AwMJJWD35L|Me~Y|?#D z#PPgI@=3&fC@4jcHw0oezt>-w07}F4C@VOQVkwcv^-hZLp^k4Mcl(^g*hGGNJWz&_ z!`H#SHHDqm>QDzVM##d3P54_Pxx4BHeeCq&aD?^WX#g znjEHNeJEZjr7INWxqI5dB}Pvg!6g(PvD=tmnIoS3d_!1}t*876X_uvXn~*X81<4yN zy6|`Z#0qXS`tfZ6i+Vl;c_Dvr0%<9gH26ej819QBG&1YArPMuSR?zfwHA!nxXm{{A z>0Tz;;Tq!=i(tAu_aH7?S)*%NE6zN$v!>K-g5Ziwwz>k2pC0xin$58`X{94R!h9%& z*zd(aCqH+{fi^yW!|A?66Jj!oj%V(cN4B~?^0~&(Blp8(g$9NkgwfZXi)c}}e!bm8l)4M^qtUK8eJ_g);wfDbls(N zlJJ*D;|PI$b-9))UMd&bywa2fs-P@}N}<4eqD-Ak1Q|HOUoCN&W3viXxjb{74cOVv z;j$3I_X}aRG;gY*cO3~z9bEAHt+N6|+L%T?S_s<%(q5UlGtee4kb_9M+;-E+*5m~9 zq0$?6Gs3~s$@n-wxG&iYIIEUKK!kA|B zv*EUXbqvquQuF78C}j*HXI?*1tmW={qTw89FHaE1shDvRT0;ph@C>^~o6rNxS*r-C zz{#@fd1@S|G|uve#vbaE8#-xPY+U7txm`vu7u;MX>4UbC2@Y^%uEIepsy8OL@_G+j^_7Z}5C zB$FHzSX<4i&*K|C>ZlkjZ9NY!nx3>n8xfhQw@03aWnns^$ zBEkgiE;M4>g94s5V~Q3Bs?W}V%w;}{(F-Uo>K9(iVJk|e#@%;}s!4RIL3@3~37mw{ zXp-r}v=kbhUm-)&ua5Isyw>Yzr3sft3C@sBLq_XR`q>$~g;0|KCgL(Q+bQ?FIz<1OfeEd6rpe{T0)7nq_aFLTxwui{1E}=8JXXS8RhH}z847F45 zQqb*YL7mZ%5^C+R5nKRnP7 zo-DSa;Iyho>k#p(>+|32Cym&*g79*@fE#lrxojnEEDzj&MUU$x@DZb!#Qri!wUinra}fJfrejzB)g3^6 z)OMMC;))j`7_&TJ6zfmnXpJ01WqBO@ejw=8+FqA!pt9npvRRC$AWBDb#fLGZ+9p+l zQJ$Px3ndcll`T|Emz{oGa3V!Dc{c@+*xmA*kZ8oswN-;bJOnhEfIdrX{#<6s0bUkj zvO39%)fjF86CZkdCcI8}`ffzsb9v#MK|- z*c^s=Rn1djy}Xz55b&7xgz4jqv8fl1ZY~OgH0-YVMr=}`m|Kg`-r`*zaDS1{5i;!{ zgQJqEF`Jhp!skL8XNPhW=*}Vv=oJ!Zckp5_Tb%_jlS^iQ0dzaE3QIg%`YvlU^V@wN zAuTpWsMDy?W6Cz?Y)hUIxs@ZD^H+*n%!Imxe_TndO-t|yu6B}b9BWYLEz8J3yt5U} z2%EK0zZJO^Bx*a(YbUIu9ZJwB9>{CEpAk08m$N9gmSp5tF}e+&9+{%JpNb<4TEx3D zFQ&GUgHc2$B1CIym%P5+b0AP~luh~CNU&9rZY>$7d8O zKt?33$%PJ;0DUecdRtR*4e?bp_Rdl)b#eh)OngAIC zCLEF|YAA`^nCR>MUK~HcQ?Ub-hNh8?n$ew~5Ft@Zrk$)F7j><~lWuOo`jwU#WZpGK zn1G-ErWHBcHVl+Z1Rs*OgrAZgB?qEc#nfXh9p=~k?sq=9KqPLiHhxzt;;-Rg^N>}1 ze2`+W5U=lEnvg+xzj?o61%ZX z2yH%yrp>#mD3Xp6+3|}|_BTS6p?yB)&)fjCG@qI6aoT=K%vyV=4muI|ITmX?+k3E$)Q=97~B_&#aDj2NB(?Gr_z z-OWlQ=ACY>BP~PSOVlmQycq(a_L~kQwyuiOWS%7eB>->kV8BOxfqaAB$<7mx4it$Vzb1`u)g^ z>8tS;b!0`peKXOJF3F7Ii(ZK2UA}2c$pe*jjPMlzHRgoR%)XIL@7B$P$Zicu|K%kV zBESdVwn#$~#FQBk`lk`3aJ~anPdoY`CKOqZvIGB>6Bm!NqQYmAItd6zFf!2h%nD{i z94aD3eZHn2(65MAbZ{Z)-$E;uopNcJyW6RxYmtHv0+SpC`N(vVJJb0f1@Qn6uT^4V z?3T6w6VZ6iUr(|?JlAjLi~rn(iL8L}mjBlPqdg~-N3 z%|*)hxw4i?MYN!$SPi_ju_uSLro0z-`q>AC;*#kTXmp53O8CA>{sb@OBPQX^UuDgq z<$ZUFjM2&HE2C(=6aDZX{sF$UtClM)UuBJF z1i(&YwQ;8i7X0;4#r6d$$w~V%81zj99Dk>*PWkZ24`PA6N0x@`% z+e&>G@tM1=Ox?W=M9OI`t<8sDF=X=`8+n;4O`JB%Idf%Yb%w~!8B9{hv84cGm@6&p z%w0TvJd7W-_T=<;-vB7lx5Rq~sN5exaRp5Q^e0C5EcM-aEnPF`M>uv+H0kRS-8Eo| z&3+aQ4GS}i3+MOHYwqjk<>>~8?NEyjE*z?@lcKglwuY7zx6q&ONd}EyT~yZ2&T6R~ z@q{n(t^p4A2rk@Ms;++X<@fmpk6#%kr!tgd7j ze>K0)8>oDFYuftg*N{{slg_Cssab$Ypq-pvNikc#I5KvCvZyEsK+o-40Xq$1&-#g` z4|2wl@{_iAwwo&g2&(D)#5g@BuV`rvzSTu|M{ZkY(3H?FLg1cCyBQ5bkVy%FGd#`e zO|)4x`ouN&EiB{w*Xt?Nz3pj7(*&oW#1BbP3yW-=SBhf+|IgQiJ@}y8Df^5=4z2`g+a_Ef-1DG=wDf1)zu3 zAwm%t8VJCdg+^&0nHm@$@Z;ILQh{EHZ4obJM=3fl^m(8k2NE}c-cfHLIzM8>e+~GZ_~I5X4)l)uf~&*c+xK;UTMhe?!UFuV zZB?W5!MXwTUHhEKYoluo_%UrY%?nRa4!^Weu9Ux2r!E`w60;ApbAdADs{#vEmZpA7 zHRz>;KEh_(Fi$5d)eA756g&`5;VSeai~gt1J>*T-*cmU)9Iin0J%T;$PD>eF;Mc-q zB*+q;qGw_q`G=Br$j2NLR-IY6W7BUX?GkXyMGqyCz#bLPaGe`+nn16Pn1?@^3X=p& z_c1km=qvW_0{7tD7&b}VNH$9zA%H%RKa@N1_B%ixkzUz5JBB?$FD#*VZB-ccXW1uK3+U+fWAZEQcrx4eb8=temSV(D0M~DUSnUA_4;TzH}CW;Y4Ji| zySOFwy)%9`>pix;O#!}Gu!L|60DcB=X*+>1_>+Z+eq}MhG7n@xMThxjO3hdv5DnK4 z=e2;<0r%(_S^@e_wNv4}-+Dzu?!5A-O5UWU^Nq!t6uyeo@0T}v%ycc9$bHiN_n|t{ z%d{g2q{nePt~2~@4WH+k!6{aRo-<_j~QSSGJ;^ zu9kpX7DLQx8Q!4+4zurzgf?yExS=v!XPGXW(A)%CZ@xdaes>3r#g@VYdKJ7Q=tL-q z0P>J?XWAU-^yzi96iwIZ6q4vvcf(L>OrBO+Gi(IwB*o&&Wdm={4rhnHlImo%k;`f1#gt{%sS( zABX=n=r0o)7?}Ty^J)0sPGDgCBlNcb<9}*m`fKO@>?H&9AEnHHZ`r5Um{|X!vi{3~ z4D|GWa7Ly-INPUD>iGXXI#~WII{vuh({R9_fcfX|zlFpn?T^GC%AfvE`=1W_@Aw}L zfAD|H{u2m)`cFFj-y8pDOFv!sFYG_V;FJ2dj{ijv{3ZJ*C_XiPYWc_hzi0ZlUH{*0 z{=Y5w$CCd)_W$_p?_KzN1%G1pZ~SlH{v8$nwYvZE_&>By%74b(pH2GL9SH#VN9sTO zKlHzO|9gr5lK;=q|6u)3(zW(ATxp)29>Dw=lFb`n>x9m;wJn81MnCi~trM9ysWK-WZ%x zPt%_7cs-V@44&T>uuTYPTve2lGsxscVdyaR9k{dL>OIx<1;oP>NzJ}m-Ts-!M$a3tF1JuZJjyN|M>B0iRlz;e{5TWUNAWfM zf#cx!MyxBGn&AE22bWS!+F?%MbEujdLlk!ub~6s$i}h}$#p3JgVD@#-D%Ef6Y1;{! z(sJu9J6N3PwoB-y9o{Cxzr*&%GoW$Pmvsi|sF{7I3}3##faL~!e$n7*Dt(7P8I5$X z@~8J1LYP*Ln8VwV9o>!@X~mdNA}sKdTqN0@F{2w1YH<+a@L5vgTKO%gPJTX;)JMJF zKu-lzRbEAkKSEv<-=I|0#m^sR@IA|4TLspZ7v?&v%k8>7Gz{m$iXKh31FIUgtI0oV z;L(5Lp7735fpad_#?S(vCl?>K6rxEB)d}nrd?*vv?2F{@x=qtb2sJ41BD>s(!bT){ zT+%?$sAF7xkq^k&zK3*QIP)*V2QQb{H`<@Iua9ElJZIT+*U+W*>tx)=n zdD{`!KUJURiy2E3?!MW(ko!~S)jxkzfy__)pl=nScD4 zOC3=`h%e!sQ#%)SYc`p9&({J7{t(G=dS0ihVh(dv)?OB0WYdHEZ5Tpu~NufDuO18D=QDNtrGT(=cYH9mLIjq}tiU-c%jc0p0O( z1$mA5y|LK%FD#HP2&!*mzv2qg`4>hJRtAB zUtH@_F?sg$Cn*W$S)^m7? z%b~mP^5=6xm8H@zd&SkC&kLV9;*f(N0hJ+0$c_?y^NhM)ust-L<7MA3a=j6p=~zE( zel5Iah}!@R#?zr)ZfjxdIeE95X?{l1g7|DX!&A!q3BkjvAgR@Wyr zckVL&AZ2aLWuIW$klEJj$e+EY69$?(Z=Jh`gEGAK^}P`j6nZ$PYfJ*p^itNij%NDH z^O0>oJ};rf<~wTR1laSKOzg1z%!7Y*gQ&T1&0@r$Y73Sh;BgG_W?0%pDV~`@Yg>-@ zO{j$eejK=+$fM=^T{7PZZ-4X!ivB`jGpvGj-Lwj?)KXagS4-nyclp(r{R2oVv_}4U zKhM&BAv!ClK>9tmEL-w2`Gb&Ri-M9d#dyBh12_bfa}lkRuz?=9IC+gXB#*ZyF= zP=h1FY(L&L-`HOwwy#Ek9@rN>(Ibwn^nq$|NsAe-sJ8WhOPjE5qZ25Tz1JztuoIyz zz_);&tW%(Cis-oluFj8P?^49*mY;k>Ct!cdzT9F3_nWso$)OK#_Bd@h2RjF|g1$1N zAl~EHv9N)w+WIp};Yi$kNM{|afFUxV;YL2>y|Rt@Wn?B$nGnN0ty0Gg)2@aMf4A4F zpafaK0(qh9Q0a4(nr>X{<+fiuFf7C>qsw?+5p8vyw1er)o7Dau$ScSoU|p1*K=>o; zO%{2%qlW*0N$Fb|nxFiq*+G%*CjOE{yC&%MynV7odc8v$l8f0ChqTV~L{3&}JPjGfWmC@e{K@d2SqfTO>qLxO=aOtzWmipP!%g4D z0|NnX7yh>`gf8G&H_Fx1{3sGT>K!eU7Xjz8Pg6H^>EeCK4`lrmDUR_TIZX>{bt72E zu)eO3EP$zq4~lw-TbQsDxZ$UjRvGI^uC(@i7R@2?pSEJFq2P(CZ=If%>Tlqg&eC4s5)3 zIx*|>>nYnG0YJ;(nKLH`4ac^vn)tUG#=+A}t)jOc5sYdiceU5Y_0KKN$1M%{!&w{b z1E~b1MGx5rCG~k8m~<tZ{}JMPFS#tZyY1H8CeuuR_Wui#7WogK|H!r z8X+LtWs3S4c#(>AlrbJ;XhtUw)VHUSx>gM;J~pq9^8K8xu+;W~q8c<&SGepqC zAIU)Ag*>xwd^>M0Gav6s^YO?t9qp5k=|m4O;yn*&5ydP@6~hTy&l`rauGETGuHau$ zfW`}TN@bE}nJO=op<1NcRLcJ`-UWn3@I+=oT8@5>t8y-P|wL!G=X+zwu&*%rg z$)zqFrc_xl@PE{z*K7w4N`t5?EdF1dKc&>ATGBfH)v z;>aySf_5mKalQdkVT<4Wdq$KR*UH1zq|Gc{nlwVPPaG+kMb}}*zimH}GdB+;X8CZ@ zgNJ}1pZsvAn(>s#XR8(5C!gEcilxjmL{l+*E-67$Dw1(TGc*S9D3>Su}h6JoKv^8G!cWLL?1d>Z);@g=Q+ueShL|ilR}Pq*1E4 z9_{t)3lL#?xn;g%YnaphC?8l>?GUH9Uv#Ne=f0nGN&PN)9y6<}S&^|Z7iP7O?V9(qvY*EfudupiT*HWG6vne(i)0%L&e~-a2y7oA zc1Uu~cZOCSUyR%a-mTR#TqFxTKc*Jl1*bp4u2yI|=QqAj@@#CaPQUQT+~c|9y;~BF z&n~vOUAzV{Lrfq#>{1)QJg+=W)N207uzQIy{s5hHyt1qRcA%R=aZokwf1$WQX_QvL zE#&DaenED-JoDIMtl|@boH)|ngUI0%!(C>I1z)1h1fi{5g5k~8 zxVU)(5p2KsBC*@_x#M8&Nh*~m|D#3e*@FJ_;B%XJ&kN_-Kx|M2wo;qvbyIS6K}c5m*FQWwqKe{#II7Z*l^u7 zo;ZJ{_XS*`HcoG-w8sOkwl*$H;+}FYfE|Ees)`Z~-&O-V!nQ>`J@0Bl#r5J!(L@c< zvVBhk*gLLEw9+U)jO*WDM)DvZ2U|o&E)0nyLn?gAWA?uMc+zno`K*@*ZQb^bT6bqt zrN7-^$?U5gj;e2^oIy$fKVbYI%?u(}}k#tWJ>|3FmNYt1tIBM5U-Q ze2IbXY#pYj?yo1UL+q_`K_yBh-&x7-yWBvkx1r`}cX0DD8)d|iEmJ4R^BGV_d)8Mt zeyD9F*#1q_RZBwEU}lm*Lz>T0yL+j9!%7hKK%WI)Py^SV;J)IksX8CNq|AK1F$Q1t!wv1N3ds$wRm<8;0j0KHLNxsk{H+r?HR#4su$317^g*f}~&jf^M zA=xpnaEu)wv`X=a2G>ECjfqW(aex4naJe=m%Umq~c@ga#=@;tSbvcFc{i-J;p8+R4 zU9T+$7A*U;07u+8XS0Lb@ual1G%|aQm8NBF+gMIYPgXY`>kX?}I`}3y)FwDXBI|-H zql+Ty40Z3-a^t}=givY|*JqZyuZu)`nRqi-dk;o91_p3>=ADkmR9dyXakV$uP zml^oT%$l-BPYc3MH+^$~nw)NBq*Zg-0AftE=HagitAfx|o@oQuq^G`4*M2Cku8&`U)>mi45C^Wcqz(+@)P%SDD{KlbBlALJx# zd+!8JLOdBB1FQlDrU8~J-=T~>q~mORA(t$WR&f^~(aYWB7h;pwuL3hN0z^hL4F;`T zx6}60=g0LhfN|lEeK2;`2!llSjk#BQuA${9CEbQ0DHDkDC@p=d@%Q#dtt|DS+TQ?g z*!N>lrFbLxHcrd%LIrl>MK)-tuG!b8e&`u}@D1F{JHvuE`kZS&mu5vJQVIq~(V?v(Ck4zQ?1Fsy$*13OY&C)jK*VHTRrR+Zb>%_^TaR6l2xZ9d0n} zr0Fu+tY$aNktbONi~1wP(jtTiRyv^fFo4VIo;9obx#n82vdv(EMND^%dLRA91h>bD z>oC*O>u5}upfr}3*M_UHsx2kIfQcVVQ`hL z0^*RpNn13LDY-szi<>DnQ7bRLj%|$&p_eUA&4DgS-C6r_n$3}P4t8Hs#2jS?i^oKO zp^QP@)IXG9Fx&VzeZ_uW#?p^1O4eu!X8}7QI$q9>r+r=qZwA%9fUkK?9N>c;D@%$^ zK-lhH5>h4d%>)`@i?(n_?9tDNcO3c{_Q)`X^_bFPw;Q7@b3B%S7U%PB9C%w2Th&ss zNB~(SJ;>r;<`=~GRHQw-TTWj>)r)d$V&RKrBy(NtH^I^iw`XQ+i#vV>7-W)lTDTfc zmP&9h8KjWyD-{sCE-}0OF+`!lC)k)#>J;AruFQTu#`b94(nu)C=uS_&lAtB09q`K< zc{;F_r6re=E@d8R9yJH;qcMOpfFWWF+aD1E@ZifH63so9I<%W1{2nyT8~EzOfaatc zKAx}NNI4OdhhaN7cB(ec1EYUbc=JTsnB%w-p(mv4u=g9gfs7e-B@JM}yd6e-y~#Z- zs~jo0r=@Hmf`!dgjF0vIuyzj7m2lCzO(hkpV%xTps@OcSZQC|Zl8SBHwpp=GY}>y7 z>%GqH-PRsscUNzH>zk8J)jWc_kUxKnMCC95yPOnpT9BN!Ph-*@4sAiX zImU)1*p@Kms0$`o|H$%Enk#CiCd%arV-m&8Utm$%x*bajl^2k!L5b9 z%Yi`&#lb$a6i0fIBacQEya_2tTuJlB9yUr^DYw{$^-QDWo|e8u#p16rbCAF;OW|Gp z_V~l}&OUw_TKH_ks_bFC%o~*QbY|gZJ^A7*q07HGq!)wBywCU%qDK2+QPVY5pa6fO z(Y{w0`x7G8mZ~bruDLw%IvshJP-rQNvf%sAB&aNG``G_?-cV>ABV!cml&@Nty{a68 z4K6_hU!u@d$ed}EKXz)wV9kn(93hI=I@UG8snOZyH2$yZPWI1|=KPKjZ14SkBGVGoX-&n z@;|KquMx|ug0+y|Az36R<|uZqX!nFpb>xK7C8k}lTlXMWC?kYHuts$}TtK0vCZ@&% zrD_P96SFyZp~cLCtc`Igi8QzL>ee<3Q#h6kj4>IS#g3vb1xq$lvzb2F!S0QFQOBes z3y;uIYLIQE(v?8DkBpf#77*_YgejW1pTcj|E2T&O@7m^kpsYdJDtt6;M{SerKZtjc z+omBaj?WNb!I8Qgf_?^J8`q&1Z_Id>iNxRtOUeQBm6@YBM-Cer(7Uej5`6{n$BI*k z6Q+W?F~i>0a6=5o?2_a@GrhL(`u&`RstBGAo-EErDPk?>S@Uftr{BUzZipg$PDO33h$5Aej2CW?8q~Sy8=iCdXdN5w%W(D^{EEdHaTkG>09l`bN0-R2-*77 z03krDwZNIC+j7n)%Y8IkpDJKn(``QI@7WPO+pb#RbJJ}N=hocFI>2Ds96TfwO}H!=zGkiuiK+K05gm>;}T7tg;Qu1!`r`s1z0|vmB=zDh(nC`kCMi#)33z~;l&9W=Gax6q;XjA~z)jLqYeE^& zBC|$mTA9=VdXkFwm-YlqpaEJW*XWDYq=D#hNpb+f)Vs7KAb>FWE+Q$sn4kPmhz41* zU4OzCaFy&7m(*FTBC|$p+MVPISe9;=nZTu~EUrp$DosiTT*W(eCSd^V5}hiOw2SCd zof?xei#|jqSZP$7FU5uJk{VPdP-(a%HR%t%XrhZgbSH#qk^wb|cU&~^MLKkcxHRfA z=}d>TH0sjnRHijaJOJ7v9g;&QnnS=u(NqenjC6TYI)E{eRYW>6=@bxEj4nl!1Ryre zNb;fa1e8dplbSXp>Chw=9Y~I+?A6$2Lbj_+h|*{RC?#>2D3qncl3Zx^i*!g$tCFIN zWB#&gO;FQV0>q`Ik`qWxLy~p@3B}}6j#5%dtg;h|G$KU^DNcWrgaA;>E$L7=#D~H( zpT!S}35x5K1oP_BM5k-o(p^bhaBDm&AEh>6oPZ zV(wy>zo-nuO6v`O$fYMd0lX_9aY0_GdYMz|;^3qw07%M`t{9lK4A7E_OqvHo$w|Z0 zpa2Gn*=ucCpyp+z(~=ZvNB}{_Kc(uWS&B7^umNr8M|VQ7Ol8ni{ntBOH%t!TA8@h3 z`h7DyR5xVYP<~+B!G$|8FGxP{JwNn=Yx@juD7b(2fI;^k?|`i$|Axp7^4&qZVR?b^ zf%y&Q-T%JR9Whnr-p{vVdxPnL*7f5lU~|Xy2G;|l3tSJ1DG;k4-T=#kkRLh+vPPB9 z(HH6hOd+_m&#ey82GVsWcSq{x=PEd7u$Cg)Z-xm7&cM-rV*@e|c7A~~l#{_7k{ef{hXizdxQ7I^eAu^_XzOv-_K{Xt+l=+urXrnkmk8>7-YREClM6Z@ z3P`N~>{8rAMhOM6te`u$Gy)vFuu%|*oS8_GII;XI4S{2v_&j}LnO*Uv&>-=lczSO>sC zl*a3$a>7jA4$dGLKC{}GZy)@B*53bmOvM<|! zq77o>KilVeU-b@V9sUM_9k}X_cpWA7Ag#jOuHlzIw! znoqRvfzY@AKK7hnDAtJls@yQg)2_>&dAF~72$zYv!@JsrE`5ZSyL*;Xu1(ThS*))Z zo(P`xjz4Et27R_N&y9tKdkA}o5U}9qc+yttBww=YFtxF?i5hxBz2>e{HE-Vh3pI@A zpp-RHK)bAp6BGIEHmrj>Mr=DB_?IjaA67S3Ziw1_W{q_D{>J|1(W{6yG#eNAb7OA7x8JudIP?n*kaEiIgAZ_KV`@?PjlQ9KrZx2o z?I(hdUh_Y7#uwagC**HuezW|ZIRzv&d7SxatLfM}0fJ7KB_ z_x_=#Wr13+etjpCrd;0lJDdDv?uu;Z_RH6rY}WpmX3cosKBKRELi%;HdCO*a+m3ED znNJds78rl!v5OgfprGVepH zvED|{xz8nuPDzdpTiJM-!4}c-+^@=QnEsjm}paU1IVAxr1?9? zME>2k`&Ly)4bb{Xg*9AEu?mmITC(Dw`KxeGOq>@ak%uxcYt(L0Z=kWXs8ut@9F1e8 zC4xXWkxs@<%}!qwE78V_gZbZ0^=#hZ2owYn8#nd16$cO3-yaz?Bt6A1sc8Xl^0D~r z%p_ZGMNHSfa^q*~Kd*Y=Q!YTm$puSRg9v{uGXW;JE?Rb%3e${Eio!lN`s`Xw3#hh^ zcOhz+(dq9SMQ@+&+Mbs)?e9;w|F(kF-p4zEmya|L`mhIYes%awBMtIZS!Yc=1q+_G zr@FPxY*0d-4Ev-WqB35SC@E7d_i7Q#ff9?@WV7qnL_ZSa9Pr*=eQB-aep0OI;G&4r zv?PU5xEr~4Egj9vi$imgty_L zHIQo7Pd=ABxty1vxn#96Ih|DHuLvk0p1ar$r^kLvJ?J-8-7g=A@o>7H|7|!qebuXN z%|sJmhnJhj5m?M>HYGdoF#8B)XmKFi>WJMwhJ&|UIWw>ToDY(mgpoxSoLtLe8aH}c zR0bF4OOaF~r-nrcrBR3Jox?2PY_fv->5aykWgXJ8`8uQQx10OkF=lprJvW~GQdLuH z^rqagtxIjXe7}Y~DmUMQ+!q?M@85E5f}eatS)!p$twuIf3NrGTX{D^ItSI-VgOS3* zoX&;FXfA%GnxZC-K7IA*{tdPn)yperhC>vd6K*#Y(6A!EtrM}zAU}ge1){bnY4AT1 z#0);x;wU{r$9$>1OJ7eVZy8ccG+dV*np@+a_t&6&Zt!L_Z^lwZtI>yCJB5nt&zkvH zs_F7QmhE4*#uR>s2k~z0Mb?b(ZoA+L9Tw!+l~1_sY#K9(BNY^cHd{6JCMB>)x3(v_ z7MrB~(@o?H^;iC{J4t^WUC;2Sv)t`N$m^TNl4t7jeUgHLDd1~KtPvZl#a4ch)6m}? z&6U)Aw$>Ov{SLpqWjwz~BVfPkZ7%+=JG^28Pdq)ib2wJ7p>VH)V5Kyp-%zN_OU&1> z@?I-kuI>9p!2R8dQo=QVPB(1%4~dq4fyL*qM7>jO5gykdpk^kQ3>=S5y#`BZ7@Lcr z5$O`k-*vs1KTic#-16Wg0YI&RZ>Ayo+dtHrS`C8QgDVih)!d99u>fw$<+sk=__x5Qy)&KDk7z@OVAE&-EKpqyB0= zoY4?x3zD@9od3KmH0_v7a`-L-?L-X!y1xc7t0rR?N3Xc*$O?t?Tu@b&8Gq|w`@~&^ zx~TlvWo>77c6jF>$=tUHPOdN_Vvh{nH78ZrC&L z3~!I;4Y3K6nHwP)NaQdg{8|Q6a)me{O)WDBHWlaSXIOUopAC}&Y?W!Zw*NKMpv+Jp z7v5%*`JXxTP{F~{*#Y)enD_DOt&%1H2&ik^6awg&L`irMl!jS!Q!RLEczzGc>25>ZX zn~<}tN;(FEH+j(O@4iY3{4a85#i78rO7|FmoAFK+79@6li_Whx?0`s7xL|=12;nJW zdJji(*d$hUx~o%f1A8tlXXodqliSoB-j3_n&#A$=Tzi{e_p1k8kP6$5GoKLtzA*dP zdCy|;R{zz(b@gCIe2MqL7v0koXdZiF5t8h0R#2*G%C}PAH3LJZ%K`-xT|^7Nnbe*f z6wmr{@0q8419pBI((CDqUzrE7>kOWK%V7Eu$4Cs4$a@d0j|S7i@xWFtSkPEG=OM(l zb3X>QE~n6^Z)flv4(ZySmr|@H2j+Fl4=tZ3rhX zm-#UaXX9sR;b#E!bO37&CHZdWd$(EQwV*_kYYTX}YHCKDKcAfOf_N|Oe@*m@MZwX5 zC@RyeU?@5qA}Zh~o_CdrX)GLjU_4{n^|<6yLwCrg)G>P{WP6lHuEX6cTr^k1lI^HP z>&PoD%KH@2#eGJ$NwFGLgL>8FoZn&%tF)+J5j~#&RkuG<+q%^d6IEdxqOhq0jN?yh zyPUtz9&2l;ujH#WIUnaPH&1kTcfvqp1Wb%(YTK)u4)0AYOkYXqaRPWwuEXr8bP?tf zP8>-I<|;|3^Rh$b80>o<;^QRJrC}S&lOhwE)2eh~${JHi#KKSbNZ)=7M?tdvMI|m(rxzq*aWy)sMb4wU)gXd6TU^8^ zZhaouNKq_31u&MhWhaeE;F3oKDTQaA?=?ND8B5;qFAsh%5q|E|l}Q>N)D3oCu>w9lcv`QYQqW7VRJQBgQLw`N=W ziWz$it=Gm+HuLHmFTa}ROb+7=t`;O&vFr9ID`#a9p4d#>dio~>Kg6wO#MuctbHj`W zv=ibjr5GfM7`0@DNnU6KC0Cgv91Qp>XeKld$dVahNU4B{B2&bFKYkz^vCY7EV2f7p z2i3v$-oN!-CXwxj%#-J1F8ndPpIbNU{{0?cHSrZ{t8ISdz7DY<&30yIMT@}5RH3m3 zkgeqGzTc5cU%N1I96XQKKD!p^>ACIkB++(M4n1iCbvNb=sXg(VsMfN4s?av0-_P6T zUA!n^Fv@L5FQPn}L9nLPS{5k2P{&nEiIvy*m86rNIu~g1%0y8>r1!x?0beunB1;l* zglYHBu+ol{Tv5wn;ed=u2@x%NmEJaIse3yI@A~>|mw~rS>*vebj}8`tc+6r`s^h34$bQ|H=2nN^lHf{2e^?DBvWKd<^r9csF0vSIJHle93T`ly#!}J-h%X(Yg2j>Re zw_?A3QpR1LA?@tJy=dSrvR6vV6)u4Uqa=_^*PV!G{$SRxIV~s%lz_R6PxD?zo&B>+ zW-)Hex;9!N{eeG!3EyDMgMM226Z>+47qO>kDI2Xx+!O&j=9~;=hQ~B48Cxx#;YtG_ zsgg1gHmZ|<8d$ONbTng=vzO!I(Vwz%4X3_!TTy3oedd6?U!$vnmZP3JIq$g^SY)mb zutGc=t@;KdS{>+KSUi0@17=YE4x`2tjpTW2Y|qkyou!BWM0 zH2(qZ$LLX+iq(Ym`pDa{PA1F-F`y&!NQ}k%V$sKux9+yf)T^#hw?&&RuIIgL^i@72 ztS)9_16kj?3EpRs!rL(Y?S}XrV+Y{8XiG}~@MUquFqIO>fgoJPm3kxT#2NHdk=<2{ zx#=<5oWIKa#^R1hiH3gyf9IW*!9Dm5WjZ#_k;+%)u732;S{wZ?q%Ec_h^&aL$ds9! ziM7tU_p@v|bCJdyr>SI~H0#KUs}7kI8`Y&UT0IGbT_|UshC3 z&*5is6#oKt-7#g$*iN0o%+#1C`tZ-5OF#7dY!pYN`OMPyhj1lAM}l zn1LxN!Xprj%>jAgCA=$NOSfW?fqNCnTCE`M1?3I>dgLi0RJ0>D-kSk-}P(`&KWjOJ0h^7*5 zIRLF?NDtasMC>NHRI*55@f?=?^^8tnruLQqihDeR9D&x(GR}}6`tn*mC}oCKe{=pR za`m0V7qd9AfdTgt%WF06qhrUjcORsbOUyP~RGa4e=PvX*c6FbkIrb9>-eVXW-)<$+ zPWDfVxn~@44EXrqBu^mt_KOi-xr>EUkt&zazv&)e(V-Erv%yDvF<3}L#WyAIrN#SO zs7)#%N^29Uud@B2@9He!ZipYrn{N9&4?P;P z;{-Vif{9vZH~3U)rU}(iRBB?`##%7^h-_p|_G=N?XIWrmFkLZC8M|r)5dvrk71iiL6c~aIP;ALwtVk%qok$TP~yP4jXsp??=D+SM=BF zeOXnuUm_Zv>w{0BGpbcu4wS-OyR<`r5zhc}yryy&y=v<|>?O{MQty9~2kKIWn6FGb zm=TMJEqj6DD*?oarUK@BA1YEP&K@PPh==chM?i`Hgs(&j#x&FKIvF;|gaMRtuem#e zV?r(0@9V3|-R3>~r?1phA1Wn(nX?W$6;U+6uu!H#9^?G^RTTCTCLJOVvZ>{~p8396 z&_WSPm{I&_9q1f9vkskAwgE%dH(_2Gw>9b6t+DaT^vIdXAv`Um%zQp+xjyA&@}0oKHEl{m{(7G=5{_?@27S zL%L0+2o{@=?P{K~e2%xu2@ffj#RVRC9{M~u7V2oK$;)7uSlLDDM|DWk`{yQC^q(!o zgp4PBTuB+LVW5X6_As8m96g>_!v3RQN9oWY&P`ib`b^!{mgX@o`w<;Ex?^&zeZGJ*L1%2`K9xvAV>00w&3w$r55gAGq8QihM z;zTenl|j4p&W=ALn11KL#k--1P=x)(cqmHjl?rQb)Q{(#=?>vn{LHCT<{5;qM+)#5sVUhG^1O*fIx)(`scH zoBo3_N!cKzs!|zFg$lgNn`R3lN`z)vZgRYWzUFgBN_(BTU*g044=G-BVxI2JX}2bn z)_&f-eJcAI$C8S=hTSD=Vk}FFXS~0*%H^{EXJW)+Z2xqGXmaELpI{r!{NT|+Au|NV zqq_P$Vr_!zuaB18odeJF$W5|_^?3rSoiG0`;Y7pX*35I!%C=l-FO&C(ir8UIU^a-h z+OMI_e1|2?)hvhW9Gk4z1((2#v4j=ZR*TOLe^u-81E)%Pws=E;cslqG_8%S^)fvY& zITkz;C2}^Jch|s%IhG%0)~{^fbBtH1((t`bo>>;8zl11pVIzNoXE^OMQL}By#%sg+ zW@o!VT2dPLL4DUyL;2m6Wfwi0Em%9O5Yq{;Ey$h1ZM>#~(sG_-ShzQ@SpIdLqvPQ% zWVlS5#qwnCBh&v%nKBd%C+U$?4pGTpOSiL?#qNMghRy;S8V{tYY0q=?AF+@AJg&Ri zm>ni3gfCXBu%)gWS{Dh?=jo!2+H`IHdvF(u)^(DVroJ7I=W0H~fWsvExi>nd-tM+N zKO4M{ajeSAk*UYidN+&Y_TG3c~VaBj(Mqr&W_z@PZYB1nd1`Xv)Gb9 zky5;|Q|n_@!DK5d9TkN^p+4jLt(YtHnF8FoxK|FiJy&x7C7N5WgDIeK*M z+{1-t`*iw6?A((6w=U|t%M%(zbRBmeDa`YZyHk#P);#zMW;19&+m$x#TWwuZw-MWT zN^wEKETN5$u<($?+@eWwjiXg_dd|EM7^f3oUgu7=G!8MIK)! z#9y~ZavUH92^VG!1D8?L%5leqdLliGx{4>z*ytm^88^}@Pn7R981NzhPw18!ftw6A2OR#0dB(0f6!S^M8S!E+bQbex$MSu=gbY((#y+s2Eyy0 z*&ed9nJjPR9X z4QsvY{f$2kNz9Pnfq?$8i;=STeR+9rNhzs`BmD&!dckw$UJ)E}ARTNC<*=xGb^U6AjM zXMCMFi3b`?SyZnizEB4ZfhhFBq{NztE2@Z6V#qO2Q@&xJzF~X5Nix6VX<&{?>edy; zX_rlmS}_qBV;^ZNp&Uv=&stB=sL7ZT;{%My?X}DN00B{wH^O0BVjf6>fB2kaT!>Wvxe{3k^C^4$ zZbhEkiwccI+c6|Gv244Jr+=)C)u`rYHRgOkg1&Aj<*qXSl^S9ViFlyUA;ft0W=!f2 z4w5`u4*$#FLtEiUuKC}#169H7v=}Zh8c_2GNv6V}2EDS_OrdL8HKBK6751|CwaydGW{WGU^+Yhq+q zU|p!~WoHQ{aaSq@`^bw(wQ}jfjGcY?ZBuyNZnNCds%wdhvBg)D6WL`Zr*VCepxNJU z53Tam&JWCYWh*)SX_U{OwsciYhHzAvQgFj8o%%k>zZIG^h?i#V6;Ij0bz0|LxP~EE zkDuBK7Q^L5m?qBm98xYuv-!WJaJqCoV5sLXTmeLI@PmHBQHq`14-0ei=qveU&bFZ#QCA4!_Dz%-V!w)JXea&-9|iYkxpXfcfK; znW1(sDeS5M-#QtZ3a~tp&oj$0FZBJv+#-aL7ElEIjw1xO1_%$|AbN=Q4aI9mW@jk> zWVZ0PkNp=@MkOdm5igr!)UVe()3u$qf*PU{_w6Y}u!nT?e({~y>OjZGWBBV`1d>Ih zNB`lkmPS1>bUKR9LA*-~dT7DYN9Vf*(JTX`S%N1LPw92p%F^}oh6;4O50hha+qHh znlNOSKkMSXV#Y)5NT`yBp^tlD&}AT)IRP5}YAmsYs{}8K4uE3p7tqMcRCjQzLry={ zPd^2T3Ch*rY~O2SJ7!d?V|sq@OW6_JpsNo6^~yJHw|St@BR<^xXbfFgzz4~@_Wf5Z z^bgPaBj~zgo->3b7LEZ?o z36+J1wHk@Zr`}T^*G}p)L6_bd28_TDhY7E^#A(*!hx=AXm)nOSTo73&7Ug=#i;2iq zYS1nw=cCLIM&e4j4&klmT0;+C7<2ReQyc39UB*<5x2Q*f^VIH^o)V|cIoB7D2vexv ztmta3A!lj|lLQ_ZEH1J6%=7p|r7p++>^tr3e3ba>?rHJ$QR7UOJBEiT*edxsw731Z zegAXQP_AmFYdT-AlO608P81(6driPiz*SjNDN=1M2#G4Qlwg^Pvj>8m|8(>S8`|xq zuZ4}j;!P0^K;VF5Ufv6W^yz2^jJe1}{fjkssYCk&BPr<54#(8BWlI zk<7GgHp4_f8*J5uugAvhepK_`;4#MijCD)k zsE}_E=g9g-u~sSc{51bB3CgM^G9{}K5u=*Tl0Q^JHm-;RHR2P7F8P8JP)YM6Nl!t) zyD4cVLAUqyY^q6M`qcN(g;9*F`;GMo)w^xDUQRUZ$HAiD6LJRxj+*27sQ(;9<$Rs^ z?(ZvBV`FikX;nEP!;9Of_xK)xeQ9&~4g=|h;k;zN|MP67D>*{w`bTZi{r-=~bs&kw zW%@1d`CqO7U1Ik6A1}kL1FCOZcQ4h7HajP8$<Ra)Zep)hHpz zlfB&rR-26m9a7x7s|S`Z7U)rCgLcC4F(Jwvy1^J9yMc*fQDVWc32}Myo%TC_j=PI% z>{s`qctB?fhI6(~rx`D1sE1$EU#g+{D}4&bsgL22apC|BTPB+(xm3KyJj;iZaqUG{t@AN{SIvBJd}Ofj)V>VA<#$Zvmek}RQ5Sow3-2N z&N7|K5KEd{XM5;QRW#B(9Vct1#gJTox{c--ow}+mEkgyWy&MI8m#8z1)ITilU>xC4 z>h1-;MW|G$r)5W(MJ2NdN%ub$^b(Zd{M0ljQj~M+aM@RuP$eo+iHs*;;ZsMV&wH-I ze{tEO?MP$PV-R=v=Ym_~=67N>N`vXUCPGI&B&mMR7iT&POnX_PH-&kfwG7;1s}sy) zOZ(OkxY_Lsubf@y_Gg=OXIWTdiC35qfF;~qsa^tk&H17vZeBT1_D4GQ{?}$H>JkIB zns>#hmrLgL;NR#|PI`Kz&9~|Qw|V2cySw4G5!il~{lV%=yC5xMYAK_3!HAS0u=QVZ zP*o_vFgk9(N-i(VS1P+hz+>kKuF>BLhu4_FC+bGvmY~W3z2;hGkZjd(8?Ric`SHrk zz93^Cm!{$)T3vbUsW$Bhhe^_sxZP?b&vDNmUz2&^10A5_BWIWh;4qz{)Gc zafTXexDubaNvI!&YHwhOZT1Mtdogra`uIpc!P&!FNuSu2u ztL7ee@<>A?^|DZ~icYmrnYo;~;~+YEkG?6ht-!gvGt44;Z0ZqNXveiBENP$mggL+N z9WrELImg6A{8tpyz4u#aEh!z|?UF95$4=T|1Y2L5cotp0SM_G*h9-)?whMg|q-VoA zUBAj|eWYr)WyB$Jr3RH?4>rEP8G^QlPJpb|fmh6rUp# zbRD{!XgmsA=@q&G^|TutJMcyR2jU#Igj<6+nJ){k<8Lpa4}uA_nf!V-tq2%xJg=Z~8%yW#cLJ(a&xID9_VsiKz~G@2H~v$sZqN zXxI{nub$uYm%gBxQ)=*ECaO=_vSI z-ag_^9?KrU1w-|36_=!>lnz|ZDQf!c@5HdaDtx;t3Ytp0uePXH4nnQ|c_Pu_Pb;#S zqh>oR%Prn@d-fkQVtJvg1)*yRVixh=Fw^%jcR%lx@rP&E_r zY4?n*3||B#d`xmwXpwAb>9raC>TGamHEZffdOdLY&*4o*?{blv(qI@GBgAfCZuwV3 z0ON*J0l153E|VBV`U$MCxlRvSTBJ0Qs|#={6* zZG>yBgR{1lsGJOZNr6)Se-uZILABqXD8Xx0&05VG-&%uYxQsPUdz1%TLrN|Q70p?@ zC_=cRE_8g8&o*pDGbJJG6Skr4YrG29S8Wk56z-DrQ`Ru%bzOQK>sJ+(!kO9uAreBI z6oMEskP)^G^ud3?Qq)}L6Wo7NG?HOtfpe#36H{mY*)r20ZDEqYSlxs6VkzZqJCbcH zNe2lZRL_++X(4VIM!e{$tQNAcn6ZV5*@GV&sL(=;T5?b(sztNIYTqP6&IQm!90p+g zAQ}ECi9hs4{0{rLoqdYUlYA0-OM?@K{N3>L@~NSLn9VsdXHrMAbx7nK)6}+|;=kS; zUfNX7rt{G1PJ-_qs@KZn=|ZisJb0fJ4ENly`m{^g4a~gN?VE$4=%8gY0yfGj8Wmow zXtsYEip0y`V0@86lRNrVJ9;-tbQ59Z+-PP3ZPE8^)fB_&zh*-GMpTU8T0f;h~>_1z+xsTH|7wJ_t znv4=gx4R1NN-!RCI8r`fhrd8RnH;q>V}D4tM;Pt4K`F}I_cbGR@E}I8S1-XHp;OJ>eHBL9Ge_PfO-+L;F%&|{hf>RGn-t`5Lj|o2t!NZ&NE)#X z>s>7DavplT>WkjdB1=n;Eu|j{Ce1$^R4=5!N>J8adXCc@cfDc9+01jgpz=F> zfL#8b(;F5}ex4onTFFe^Zl@;BdP0H44kKwPQ0hXfpS_m?@GjaC$@g0eiI1&c3AR#x z2{xJukM)P&>TVa@@}JSh$LrjYHgqkhYfiTR<@!lUl%eyWrob@2sne83NVqJB%%NBI zPkL{QPKOlCC&iN-fh-%%-=RUBNetT5z*WBc91|Xv%?1agPWrAG^%Ns*m&tFJsFR!A9&VjdBo{aE|qD62+{tjIesxO`*V#|rB{_} zg{DUj460EMVA0hPGg2BeJH~~kpR!Vz)u*MY;*GCc`DhLQWB0$ldA2C@IC;b9NYof1 zuJX3Z$58fqG$;r<28u&+iaHhQ)O5q?m_cUOjGa8N`V3pu|J*y!E||seXKt%oUgJc) z!nSL`SHt4+XmusPEVgERZ?NY(u}j~Yo2 zjp+$UNBGVum1NA<#QeP%N+cG^DkHJ=5|BJ#7LDMtUVeO|_01)CNv=q@pIg=>IZRK7GR|6% zu#+Z)({dgZIWSTml)J(1dXtVnryp$kFYj-#A(T0Wm@E}ry4SPA*(Trb!+U?!qr^Zi ze<@#@X78CtdKsES1-#aip&k8s=3xLSxn2ltmexVL=uykBU8GYsOT9?)Ju&|2=(6Fq zJ~Y3 z_u-bhsuBTq=c;eyv_sZzMT&IXkcEm7i!vqVCun79J!=i14TU#6C_d^v#!`6xvH1ADIb-< zmG&;iy1Xvq(hM+3xwglSp?`Qx z#&~qSSkZwR(isHsx2n?aH2ma~zZpj{t!uFsfkou`fYtxPQQ?hcKEq4f)bh64ho`zn7D4QI0U!*`C9 zD&u&c*!1Q1Ru;Gebp|7eSYW6ju-b`O22ylfkQG4O6z_5-`Zn7P>jep})8E}88dVlk zO*@}_0idVBqM}%!s*@}7AKHLJ@k|fk_nA-97gwhJ7HnYCaeqs*I5D{t%swn^X7DU7 z?(vby%(jMGOVfYe`hB9TJmnonq44%r+EI#R5nT~QO`eJ~*-Dn53%zP~^PKe&<18O| z^EjwhdSY3DQ{qYGs}RkZN@HU2fokG66M-#0Edc#9GjIdnL*3l+($?e_pWKOml)HBJ zStkafC*3;vFXJAo6@XTLqI$Cs^>`Ji8R%8B#4SLob_6C?dX^4>BGXnbPQ*R;@!EU) zBR+_^zy?CoxMIfKF@Yp43nSTD;cPwkw`zs}oOInd343Hn{v3eTeQ;}P{9wgq-6 zPo6zu>u9tr%8FZmZ!?Nx37K1t;jRbOUq81uq<$ss1~|na)_mjEf0ZnABiGP!Rq(`? zA^JRi_1>9ochR?9o*yq>z01DNPU@7UqGfP?vFu`v+k%D&n~$4DQlSLHyA77h@DR>2 zD7dXYChvMWYg;UGmdyJShXG};4m0|!Cduf>eVFW?xSV`TD)*kwT`tf>;Ph{_(t1o( zk6=J#Qdlb@k(1dumxno8@sz+dEZoH7)FqEM7Ltns&;~Hb=rp(>Vej?(rqqb$o+;$vP@+XAGEvry0g1y+qP}nwr$(CZFkSMZQHhO+xD5iwa(gipB-@`?)~@1 zn_pzs7@3ik^;J}4Rb`DQFgR1EXc4c0yhyFWNTOn^vyj9w8SA@{?)FmMF@vs6+jr0X ziP8uk=GEdsL2HX*hN3TlL7j0^T2addDu$BtH>pN5;hN6sXckY3BFmfxqnf3^`?TEU z^9IC3D^LN!(euHVjY`5hHzOBH^{q`u6)D05{YQi%uR3TS!afidtvps0;ig!H-Yoc2rk{bd@VjQMSe2Vr}fd781hBr0_ByKG7vMu#f}Etx@sF$Br8 zaxt425`qBor`S~9_aNVqy6On{*bl#*1@!^-^7aWYL7MlG^vw~RH4auZxub2H*(2pg z<)!B&bSIBoI?z`-!fN0w44Ip6R_cmFF{-}yBDwLS1Y+a~i8JE3Vfns#ecg;pw;sE% z4%+*IB4J`vVoCd`@{-hjbR$M|Jk~4~R0H_Y^6)ftE3aRk6u9np!Y|8rDJeM501ntJP!t+1|)Fzn2JH5^mX1Bj|xn9kkMCLY(;IN{mtC1uKzadsNv%3kt1_~|IbwkSZ zC~U*GFyB}AykRk*1Vwf@I~}8W_q|~Xl!bkwJEU$dIy~24A8P^UM`xH)z(z1f2S>+d-CLMe=H;1Nzz9NfrF-}nY zhxWN{8lx2e10-jo#dxW4oK8t%dVKsmgNJ{Es=pzZJ*7V4ni0T@hGe!RZONV6b}@ap z@g6EP)b_c{Cfx2|=9}_ebKV9=Ba5_6e4o57GEAiCYI^P3P&i-pSRya8H^)BJ!g!BGv zP3%64EEl=570-d9vJqbud5b5M4*KM)e63-cZ+NjbF!mH`_sEFN3cjSMZp*cC>U&Sw zAc0T7!L8ldB3Hm6c96cJhE%KY5FkWj`gluI;oTI93Han8%e{8Mx25BL4_^RFOY|$U z<v;8>n5k9I~zbpwa9I9R9fko~X>zRGw>DOjB z{F0ug2xc-|BR{xZeKs8>8|O9-?4H{r`oBN7IJLH5;x&s}FQ$j%VN$i)R14@qSwBKu z%9wk@r|@XA5wRS!$62W<9378T&pNOXfSl?uo#nRjZzU$}P)BuQ8j}8k`&P zd5ti3WM8mgY@m^`%BmcmQ_?M{v;!+EpJtEqvF<8XCJ03%Hl9OYZc42`>$JK-C5zzA z?gFm$YG&-$+VK#?+k`8rCm_xUvWJ01D#0iUR!&xf!S%gICMAomM`3L!$T98PVN^Fg zTO1eUysx%vxrd!>%&$$>a+~FJ6m%<>#5|4HgAteFrxipWY-BxuSiEY;mxvhXod5C} zqYcsLA~~jzcf+;nd&{fmOZv8VqWjeY@RxZg@1Zu{+o_L6bUdv;!%2PA)+lsFzvOIn z aXZ9iE1z!(=*NT5;{A}Hh_&qa=?DIN@@6+iMNDO7wr(<}q-9KflQQg`an@ECeC zJoh3ZE>;Bcke`?*e_b+#H+Yd0+yT8xEe#qYL^f!Xyq8cc9r9Oe5G#5-+^q8$SjSk| zY$>Ly9yb2DB=OE}YUeCNID1|E-bJPz=%bvKso)g9>0y;Ez3o+yGy9M3IK zD=t`?L!z4HoTwT~F5dJT(n&6NBfp_TXV1tg|`%8?o3I(0aE)u zd~6%HdcOCe;rA=KU~?opL%KfEj6jD$znq93nLRo=sUX*IdIVPSVfM<%@b}ds1f6V# zN8-0i`9ZWUpSEWn4aN?7v=pk4LVw*ukgn?;K-}79{#hvbqn*V|;(n+{bI$xs+D_h_=8E3Pi^p*<@jH#99+-NdKXY9(vfX)UvK@7K;^d*9`ovs zG*lbbmCnvktjoA&loCB7ARJxLG*`XU*&%-e9ND0)?YlHWhwgO^)|tC4_;WYQY2NvF zEz5D*_%bfQxZt+j@MTZTb`z-5j0kCg6wF8vv0J@k0iV&Fxuh2(3_$DXE_0pUU{se= z;qK77T$SBgS2g04l^=Nk9k+kxd(^}?S;sVQAlJFP%p7ZcDd8nee+qb zsWnywtODL6xkwR}>;$@u$8`uFCu%xZcA3RD?1zgG=uYZ4m}>2{CC=XQdM)t3=OKRS zh&$GN+_L<{^)!={_T(?F9nP#I!#1Kkw{?A`3`3anw~@}CFGt|Gt{|5=cO*$*mdu_( zOQ`e9kD5chRT`Y-N{u6v^%~2UR#xN5ZQ?2PO}e2!&My&6$DYd=alz1r2OiB2^ zAk_i8#MyH$((<=EMje`?y;HURDXHToFqE*PBmcP6R@n$G(a8eld^z&RUTj7JF4C5w zL=2fXF&f}hc|4owbY`|-lpO(vNNb}1fa)oP)+I09ZfxK`QyRynNGvH7d+1tAd|N)4 z6_XZiEQ-r_nQQD<(GmLMajve(ImbucR_w|15uBMtJ!|NbO4Gk0(O(w@19tTu{AXCtetx_0X$+|4&jw4e88z`1UO8a4LVa&zWdhY1#E$dui6a ziG9O;(1mOTcm}2c*#fC1vUOYsUSr7~ErY%H26tD#| zmq&q#oJ`UXi)^@eoUN162uUl(_VFbAG}_!FnX@FJ&+I!_UF^SdMn zcMZC32Fx5ne3dnrDcWIB2Y(7Lt5r)ZvffUu5UvH%++M{8#$vLNc(m6?T1+xtBhyAT zMq&swxDgP2@CDqQ_^I>N9=}RpDM8*W3qc&%=iWR!$1DAWH0StjtVCFyjf?lKk&QCv ze;eHYPrtj54}d*c(D-f*gaS8Nh{!CHL@xR(XHY`i3P_nblduk99Ae-i5kxO*71$2e zAiob(be{dIO^8=SzbDdmZ2twBUUNJ`>oD7*pS{5$GoY`y!e5h2?*|hHpnI$F=M>vv zwd?}g8yBQfQJv_nHz1E_ZV_4@o?r982b$iH_!R4kBJCqhMsrrsJ$ha$R=K~&xE%X= zh=)jA#;zNlZ_;dY@M0=nBMCrp?lfL6mj5p~C<?|9QN^6L8fLFwo-M5cg4R|4+d} z>&dC6rs4-!fN#3J_joDq3w!erJ2MCm6vBjE24=mpD}1a&u$_}V?-a4<6HeM3CYEVQ zI2^+@1MDyY+&zHd41&xIQuaR|ty$y(*t#x4rNk6o$IN;Wx9AU~;@ty9A>i2NZ6o5b z`9d6A+Fs%41P^B!4lu^=yB~ZUOu{^|2yG zoi11xmg0(lT1mZh6AHT;f%~2wys?8b+24?m z6|+!JIdP*g%3KcPF3WE>OG6~6S8;O-7yzc$ed~FIf|t0jeYaiqmP3Oj`0UchQlrzL z;}=}7qPEJZmquKKM@x~KAcQ7pX&QJ?&|`}aBP0S;Gvk~T8yX&rq1$k9AZGDkU_&<& zlb9mhjY6Sj>=9@=a4=wGg-PM=A%wObx-nH0*PRVsB2D9nOGycn7|x~jKA)~yaWQ1| zm7UBtT}8>v@Fi+V=T>%lx(s5c-Psk$+e|nS#AC^ll$;0lwY9bN3bh;kqJSnjDM2t? zh%dW)+ZDT)GhkpR%-w&?8@+Y91RK^QHaBsxpumZYaX`cGy%L{vK#X^{aUp?d6!f)Y zNtLoAT$mGMI6n|EWK_Hx?Tn3-ig8d}pnrAwEth`_1NA3DMs_ekywt`m`;i+5GhAlB zeK!)FhCWLMtDpMS=6H6|$-v2l=dfL2AKPvWKjqlzZRsE&fGW`-|fGPyQd@e?VXV#&rFI;`&QH`wRd2H*D)4>wj2m|7r0*^M8;~|3JC^ zd;U-F|JeTd{nNugP^5pu!r0i@|I72A8jOE!|M>sY>YwtzTl~+y{{OK3@6r7;lK&d# z{{e#f7c=Ys1qh0jk&flRKv1mojO_mt1mz9srL@%avhJC~IKxG9gUUtuYP^b@;ACN*NEP?<48MjEmGMvne|HvqS4P0*jL!R2LiQ41J+NWDr zt4p~oCn4=)#}s7HXSsRD)%NRYTYX^Nl6TqnrfEgTGsl|*E(~5C5FwbuWD-L=bGf@Y zFpOHn`k`M^W4-a-Iu%j68x0~Rm~x}(_V3$Vai$%-dX}6Wmp)?eH*wCe>CS*rhO>ak z8m^yddCqOT_FiG9>$ssR>d0=1jvM61CwR9{10|xGX*0o}0ePJ6*Mw)vZVg2{^zj)4 zf?#-jw4VH95B z$6*bEpJ;2z)Rs`lGgq_N#CPk|rsi<;16-1H6e5ZD%tNiJ@CTTlvx@W_weL3S36X^6 zE_EKIao5El-=F)5gFu>kZ4m8_GFtF?4sxa}l8(~}+8(W={6uPbiB{cZqL0!*two^# zwnMXYAT8{__ygp9USdeqyHcA+J^@1$zeN61GR zAQhkuo%({*P3?8xY=rbAC%JyRTSfoNag&G^{9&bs8fXWg`154w4r~h&;5`n+!T2{8Cxjn7EL`1tV6ZQ7cK}r$*dFncqe_fNGZsg z7g)X>IaN_Tct~(H!Isf(9QGK?!wA0YoXH%ar`l6g_{4J z3!RJ3XQco(4*Aane2a{#$BoRoAZHaB@!trp7jBBcEQDAmeZEY~t~)ZoW=Mf&eD82S z@5s)J?zBMDzn>7a#r~ATJiqbwCi#+OvE}^hNc`R1cU>6>HNQ|%U+n?=q__htgcq}K z45pae?mtY23Fc^7GD}R*Gjr^Ve3E9w+Wgv$p#hIjUE(+MA68Jl$vrl6%pEN29|3P4 z@JxCXWho|q{-Cs<*68=!`b?PWzba$kbaQT^chh&$c!0csVL`T}q>5rlX?+?xyK*&5B2&iPQ_XfoJ%ktfd$B3djlH<`+$ zd8ll`n!QK8H<(VRdrxONVq%5(@SNAiVtBX;9>jUbyazK*EJa+OUw(f}!-O5IB%o6vJ(98dMziN063{gAgA=o}V8 zqZI#K0yE?f@4?Y;>qg=3|3sA9!qF!^Z<>Qoale6tcgdHo;V^C1p+Aed4=+iTCc{=` zXm*S8eJst7@9u{Oeska6%hqp8|7gq;bf|A&V4Po2P)tbF3J1TlmkHf6X)2A_jS^XQ zv&dj3tXY%0=Ta<28mlh>My4)NV#35Ix9BM|G@B|;f0mZUPCxlcE5?~yl(k?;RgpOo zCTzlpHR)8lnhUVHUV|RpQMxi;j4CkkNl)7`IzEV>nF8jk7}L_KLa$V>ptn$^xvn&? z7IEE@88MnUR1uvMJ9lA=cP213a$9d#A0{#}aTA$3+sd@MqZ=Q6R$4(*&$cl4$M0mO zj{N)AU|N(rIyhoRieOmX3(Nopaw&`C&nV@M9n*yv*XuKf&ZUg5&4N^U8u^|K=SHG1 zee9#Sqclxg(g{fxU=fam+E+wJDcc(RXbYW;C1NgTt)~-mSTdxV)vvC#$uPdp*swJ* zs}@H`p8}~}S6N?I4$Wfj4YsReh#ai2vY=ghK$VT9jj9VDubh{~Gf$yfgls7usc*XL zGp3MqEaaxC9)r|3JZ<0pGF~yuYQW@b6A_LnsVnjjbER;e?yM)3T&eF~Q(&s!NJg@^ zauqOOLPYhoD1I`^NMl?wuC_TApJ6bMH>CjVpGC2#E$j`u5tYGhZQ>(2PGk}AdukVzt17Bg;j9;@Mz1ilORU-ZwoOC0fozU1JaCC1DYX;tgGQeS5+*sW#j1h_UTWvB|#|Z3q zk$PXF#=sI6aRLj6gX0eYiCI$?AVdgnJei^wbfkuP=Jkh(`#$UVPAyHfVy_Ny?8;R3 z#@gAdh$b~m?)0=qtjfmP<{AQ&E7h$TjjSncPtMNHn#zKg0e|3LH#ED*)v$<^0X-*n z-*{R1#()i1ho&j(bo$2ZMSr;G!DYxVQ!fWcWy2S{M&`$T_ri=Ot>tkE^;skTbF_V~ zUh85X8QPdTm-)m;9Kye!SSlHUgldG@!HARVpZoc$BZ8<^T@O!fL{N zZMdc8ydh%E_pphIT&t)!E+&${rxY{RB=@pf(1;n<6iUZYXzYOF5;YDAtUUXnAa_dM zb z&fbo`&|V5^;dW#!5Xi&kJZnt*I_XVByEg4)m^()epyU0Lq*1iCxsNPvngv!(4+O=C zCyCTo`lUd2bOVg1K=$@7?EhPFK7QjB1Bi^UR{K-$5w{Oi@jjVuy7T7a`)Q83Vhi>Eim#_4}UeGyW%Vl+5C!D zpaW&TE`h>5k|UI40_+yDmQs|pB*XG5Fe0I0b$Bu|s^KWoNGU~P`9VeSLTNRiAsg$Q zS?QCVRYO;a4;!e~`3Tssi{NC1yd{7tDA#>}z20$-$+XGqx)D^{@+iax`l&U$NK6(6 zw>O1`0!g6LQU>wGF#M#L=Ey-giNH$alC*mtdsrHO8+%vTZ8+B-9{Y~)c4?Q0JyKx- zO9fdk0ej!S58wnU@NI?xH?3g0%Sden=#`Jd0q+CZy*feKm`!F~30A_DF9s!?4T?DQ zl}Phr2R#}qL#NH^BGw0w8o@h}-O*Ld0oWQ>CZTA_u94LV5M%7G0x8ig8ayw-=j$i= z93sk6)(}(WwCwq)L^EkcbK1>r#zv`h+GUZ_717rc(+B~q0fH_7;^EaXek~j&V8IFQ zf}Q=z+i&Dc-zaq|paNQpRxLbv(?28nX84KLvhe{kCig{7rWHA=A!OK`ffZ!|{y`-N zr3k%knOpPv_FRv^I{A1-JOtfk#z|7yWzI{bz>Cf81gl4Npfb|Mdr^0d1NFqy`_T4( zbz)cl7V2f?NVoSaNQ=reBVq4qou9xxiJgOUy5k((i;KPU=>%?l;~$k8_1Tu05#R|> zv?MspF7-;dNyVSWJ@PaMG?K;RPuKfhsc;FSR_MmY7q9ST_TuwRw!u8odtc zaUarw_>#vv-16|;E9dG^6u2{hv1%%Jo=>Qtz_5ctmYih;ak;YD<%PIYXqs}a({b_=Qr^)_IyW`iG#1wN1aIq`x<;_RVOaV$y%1$eTE!Huq z#W>HxC@Q8+s>w_5i2Ui14sd*G$m|B)3giWigP8G3ajU$G>tDlSx!yoQ&DRJpXF{+(Ij8DJoR1!6f|u@Z#sWXZ#{@J7CDafe_sjw`XGL)CUkaL! z6Xt;^4=cnC_r$lD5$1vS*T@Rb0r!vE!)N{fCFvlTzp^3t5Hd zhk>f@tM?gb1J5GtHDNj_v44PLw7 zDFeK=$!_&8$-y1mB;S|$TT8V-=oyXeK9N5T(B4MlOHPAHs){Y-zCxT*?#SAfli8M^rS~OfW zVKtVvmH{Yb8z*b6nl78M8cREC%Rq2|EpJ=>R4n zJ5nL;kRW|?bY~?xqj+bOdT@yPtA#f1CKL?h-jt6EV^nvyy0M`viPl#5625YiW^@bS zMbpfOi~t9!^@X1c&MuXY3HV-=KNEG-cTO99W*WbRAg`go0p*zbXwuIUW^o#(Rg~`) z@XcMJ0}jZ2jw671Rx@s!%)-rxNVb<7-axHao21D)i54|my>ygG7Fd$pLvMDZtCC;8 zeZhOtyW@kfmp8}|9;e2FA9z2Gb@ccY=H)v-N3d-kyVOa{n+vzSU_S01{srd7<3;;4 z(*32qrB$@emIh*7&KvPk#})?OPB(I8O4}o%W8liQ1-0u)=bC3?Yd`zU=02ZiH~VCp zcB)Whx!3i|d8J^W>`yT3F3$oclXRuC?%DFz+ckrA&m*mL*rTRKqppWv1YtpJkS}Bx zKl;H*5{P<&j@j2>`|(>TZ6c^yncwh6X{bQAH0R`YMo%qjhbH7b^&VQV8v=nzn+ zoKGL-rTv6{0^b4#a+B>LV~U=LzM zB@yUj&PuEu<9~K5ATpqE>{S%lySWo(gEuOfg@GF`Y~>cNIL^>drmHO?u!xab(DCs8Z=8;3Ktac$Avn2OTos~Q> zjd7@koV+uyMV>)J+#odU?Jd|6U5-II6FG`=?SvPS;pl)_F|Tc% zXvFs-GdZP|wNL`*>6h6pLO88xJQ!%KC~quZWYy1;C=Gr;F>0oGd>-D$?tSTRmI%2d zo=w5Ar;#AxJlxuF5w)rAh-p-c<}uMyv*sXf;z+VYRW^*9v>`owq9m>HUA6da-Dofr zam1j$rzxXkiZor$IGh75S*ZfGq_q{+E>C(MM0e4C^_;HPiW|*?*0+9S$8ciE<0j|2 zTQxD>L8Yxiql>)mSs#i*%k4+{ib1!1@OtDwcYMLv(`ougwY#VBB?<>M-c381AudZZ z3f@BkAbB1$-a`SNA)_-Rf+g9-y(6Msg1U6811R|#`(7RctxHg@!}bka6mMf2%yfW? zeVf}KUvSF5!)W;LVsdKJ;+}6?(*{Ht-m}%CIi1vut(F|MII*vLGg7ti(Njv9UKJsJ zlLH7uq%$G~jx;anD@r5P`5d;QB_0fe`!6P;v4IYh;2n;>JmS3;ql~0apv+T2Lad@+ z++sb3z)}S`&&0;Fs8o^Bd1Ybqyn7HW86amPZc{nZ8R`t%Tk~gF>ElSJuV#cDl${Kp zJX!oLbIrE}m+QAOjXMIe3#hfbJGX;WMQ5|Tp&Do2cPz{yz7i}NqB5NuQ$}Zt0zE?~ zj;nT@Tl_snyb!+r@``5~kE$}sO6WdRUYQW)1@+?G2EIWyMc$JLI%qlp8S`}Khkpb3 z4fy7ASrmGn@vj~levKoI7e$_Wv?m&em&AY2Z;9fgjARTNJ>iL@OE{h?v}badlXA}_ zsCRh?eGz#nziJB^LLm1R6F>4`juhWrNoct#yfSmddYq|Gs`|VqKO}uDJqRH0bqvHF zP=CrF>wT;lTGV+w>@x3p_QNV-U7`cUN{z5Vo^p6ZWf%P9^4>G=F*605=pUA>$`#jf zxqY^9Mb;cw3{5aWG*|~TQJPVrtHtxIi-PWE_!a?pWE9n7xu9tF+8FyD+IzwC5?v#B z4pKrW`T+QPjzZ!#Ex4>p(#}RYg_WJ|%$%;f3k0PHJpJqMlAQXyePn4<>@Zip2j|sV;j=2_*0v_&;>%*#=!zjJrtAAo+z8ab}P(vLg15OxG|4`v(C*1^(AGVl((3h6}4r`}KOmRP=BeZoZf zi1{4!=^pf);NH&!Q-p-y8Kkv}q7jh6M7koSCX_@e43jY?wMn2+DpqEfO)KEaxt7IN zg%JNKS;>T`BY4vg^!4uhXtJ0lPbHIAX z{rY%!@@o4A?}LW3s>cHHkt9>>>egLthhPMnQpS zKA$q=^E%+F!2Xh4!*EaFn-m`iNw4Ms;0W9Yn<$bC<(P^Mj^+WYj=YUpS zBhRER|FpmpgUY28xv!qVWD*apErL0<>v6yv^^OSlXnL>a2tkDh$ z{{@o2=8sNQn0gBIj@Y1YU2jH|+`A24+vQJl9KHWb+GK4pgj>_kv6F%$@#yqO!|ld& zbq2JSm8XxLKGsKE6wMU<&zwM%tst-S?Sq%uO)rGbh=*JQScwb1cbGeAEw~#!t~iIs zdX!=XE=!lpP49;OLUlYFYWW)0;}a=IDvEy4ozhHu9iV3!6scvC8^AZtw3gCPNZp5@ zDY@@)w|GJ|$(<(zx;X)41yKjGovKD7;ZoId^6-)iPeqtfH8*Kme8s z@gUiRiM*iJrK!8oNMD+G8qZ4Iec%0A^~`t)bc!PIef5FZ@T+5@Lt>}c?Xd6&^PI1i z>KWyHlFu8fUO?E<%k8Afn2hSvUESf~Q2R{Yx%zd2&{CCybG3DVuwA_-RtsAiBPF5c z>H}3XS4F~z5KhbE9j9GeyI}0W{ofUsh*T4MPz;)~*PI=ZOWHto%SeS~xQcNcDNvU7t&}yhD3_9DAj<;OGhzNt-=cya7&D~fq#fBsI4Q9D z;f++!VDj3gNK^IryW2=@K8((~iVFgz18XOpnaB&Gh>25^#Y4PPsU_hL92N%E7`J$0 zkYnGaNb)m?z3+ri55y?C-mp5~K&J8A910$>=>65H5RpQ?TXr`SMwpghL$8~=S|_k% zY2@GN%h@#SouYV!Ww1XpGbyT(?i6r-7Bk%rZ7KGUt_6EwRMsSyZ{fFeG*k@8`3#a- z%ae#%7)_)Tu&!@v`NK{~!wYbuy&I40Zc_@% zK7?0R-oBNrHOj@VyLKj81_C+2+Ju~-_CAmo>Z9V${e`g_MIu&{d5N+YlRa-Tbrk6t zRkL!hWw8)Oe=N9^+%+~T&CuC8L`X|MG3>LyS*O5ov0XsiyN=Ycy}j$XrVIkViEyv* zO5iHS7zrq2mvn=}g*d(`5M$6c>A{WFvcU zBF#YH_?F&+o-AiE-xp<3wN0opOZMEmH`$(cQa{kT&8GJFPUQnl>|1NOb3Ddb@V-o* zJ6@ja*`7jpYArmLy9eJ-?v0PpQAjHKA#LK$}|N`^$HpkN3Q;gvBGk& zb5}44C&vy;#mlHLH9{e8{>_cwMwDmcLy!?aityApnJ|b)IU7oh$}Y4&0`7Qf$j~wV z4683Vf8S9_HFw_pUIqB~Za6!0Y0kONLdumH+p<4jPa1^xftlKKg0yIoDf&(k(bB@g zpj`OQEa@;`i<*#IFqkk@JwfZtFmi+vc~TqMoZW$M&DYfbg@6q@)FqUPaVGl;g43c0`*hZrumb%x4m$5+E zJX#Twah##JkhPdrMs0OS@D}Z5i83m&(*}3wGmIUvX5kb-jH?bkP1$)W(=eRqbSNcLw>b z)X`e{!p)(~Q1xEi!$fXxCRWMxm{$M2IoY4slD>PRwhFx92u9N=Un>P0jp{L8cu`Q& zd*MJ{#gA?<>mP$#74r?i!5e_Vnh?f(mwt2SlBv4t#;T}}?Piwi_oyaz7s(J}Qg%Z2 z61+0J2E2}4rVXSM_i?Kat&`w!&qMajDswW}bT)vy;HDF4iwdNdrUC5FN*xbGOf{muJ?ij7i6DCbZ0s zZ+e%$3EKwYhm!6uud^Xl+z-Rkidj`f$#*(g_jMG5;4G(_>n6%_LA#ED3ZfenDX{ud zij;Ya2KKSyb*2e?^bW5;9;w)xSlbv>>pSc_t;m-V*P)jY%DYW;^-CYD1BbSQ;mTJN zFkP2Qat}=gTn@3r^bF9lUT}^^DIpjI@@^inDK|EF1XA7%b$}WZ;5@>>kEQOuz|Q*# z4WFr9j7f9CenZ$}A~SQ^vQ>*AV&d{m2b>!6r-;QU1O?|Nohvy|rOo#AC6*^d)4ZqtCNk?L0Wriag8I5n|h97wi7u|YxRbt!&sk}Rbp|qn?#r1k)~=o-K^Y#I5r2bFex=}+o#@7K*J15 zQ$2cnQe1i&FEppE7YndWOT>&Rz#d2s)`w2(l`O=gsKATlEMVMg>C}dS zIZ==}(GkJAqQWVgaB1;@pGI;<_!PH~lVpT+)r4t0M;0rICc`9*EUIl+;`j^qqg{va z9mi3${EJ+;wMPd&1hiyKZTvIi4Tl@7N8+eEAb@9%!_0GK%gwmZ==IrWxptUF`&vZr87>d$O zAW>Srp-L!(#p@vc3RfSG>>V{7|F-DO#<-!Dl|Ua7#==X0Ni2Qt^^e$NjdKN+=-HZ#RBq%UbVtaW53?WPpMbBb;cx(`V!)%_)?&}isStlHl1Xl`D zr5G6f!bG7gd@};zl+_}mCY)Fk9ULhN+?PMAe6=0xh}mjYtUnt)4TQVNrk_m`gNgaP(#|ZBhmDdZrgPOF)L6W)r2e&e!mJ zDrIui2`JRX0!AYq7~~a3O%xnl$rbawY&ab#Z2*}&hSlTLKj6Asf1U3&g^n`JfO()C zY5+k!p&T7h8L&mak_untj}ieb(nf6+CMD=;!cBoun0~j(kyOJ`1{bkm*)g+ElV7l5 zg+5ZunEo1siDB3=pl1=kU?D%DlD3pT?WRZ`%Bp>F0coc2hN~S&0q6i0Ke=8SOKZ(l$3kR9YIzK^pTBog;g5t<;%V! zd67U9hatCm*QVA&pBJdgBhH(aQw@U*l?fFajlUFFh3T!Dzm<2zCDVH92R^aNt<3bV zhl&ByzWP;|4S*%nRRc~8uTu5^><1jf###Sv`oN0S8~qC7o$MpUn_ttfoWuRJn8U;6 z_4?cWVDOpvLs9Ta_#Ng8-<$r|II!-8x3~AnjJKfuA<79F0-vBhAiK;G zygD^MzLk3@BS;_IjcTo@*G+3glkom z7&G-;vEkP3UgW-34>@wbk7P^13j)qr80Qk#l32}awKGi=9m=YvD%VO?RwOVjF)DDB zQha6vvbJegNCX=pwh7wZ)CbA{$_SO9n?qJM*54~3xVx^Snpv@Tp`ElIJNF!xkS1&! zZ!TvjjeuAPII-0ND}*f9(IkoXw6SP0L=XU(#2sU=f#KnE{segl4V;XHn6k^4lbHE` z#E?J}3f*nlU5<**7@CT-C~c{A6}Hv4m8XLzWK$FG_l;7KH>!q~B`5{K5B=+JEFedp zjA-##7ww8kk4~5PZKY<6hZ*ERbJ<=Gds!dw7Kv77#NeDahyx!y4)YodS>h|7G|FvG zJM{}qI6KX2z8$|*^qvjv5p|Rm%Wkz;tp?j7vicaLX1?7j7@ z!w|d5&6+kyHf-vv>a3f;22JXMkt@o{R63RlUOS0xLL)SnXp&1QAVfx!8ZS9&B1-O;l+j4D6{HM-38WOFprCkekq%m}kdghFlOh)Afs8fD}2f z^n$OZo$QRF0m(gC2rl?Ld~Cv-z=aQGPDj4OFnQkTZeMzRr&dv-;>`L!F3*#zwk-Ah z9QVJ68}7^I)MI76UTpZhyX?g^$)4uyhLG0BZ)Z&gy+%!pOx0_YmSC;Cpnt&5egnKR zU)2*%U(e>vqk`7*rE~vH--}!mo%5J&N8`~&HyS$;>J9Ts8;fQv&Asc#>A2>!S-g7c zX4YnwgXasR`rBZcI28iVSro zW$NZpF;Ub?$3rS)Ekdkht|GX}E|i}6HhvBes7D3>@V;Y6H}*$hQ63Zn!)#z@MytxgYFmlQ2}M zU%Jn4wX%*M4BaL?X<#q`><6o$Sk36ZXH1}Ubhl5P%of>P2jqp&V;FNu;4uwy#VHC9 zJMkQel37d>MP-R}{!*}pepO_n1$azcBK(Q!-{w6XG!Xm#fX_N>H~F45<}RvP2rf(F z!y_`W-qmZBc2Z@DpN|LCA1X1d8KD5#dGm5bBpN^tSQSxGEJHZSIFS-$saK?H#5y(R z_ld){(k8dU48x8%ODD$AQ~7rx_H0<2O*RKzsnIi7Z<-hFOS(KE%9jUEuA7vq+Z@dX zy>Oj3uB=;Iq1g*Rl|8|{RGqG05#niez}5T%#Z|9w38?Geca}v5>FH+L#fAA-A%?D1 z@4FmPWj@fqZ0!fb#AucbkPV16&CTIc!?N~C;!kRWlWL*%;_=?S_+6TSHQVp_r1xLx zs)d~VnYNr6no;V5oD!sihyn6m#y(4>((jP`<>_19Y`LvOrYyPAUQnJun0~+L2&S=p zi}eJ2Ha)ay<-ZtK8QX>=7 zv{?&jNdR5FO3`MLpmIb81P-X`XP5+yOZ|J%u%W`3k`C2j2nwQC$a^5UqsuRlVoj?U z!>X0v#nO|Z5*@>8kI8`P`ip6>BK?5(3@~1^p|YH}C{BC}V+ZgB)xMs1j;m1&$q>+U+a9{FoE~-XBJ`Q2lfGUALsj zKd)J%U8c8SpWJk_f}TI-?C(>Mm&SZBQ?4}BiO}E7QXh4R)Nv$rjKt6269u1$`-Jx& zuh7{Yg4;2yjGR;(uh{!0?yx>V@PVrXsr$ckh8A8q|8P>fMi_v=9*FZnG$1Pwa_XcL zSLxrRTpPQJYhPks;u067U=wW3@Q`&4b`x|B-y^`YrhZ6t7#mp@oa4%^Z41Q)dVAw2k$pB+);xQ`v^aqM7puX{LyDni$Wbp zsTO&Y8s5mWqaq3=+XXxYlEo*!5`#_y{PD9}89$84aeQ2{E zEJSch94lM3p48rI!V5Mo`%s^Maqpv$rk;<1^?rBnSD$F0D1;6~x0c=!+t9%g;5L!! z@fb)-eAA))HQoz>sW;6-bzso`lJYcwgq9@q_+XP6xfNjEhm=5CBh=z-~-8MfXZPnL) zfa!8SOmbX;QUs^K7}r$5x>Jog{;#oKKc%J<(bYA<{k zz1NyAgsnXNKz!z${y6~-PbY+c%U|8)sF9?ls!8AO*IH~pS7^Rr{$aJP`a`uJUh!v} zCA3K0ghYdgTWON4&KA#DmdD?YIu_i8psk8c|PK+ds0$x~C=gk0!>h@i#d# zTJ@)4g1kPV8hK^YkKn0Lz;|`5=0CYlM?El>R<+MJl0MM z*KA4JT%+Z!EtvsPQN~o`llKYc>6!hR_}1icljw=h+wAyfCtUXj!cZ`ni)MX>MjQp8 zgTuXZxe(6DxUDkLIc=pLS7|qsoR;E*nouw;(z%oj1#_5LW7r)V1R*rvqBKIPNu*)q z*95#E!Skq<#5YChx>b2QqJ^wbM97KK?rA~n&jM`2CzFH%>?L$3Ve`Y4BA1jh8ZBC! zu=zbMNUH@1)OQ7C{llZpNH>cE{db)A=l7OvzM6{s>tv48#qn;J^+|R0OwdTve*cL9 zd!LcrU5xj;xPIOZuyCU3o{`bmj#1eAo+D z13+~)C=D;ayX$!0}4?P5GU>$+xI*rw;C0Eg`v>EX#BY>*d^{#4L&{r7W5| zuEzBr6rWBi%h!&%HLsDOq`?Ob=$EWVy#88}bdC6Kw>jt3)UKH{;1ozNZ-#`KH!r`P zc}_fOGj0(+kp(!hn&0QXAkisD$!FPxdrHP9IRE{fiN4GUZRq)X&r*;0UWmCs7=ahI zo}#L>zQ_hb+Z=dw4Y+|!RRGy}n7@0}v+EyjySbZ75)t_B%7gpGdqu8gp|KT~gxzk3 zd1;G}t2cdx$21{d$fzf)p z`T9}OC7|&0vTzyz5Mga9`;_$p+F78Ac9B$bZvyT)5?Xkp@kB32d`uTSP0GZJUfy{b z@g|>c-s#zYzUaV)P9xBGV)i*17f{Au!P8sYmz;^3`kW$z5bft$X zHpQu7`UPlqnvJP!AqkW^t>mvDx743JX;K{;ysdpY8TdTKo3fsKKJC4oSKVH!ck6AO zOOFs_=ZCgXy4(lK;;lWACfi?`YEH)ZJRJ4C5IrvaK9AwP+r@uxJS9N<BA!we~Ff#)9r1v4~8@lnIlG9vPYKjH;jhx81*r1 zcyMumPFS*TiLx5KW6Lji+s#NH>&hTKj?7RFo*v0`+f9Y=$CH#G2GKPv4&u#w`%(H$t!;%%L+44^2|DP$M4e>f327=n$$#lu?G(eXIVKD4h*}|dX1`;YruE1ouCykd zt)B&+$&A6Cud5HB(o(GWX{Q4$~R@?=)nfIylxpc|kQhItD&r!NMV@gC|IjPSv#MLsoKmkUKR~u* z_|AL8a+$5GK4>h5G2&2GkX!nrOeJ|M4JDcP_DO-9xnh&icm?eT5}dF;NO}!i;Za}O zaVCHT2Zt4cmYcc~YspxfNLQfHfeW9IN~d$H#X_R>)eQ!>HX&m4)s;vhhY<^Fy&_3q z1B9${i&!zUxuB#IRv@3|`^mD$iJS6Lj49*0C4CZ&&PH~OOyl`S?CKepQ;ZV)XL%PM z6H2|@MjDguO?x*3B{^5)3WPuBS-!!VG`9LCP0yk_OS11ryOHzWU{Q}8vJ^Yq^eJ_* zSE%YARPS1AQf1(EofL;i>f40GC;X(rEX<*_!2Ds*%lD3&(fU|Np;#i5=B>Q@utA;n zUci5Ec(GEBGJCf*4lHencf@UL^0Y)9nTByb`}S+~Fq?uJamfa~-;4u0SmYH09>z44 zmCRO=y@DDjGDDlSGYl^-ge~^vs;_3CcB`*r~ck}*i4KA z(I%t$&)A;nI`wxYFNO6b97R2HYS(vy7DEFD3J-^k*kqR1djr@*r+D zX4hx#@Th+82;PkS?C%8pGHY5Rw%Ymw896sCuq-NESpQ377(x>!88!BsCa8Brb}I1{ z==uAGtd(mcu!-$sLuxxurlYCWG|is8|JEQ1Ny<|j)#cgxvuoyz?(?bli>m9+)gR%A zW-IkZGpewSs5Y+ql&-gqkG}LSZr+uK63fGE?#6D{%aZlG*B^D0hGi7^5Pqe-OU4x{ zj}v_WF4B}<_6RS)R!NbNJ z>tl1%;?*y%`EaDyjctFd*TA4 z`IkQXLV0tgVKivTaC@<owHU1TJKV))Yc(d*N$;2Y0WOc57wskg-r@1g z`|0kC)DqwN(`|20{5KBaqfR5+f+U*;y_uh6nk&?7>Z#C+u&duzkuj*km|KKd zHC`(VdpTz$D%XwDREBQp`?31Oi^QBllXj5f&;l#8JG6|NEt@QBPFXNc9dbcUVBr9T zD6aYeAd^3dS`qIdSZSq>8rpi$ND(3Qd)}%mSd;QVST1j+i)*@Ol;aiz`A`UlD!YJ2 zILI#Y9XX1zuL0;5kvKONrWoSu>y=8Q* zIxIWdY1)dk_7+H!TC?S&mvq9tu(?Vyqa2J!m5j6Ng*OgXaJdckMF$$)3=MqlRt7j1 zIi0QV9$M8hFb7#ib`!%Mihu9R^I*%8T2!$hrQ?@|-DVouha+{zH;NR@VB*JEfOMH^ zS65%HX}KsqdrYO@;AYe8%ZlbiQ)IanT-tns?8hWs;`BTd>d+vMxi$6eNY!EZf)|#ufiY&Aiu+SoJoo3-S(v6Ui@}uTm2? zw_}Z1E>>tG2@RCC!BazdD*)>HOgJUarAXV9n?yQh9V?Bi51)HSfe7C{@Bx^eS%S&6 z$@B&mNY9*)9vs!=+X}Ub&3~1hMch}1lTh?=>|v_~KBTCv!>1X+R4+wz^9`E#fw~&b zb!|QlnD$IhGKfmHp=%sa>-bjZ#_E|)(g>piXLBz7%G_Eb8<_FZn7$GjGi2heA&zCW z)#ajFhXiiytr&>?MCXav0{R+2yy;HFVEi_XR@?^P{>Q28D-$Kk5ohEuhIrc>zqh*X zQe~ERVOf&Y2XS~hB`BIK*EltE?oK)={&-8&3b`5csoz2xr$&jzvwIk6jWf;%-<~B# zdGGgQR?-LytQKBelhe%i7&?)i)VSX$x$KlL_ubFrHncZxrT6zu%d$ThbM>?rJ{c`5 z3P#h9IkCdyIrrzv={&41-%i2sG!)M6Yqp=Ty!7w>NFFWvV(j)tcMK!U|waYB;yfLoGF^T!Td)V#uTY5GqU|;Hx!v`KDPZcihrF7bC zXA-MId@+%Igl2f(NZ%pT_X1&PQ-Lg^Wg|S$YA1+izgbDY*K_|^yr8^!YcuPqpXdcV zb_$_)RLTU!FV@AP>v-~cvI<>f@0%8PnJa68$st7_&f3(-s1C8ak}@(!OZt@?B1R4L5np>bZ;zlA|yvK0XQcTW$jV1eGD^NEb&U}FhYXVz;-oJ!{PMqT4o}u zL5ZS#iprf^FkFQ|<{7mo#j+9VvV>)D7s9bxQE?K!d6@~DDt>lY(x6`8)0+0114^!!`Co z5XbjE4t9yITYL&eqw~;=)+|Qt+8q^%zG)<@&qPlkZ=}8&b&y`M_l`wcG}wfZz3Oiu zUMO@VcH#JuAve-)-@CPWhO>4Dky(3FB(Ypdnd7qnojuUIMHE&l?a7_|8qul}8Ov2~ z^2gE-AjiK7>8TNj7sh@rJ5Wrpib+=<>4VJ5nFly!HD+*+cM)b83*&(w`i;qo-eiIJ z1(-EU->PL{jZ6H92$hEv(LtR2PKlFx%!K})HGQ_>_ETJI9PL0hCz+wjf`^@kjCKUy z;rWfAO`kUfokB1<*c~NFdlV{I9@8VW(AZK?EyrH|x1Tm*Zpe{it7y^zF{ z7lD~*lJW(w$Ftm0H~KGnDc+^$@%!mFZm|mn&3Xv&JM=X20D~e&0l=?Uo8YwL3w+FW z`k;wDd~XTdVlJB5B0kKTjx^{8Wi60w6;>*#<;al=GACzN@0Ku@>>2Slvw}SE6a{C^ z1~_$Kfo7=ipaC($v>h962sOM!VBB1SS&Elff#7>gCK)-V|Fs4asaQ!xk>F zF=gyqwk=78h`KIr>@vBJ6j!*;3M8ritp9eD7N?9+T~@ss{yCti;m2QyB89U8+IVOS zFu9=>@l?OfVk6oDimxhRP{mgCM60l?D$EJ)Ain#KweAZ3EEiEsyrALSI7+;R#is4x z+>L(ggZz`DTmEH$_S#Xa*RL0?l0}4l4Bw6SCL!+N^`~m)IF_WlnW$qY865Fm^@t!= zxVDA4?J*AJR@Q;+STSM0s2Ncpib?e^>+#IMmJ_x$>X7#^vnCJe-uy5oiwu3{=pE!= zXm^qPABLg&=)g(a+YNUVFx7UM{6i~JFF*HHxZ#@{LAk~)m!>{G7i)9gU*X35A{O=T zbiEK}V|I!o79l?eS-!oTlur9jFtrX`>uT-t>GJGi8h@Rw%}BW-vi#gY$zjk6h|Bq! zN`a$9P-dK%aVA6~W*8f>J1eytwqBAc6syzg-XH`|Qr6TsbMIH4o)}c}xua6Sd9Q`FFtzl^uX z(_n1%SMuYlyry%78sU`Rt{2+FRdqd+u>XXILl0ZlaOyb!;a|9>tkQX5SlNbdaC-Y= z^_eyFIfF->JCuKSX+U;c#HIUkWBlp>;zZ6`kBo^CD z-;BJRV12i|LhMoHvl&};&tn#CNHh01`-FLduswvygp!|zunTg_6bB^&o{%MNJE%Go z57d*jZ!QU&bqK_IcA;NdrjMrFsZh^*GFMnU{>8&v68O+)IHJC&Z$qmnb<_sy5(IF75v(|JeQdl z;W9J(kGr2)jt4wWw|jnEXPHjBt*N$G$^~~Z-ho=pquYkafkR5~n-5)}T~#OJkC~n| zg;XPd3?y}oUq>VMgqw}2nj1vJY3RaQe!F~6NEK?{QwuY6C*~m4hw_9BiY@dHV##W5 zoJ&sbyOIz)wuLq%rwL}X#+NKf1V8iswLd5O z*89_zW32O#v!?YlUH^Ff<`f74=59QtL$!3YehgFRHxj0@j0?OW1cE{Ax&gy-a9<#h zl>Qfw*2P|3HXUKkWIbwf+Ju(9Gj@jRMA@-Hmc977dH5GWl*i2m46u3Yr1%3yJXzZAX5Zz3VovVjG@eLwP6&7RHidl~Q z2Q$B<`#o2R=`{Tj-l@NP>0X&EA}0b?ZTU=6$+4yQ(!zy>KJ4y>{`p z0xRv+)>GX525hJu*BeU;$w~G_uC45hUSX;y7o{>$t>L^P?4+S9sU%SXY4qOslfWzN zv_z>hQ+)=_x2c_~(Q9r&d+=J-UL?-;Rl2KK&CSRf%~SfS*hc^PfzFEkwQy%|3L8RI zP1?DOs+)tvp`RCMjXFT58?|*rmsqbu&L4`Ga|?D+ zPZ>3Fw1$kwF><^dY{nAylqVYC@Vy6kVhXi&hx0_>UT|6teeh(j)C%~2g=5TPCumeb z=~wcn&h+phm`m-+CeD}DC^A_%r$f_kWq$83!&I3_Ak(cZon2N>B>{$aY!FdBudQ&g zO#ORZ+$|+{G#3I>VkgQ zlbVRQW7twi1E4H|);#unGhV(~eKislLy=j6y+N%RI(XuCz2FpR4ysjzpe4f**zhQ9 zXY`?U*Zdvh#`6&1H=|IpyPJa*^=Ffk&6_+a0PI#+Y}hyF2-KAXvI@6--Tq z$_cX`Y^ARsJaRH`=zRf2`p_y-plqgQ45i*+qBmb#j<@qld15gj5E9liAQ8ggok&}a z9y*hm6s6+8I3%P49iF59el--OyRvZ7xkFVUxqhlW>91^DRPkqV5usJGElISR^LFf& z72bK{dN!)%mz(D48kfK3izL-c^&Q;iRzuR`IHF|-o=Nxa{7RqARO10DTmyeITZWl! zeBE+s9ZMX1bb30R{lJnM{0jy>D~XmKivf^$BWrC7m}N&|mHqBD-7vziZ^*4Z8D3<} zPRfqV;79{+55izm(6FN|1Cx`jP||AY0zfkhD&kbGQd}^(7H_WDGOvR~ZPrKcLbyJ_ znwh_{M9{6i)u=`vC-O;0rgybGYMv?R3#c^ z6W?4lcEKG-Bm8AOc?Y>(d8T@N5H*p97J&c0#G!`81(p5ll@f6vK;U9+O@lVngU-O_2u^0-mznp z5TP^^NmaP{1zWq6r0OGAP6*A>T_%o=na&hz%SX8H-p8Y-K35Ymx{_3#2xqeoJZ3(q`&PP8F4G zMTi$ zUXh_ZK8aliLl`-el#nXE+19dBMM>3M*W9}v9Uma;SDJTRuY%RAUTRIZ7tp)Da@5kOZNtCk}1F9)ijbLii zWUf=da_g2wk4MIqLpx?l*4F2vM`tw8f|I!uSsLc3X}55QKw(aTmMk6=)`+7e69c(9 zMpRW0i==Ly9|26|w47E8W#6BM>ekK&Vy;J?s%q8|p5KRe@I)A# zZ3PS1Ivmu^$+ec-CKT-~*}KBmKMjA%v#y-eg7pu0ZYbW2;qpX8bEGROv^Vs3{b~3u z=Vaz&>{cs2=zw`7_xf>|j-RPQq!&i%*vkKmu~CWoD2^h%O8l^Y5W z%zJe>W+l|GTwi}~*|tD{^Z~LW>8CHfB}1Q`noFHNC7YNu>%mwTi8WZ9c3E9mhmH8# z+Svs@zD~=qIe&RwvZQ zo%WJ;y1M3i5OtFIEp&bxzf+#m*Oo0R^z1d(k^|8v4eE?nf?!R-6Zkg?C#H>%NeINn z8FThf*g)R}lF!@%{2|{NL9yBr$YoKTVW*mh&$_rzE7)8PT5Pk`QECSJwxhR61T<7! zno1Zw3sNG?8kdP&f7I zJz7)3GVfdSq9p^#%f9}}3#IGYptOB|>LOcr1_)^ywaDnm>y6$S9x&B0_hlri1RD$r z94h+7t+8Y3vK+g6zTPyL-jK8%J|*t(KtC)07S(=kRIBT6Phj|J0H&jAUmAc!nTd7* z-JG9aK23Sy$!03=8sP76@tC&^OJC)n^o`Td9lP_iY|oMI4;d!L_sD#I{#Kn-J8eC~ z+!62tydim{Rad78Tw9*$P4BZPK{`SDs880*a5x%8Zg^2!x2C3s^{n7accP2hs%kpS zU!P5< z`9E6Hku(Jgbu7>;=vfR>)F3c89R1c5?JuK zYk*usC!s5Xt7d&b<$8Lg=6j|``V6!{jvOLGt8*UdSGJlhKNR5PxPcoIRZLV+)-4)Q zwEfol6YVK#>3d_v@9gRA(^XL=v6J?y8a@km!KyP@rp#K@SExkk23B%_YJr)@Kn($6 zf{w^+Z$YIx%n-`d)0!r!x_Z)xX|DgX+PKLAydn`BZa6%W6t*x#Ho>16Yk8~WA)XGO z4o1&h{-`fI8Il)K=8B@RRP2Zd6Vy)dx;b2QV+bZRV=q zr=99sM*OhXw>2s~@U)KLemoo8;m`sPnZZ$PcBr@7+sQV)jCg~IY2@8)LQ_*2@lbLx zgoKD{*GHo?oQZK{_;%yWp*$_ec7+<{WfzKTRuY}+GbP3a2`hgcrGk;ow6MUmrKIJ| z5meJ1sV7~C&e$1qqsTTBnF>YB#5BKg!((%c!1;(tlj#eTp>rn~PWAHK0dtIXdPvxv zBr8Qyh7M%dFuM%(-zE(lnGywa**zhL3vKm7hDy7q>u??JzmkW40w;=%l04()WpObB?ZTQ z_$G_g1hcl7;O4kO5*SWV+jGTW)gxQGgy_q)G58G!cYN0ro*rT1K7q%7Ajg&9Bleju zr^CZu-4koh$hv_sf@y^*6&{_=xNfMu8j)-gp%7Pnfb4Ds8TlO*@h7q;^a=V7b;tnL zjWXp9fJJKAeJ()yG`g$(@nG@d;S$Sx7Eh? zU9INwLU*J7vZ>(o>q~6^ArA+lq8!|Ih25#2Px$!-XTK}zsTK;Yz@$akkVOsjXjv@+ zEXl-}lu&Hr%9EO?zmP$f(JF#K;ND@2_q8iJ=e7@YxZyo=0TF-9vM8ho^MV>bo5$N7)J~X;xRWqx# z46-G?mJOxFlAc-48rVnE(O!|Zd4ETXc1)wK4%9VqYfvwsa{;0POO^$_JZnM9`Q9Kx zSefR%OzO7BBL z^^6!Uxy)+EgtBcemY3mY8D)l*Q3!{7ot#RUKH{%^1j}Rz&6Y{wD2lc`d4+g>AC3_r0TK1orjky4$T>5^2YZ7qO?XxUy4JrJej{ z3IFD|a$&Bure%|ci9~%yV@$?yDbgsk)hL%C8&T&bVFS><-;l1xrm5ku9^7C>rKLkg z=*%ooD)f?Ec$leJBrWmhd0oj?z%}{TacUw2L@|65PMd67aE8b6$c836rA-4~k=t*t zZxuAp&j)DRW9b6b3D>q>%ll(<`!El`#sjTSZ@;5eQwnw;+R*0H*-+ATuPJvL*Aitg zLA395zqWe*?x{eN@`Ac$Nmi_)cBNKf9M5pleKMbff0zq;XyYPuv7)PPzIj%O*yq9B zDDD@e9~S)OkKx8Pb?tS4Y-DXg*&ME0U`Z&aHj>jkU)L@y{LF%GRksvz#!*P-;}q8% z3KUQD++ZKZ?m>9TCr79+?wT4TnhPfv9wg2PRfrEoWH!x4RLGJ4OE^5A6hyR}HVA}+ zi#4%@4LV6?&iEV$VZ}g_NH^nYuw&>6XXqn)ga0v7@A;p4NvBQJr@-d%#Fi+F?7T3Qgr(S2#tD@EVtx~>rviD z!7W3E=BFaOql9LvmzG$Oya6G#WFs#dOy_Jwzat&&T#a=&S3*sgZ^FC$8-dGanWCv_ zZgC(Y4I96-@$ciw@kT6I7fgdhjxkIv6KcazH%3OPY3!nnl$Vs&M9~gq^L#1Q_G1_{ zpg?wPkXCGX1ONTT-L8`bWsXb%e=wek%G_LQ#X{@HP3h`s%Q=A4W`jq8xwxi?yt~VW z2pdY8C%N6+LGq4(slCX(#wX$7Hk{EIhTD*IPXW?fUsCl#>D+=whKTiSDNDKwiwX`$ z5p<=1x#Be`+7wPqzm{S!XsRz^6S@dO&uhUi4M9(g9Yl7Fd3ZyxR2kaJh*&7nEbUP& z2ez=03fO6G##7XjJJ@x{R@x0@UKHCwY2F3nBcLmB&I}{$$?rh!*>PVKcn8Fx9a#^d zt7}v&5;O%W6ivh$l)uUBQ6n|-lqb7Ea)Wf_tasH>rm)4P!e+6+uPq7}$kAY_;JnFS zTmJ|zmg{c`;_ad&!`)2k+3g3F++jVby}7JC+(<6rY$Dtd`Mwd&jiCs0bYL#1)K=m^$7vjUiK>4qvcbs7G=w=JX);JG9*)G>Z7kF{CdUv7ZoU(|vnBkz zF?BXDml2k96fZiy{hkCaAuIXA7q|w){ow@_dTaR=w81RzhxbOF2JpVC#9HRS>cGvz zXd^SZtt=(og}N8)s4mR{0xNlPjoG}TT$jODfe{cb^TgTnqx(;_zVyO7!2=1_kWSoL zUdKhdZdGhbv41`(697g>2GT=3UgrQAcgDCx^HX8IUnuu z!T!ubbIP$ki4tXB$K9t`_j^QPo03FPcb8JEww!7$Ymx%Re?2}jm0Ed{J}l<;|EAP7 zt10|0{0J?PC%HuN_It(r9lg+TP%=UmwVR~KHG-EqIz=k^t;21OFpESjh9F7j&pdWTp2#Ph8$+HT zh&qM~Q^{qXgFvnlGB=@~p;ThiMTl`=jFD`_cAq##&Z;&T=PAW%bYCpwC|2r`HmhFvD=^MX;X_d3F#ZNULP*d(#DDf#s3dwH8IIGL zokE+>`Gq8mloQQaRcP#$I6cv}$cbX6%up+zza&~Zv6}day*kOPXBer#iKXf=R4Bp2 zhJZQ8kN2+wfRO`G)ev}Y47xFg@(@qBzi}qc_1FvYjpDm6utyCt#DFdkf5H&#m`0s2 z|JeXJy|;+)OrVhuUhOswN(}rAu%%kV7nJuv4>ClQxD46^qDg}OcM>@7$h|Nx74`GzmotTo`o;}?hpH7Z-a<6)cqv4iYjtZX{hQzYN)S#_PQ}mV zeHy;Z5Y;+mwA#PH{x^wgiU07?FDcLv#3aL=__s)TYS2dPNzpCnk$Xkrcw^L4D9L!} z#w7DrPKkXIQ$x8&*fpFyjyX9>zBm+s?jKQzE1_wqnO9PZ<}ZF8ehtD<7rqS8W1x35 z;^@8lZxb;B@V&McI)N|_v8N1MgrKe>z+eb86cWUe+a*7cU?8W_p{7>LoD6fuJSOSd zn9CLx?2H3O72rt(6CbaB7@(PP3sZ2?iKEOPfpY1T7I<_;({5exF8*P;(;eOT>{xmY zcB#!RLbrfol5;`60(g86M2$BpFJ21q&q1`#%h16pNj?l|53}(oBN0AILcA~FdD4!} z(o+dZ&K=C1fw9i|%?WLKv`%~2i|bGlU}5!hhgpLA&>(d}EdFPythC%^ z9QV7|*$6jEB4x-8ns~Fv^@=ba4_$LuHel5>Xr;@_w*FjK>Ra1A%(F(a9*7XKd@`oB zY;+XT33o^7du&aX^la#Mcj4FjI?oA}5_fO4cT^H2v&7)i;&GvNRF;a>A&JD-7Y$ZQ zkt+%DCg>I^+{DCr-}CAiguK~FkCP)+_jC**J#Imh6ht_d%W;*Md?X-K=kot;LVA;E6{{DQj?;vP{!DfB)Fn%AXPdo;Z_u2%-)qZ!rVo zFRQAWPbMSjoBa;v z#1IpkGm4k|@}0v6)NGPjkLcxhAEsz-=Sq+f(>`NN${!lScQu6=_?2q09&$dU z$GpfKdHkp@0*@T6mz|jjdmX2QI^Iv4fOJe*l>Ovq4NPkqw+R7R;?rsVUGLOpMpj=u zsaizj79lv{yyjp$l|7H38{|Zf@?&>x(jjE|kf!*df_X{O1NYc>&gJ9}FC`RDH~G%pM(oVQukgYSF}J&L0!*u!KWrwbi|7ICS}pPon9_f3mqt2# z$ETD*ifD3hcyZ@R_QDwbD$cVWEE3sJ;znB@n@UJKO3Xp`*VMhPKtY`QalyO zMi{rPLCl9c4d2D)N?eN0Ie(QMD`aO;%z+ah9#+fC)~E-&rT-fG?>fOSp5yi`3yatX z!E-qqzXy8*T%M=%pSJ|qyBcwPQzOe7~do5_Pb!-RMYjGOsDKPu^8-O>Rm9+i-+3Ym$1E5U6wojJz6T5=tGVChdPlPDJiI~ z*{K^ff)FyjgebdO-8z{v`FjRlx7Q3JR{0{r>t0aBrD z0su_hsG#Cdzs4v&`U3=MAdP@qRlYdb&>@{rnG36TDaMeko($s zGh(S%%5^-LVezc#7MW3vLM#+B3%t4*!PO)?IZ)CzXdXjDe!O`aSHYqJ2?DNWPCtHZ z$8ZMe_);Td!#xM{@Fp=*&eN!C@oq*UW@V?RL;8CeZjY!%(W7RgyF9r<%>m3K^(5+p z)zmPfK;P?f1tEj-75y3nuVlf4%ETb6qXpss-{0l+DKVFk!UuEPy4>S(- zp`hqnd_$gx*B`(K)9FvR1*f6P^ZJ3Fp`oFPlUbPtWsM*W214K~&0gP>oVg${G7*2Q z>gQ3WE(B#*m>%(ZGU# zWMu2>^l~Qzb)jsQJ_!#cK-0*T5h8Yi1aRLtMRnMh?Ig6D9OW#i}4e3T_qsvVw^o zG6K=8HKhOG3+3EUm>GB{+Ro|?@&Vq#x5pYqu&*xzlZ>Mk9)1Tbh|!NYzC#6wh~3U$ zVxmBGZvDzH@I?n2*j(up(3Q3P%pE>N<9rFy(VcpUMz@X7d%WvyTPJ{ZxodfZy@QeK zJ3PL?NA~!l`Fo75Z7%2-ByeN9d->(gssNCn@ThQHyEKoricY{YkJ!Nrm4Zb9M0XGZxT@XI6sh~S!ei=H+BShNnpK62F*3}oT5!yY#l{*vC`#gJ6W+l-SsP$3)Bi+*@#0%H@ z{9V9P)-(cxQPtC*Y3f9#W(gH?VeR<%$N`cjn^ra>Wn%%P>B)>CtfSY3ZX)2}$Ca<6 zuc#~=QbAiu$0kGxeC+HZukWdzXwD*mn-{J(Xsn7Zf|Bf35SND;iJfkXYGp7V4Y4_B zcbv^g3n zvQ=X__!EkjwwHQxS9r(7Vg}hkImtDwOi_|%i{Iw5HgoZk*R{i?? z)6wM)y8X#ljWao69mqdyqzX9;4edB-@3PB-kc$*o;I{gAkf2!vXbV@6jIY%!%5@BT z5KK*+vqT+e6*nAZR~p9`oi!xGE0T$g>4gZjBZ4Es^dsZrH*b7L{OLy~1R(FsXVu)# zsAtS0Sv8_QEAR||HP!@cu=Xfz7$-7MxM|NN?oj<2<4uSckO%5%N2~8gj%OSL4xtf* zqpVS^+#0!!Ak6)V z(N4b2{56QXs`45@rH9--z(847`JFDed(0EDzviGufZCS@VfNbd>D8|*#~0&>_vRAh z-5*;+9-+(>v7$?$LPV`%rkF^jVlEj(JI@)oQ|y%`_fzJTetL6;2kdTcMLjoXh6mwJ zwvi4X@|ypEI&EtyEBMi|!VCP~^&ptbJL>~_H>|;@^BVS`m^K@*-<~Nkn=c=mgStJanfuP<&Wa1K~mc?vTrNnybA&*It5D@G9yJ-}j=`iN^3^ z&xz=O_VG4r9kAhnFz@efNYr~uyP(x zNb-w-eU=As36lBhK26%~50Uvi<`x_Lcw2DAH6+`6DdU6jb^(rybgl)_y?DDQI0^R# z$G7==6ZTHW2YB38d3fNU9i1GE^{wHcfAG55=^I)aI{^%gP0ek%2`@Xk z2?6Ft+=Qwu()7}Hg2ra%Vjd30iXJjbh8~uN97cq^JU`qx-K_1bzlH(0SzFmSa=LL7 z>KoY_7;}EL|LUeA1pH0J$&#CpiI#zejh=y!2l|JDkqM`QkjTF)eO+-AnmIYyanjMb zy1LT3GSS*Pn9?zDaB$GkGtx0K(tJtKIJ(<7>ATU`I1>Gb1tDWcLkDv^Cv#gHz+V>i z4Q!pAxCsdX|8CdrAMDyV(*6w?t)ZKj{7%`5%z{`|SRq=fA~fWcc?f+c`T} z{cXZVhIGbO#@5C*PL5yj82$#&$dJ?UYk{0>9eDocHAQ3le`{Eo|Ap2}-`EOZqz@3Z zb^KZbL45}!TMyfR!}KNaZ;ijH(EX#K{~_kDrT#nU|Kv|MeY-Ef^nU~VCB!NCZ-|v_ zZLR+AT>4*?e*C?|SV-T=n1_*`k(Gv?m4=a3iGh)mo`sX)>yn9+p8f~jf06u~rnIe* zxrzJ#jpjc{{!dr(3xKrr|K2aw*8kxeX=zR&TSMo+)>vGK$JyE3h?Aayo|RuvP=JP+ zm5qspnTeH^hF_3{iH1RlO@x(2h*eOCjpJ|A{Li|7P4ufr+~#ZHZ48b7C)WN!_rIAu z;Qzq~`v1WgfPW40PkH%Y?)sOz{wWXqQ^Nmxy8h*^f64>@l<>cvu7A#5|Ms*06^Va^ zihmE~dHz=_U~Kah|GRz#`u{%}2jf3v975)fc2@fCU*V0Cxs#PKz{$bc_^$-x%f{c5 z5=NH43MK!i{6tN`*o02e+`|~~b(2n26Tkpq1~6&^=oD;ioxU#U0Sa{DHYTGyN4S{bQ)a^M9838bHL{%E{P)PQ>afx)d@tv^6sR zkGzP5iTNMXBDYMId2`K3ebk<7)1T;t%nB3C3sQ4J1O)0LY3Sf(LK_d6@D}rns`6UQ{rLPiB%r9!{^0yI zAP|Yr;Ig59n*y;q^o)9EXaOC;h$V>WG^uM~4G{AJ6^O~u_W?yRke3)%zQJx0dTbR` z6=Rx7w$lik+YnyCpIN*MZ9>Cx#lSRb3mWN;r3c{sdhBiiTy8z&n7~B6IKT9mp8^mf zfqAEU)gb-sw-CS~LAgN%5ixsV3G@QwAVcVDd)z|(pkjfp>9O(w$fUhOso;tL>|z1f z@gTKSczb$~gFdTNFxFI%TRoCJerVzV+;{>g^xki&2r#|C$Fa~61aQec4rLc0DM*fV5`81S}16J7}Q>sB{(iUu0%eDvw$o+ z$cH|>C4|>4z;{2!uU!sy-$%D0m3rh$gADDk%k!XR`k>gsSnSXzdx^|K#^fLlzY4ee zd9;GEZ9y~h5oraV+QD7)(zHTJ+u>aF;hF`$PQxN>12+o5IQ3{A1*cqr;`KARLDOw< z-0@*J39$VNV7tQJ==FR8!rw-D3z(XQ(A7$5=8tys$dUU)o@fe<<}Oe)5=`1ly0ovD zzzisCg*xZV>t794x%~f%J9`|tiYSccBTeX-PArG6d+YZ}q>zH-gw(-UvSb{zMu=6g z5!S#W*a8y!;Csy2rt3Q?awqm&S69_}^>gZ7k5sRI`u)Y<-~9E*Z-3%)`1Shm?*6W5 zE;s%0{Hf(KdawO*{?qd(=H=hNT|Rz}#po7GeQPh*d)s6EIJB_N&zJf}Ox>2f#aifA z@jW)R%C=ZfYvWvLR&E>~Tf_TuRO6kobYJ;xyR&vLH;PvNR&Flq^D4S^eJi)(Fy7U= zJ=4{fmmB`9)|}Rtwa!iLwtq(yYvX7(=AlWkw;O$&Nt~PdV)%Depz5aheO^`FJZ~}G zDsQm{u?{iaQg7d#5L*z_o%Q>78?1%4_gSMB_DoA@z|nxC0Y?Ll1{@7I8gS^kd1KSK zd2DCXxM{w4IpEMd^>Vc{CUgb9^ZQJ`eDDfX@Sb9^mr;p9lCnz~=!zj}G$+d>%TZ z+cO29>$C^uU_5kA^!i{tz~=!z5Ab<_&vo+jcJ-cOeF2{b_&mVp0X{$9U91i8c}y4& z@OgmG1AHFf^8lX*_&mVp0X`4#c`V=np9lCnz~|??l|8=&^9p<(3+5I0JizAxJ`eDD zfY0NAc?CWX@cH?kXS|1VB=|hR=lah^t1pZ1y9qu|@Oc&=j|875_&mYq2|iEod4kUq ze4gR!Ji+G)K2Pv@>fL!Cj|875_&mYq2|iEod4kU~eO@K_Ji+G)K2Pv@g3l9tp5XHY zpC|bI%mvvqtuU{^=UII`^d`F1r&VOR8?6q@OI1ycJ?~SqTMnw$u=n^{yBu()H@@Vm zmUO=7eU_H$o@-e-ERVGEkk4=G(*d~exuwuae$Q1!olX1I?RQ%l+S~Wo{BJ(&@%r}t zhli_vU*_9Snd|nyr>mb5(r>SiPbo$nvf7CIyO%G&`{DchyH9z?a)0+#diVOr>%%XX ar%t^-Q~GZnf7Xvh&&@hM^b0`G9sdXCgDQ9c literal 0 HcmV?d00001 diff --git a/README.en-US.md b/README.en-US.md new file mode 100644 index 0000000..00511a4 --- /dev/null +++ b/README.en-US.md @@ -0,0 +1,1536 @@ +# Agent Framework OCI Evaluator — LLM-as-a-Judge + +**Language:** English | [Português](README.md) + +## 1. Overview + +The **Agent Framework OCI Evaluator** is the batch component responsible for asynchronously evaluating the quality of conversations produced by agents built on the Agent Framework OCI. + +The service reads sessions and traces from **Langfuse v3**, deterministically samples sessions, calculates quality metrics with an **LLM-as-a-Judge**, persists state and results in **Oracle Autonomous Database**, generates the official `.TXT.GZ` artifact, publishes that artifact to **Google Cloud Storage (GCS)**, and audits the evaluator job itself in Langfuse. + +It is designed for **D+1 execution** as a **Kubernetes CronJob**, with no runtime HTTP/gRPC endpoint. + +> **Recommended deployment model:** one evaluator instance per agent/squad. The code is generic and supports multiple agents in configuration, but in production each deployment should normally enable only the agent owned by that squad. This keeps credentials, ownership, scheduling, and troubleshooting isolated. + +--- + +## 2. Why this component exists + +Running generative-quality evaluation synchronously inside the agent request path would increase latency, cost, and coupling. The evaluator moves that responsibility out of the online interaction path. + +The architecture is: + +```text +Online interaction + | + v +Agent Framework + | + +--------------------> Langfuse v3 + | + | traces / sessions + v + D+1 Evaluator (CronJob) + | + +------------------+------------------+ + | | | + v v v + OCI GenAI ADB GCS + LLM-as-a-Judge state/results official .TXT.GZ + | + +-----------------------> Langfuse + audit / optional scores +``` + +This separation provides: + +- no evaluator latency in the online interaction; +- replay and auditability; +- configurable sampling per agent; +- isolation of evaluation failures from the online agent; +- support for dedicated OCI GenAI models; +- an official artifact for the TIM analytics layer; +- no code coupling to specific agent names, buckets, or credentials. + +--- + +## 3. Core concepts + +### 3.1. Trace + +A **trace** represents an interaction evaluated individually. The evaluator calculates these trace-level metrics: + +- `judgeScore` — overall response quality; +- `accuracyScore` — factual/contextual accuracy; +- `alucinationScore` — degree of unsupported claims; +- `promptLength` — raw character count of `trace.input`. + +### 3.2. Session + +A **session** represents the complete conversation. A session normally contains multiple traces. + +Session-level metrics are calculated once and replicated to every trace row in that session: + +- `inferredCsiScore`; +- `resolution`; +- `conversationPrecision`; +- `loop`. + +### 3.3. Session-based sampling + +Sampling is performed on **sessions**, never on isolated traces. + +The deterministic seed is: + +```text +agentId:reference_date +``` + +Example: + +```text +billing:2026-08-13 +``` + +Two executions for the same agent, date, and percentage therefore select the same sessions. + +Once a session is selected, **all traces in that session** are retained. + +### 3.4. BRT time window + +`run-agents` calculates the reference date using: + +```text +D - daysBack +``` + +with explicit timezone: + +```text +America/Sao_Paulo +``` + +With `daysBack: 1`, a run on August 14 evaluates the full August 13 window, from 00:00:00 through 23:59:59.999999 BRT. + +### 3.5. CONTEXT_TURN_LIMIT + +`CONTEXT_TURN_LIMIT` controls how many previous conversation turns are included in trace-level judge context. + +Default: + +```env +CONTEXT_TURN_LIMIT=6 +``` + +The variable exists because a response should not be judged outside the context of its conversation. Increasing the value provides more context, but also increases tokens, cost, and judge latency. + +### 3.6. Loop + +The `loop` metric is not simple text repetition. It represents the maximum consecutive **`metadata.intent` streak** in a session. + +Example: + +```text +billing +billing +billing +payment +payment +``` + +Result: + +```text +loop = 3 +``` + +--- + +## 4. End-to-end execution flow + +The production flow is executed with: + +```bash +python -m evaluator.cli run-agents +``` + +or through the installed entrypoint: + +```bash +af-evaluator run-agents +``` + +Steps: + +```text +1. Load agents.yaml +2. Compute D-daysBack in BRT +3. Resolve Langfuse credentials for the agent +4. Read ALL native sessions in the period, paginated +5. Read ALL traces in the period, paginated +6. Read observations in bulk/paginated mode +7. Recover the real session_id from metadata when a legacy trace has no native sessionId +8. Filter agent traces/aliases +9. Deterministically sample logical sessions +10. Keep all traces belonging to sampled sessions +11. Persist items in ADB +12. Run trace-level judge +13. Run session-level judge +14. Calculate local loop metric +15. Generate .TXT.GZ +16. Upload to GCS +17. Delete local file after successful upload +18. Update judge-job-{agentId} audit trace in Langfuse +``` + +### 4.1. Why collection avoids N+1 + +The implementation avoids the old pattern: + +```text +1 trace + -> GET trace detail + -> GET observations +``` + +repeated once per trace. + +Collection now uses bulk pagination: + +```text +/api/public/sessions page 1..N +/api/public/traces page 1..N +/api/public/observations page 1..N +``` + +Therefore 10,000 traces do not produce approximately 20,000 additional requests. Request count grows with the number of API pages instead. + +The current `PAGE_SIZE` is 100. + +### 4.2. Compatibility with traces without native `sessionId` + +Some historical instrumentations store the session in trace/observation metadata without promoting it to Langfuse's native `trace.sessionId`. The evaluator recovers the real session using this precedence: + +```text +trace.sessionId / trace.session_id + -> trace.metadata.session_id / sessionId / conversation_key + -> observation.metadata.session_id / sessionId / conversation_key + -> business_context.session_key (last metadata fallback) +``` + +`trace.id` is **never** used as a `session_id`. This preserves session sampling and session-level metrics for historical traces without creating an artificial one-session-per-trace model. Collection statistics expose `native_sessions_read`, `metadata_sessions_recovered`, and `traces_without_session`. + +--- + +## 5. Component architecture + +```text +agent_framework_oci_evaluator/ +├── evaluator/ +│ ├── analytics/ +│ │ └── vloop.py +│ ├── collectors/ +│ │ ├── langfuse.py +│ │ ├── langfuse_credentials.py +│ │ ├── agent_framework.py +│ │ └── mock.py +│ ├── config/ +│ │ ├── agents.py +│ │ └── settings.py +│ ├── core/ +│ │ └── models.py +│ ├── identity/ +│ │ └── resolver.py +│ ├── judges/ +│ │ └── llm_judge.py +│ ├── llm/ +│ │ ├── client.py +│ │ └── profile_resolver.py +│ ├── output/ +│ │ └── legacy_exporter.py +│ ├── persistence/ +│ │ ├── oracle_store.py +│ │ └── repository.py +│ ├── prompts/ +│ │ └── loader.py +│ ├── publishers/ +│ │ ├── gcs_export.py +│ │ ├── langfuse_job_audit.py +│ │ └── langfuse_scores.py +│ ├── cli.py +│ ├── engine.py +│ └── time_utils.py +├── configs/ +│ ├── identity.yaml +│ ├── judge/ +│ │ ├── agents.yaml +│ │ ├── storage.yaml +│ │ ├── trace_metrics.yaml +│ │ └── session_metrics.yaml +│ ├── langfuse/ +│ │ └── langfuse_credentials.example.yaml +│ └── llm_profiles/ +│ └── llm_profiles.yaml +├── deploy/ +├── tests/ +├── .env.example +├── Dockerfile +└── pyproject.toml +``` + +### 5.1. Collectors + +Collectors transform external data sources into `ConversationRecord` objects. + +Available sources: + +```text +langfuse +agent_framework +mock +``` + +`langfuse` is the default production source. + +### 5.2. EvaluationEngine + +`evaluator/engine.py` orchestrates: + +- run creation; +- collection; +- sampling; +- persistence; +- judging; +- export; +- upload; +- audit. + +### 5.3. Persistence + +Execution state and results are persisted in Oracle Database. The schema is created automatically by `init-db`. + +Main tables: + +```text +EVALUATION_RUN +EVALUATION_PROGRESS_EVENT +EVALUATION_ITEM +EVALUATION_RESULT +EVALUATION_METRIC +EVALUATION_FINDING +``` + +The table prefix is configured through: + +```env +ADB_TABLE_PREFIX=AGENTFW +``` + +### 5.4. Publishers + +- `gcs_export.py` — uploads the official artifact; +- `langfuse_job_audit.py` — writes the `judge-job-{agentId}` audit trace; +- `langfuse_scores.py` — optionally publishes trace scores back to Langfuse. + +> **Note:** production job audit uses the agent's dynamic credentials. Optional score publishing (`PUBLISH_LANGFUSE_SCORES=true`) still uses global `LANGFUSE_HOST/PUBLIC_KEY/SECRET_KEY`; keep it disabled unless an appropriate global project is configured. + +--- + +## 6. Agent configuration + +Default file: + +```text +configs/judge/agents.yaml +``` + +Example: + +```yaml +agents: + - agentId: billing + enabled: true + daysBack: 1 + percentage: 1.0 + langfuse_agent_aliases: + - telecom_contas + gcs_prefix: agnt_ai_contas +``` + +Fields: + +| Field | Description | +|---|---| +| `agentId` | official logical agent identity | +| `enabled` | enables/disables the agent in that deployment | +| `daysBack` | number of days to subtract from the current BRT date | +| `percentage` | percentage of sessions evaluated, from 0.0 to 1.0 | +| `langfuse_agent_aliases` | technical agent names accepted in traces; when populated, it is an explicit filter regardless of the Langfuse credential source | +| `gcs_prefix` | agent prefix inside the bucket | + +`agent_id` and `days_back` are also accepted for compatibility, but `agentId` / `daysBack` is the recommended form. + +**`langfuse_agent_aliases` semantics:** when the list contains values, only traces whose agent identifier matches one of the aliases are eligible for sampling and evaluation. When the list is empty, the evaluator does not apply an agent-identity filter and treats the configured Langfuse project/endpoint as exclusive to that entry. + +### 6.1. One evaluator per agent + +For the recommended one-instance-per-agent deployment model, keep only one agent enabled in that deployment: + +```yaml +agents: + - agentId: billing + enabled: true + daysBack: 1 + percentage: 1.0 + langfuse_agent_aliases: + - telecom_contas + - billing_agent + gcs_prefix: agnt_ai_contas +``` + +The same evaluator code can be reused by another repository/deployment simply by changing configuration files. + +--- + +## 7. Dynamic Langfuse credentials + +There are no hardcoded variables such as: + +```text +LANGFUSE_BILLING_* +LANGFUSE_SALES_* +``` + +The evaluator reads one declarative definition: + +```env +LANGFUSE_CREDENTIALS_CONFIG_PATH=configs/langfuse/langfuse_credentials.yaml +``` + +Create it from the example: + +```bash +cp configs/langfuse/langfuse_credentials.example.yaml \ + configs/langfuse/langfuse_credentials.yaml +``` + +Example: + +```yaml +langfuse: + agents: + - agentId: billing + host: https://langfuse.example.internal + publicKey: pk-... + secretKey: sk-... + + - agentId: another-agent + host: https://another-langfuse.internal + publicKey: pk-... + secretKey: sk-... +``` + +The resolver iterates through all entries. Adding a new agent requires no Python change. + +### 7.1. Environment/file references + +To avoid storing secrets in Git: + +```yaml +langfuse: + agents: + - agentId: billing + hostEnv: BILLING_LF_HOST + publicKeyEnv: BILLING_LF_PUBLIC + secretKeyFile: /var/secrets/langfuse/billing-secret-key +``` + +Supported fields: + +```text +host / hostEnv / hostFile +publicKey / publicKeyEnv / publicKeyFile +secretKey / secretKeyEnv / secretKeyFile +``` + +Resolution order for each field: + +```text +direct value -> declared environment variable -> declared file +``` + +### 7.2. Global credentials + +These variables remain available only for the generic manual `run` command: + +```env +LANGFUSE_HOST=http://localhost:3005 +LANGFUSE_PUBLIC_KEY=... +LANGFUSE_SECRET_KEY=... +``` + +The production `run-agents` flow resolves credentials from the per-agent declarative file. + +--- + +## 8. LLM configuration + +Supported providers: + +```text +mock +oci_openai +oci_sdk +openai_compatible +``` + +The recommended provider for dedicated OCI models is: + +```env +LLM_PROVIDER=oci_sdk +``` + +### 8.1. OCI SDK + +Local example using `~/.oci/config`: + +```env +LLM_PROVIDER=oci_sdk +OCI_AUTH_MODE=config_file +OCI_CONFIG_FILE=~/.oci/config +OCI_PROFILE=DEFAULT +OCI_REGION=sa-saopaulo-1 +OCI_COMPARTMENT_ID=ocid1.compartment... +OCI_GENAI_BASE_URL=https://inference.generativeai.sa-saopaulo-1.oci.oraclecloud.com +OCI_GENAI_MODEL=meta.llama-3.3-70b-instruct +``` + +Authentication modes supported by the code: + +```text +config_file +instance_principal +resource_principal +oke_workload_identity +workload_identity +``` + +### 8.2. Dedicated endpoint + +For a dedicated endpoint: + +```env +OCI_GENAI_BASE_URL=https://.inference.generativeai..oci.oraclecloud.com +OCI_GENAI_ENDPOINT_ID=ocid1.generativeaiendpoint... +``` + +The dedicated endpoint OCID may also be provided in `OCI_GENAI_MODEL`. + +The provider selects `DedicatedServingMode` when an endpoint OCID is available; otherwise it uses `OnDemandServingMode`. + +### 8.3. Judge profiles + +File: + +```text +configs/llm_profiles/llm_profiles.yaml +``` + +Example: + +```yaml +profiles: + judge: + temperature: 0 + max_tokens: 900 +``` + +Selected through: + +```env +LLM_PROFILE=judge +``` + +--- + +## 9. Prompts and metrics + +### 9.1. Trace metrics + +File: + +```text +configs/judge/trace_metrics.yaml +``` + +The judge must return JSON such as: + +```json +{ + "judgeScore": 0.0, + "accuracyScore": 0.0, + "alucinationScore": 0.0, + "rationale": "..." +} +``` + +If parsing fails: + +```text +1st attempt + -> parse fails +2nd attempt with retry hint + -> parse fails again +trace-level metrics = None + -> exported as "0.00" +``` + +### 9.2. Session metrics + +File: + +```text +configs/judge/session_metrics.yaml +``` + +Expected result: + +```json +{ + "inferredCsiScore": 0.5, + "resolution": 1, + "conversationPrecision": 1, + "rationale": "..." +} +``` + +Session-level values are replicated to every trace row belonging to that session. + +--- + +## 10. Identity normalization + +File: + +```text +configs/identity.yaml +``` + +The identity layer converts channel/domain-specific identifiers into canonical keys such as: + +```text +customer_key +contract_key +interaction_key +account_key +resource_key +session_key +``` + +This prevents `cpf`, `msisdn`, `invoice_id`, `order_id`, `ura_call_id`, and similar names from being spread throughout evaluator code. + +Source ordering works as fallback priority. + +Example: + +```yaml +session_key: + sources: + - business_context.session_key + - session_key + - conversation_key + - session_id +``` + +--- + +## 11. GCS configuration + +Buckets live outside the code in: + +```text +configs/judge/storage.yaml +``` + +Example: + +```yaml +storage: + buckets: + dev: tim-bigdata-dev-ca1f-raw-standard-agnt-ai-atendimento + fqa: tim-bigdata-fqa-60ce-raw-standard-agnt-ai-atendimento + prd: tim-bigdata-prod-e305-raw-standard-agnt-ai-atendimento +``` + +Selection: + +```env +APP_ENV=dev +STORAGE_CONFIG_PATH=configs/judge/storage.yaml +``` + +Operational override: + +```env +JUDGE_GCS_BUCKET=my-bucket +``` + +Enable the production flow with: + +```env +ENABLE_GCS_UPLOAD=true +GOOGLE_APPLICATION_CREDENTIALS=/path/GCP_ACCESS_KEY.json +``` + +`run-agents` treats GCS as mandatory. If `ENABLE_GCS_UPLOAD=false` or upload fails, the agent run finishes as `FAILED`. + +--- + +## 12. Output file + +Pattern: + +```text +AGENTE_{agentId}_LLM_JUDGE_{YYYYMMDD}_{HHMMSS}.TXT.GZ +``` + +Example: + +```text +AGENTE_billing_LLM_JUDGE_20260814_020015.TXT.GZ +``` + +Header: + +```text +"judgeScore"|;"accuracyScore"|;"alucinationScore"|;"promptLength"|;"loop"|;"inferredCsiScore"|;"resolution"|;"conversationPrecision"|;"uraCallId"|;"channelId"|;"sessionId"|;"messageId" +``` + +Final line: + +```text +"TOTAL"|;"" +``` + +After a successful GCS upload, the local file is deleted. + +--- + +## 13. Local installation + +### 13.1. Prerequisites + +- Python 3.11+; +- access to Oracle Database/Autonomous Database; +- Oracle Wallet when required; +- access to Langfuse v3; +- OCI Generative AI access for real judge execution; +- GCP credentials for production upload. + +### 13.2. Create the virtual environment + +Linux/WSL/macOS: + +```bash +python -m venv .venv +source .venv/bin/activate +``` + +Windows PowerShell: + +```powershell +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +``` + +### 13.3. Install + +Recommended development mode: + +```bash +python -m pip install --upgrade pip setuptools wheel +pip install -e . +``` + +Validate: + +```bash +pip check +python -c "import oci, httpx, oracledb; import google.cloud.storage; print('Dependencies OK')" +``` + +--- + +## 14. Local configuration step by step + +### Step 1 — `.env` + +```bash +cp .env.example .env +``` + +Edit the real values. + +Typical minimum configuration: + +```env +APP_ENV=dev + +ADB_USER=admin +ADB_PASSWORD=... +ADB_DSN=... +ADB_WALLET_LOCATION=/path/to/wallet +ADB_WALLET_PASSWORD=... +ADB_TABLE_PREFIX=AGENTFW + +ENABLE_LANGFUSE=true +LANGFUSE_CREDENTIALS_CONFIG_PATH=configs/langfuse/langfuse_credentials.yaml + +LLM_PROVIDER=oci_sdk +LLM_PROFILE=judge +OCI_AUTH_MODE=config_file +OCI_CONFIG_FILE=~/.oci/config +OCI_PROFILE=DEFAULT +OCI_REGION=sa-saopaulo-1 +OCI_COMPARTMENT_ID=... +OCI_GENAI_BASE_URL=https://... +OCI_GENAI_ENDPOINT_ID=ocid1.generativeaiendpoint... + +CONTEXT_TURN_LIMIT=6 +BATCH_SIZE=50 +MAX_ATTEMPTS=3 + +ENABLE_GCS_UPLOAD=true +STORAGE_CONFIG_PATH=configs/judge/storage.yaml +GOOGLE_APPLICATION_CREDENTIALS=/path/GCP_ACCESS_KEY.json +``` + +### Step 2 — Langfuse + +```bash +cp configs/langfuse/langfuse_credentials.example.yaml \ + configs/langfuse/langfuse_credentials.yaml +``` + +Declare the agent that will be executed. + +### Step 3 — agents.yaml + +Adjust: + +```text +configs/judge/agents.yaml +``` + +For a dedicated instance, keep only that squad's agent enabled. + +### Step 4 — validate configuration + +```bash +python -m evaluator.cli show-config +``` + +### Step 5 — initialize the database + +```bash +python -m evaluator.cli init-db +``` + +> `reset-db` drops and recreates the evaluator schema. Use it only in a controlled environment. + +--- + +## 15. CLI + +The actual help for the current version is: + +```bash +python -m evaluator.cli --help +``` + +Available commands: + +```text +init-db +reset-db +show-config +run +run-agents +progress +runs +``` + +### 15.1. `run` — manual/diagnostic execution + +Syntax: + +```bash +python -m evaluator.cli run PERIOD_START PERIOD_END [OPTIONS] +``` + +Example: + +```bash +python -m evaluator.cli run \ + 2026-08-12T00:00:00 \ + 2026-08-12T23:59:59 \ + --source langfuse \ + --limit 10 +``` + +Options: + +```text +--source TEXT default: langfuse +--limit INTEGER +--show-progress / --no-show-progress +``` + +**Important:** `run` is a generic manual path. It collects, evaluates, and persists results, but does not execute the full production per-agent contract (configured sampling, GCS export, and job audit). Use `run-agents` for certification/production behavior. + +### 15.2. `run-agents` — production flow + +```bash +python -m evaluator.cli run-agents +``` + +One agent: + +```bash +python -m evaluator.cli run-agents \ + --agent-id billing \ + --source langfuse +``` + +Limited test: + +```bash +python -m evaluator.cli run-agents \ + --agent-id billing \ + --limit 10 +``` + +Options: + +```text +--source TEXT default: langfuse +--agent-id TEXT +--limit INTEGER +``` + +`run-agents` uses the agent's `daysBack`, `percentage`, credentials, and `gcs_prefix`. + +### 15.3. List runs + +```bash +python -m evaluator.cli runs +``` + +or: + +```bash +python -m evaluator.cli runs --limit 50 +``` + +### 15.4. Inspect run progress + +```bash +python -m evaluator.cli progress +``` + +With more events: + +```bash +python -m evaluator.cli progress --events 100 +``` + +Typical production events: + +```text +LANGFUSE_JOB_AUDIT +RUN_CREATED +COLLECTING +COLLECTED +SAMPLED +ITEMS_INSERTED +BATCH_STARTED +ITEM_COMPLETED +SESSION_JUDGE_COMPLETED +COMPLETED +EXPORTED +GCS_UPLOADED +LOCAL_FILE_DELETED +LANGFUSE_JOB_AUDIT +``` + +--- + +## 16. Tests + +Install `pytest` if needed: + +```bash +pip install pytest +``` + +### 16.1. All tests + +```bash +python -m pytest -v +``` + +### 16.2. TIM specification tests + +```bash +python -m pytest tests/test_tim_spec.py -v +``` + +### 16.3. Extended compliance tests + +```bash +python -m pytest tests/test_full_tim_compliance.py -v +``` + +### 16.4. Hardcode regression tests + +```bash +python -m pytest tests/test_no_product_hardcodes.py -v +``` + +### 16.5. Recommended release suite + +```bash +python -m pytest \ + tests/test_tim_spec.py \ + tests/test_full_tim_compliance.py \ + tests/test_no_product_hardcodes.py \ + -v +``` + +Validate syntax/imports: + +```bash +python -m compileall evaluator +``` + +--- + +## 17. Recommended local functional test + +### 17.1. Mock smoke test + +Configure: + +```env +LLM_PROVIDER=mock +``` + +Then run a small period using the mock source when desired: + +```bash +python -m evaluator.cli run \ + 2026-08-12T00:00:00 \ + 2026-08-12T23:59:59 \ + --source mock +``` + +### 17.2. Small Langfuse test + +```bash +python -m evaluator.cli run-agents \ + --agent-id billing \ + --limit 5 +``` + +Then: + +```bash +python -m evaluator.cli runs --limit 5 +python -m evaluator.cli progress --events 100 +``` + +Confirm: + +- `COLLECTED` > 0; +- `SAMPLED` lists selected sessions; +- items finish without `ITEM_FAILED`; +- `EXPORTED` appears; +- `GCS_UPLOADED` contains a `gs://...` URI; +- `LOCAL_FILE_DELETED` appears; +- `completed` audit was confirmed. + +--- + +## 18. Docker + +Local build: + +```bash +docker build -t agent-framework-evaluator:local . +``` + +The production runtime is batch-only. The container does not need to expose an HTTP port. + +In Kubernetes, the `CronJob` runs: + +```text +python -m evaluator.cli run-agents --source langfuse +``` + +--- + +## 19. Kubernetes / OKE + +The **evaluator itself remains CronJob-only**, as required by the specification: it does not open an HTTP/gRPC port and evaluation processing continues to run exclusively in `evaluator-batch`. + +The web interface is provided by a **separate read-only component**, `evaluator-monitor`. It does not execute evaluations, mutate runs, or change the evaluator batch contract. It only reads run/progress/result tables from Autonomous Database. + +```text +Evaluator batch (evaluator-batch CronJob) + │ + ├── writes EVALUATION_RUN + ├── writes EVALUATION_PROGRESS_EVENT + └── writes EVALUATION_RESULT + │ + ▼ + Autonomous Database + ▲ + │ read-only + Evaluator Monitor (HTTP Deployment) + │ + └── /ui + monitoring APIs +``` + +Kustomize resources: + +```text +Namespace +ServiceAccount +ConfigMap +CronJob evaluator-batch +Deployment evaluator-monitor +Service evaluator-monitor (ClusterIP) +``` + +No Ingress is created by default, so the UI is not automatically exposed outside the cluster. + +Current CronJob: + +```yaml +schedule: "0 2 * * *" +timeZone: "America/Sao_Paulo" +suspend: false +concurrencyPolicy: Forbid +``` + +### 19.1. Prepare deployment files + +```bash +cp deploy/env/evaluator.env.example deploy/env/evaluator.env +cp deploy/langfuse/langfuse_credentials.example.yaml \ + deploy/langfuse/langfuse_credentials.yaml +``` + +Place the Oracle wallet in: + +```text +deploy/wallet/ +``` + +GCP credential: + +```text +deploy/gcp/GCP_ACCESS_KEY.json +``` + +### 19.2. Create secrets + +DEV: + +```bash +./deploy/scripts/create-secrets.sh dev +``` + +HML: + +```bash +./deploy/scripts/create-secrets.sh hml +``` + +PROD: + +```bash +./deploy/scripts/create-secrets.sh prod +``` + +Created/updated secrets: + +```text +evaluator-secrets +evaluator-wallet +evaluator-langfuse +evaluator-gcp (when the GCP file exists) +``` + +### 19.3. Build and push + +```bash +export REGISTRY=tim.azurecr.io +export IMAGE_TAG= +./deploy/scripts/build-push.sh prod +``` + +### 19.4. Deploy + +```bash +export REGISTRY=tim.azurecr.io +export IMAGE_TAG= +./deploy/scripts/deploy.sh prod +``` + +By default the script also runs the database initialization job. + +To disable it: + +```bash +export RUN_INIT_DB=false +``` + +### 19.5. Status + +```bash +./deploy/scripts/status.sh prod +``` + +or directly: + +```bash +kubectl -n agent-evaluator-prod get cronjob,jobs,pods +``` + +### 19.6. Immediate execution outside the schedule + +```bash +./deploy/scripts/run-now.sh prod +``` + +The script creates a Job from the CronJob. + +Follow logs: + +```bash +kubectl -n agent-evaluator-prod get jobs,pods +kubectl -n agent-evaluator-prod logs -f job/ +``` + + +### 19.7. Web monitoring interface + +The UI is served by the independent `evaluator-monitor` component: + +```text +GET /ui +GET /health +GET /api/runs +GET /api/runs/{run_id} +GET /api/runs/{run_id}/results +``` + +To access a Kubernetes deployment locally without creating an Ingress: + +```bash +kubectl -n agent-evaluator-prod port-forward svc/evaluator-monitor 8080:8080 +``` + +Then open: + +```text +http://localhost:8080/ui +``` + +The page shows recent runs, `RUN_ID`, agent, status, progress, events, and results. The browser auto-refreshes every 5 seconds. + +To run the monitor locally: + +```bash +python -m uvicorn evaluator_monitor.app:app --host 0.0.0.0 --port 8080 +``` + +The monitor uses the same `ADB_*`/wallet settings as the evaluator. Functionally it is **read-only**: there is no endpoint to start, cancel, delete, or modify runs. + +> Security recommendation: keep the Service as `ClusterIP`. If external exposure is required later, enforce authentication/authorization through the corporate Ingress/API Gateway; do not expose the monitor directly to the Internet. + +--- + +## 20. How to release a new version + +Recommended flow: + +### 1. Update code/configuration + +Never commit real credentials. + +### 2. Run tests + +```bash +python -m pytest -v +python -m compileall evaluator +``` + +### 3. Test locally with a small limit + +```bash +python -m evaluator.cli run-agents \ + --agent-id \ + --limit 5 +``` + +### 4. Validate the run + +```bash +python -m evaluator.cli runs --limit 10 +python -m evaluator.cli progress --events 100 +``` + +### 5. Confirm GCS + +The event stream must include: + +```text +GCS_UPLOADED +``` + +with a URI such as: + +```text +gs:////AGENTE__LLM_JUDGE_.TXT.GZ +``` + +### 6. Update environment secrets/configuration + +```bash +./deploy/scripts/create-secrets.sh +``` + +### 7. Build/push a new tag + +```bash +export REGISTRY= +export IMAGE_TAG= +./deploy/scripts/build-push.sh +``` + +### 8. Deploy + +```bash +./deploy/scripts/deploy.sh +``` + +### 9. Trigger a manual certification run + +```bash +./deploy/scripts/run-now.sh +``` + +### 10. Validate logs and artifacts + +```bash +kubectl -n agent-evaluator- logs -f job/ +``` + +Also confirm the GCS object and the `judge-job-{agentId}` audit in Langfuse. + +--- + +## 21. Troubleshooting + +### `Collected 0 traces` + +Check, in this order: + +1. processed date (`daysBack` uses BRT); +2. Langfuse credentials for the correct `agentId`; +3. Langfuse project `host`; +4. `langfuse_agent_aliases`; +5. whether sessions/traces exist in the period; +6. whether the logical `agentId` differs from `metadata.agent_id` in traces. + +Example: + +```yaml +- agentId: billing + langfuse_agent_aliases: + - telecom_contas + - billing_agent +``` + +### `No Langfuse credentials configured for agent ...` + +The agent exists in `agents.yaml` but not in the file referenced by: + +```env +LANGFUSE_CREDENTIALS_CONFIG_PATH +``` + +Add an entry with the same `agentId`. + +### `Langfuse credentials definition file not found` + +Create/mount: + +```text +configs/langfuse/langfuse_credentials.yaml +``` + +or configure another path. + +### `TIM specification requires GCS upload...` + +`run-agents` was executed with: + +```env +ENABLE_GCS_UPLOAD=false +``` + +Set it to `true` and configure GCP credentials. + +### GCS upload failure + +Check: + +```bash +echo "$GOOGLE_APPLICATION_CREDENTIALS" +ls -l "$GOOGLE_APPLICATION_CREDENTIALS" +``` + +Also validate the bucket resolved through `APP_ENV`/`storage.yaml`. + +### OCI SDK dedicated endpoint failure + +Check: + +```env +LLM_PROVIDER=oci_sdk +OCI_GENAI_BASE_URL=https:// +OCI_GENAI_ENDPOINT_ID=ocid1.generativeaiendpoint... +OCI_COMPARTMENT_ID=... +``` + +Do not use `/openai/v1` in the native `oci_sdk` endpoint. + +### Database schema missing + +```bash +python -m evaluator.cli init-db +``` + +### View progress without a UI + +The service exposes no HTTP endpoint. Use: + +```bash +python -m evaluator.cli runs +python -m evaluator.cli progress --events 100 +``` + +--- + +## 22. Main environment variables + +| Variable | Purpose | +|---|---| +| `APP_ENV` | environment used to resolve storage | +| `ADB_*` | Oracle connection and wallet | +| `ENABLE_LANGFUSE` | enables Langfuse collection | +| `LANGFUSE_CREDENTIALS_CONFIG_PATH` | declarative per-agent YAML | +| `LANGFUSE_*` | global fallback only for manual `run` | +| `PUBLISH_LANGFUSE_SCORES` | publishes trace scores back to Langfuse | +| `LLM_PROVIDER` | `mock`, `oci_openai`, `oci_sdk`, `openai_compatible` | +| `LLM_PROFILE` | judge profile | +| `OCI_AUTH_MODE` | OCI SDK authentication mode | +| `OCI_GENAI_BASE_URL` | OCI GenAI endpoint | +| `OCI_GENAI_MODEL` | model ID/name or endpoint-OCID compatibility | +| `OCI_GENAI_ENDPOINT_ID` | dedicated endpoint OCID | +| `OCI_COMPARTMENT_ID` | OCI compartment | +| `AGENTS_CONFIG_PATH` | agents file | +| `TRACE_PROMPT_PATH` | trace-level prompt | +| `SESSION_PROMPT_PATH` | session-level prompt | +| `BATCH_SIZE` | internal processing batch size | +| `MAX_ATTEMPTS` | configurable attempt limit | +| `CONTEXT_TURN_LIMIT` | last N turns passed to trace judge | +| `ENABLE_GCS_UPLOAD` | requires upload in production flow | +| `STORAGE_CONFIG_PATH` | bucket mapping by environment | +| `JUDGE_GCS_BUCKET` | bucket override | +| `GOOGLE_APPLICATION_CREDENTIALS` | GCP credentials | +| `OUTPUT_DIR` | temporary export directory | + +--- + +## 23. Security and operations + +- do not commit the real `.env`; +- do not commit the real `langfuse_credentials.yaml`; +- do not commit GCP service account JSON; +- do not commit Oracle Wallet files; +- in OKE, prefer `oke_workload_identity` where applicable; +- mount credentials as Kubernetes Secrets; +- keep `readOnlyRootFilesystem` enabled in the CronJob; +- keep `concurrencyPolicy: Forbid` to prevent two D+1 jobs from the same deployment running concurrently; +- never run `reset-db` in production without an explicit recovery plan. + +--- + +## 24. Certification checklist + +Before promoting a version: + +- [ ] `pip check` passes; +- [ ] `python -m pytest -v` passes; +- [ ] `python -m compileall evaluator` passes; +- [ ] `show-config` points to the expected ADB/OCI/config files; +- [ ] `agents.yaml` contains only the expected agents for that deployment; +- [ ] Langfuse credentials exist for every enabled `agentId`; +- [ ] `run-agents --limit 5` finds traces; +- [ ] sampling operates on sessions, not individual traces; +- [ ] trace/session judges complete; +- [ ] output contains the 12-column header; +- [ ] filename matches `AGENTE_{agentId}_LLM_JUDGE_{YYYYMMDD}_{HHMMSS}.TXT.GZ`; +- [ ] `GCS_UPLOADED` event is confirmed; +- [ ] local file is deleted after upload; +- [ ] Langfuse `created/completed` audit is confirmed; +- [ ] CronJob has `suspend: false`; +- [ ] CronJob timezone is `America/Sao_Paulo`; +- [ ] no real credentials are present in the repository. + +--- + +## 25. Quick mental model for new contributors + +If you need to understand the project quickly, think of it as: + +```text +Langfuse stores conversations + ↓ +Evaluator selects D+1 sessions + ↓ +OCI GenAI judges traces and sessions + ↓ +ADB stores execution state and results + ↓ +GCS receives the official artifact + ↓ +Langfuse stores the batch audit trace +``` + +Extension points are configuration-driven: + +```text +Which agent? -> configs/judge/agents.yaml +Which Langfuse project?-> langfuse_credentials.yaml +Which bucket? -> configs/judge/storage.yaml +Which prompt? -> trace_metrics.yaml / session_metrics.yaml +Which model? -> .env + llm_profiles.yaml +Which identity rules? -> configs/identity.yaml +``` + +The most important architectural rule is: **the evaluator code must remain generic; agent, environment, credential, and storage details belong in configuration.** diff --git a/README.md b/README.md new file mode 100644 index 0000000..3c798cc --- /dev/null +++ b/README.md @@ -0,0 +1,1551 @@ +# Agent Framework OCI Evaluator — LLM-as-a-Judge + +**Idioma:** Português | [English](README.en-US.md) + +## 1. Visão geral + +O **Agent Framework OCI Evaluator** é o componente batch responsável por avaliar, de forma assíncrona, a qualidade das conversas produzidas pelos agentes construídos sobre o Agent Framework OCI. + +O serviço lê sessões e traces do **Langfuse v3**, seleciona uma amostra determinística de sessões, calcula métricas de qualidade com um **LLM-as-a-Judge**, persiste resultados em **Oracle Autonomous Database**, gera o arquivo oficial `.TXT.GZ`, publica esse arquivo em **Google Cloud Storage (GCS)** e registra a execução do próprio job no Langfuse. + +Ele foi desenhado para execução **D+1**, como um **Kubernetes CronJob**, sem endpoint HTTP/gRPC de runtime. + +> **Modelo de implantação recomendado:** uma instância do evaluator por agente/squad. O código é genérico e suporta múltiplos agentes na configuração, mas em produção a recomendação é cada deployment carregar apenas o agente de sua responsabilidade. Isso mantém credenciais, ownership, scheduling e troubleshooting isolados. + +--- + +## 2. Por que este componente existe + +Avaliar respostas generativas em tempo real dentro do fluxo síncrono do agente aumenta latência, custo e acoplamento. O evaluator separa essa responsabilidade do atendimento online. + +O desenho é: + +```text +Atendimento online + | + v +Agent Framework + | + +--------------------> Langfuse v3 + | + | traces / sessions + v + Evaluator D+1 (CronJob) + | + +------------------+------------------+ + | | | + v v v + OCI GenAI ADB GCS + LLM-as-a-Judge estado/resultados .TXT.GZ oficial + | + +-----------------------> Langfuse + auditoria / scores opcionais +``` + +Essa separação traz alguns benefícios: + +- não adiciona latência ao atendimento; +- permite reprocessamento e auditoria; +- permite amostragem configurável por agente; +- isola falhas de avaliação do agente online; +- permite utilizar modelos dedicados do OCI GenAI; +- mantém um artefato oficial para a camada analítica da TIM; +- evita acoplamento do código a nomes específicos de agentes, buckets ou credenciais. + +--- + +## 3. Conceitos principais + +### 3.1. Trace + +Um **trace** representa uma interação avaliada individualmente. Para cada trace são calculadas métricas trace-level: + +- `judgeScore` — qualidade global da resposta; +- `accuracyScore` — aderência/factualidade em relação ao contexto; +- `alucinationScore` — grau de afirmações não suportadas pelo contexto; +- `promptLength` — tamanho bruto, em caracteres, do `trace.input`. + +### 3.2. Session + +Uma **session** representa a conversa completa. Uma sessão normalmente possui vários traces. + +As métricas session-level são calculadas uma vez e replicadas nas linhas dos traces daquela sessão: + +- `inferredCsiScore`; +- `resolution`; +- `conversationPrecision`; +- `loop`. + +### 3.3. Amostragem por sessão + +A amostragem é feita sobre **sessions**, nunca sobre traces isolados. + +A seed é determinística: + +```text +agentId:reference_date +``` + +Exemplo: + +```text +billing:2026-08-13 +``` + +Com isso, duas execuções para o mesmo agente, data e percentual selecionam as mesmas sessões. + +Depois que uma sessão é selecionada, **todos os traces daquela sessão** são mantidos. + +### 3.4. Janela BRT + +`run-agents` calcula a data de referência com base em: + +```text +D - daysBack +``` + +usando explicitamente: + +```text +America/Sao_Paulo +``` + +Com `daysBack: 1`, uma execução em 14/08 avalia o dia 13/08 completo, de 00:00:00 até 23:59:59.999999 BRT. + +### 3.5. CONTEXT_TURN_LIMIT + +`CONTEXT_TURN_LIMIT` controla quantos turnos anteriores são enviados ao judge trace-level. + +Default: + +```env +CONTEXT_TURN_LIMIT=6 +``` + +Ele existe porque uma resposta não deve ser julgada fora do contexto da conversa. Aumentar o valor fornece mais contexto, mas também aumenta tokens, custo e latência do judge. + +### 3.6. Loop + +A métrica `loop` não mede repetição textual simples. Ela representa o maior **streak consecutivo de `metadata.intent`** dentro da sessão. + +Exemplo: + +```text +billing +billing +billing +payment +payment +``` + +Resultado: + +```text +loop = 3 +``` + +--- + +## 4. Fluxo completo de execução + +O fluxo produtivo é executado pelo comando: + +```bash +python -m evaluator.cli run-agents +``` + +ou pelo entrypoint instalado: + +```bash +af-evaluator run-agents +``` + +Etapas: + +```text +1. Carrega agents.yaml +2. Calcula D-daysBack em BRT +3. Resolve credenciais Langfuse do agente +4. Busca TODAS as sessions nativas do período, com paginação +5. Busca TODOS os traces do período, com paginação +6. Busca observations em bulk/paginado +7. Recupera session_id real via metadata quando o trace legado não possui sessionId nativo +8. Filtra traces do agente/aliases +9. Faz sampling determinístico das sessions lógicas +10. Mantém todos os traces das sessions selecionadas +11. Persiste itens no ADB +12. Executa judge trace-level +13. Executa judge session-level +14. Calcula loop local +15. Gera .TXT.GZ +16. Faz upload para GCS +17. Remove o arquivo local após sucesso +18. Atualiza auditoria judge-job-{agentId} no Langfuse +``` + +### 4.1. Por que a coleta não usa N+1 + +A implementação evita o padrão antigo: + +```text +1 trace + -> GET trace detail + -> GET observations +``` + +repetido para cada trace. + +Agora a coleta é paginada em lote: + +```text +/api/public/sessions page 1..N +/api/public/traces page 1..N +/api/public/observations page 1..N +``` + +Assim, 10.000 traces não produzem ~20.000 chamadas adicionais. O número de requests cresce com o número de páginas da API. + +O `PAGE_SIZE` atual é 100. + +### 4.2. Compatibilidade com traces sem `sessionId` nativo + +Algumas instrumentações históricas registram a sessão no metadata do trace/observation, mas não promovem o valor para o campo nativo `trace.sessionId` do Langfuse. O evaluator recupera a sessão real com a seguinte precedência: + +```text +trace.sessionId / trace.session_id + -> trace.metadata.session_id / sessionId / conversation_key + -> observation.metadata.session_id / sessionId / conversation_key + -> business_context.session_key (último fallback de metadata) +``` + +O `trace.id` **nunca** é usado como `session_id`. Assim, o evaluator preserva sampling e métricas de session mesmo para traces históricos, sem transformar cada trace em uma sessão artificial. As estatísticas de coleta expõem `native_sessions_read`, `metadata_sessions_recovered` e `traces_without_session`. + +--- + +## 5. Arquitetura de componentes + +```text +agent_framework_oci_evaluator/ +├── evaluator/ +│ ├── analytics/ +│ │ └── vloop.py +│ ├── collectors/ +│ │ ├── langfuse.py +│ │ ├── langfuse_credentials.py +│ │ ├── agent_framework.py +│ │ └── mock.py +│ ├── config/ +│ │ ├── agents.py +│ │ └── settings.py +│ ├── core/ +│ │ └── models.py +│ ├── identity/ +│ │ └── resolver.py +│ ├── judges/ +│ │ └── llm_judge.py +│ ├── llm/ +│ │ ├── client.py +│ │ └── profile_resolver.py +│ ├── output/ +│ │ └── legacy_exporter.py +│ ├── persistence/ +│ │ ├── oracle_store.py +│ │ └── repository.py +│ ├── prompts/ +│ │ └── loader.py +│ ├── publishers/ +│ │ ├── gcs_export.py +│ │ ├── langfuse_job_audit.py +│ │ └── langfuse_scores.py +│ ├── cli.py +│ ├── engine.py +│ └── time_utils.py +├── configs/ +│ ├── identity.yaml +│ ├── judge/ +│ │ ├── agents.yaml +│ │ ├── storage.yaml +│ │ ├── trace_metrics.yaml +│ │ └── session_metrics.yaml +│ ├── langfuse/ +│ │ └── langfuse_credentials.example.yaml +│ └── llm_profiles/ +│ └── llm_profiles.yaml +├── deploy/ +├── tests/ +├── .env.example +├── Dockerfile +└── pyproject.toml +``` + +### 5.1. Collectors + +Transformam fontes externas em `ConversationRecord`. + +Fontes disponíveis: + +```text +langfuse +agent_framework +mock +``` + +`langfuse` é a fonte produtiva padrão. + +### 5.2. EvaluationEngine + +`evaluator/engine.py` coordena o pipeline: + +- criação do run; +- coleta; +- amostragem; +- persistência; +- judge; +- export; +- upload; +- auditoria. + +### 5.3. Persistence + +Os estados e resultados são persistidos em Oracle Database. O schema é criado automaticamente pelo comando `init-db`. + +As principais tabelas são: + +```text +EVALUATION_RUN +EVALUATION_PROGRESS_EVENT +EVALUATION_ITEM +EVALUATION_RESULT +EVALUATION_METRIC +EVALUATION_FINDING +``` + +O prefixo é configurável por: + +```env +ADB_TABLE_PREFIX=AGENTFW +``` + +### 5.4. Publishers + +- `gcs_export.py` — upload do artefato oficial; +- `langfuse_job_audit.py` — auditoria `judge-job-{agentId}`; +- `langfuse_scores.py` — publicação opcional de scores de trace. + +> **Nota:** a auditoria produtiva do job usa as credenciais dinâmicas do agente. A publicação opcional de scores (`PUBLISH_LANGFUSE_SCORES=true`) ainda utiliza as credenciais globais `LANGFUSE_HOST/PUBLIC_KEY/SECRET_KEY`; deixe-a desabilitada quando não houver um projeto global apropriado. + +--- + +## 6. Configuração de agentes + +Arquivo padrão: + +```text +configs/judge/agents.yaml +``` + +Exemplo: + +```yaml +agents: + - agentId: billing + enabled: true + daysBack: 1 + percentage: 1.0 + langfuse_agent_aliases: + - telecom_contas + gcs_prefix: agnt_ai_contas +``` + +Campos: + +| Campo | Descrição | +|---|---| +| `agentId` | identidade lógica oficial do agente | +| `enabled` | habilita/desabilita o agente naquele deployment | +| `daysBack` | quantos dias voltar a partir do dia atual BRT | +| `percentage` | percentual de sessions avaliadas, de 0.0 a 1.0 | +| `langfuse_agent_aliases` | nomes técnicos aceitos para o agente nos traces; quando preenchido, funciona como filtro explícito independentemente da origem das credenciais Langfuse | +| `gcs_prefix` | prefixo do agente dentro do bucket | + +Também são aceitos `agent_id` e `days_back` por compatibilidade, mas o formato recomendado é `agentId` / `daysBack`. + +**Semântica de `langfuse_agent_aliases`:** quando a lista possui valores, somente traces cujo identificador de agente corresponda a um dos aliases são elegíveis para sampling e avaliação. Quando a lista está vazia, o evaluator não aplica filtro por identidade de agente e considera o projeto/endpoint Langfuse configurado exclusivo para aquela entrada. + +### 6.1. Um evaluator por agente + +Para o modelo recomendado de uma instância por agente, mantenha apenas uma entrada habilitada naquele deployment: + +```yaml +agents: + - agentId: billing + enabled: true + daysBack: 1 + percentage: 1.0 + langfuse_agent_aliases: + - telecom_contas + - billing_agent + gcs_prefix: agnt_ai_contas +``` + +O mesmo código pode ser usado em outro repositório/deployment apenas trocando os arquivos de configuração. + +--- + +## 7. Credenciais Langfuse dinâmicas + +Não existem variáveis hardcoded do tipo: + +```text +LANGFUSE_BILLING_* +LANGFUSE_SALES_* +``` + +O evaluator lê uma definição declarativa: + +```env +LANGFUSE_CREDENTIALS_CONFIG_PATH=configs/langfuse/langfuse_credentials.yaml +``` + +Crie o arquivo a partir do exemplo: + +```bash +cp configs/langfuse/langfuse_credentials.example.yaml \ + configs/langfuse/langfuse_credentials.yaml +``` + +Exemplo: + +```yaml +langfuse: + agents: + - agentId: billing + host: https://langfuse.example.internal + publicKey: pk-... + secretKey: sk-... + + - agentId: outro-agente + host: https://outro-langfuse.internal + publicKey: pk-... + secretKey: sk-... +``` + +O resolver percorre todas as entradas. Adicionar um agente não exige alteração Python. + +### 7.1. Referências a env ou arquivos + +Para não manter secrets no Git: + +```yaml +langfuse: + agents: + - agentId: billing + hostEnv: BILLING_LF_HOST + publicKeyEnv: BILLING_LF_PUBLIC + secretKeyFile: /var/secrets/langfuse/billing-secret-key +``` + +Campos suportados: + +```text +host / hostEnv / hostFile +publicKey / publicKeyEnv / publicKeyFile +secretKey / secretKeyEnv / secretKeyFile +``` + +Ordem de resolução de cada campo: + +```text +valor direto -> variável de ambiente declarada -> arquivo declarado +``` + +### 7.2. Credenciais globais + +Estas variáveis continuam disponíveis apenas para o comando manual genérico `run`: + +```env +LANGFUSE_HOST=http://localhost:3005 +LANGFUSE_PUBLIC_KEY=... +LANGFUSE_SECRET_KEY=... +``` + +O fluxo produtivo `run-agents` resolve credenciais pelo arquivo declarativo por agente. + +--- + +## 8. Configuração do LLM + +Providers aceitos: + +```text +mock +oci_openai +oci_sdk +openai_compatible +``` + +A recomendação para os modelos OCI dedicados é: + +```env +LLM_PROVIDER=oci_sdk +``` + +### 8.1. OCI SDK + +Exemplo local com `~/.oci/config`: + +```env +LLM_PROVIDER=oci_sdk +OCI_AUTH_MODE=config_file +OCI_CONFIG_FILE=~/.oci/config +OCI_PROFILE=DEFAULT +OCI_REGION=sa-saopaulo-1 +OCI_COMPARTMENT_ID=ocid1.compartment... +OCI_GENAI_BASE_URL=https://inference.generativeai.sa-saopaulo-1.oci.oraclecloud.com +OCI_GENAI_MODEL=meta.llama-3.3-70b-instruct +``` + +Modos de autenticação suportados pelo código: + +```text +config_file +instance_principal +resource_principal +oke_workload_identity +workload_identity +``` + +### 8.2. Dedicated endpoint + +Para endpoint dedicado: + +```env +OCI_GENAI_BASE_URL=https://.inference.generativeai..oci.oraclecloud.com +OCI_GENAI_ENDPOINT_ID=ocid1.generativeaiendpoint... +``` + +Também é aceito o OCID do dedicated endpoint em `OCI_GENAI_MODEL`. + +O provider seleciona `DedicatedServingMode` quando há endpoint OCID; caso contrário usa `OnDemandServingMode`. + +### 8.3. Perfis do judge + +Arquivo: + +```text +configs/llm_profiles/llm_profiles.yaml +``` + +Exemplo: + +```yaml +profiles: + judge: + temperature: 0 + max_tokens: 900 +``` + +Selecionado por: + +```env +LLM_PROFILE=judge +``` + +--- + +## 9. Prompts e métricas + +### 9.1. Trace metrics + +Arquivo: + +```text +configs/judge/trace_metrics.yaml +``` + +O judge deve retornar JSON com: + +```json +{ + "judgeScore": 0.0, + "accuracyScore": 0.0, + "alucinationScore": 0.0, + "rationale": "..." +} +``` + +Se o parse falhar: + +```text +1ª tentativa + -> parse falha +2ª tentativa com retry-hint + -> parse falha novamente +métricas trace-level = None + -> exportadas como "0.00" +``` + +### 9.2. Session metrics + +Arquivo: + +```text +configs/judge/session_metrics.yaml +``` + +Resultado esperado: + +```json +{ + "inferredCsiScore": 0.5, + "resolution": 1, + "conversationPrecision": 1, + "rationale": "..." +} +``` + +Os valores session-level são replicados no arquivo para todos os traces da mesma session. + +--- + +## 10. Identity normalization + +Arquivo: + +```text +configs/identity.yaml +``` + +A camada de identidade converte identificadores específicos de canais/domínios para chaves canônicas como: + +```text +customer_key +contract_key +interaction_key +account_key +resource_key +session_key +``` + +Isso evita espalhar lógica como `cpf`, `msisdn`, `invoice_id`, `order_id`, `ura_call_id` etc. pelo código do evaluator. + +A ordem das fontes funciona como fallback. + +Exemplo: + +```yaml +session_key: + sources: + - business_context.session_key + - session_key + - conversation_key + - session_id +``` + +--- + +## 11. Configuração de GCS + +Buckets ficam fora do código em: + +```text +configs/judge/storage.yaml +``` + +Exemplo: + +```yaml +storage: + buckets: + dev: tim-bigdata-dev-ca1f-raw-standard-agnt-ai-atendimento + fqa: tim-bigdata-fqa-60ce-raw-standard-agnt-ai-atendimento + prd: tim-bigdata-prod-e305-raw-standard-agnt-ai-atendimento +``` + +Seleção: + +```env +APP_ENV=dev +STORAGE_CONFIG_PATH=configs/judge/storage.yaml +``` + +Override de emergência/operação: + +```env +JUDGE_GCS_BUCKET=meu-bucket +``` + +Para habilitar o fluxo produtivo: + +```env +ENABLE_GCS_UPLOAD=true +GOOGLE_APPLICATION_CREDENTIALS=/path/GCP_ACCESS_KEY.json +``` + +`run-agents` considera GCS obrigatório. Se `ENABLE_GCS_UPLOAD=false` ou o upload falhar, o run do agente termina como `FAILED`. + +--- + +## 12. Arquivo de saída + +Padrão: + +```text +AGENTE_{agentId}_LLM_JUDGE_{YYYYMMDD}_{HHMMSS}.TXT.GZ +``` + +Exemplo: + +```text +AGENTE_billing_LLM_JUDGE_20260814_020015.TXT.GZ +``` + +Cabeçalho: + +```text +"judgeScore"|;"accuracyScore"|;"alucinationScore"|;"promptLength"|;"loop"|;"inferredCsiScore"|;"resolution"|;"conversationPrecision"|;"uraCallId"|;"channelId"|;"sessionId"|;"messageId" +``` + +Linha final: + +```text +"TOTAL"|;"" +``` + +Após upload bem-sucedido para GCS, o arquivo local é apagado. + +--- + +## 13. Instalação local + +### 13.1. Pré-requisitos + +- Python 3.11+; +- acesso ao Oracle Database/Autonomous Database; +- Oracle Wallet quando aplicável; +- acesso ao Langfuse v3; +- acesso ao OCI Generative AI para execução real do judge; +- credencial GCP para upload produtivo. + +### 13.2. Criar venv + +Linux/WSL/macOS: + +```bash +python -m venv .venv +source .venv/bin/activate +``` + +Windows PowerShell: + +```powershell +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +``` + +### 13.3. Instalar + +Modo recomendado durante desenvolvimento: + +```bash +python -m pip install --upgrade pip setuptools wheel +pip install -e . +``` + +Validar: + +```bash +pip check +python -c "import oci, httpx, oracledb; import google.cloud.storage; print('Dependencias OK')" +``` + +--- + +## 14. Configuração local passo a passo + +### Passo 1 — `.env` + +```bash +cp .env.example .env +``` + +Edite os valores reais. + +Configuração mínima típica: + +```env +APP_ENV=dev + +ADB_USER=admin +ADB_PASSWORD=... +ADB_DSN=... +ADB_WALLET_LOCATION=/path/to/wallet +ADB_WALLET_PASSWORD=... +ADB_TABLE_PREFIX=AGENTFW + +ENABLE_LANGFUSE=true +LANGFUSE_CREDENTIALS_CONFIG_PATH=configs/langfuse/langfuse_credentials.yaml + +LLM_PROVIDER=oci_sdk +LLM_PROFILE=judge +OCI_AUTH_MODE=config_file +OCI_CONFIG_FILE=~/.oci/config +OCI_PROFILE=DEFAULT +OCI_REGION=sa-saopaulo-1 +OCI_COMPARTMENT_ID=... +OCI_GENAI_BASE_URL=https://... +OCI_GENAI_ENDPOINT_ID=ocid1.generativeaiendpoint... + +CONTEXT_TURN_LIMIT=6 +BATCH_SIZE=50 +MAX_ATTEMPTS=3 + +ENABLE_GCS_UPLOAD=true +STORAGE_CONFIG_PATH=configs/judge/storage.yaml +GOOGLE_APPLICATION_CREDENTIALS=/path/GCP_ACCESS_KEY.json +``` + +### Passo 2 — Langfuse + +```bash +cp configs/langfuse/langfuse_credentials.example.yaml \ + configs/langfuse/langfuse_credentials.yaml +``` + +Declare o agente que será executado. + +### Passo 3 — agents.yaml + +Ajuste: + +```text +configs/judge/agents.yaml +``` + +Para uma instância dedicada, deixe somente o agente daquela squad habilitado. + +### Passo 4 — validar configuração + +```bash +python -m evaluator.cli show-config +``` + +### Passo 5 — inicializar banco + +```bash +python -m evaluator.cli init-db +``` + +> `reset-db` apaga e recria o schema do evaluator. Use apenas em ambiente controlado. + +--- + +## 15. CLI + +A ajuda real da versão atual é: + +```bash +python -m evaluator.cli --help +``` + +Comandos disponíveis: + +```text +init-db +reset-db +show-config +run +run-agents +progress +runs +``` + +### 15.1. `run` — execução manual/diagnóstica + +Sintaxe: + +```bash +python -m evaluator.cli run PERIOD_START PERIOD_END [OPTIONS] +``` + +Exemplo: + +```bash +python -m evaluator.cli run \ + 2026-08-12T00:00:00 \ + 2026-08-12T23:59:59 \ + --source langfuse \ + --limit 10 +``` + +Opções: + +```text +--source TEXT default: langfuse +--limit INTEGER +--show-progress / --no-show-progress +``` + +**Importante:** `run` é um caminho manual genérico. Ele coleta, avalia e persiste resultados, mas não executa o contrato produtivo completo por agente (sampling configurado, export GCS e auditoria de job). Para homologação/produção use `run-agents`. + +### 15.2. `run-agents` — fluxo produtivo + +```bash +python -m evaluator.cli run-agents +``` + +Um agente: + +```bash +python -m evaluator.cli run-agents \ + --agent-id billing \ + --source langfuse +``` + +Teste limitado: + +```bash +python -m evaluator.cli run-agents \ + --agent-id billing \ + --limit 10 +``` + +Opções: + +```text +--source TEXT default: langfuse +--agent-id TEXT +--limit INTEGER +``` + +`run-agents` usa `daysBack`, `percentage`, credenciais e `gcs_prefix` do agente. + +### 15.3. Listar runs + +```bash +python -m evaluator.cli runs +``` + +ou: + +```bash +python -m evaluator.cli runs --limit 50 +``` + +### 15.4. Progresso de um run + +```bash +python -m evaluator.cli progress +``` + +Com mais eventos: + +```bash +python -m evaluator.cli progress --events 100 +``` + +Eventos típicos do fluxo produtivo: + +```text +LANGFUSE_JOB_AUDIT +RUN_CREATED +COLLECTING +COLLECTED +SAMPLED +ITEMS_INSERTED +BATCH_STARTED +ITEM_COMPLETED +SESSION_JUDGE_COMPLETED +COMPLETED +EXPORTED +GCS_UPLOADED +LOCAL_FILE_DELETED +LANGFUSE_JOB_AUDIT +``` + +--- + +## 16. Testes + +Instale `pytest` caso necessário: + +```bash +pip install pytest +``` + +### 16.1. Todos os testes + +```bash +python -m pytest -v +``` + +### 16.2. Conformidade TIM + +```bash +python -m pytest tests/test_tim_spec.py -v +``` + +### 16.3. Conformidade ampliada + +```bash +python -m pytest tests/test_full_tim_compliance.py -v +``` + +### 16.4. Regressão de hardcodes + +```bash +python -m pytest tests/test_no_product_hardcodes.py -v +``` + +### 16.5. Suite recomendada antes de um release + +```bash +python -m pytest \ + tests/test_tim_spec.py \ + tests/test_full_tim_compliance.py \ + tests/test_no_product_hardcodes.py \ + -v +``` + +Validar sintaxe/imports: + +```bash +python -m compileall evaluator +``` + +--- + +## 17. Teste funcional local recomendado + +### 17.1. Smoke test com mock + +Configure: + +```env +LLM_PROVIDER=mock +``` + +E rode um período pequeno com source mock quando desejado: + +```bash +python -m evaluator.cli run \ + 2026-08-12T00:00:00 \ + 2026-08-12T23:59:59 \ + --source mock +``` + +### 17.2. Teste Langfuse sem volume grande + +```bash +python -m evaluator.cli run-agents \ + --agent-id billing \ + --limit 5 +``` + +Depois: + +```bash +python -m evaluator.cli runs --limit 5 +python -m evaluator.cli progress --events 100 +``` + +Confirme: + +- `COLLECTED` > 0; +- `SAMPLED` mostra sessions selecionadas; +- itens concluem sem `ITEM_FAILED`; +- `EXPORTED` aparece; +- `GCS_UPLOADED` contém URI `gs://...`; +- `LOCAL_FILE_DELETED` aparece; +- auditoria `completed` foi confirmada. + +--- + +## 18. Docker + +Build local: + +```bash +docker build -t agent-framework-evaluator:local . +``` + +O runtime produtivo é batch. O container não precisa expor porta HTTP. + +Para Kubernetes, o `CronJob` chama: + +```text +python -m evaluator.cli run-agents --source langfuse +``` + +--- + +## 19. Kubernetes / OKE + +O **evaluator propriamente dito continua CronJob-only**, conforme a especificação: ele não abre porta HTTP/gRPC e o processamento continua sendo executado exclusivamente pelo `evaluator-batch`. + +A interface web foi reintroduzida como um **componente separado e somente leitura**, `evaluator-monitor`. Ela não executa avaliação, não altera runs e não muda o contrato batch do evaluator. O monitor apenas consulta as tabelas de execução/progresso no Autonomous Database. + +```text +Evaluator batch (evaluator-batch CronJob) + │ + ├── cria/atualiza EVALUATION_RUN + ├── grava EVALUATION_PROGRESS_EVENT + └── grava EVALUATION_RESULT + │ + ▼ + Autonomous Database + ▲ + │ somente leitura + Evaluator Monitor (Deployment HTTP) + │ + └── /ui + APIs de acompanhamento +``` + +Recursos Kustomize: + +```text +Namespace +ServiceAccount +ConfigMap +CronJob evaluator-batch +Deployment evaluator-monitor +Service evaluator-monitor (ClusterIP) +``` + +Não existe Ingress por default. Assim, a interface não é exposta para fora do cluster automaticamente. + +CronJob atual: + +```yaml +schedule: "0 2 * * *" +timeZone: "America/Sao_Paulo" +suspend: false +concurrencyPolicy: Forbid +``` + +### 19.1. Preparar arquivos de deploy + +```bash +cp deploy/env/evaluator.env.example deploy/env/evaluator.env +cp deploy/langfuse/langfuse_credentials.example.yaml \ + deploy/langfuse/langfuse_credentials.yaml +``` + +Coloque o wallet em: + +```text +deploy/wallet/ +``` + +Credencial GCP: + +```text +deploy/gcp/GCP_ACCESS_KEY.json +``` + +### 19.2. Criar secrets + +DEV: + +```bash +./deploy/scripts/create-secrets.sh dev +``` + +HML: + +```bash +./deploy/scripts/create-secrets.sh hml +``` + +PROD: + +```bash +./deploy/scripts/create-secrets.sh prod +``` + +São criados/atualizados: + +```text +evaluator-secrets +evaluator-wallet +evaluator-langfuse +evaluator-gcp (quando o arquivo GCP existir) +``` + +### 19.3. Build e push + +```bash +export REGISTRY=tim.azurecr.io +export IMAGE_TAG= +./deploy/scripts/build-push.sh prod +``` + +### 19.4. Deploy + +```bash +export REGISTRY=tim.azurecr.io +export IMAGE_TAG= +./deploy/scripts/deploy.sh prod +``` + +Por default o script também executa o job de inicialização do banco. + +Para não executar: + +```bash +export RUN_INIT_DB=false +``` + +### 19.5. Status + +```bash +./deploy/scripts/status.sh prod +``` + +ou diretamente: + +```bash +kubectl -n agent-evaluator-prod get cronjob,jobs,pods +``` + +### 19.6. Execução imediata fora do horário + +```bash +./deploy/scripts/run-now.sh prod +``` + +O script cria um Job baseado no CronJob. + +Acompanhar: + +```bash +kubectl -n agent-evaluator-prod get jobs,pods +kubectl -n agent-evaluator-prod logs -f job/ +``` + + +### 19.7. Interface web de acompanhamento + +A UI é servida pelo componente independente `evaluator-monitor`: + +```text +GET /ui +GET /health +GET /api/runs +GET /api/runs/{run_id} +GET /api/runs/{run_id}/results +``` + +Para acessar localmente um monitor implantado no Kubernetes sem criar Ingress: + +```bash +kubectl -n agent-evaluator-prod port-forward svc/evaluator-monitor 8080:8080 +``` + +Depois abra: + +```text +http://localhost:8080/ui +``` + +A tela mostra runs recentes, `RUN_ID`, agente, status, progresso, eventos e resultados. O browser faz auto-refresh a cada 5 segundos. + +Para executar o monitor localmente: + +```bash +python -m uvicorn evaluator_monitor.app:app --host 0.0.0.0 --port 8080 +``` + +O monitor utiliza as mesmas configurações `ADB_*`/wallet do evaluator. Ele é **read-only** do ponto de vista funcional: não há endpoint para iniciar, cancelar, apagar ou alterar runs. + +> Recomendação de segurança: mantenha o Service como `ClusterIP`. Se futuramente houver necessidade de exposição externa, coloque autenticação/autorização no Ingress/API Gateway corporativo; não exponha o monitor diretamente à Internet. + +--- + +## 20. Como subir uma nova versão + +Fluxo recomendado: + +### 1. Atualizar código/configuração + +Não coloque credenciais reais no Git. + +### 2. Rodar testes + +```bash +python -m pytest -v +python -m compileall evaluator +``` + +### 3. Testar localmente com limite + +```bash +python -m evaluator.cli run-agents \ + --agent-id \ + --limit 5 +``` + +### 4. Validar o run + +```bash +python -m evaluator.cli runs --limit 10 +python -m evaluator.cli progress --events 100 +``` + +### 5. Confirmar GCS + +O evento deve conter: + +```text +GCS_UPLOADED +``` + +com uma URI como: + +```text +gs:////AGENTE__LLM_JUDGE_.TXT.GZ +``` + +### 6. Atualizar secrets/configuração do ambiente + +```bash +./deploy/scripts/create-secrets.sh +``` + +### 7. Build/push com uma tag nova + +```bash +export REGISTRY= +export IMAGE_TAG= +./deploy/scripts/build-push.sh +``` + +### 8. Deploy + +```bash +./deploy/scripts/deploy.sh +``` + +### 9. Rodar manualmente para homologação + +```bash +./deploy/scripts/run-now.sh +``` + +### 10. Verificar logs e artefatos + +```bash +kubectl -n agent-evaluator- logs -f job/ +``` + +Confirme também o objeto no bucket GCS e a auditoria `judge-job-{agentId}` no Langfuse. + +--- + +## 21. Troubleshooting + +### `Collected 0 traces` + +Verifique nesta ordem: + +1. data processada (`daysBack` usa BRT); +2. credencial Langfuse do `agentId` correto; +3. `host` do projeto Langfuse; +4. `langfuse_agent_aliases`; +5. existência de sessions/traces naquele período; +6. se o `agentId` lógico é diferente do `metadata.agent_id` dos traces. + +Exemplo: + +```yaml +- agentId: billing + langfuse_agent_aliases: + - telecom_contas + - billing_agent +``` + +### `No Langfuse credentials configured for agent ...` + +O agente existe em `agents.yaml`, mas não no arquivo apontado por: + +```env +LANGFUSE_CREDENTIALS_CONFIG_PATH +``` + +Adicione uma entrada com o mesmo `agentId`. + +### `Langfuse credentials definition file not found` + +Crie/mapeie o arquivo: + +```text +configs/langfuse/langfuse_credentials.yaml +``` + +ou configure outro path. + +### `TIM specification requires GCS upload...` + +`run-agents` foi executado com: + +```env +ENABLE_GCS_UPLOAD=false +``` + +Altere para `true` e configure a credencial GCP. + +### GCS falha + +Verifique: + +```bash +echo "$GOOGLE_APPLICATION_CREDENTIALS" +ls -l "$GOOGLE_APPLICATION_CREDENTIALS" +``` + +E valide o bucket resolvido por `APP_ENV`/`storage.yaml`. + +### OCI SDK falha em dedicated endpoint + +Confira: + +```env +LLM_PROVIDER=oci_sdk +OCI_GENAI_BASE_URL=https:// +OCI_GENAI_ENDPOINT_ID=ocid1.generativeaiendpoint... +OCI_COMPARTMENT_ID=... +``` + +Para `oci_sdk`, não use `/openai/v1` no endpoint. + +### Banco não inicializado + +```bash +python -m evaluator.cli init-db +``` + +### Ver progresso pela CLI + +O `run-agents` agora imprime o `RUN_ID` assim que a execução é criada: + +```text +[telecom_contas] RUN_ID=8d9a2d1e-... + progress: python -m evaluator.cli progress 8d9a2d1e-... --events 100 +``` + +Também é possível listar runs e consultar qualquer execução: + +```bash +python -m evaluator.cli runs +python -m evaluator.cli progress --events 100 +``` + +### Ver progresso pela interface web + +```bash +python -m uvicorn evaluator_monitor.app:app --host 0.0.0.0 --port 8080 +``` + +Abra `http://localhost:8080/ui`. Em Kubernetes, prefira `kubectl port-forward svc/evaluator-monitor 8080:8080`. + +--- + +## 22. Variáveis de ambiente principais + +| Variável | Uso | +|---|---| +| `APP_ENV` | ambiente usado para resolver storage | +| `ADB_*` | conexão e wallet Oracle | +| `ENABLE_LANGFUSE` | habilita coleta Langfuse | +| `LANGFUSE_CREDENTIALS_CONFIG_PATH` | YAML declarativo por agente | +| `LANGFUSE_*` | fallback global apenas para `run` manual | +| `PUBLISH_LANGFUSE_SCORES` | publica scores de trace de volta no Langfuse | +| `LLM_PROVIDER` | `mock`, `oci_openai`, `oci_sdk`, `openai_compatible` | +| `LLM_PROFILE` | perfil de judge | +| `OCI_AUTH_MODE` | autenticação OCI SDK | +| `OCI_GENAI_BASE_URL` | endpoint OCI GenAI | +| `OCI_GENAI_MODEL` | model ID/nome ou compatibilidade com endpoint OCID | +| `OCI_GENAI_ENDPOINT_ID` | dedicated endpoint OCID | +| `OCI_COMPARTMENT_ID` | compartment OCI | +| `AGENTS_CONFIG_PATH` | arquivo de agentes | +| `TRACE_PROMPT_PATH` | prompt trace-level | +| `SESSION_PROMPT_PATH` | prompt session-level | +| `BATCH_SIZE` | quantidade processada por lote interno | +| `MAX_ATTEMPTS` | limite configurável de tentativas | +| `CONTEXT_TURN_LIMIT` | últimos N turnos enviados ao judge trace | +| `ENABLE_GCS_UPLOAD` | exige upload no fluxo produtivo | +| `STORAGE_CONFIG_PATH` | mapping de buckets por ambiente | +| `JUDGE_GCS_BUCKET` | override de bucket | +| `GOOGLE_APPLICATION_CREDENTIALS` | credencial GCP | +| `OUTPUT_DIR` | diretório temporário de export | + +--- + +## 23. Segurança e operação + +- não versionar `.env` real; +- não versionar `langfuse_credentials.yaml` real; +- não versionar GCP service account JSON; +- não versionar wallet Oracle; +- em OKE prefira `oke_workload_identity` quando aplicável; +- monte credenciais como Kubernetes Secrets; +- use `readOnlyRootFilesystem` no CronJob; +- mantenha `concurrencyPolicy: Forbid` para evitar dois D+1 concorrentes do mesmo deployment; +- nunca use `reset-db` em produção sem plano explícito de restauração. + +--- + +## 24. Checklist de homologação + +Antes de promover uma versão: + +- [ ] `pip check` sem erro; +- [ ] `python -m pytest -v` passando; +- [ ] `python -m compileall evaluator` sem erro; +- [ ] `show-config` aponta para ADB/OCI/configs corretos; +- [ ] `agents.yaml` contém apenas os agentes esperados para aquele deployment; +- [ ] credencial Langfuse existe para cada `agentId` habilitado; +- [ ] `run-agents --limit 5` encontra traces; +- [ ] sampling seleciona sessions, não traces individuais; +- [ ] judge trace/session conclui; +- [ ] arquivo possui cabeçalho de 12 colunas; +- [ ] nome segue `AGENTE_{agentId}_LLM_JUDGE_{YYYYMMDD}_{HHMMSS}.TXT.GZ`; +- [ ] evento `GCS_UPLOADED` confirmado; +- [ ] arquivo local removido depois do upload; +- [ ] auditoria Langfuse `created/completed` confirmada; +- [ ] CronJob está `suspend: false`; +- [ ] timezone do CronJob é `America/Sao_Paulo`; +- [ ] nenhuma credencial real está no repositório. + +--- + +## 25. Resumo para quem está chegando no projeto + +Se você precisa entender o projeto em poucos minutos, pense assim: + +```text +Langfuse guarda as conversas + ↓ +Evaluator escolhe sessions D+1 + ↓ +OCI GenAI julga traces e sessions + ↓ +ADB guarda execução e resultados + ↓ +GCS recebe o arquivo oficial + ↓ +Langfuse registra a auditoria do batch +``` + +Os pontos de extensão ficam em configuração: + +```text +Qual agente? -> configs/judge/agents.yaml +Qual projeto Langfuse?-> langfuse_credentials.yaml +Qual bucket? -> configs/judge/storage.yaml +Qual prompt? -> trace_metrics.yaml / session_metrics.yaml +Qual modelo? -> .env + llm_profiles.yaml +Qual identidade? -> configs/identity.yaml +``` + +A regra arquitetural mais importante é: **o código do evaluator deve permanecer genérico; detalhes de agentes, ambientes, credenciais e storage pertencem à configuração.** diff --git a/configs/.DS_Store b/configs/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..08521bca051d31fcb8a2c300f287847e40dff155 GIT binary patch literal 6148 zcmeHKy-ve05I(m>N=1hbNHBV0?;DsZY;1jiLP=Xd35lq>F=S)s4R{+~fR|u_N8mf# zg#0xyA%yOv^Jib~?8_G?7ZZ`1Zlej&n1})>V{eG2Mff^vLwepa0xCPlGG0$+muZqU ze8uoL8Q^EPM>n*kh}Kkle#2GuYLbTGB2E{u2FFkDuP=wsCv}hL8;|U&yh9AavAJTp zpj)iJLLQrX7X5jfPdS%^`?t?r8gnJ-S4Lfa)x7w9pb_$lDWN%KbVn(&yT*7)o7yga z?D8$=`EXWnwsY=szCZDA$(wT*5e9?-VW5iv)NGbu-=Va^fG{8o6b$h9K|>jh$K0X) zbfB?E0N?;-E9l#QC!l5n(0I%p!U9n)6zD>gKVm2s4!ib##$)c#g_E)?V;;Y<@+TB! zSBG6&cT&cow8DTeP-UR+hGVY(r@PPp)gT!O1H!<+Vn7YTSvbWl`P$mJIj*%DdJbh_ nzuaLFf{rc4@a0l`1hoRY<_^$!%pJl4kskp|gEYdxuQKof, "resolution": <0|1>, "conversationPrecision": <0|1>, "rationale": ""} + + rationale deve ter no máximo 200 caracteres. diff --git a/configs/judge/storage.yaml b/configs/judge/storage.yaml new file mode 100644 index 0000000..6893939 --- /dev/null +++ b/configs/judge/storage.yaml @@ -0,0 +1,5 @@ +storage: + buckets: + dev: tim-bigdata-dev-ca1f-raw-standard-agnt-ai-atendimento + fqa: tim-bigdata-fqa-60ce-raw-standard-agnt-ai-atendimento + prd: tim-bigdata-prod-e305-raw-standard-agnt-ai-atendimento diff --git a/configs/judge/trace_metrics.yaml b/configs/judge/trace_metrics.yaml new file mode 100644 index 0000000..6e73b5a --- /dev/null +++ b/configs/judge/trace_metrics.yaml @@ -0,0 +1,16 @@ +trace_metrics: | + Você é um avaliador imparcial de respostas de agentes conversacionais. + Vai receber o HISTÓRICO da conversa (últimos turnos), a MENSAGEM DO + USUÁRIO e a RESPOSTA DO AGENTE. Sua tarefa é atribuir três notas + numéricas entre 0.0 e 1.0: + + - judgeScore: qualidade global da resposta (clareza, utilidade, tom). + - accuracyScore: factualidade da resposta frente ao contexto fornecido. + - alucinationScore: grau em que a resposta contém afirmações NÃO + suportadas pelo contexto (alto = mais alucinação = pior). + + Retorne SOMENTE um JSON válido, sem nenhum texto adicional, no formato: + + {"judgeScore": , "accuracyScore": , "alucinationScore": , "rationale": ""} + + rationale deve ter no máximo 200 caracteres. diff --git a/configs/langfuse/langfuse_credentials.example.yaml b/configs/langfuse/langfuse_credentials.example.yaml new file mode 100644 index 0000000..3b061d9 --- /dev/null +++ b/configs/langfuse/langfuse_credentials.example.yaml @@ -0,0 +1,22 @@ +# Copy/mount this file as configs/langfuse/langfuse_credentials.yaml, or point +# LANGFUSE_CREDENTIALS_CONFIG_PATH to another location. +# +# Production recommendation: mount the complete file from a Kubernetes Secret. +# The evaluator loops over all entries; adding an agent requires no Python change. +langfuse: + agents: + - agentId: billing + host: https://langfuse.example.internal + publicKey: pk-example-billing + secretKey: sk-example-billing + + - agentId: sales + host: https://langfuse.example.internal + publicKey: pk-example-sales + secretKey: sk-example-sales + +# Alternative: keep only references in Git and source values from env or files: +# - agentId: collection +# hostEnv: COLLECTION_LANGFUSE_HOST +# publicKeyEnv: COLLECTION_LANGFUSE_PUBLIC_KEY +# secretKeyFile: /var/secrets/langfuse/collection-secret-key diff --git a/configs/llm_profiles/llm_profiles.yaml b/configs/llm_profiles/llm_profiles.yaml new file mode 100644 index 0000000..3caa69b --- /dev/null +++ b/configs/llm_profiles/llm_profiles.yaml @@ -0,0 +1,9 @@ +profiles: + judge: + temperature: 0 + max_tokens: 900 + mock: + provider: mock + model: mock-judge + temperature: 0 + max_tokens: 300 diff --git a/deploy/.DS_Store b/deploy/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..01a15a8ee2b3a27606f6fa181a7d7ca2d03516ec GIT binary patch literal 6148 zcmeHKJ5Iwu5S_)5u%Jl+0y84p1n(*u4^Zj*idg@yHYpnUDyDFzc zzH2(6v7fKCQ(D}7o#goN@xbQOaigrvSGD^2-Z)=KDV<}7L%N^|jcMR#4$hYOxHUYd zAG1AI)^e*^=jYd@dDuiPJ-VeU%IR*AkKHM%TJUgddU}~Zhl*Fp&uTBf9{FhV59OZ; zdNxbAZBS-mKo}4P<_z%nA;MscEfxm#rvsBd0sw1JYr(bdZNP{Xz}RA85E_Uwp+FO= z>=8qmaOflJ7h5b0ns8D&GuE-0l|7*-ogMl}yOW9y$}9{B13m+Rxvg{mZ>>K6`$2Li z3lQ;71ww1}n6PH2?qr literal 0 HcmV?d00001 diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..c5836fc --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,125 @@ +# Deployment Kubernetes — Evaluator TIM + +O **evaluator continua sendo batch/CronJob-only**. O componente `evaluator-batch` não expõe HTTP nem gRPC. + +Para acompanhamento operacional existe um componente **separado e somente leitura**, `evaluator-monitor`, que consulta as tabelas do Autonomous Database e disponibiliza `/ui` e APIs de monitoramento. Essa separação mantém o processamento em batch e evita acoplar uma API HTTP ao evaluator. + +Recursos aplicados pelo Kustomize: + +```text +Namespace +ServiceAccount +ConfigMap +CronJob evaluator-batch +Deployment evaluator-monitor +Service evaluator-monitor (ClusterIP) +``` + +O CronJob executa diariamente às 02:00 em `America/Sao_Paulo`, com `suspend: false` e `concurrencyPolicy: Forbid`. + +## Preparação + +```bash +cp deploy/env/evaluator.env.example deploy/env/evaluator.env +``` + +Configure ADB, OCI GenAI e GCS no env file. Copie `deploy/langfuse/langfuse_credentials.example.yaml` para `deploy/langfuse/langfuse_credentials.yaml` e declare ali quantos agentes/projetos Langfuse forem necessários. O script cria o Secret `evaluator-langfuse` dinamicamente a partir desse arquivo. + +Wallet Oracle: + +```text +deploy/wallet/ +``` + +Credencial GCP opcionalmente materializada pelo script: + +```text +deploy/gcp/GCP_ACCESS_KEY.json +``` + +## Secrets + +```bash +./deploy/scripts/create-secrets.sh dev +./deploy/scripts/create-secrets.sh hml +./deploy/scripts/create-secrets.sh prod +``` + +## Build + +```bash +export REGISTRY=tim.azurecr.io +export IMAGE_TAG=0.3.0 +./deploy/scripts/build-push.sh prod +``` + +## Deploy + +```bash +export REGISTRY=tim.azurecr.io +export IMAGE_TAG=0.3.0 +./deploy/scripts/deploy.sh prod +``` + +## Status + +```bash +./deploy/scripts/status.sh prod +``` + +Ou: + +```bash +kubectl -n agent-evaluator-prod get cronjob,jobs,pods,deploy,svc +``` + +## Execução imediata + +```bash +./deploy/scripts/run-now.sh prod +``` + +Acompanhar logs do batch: + +```bash +kubectl -n agent-evaluator-prod get jobs,pods +kubectl -n agent-evaluator-prod logs -f job/ +``` + +O comando `run-agents` também imprime o `RUN_ID` assim que cada execução é criada, permitindo: + +```bash +python -m evaluator.cli progress --events 100 +``` + +## Interface web + +O Service do monitor é `ClusterIP`, portanto não fica exposto externamente por default. + +Para acessar via sua máquina: + +```bash +kubectl -n agent-evaluator-prod port-forward svc/evaluator-monitor 8080:8080 +``` + +Abra: + +```text +http://localhost:8080/ui +``` + +Endpoints read-only: + +```text +GET /health +GET /api/runs +GET /api/runs/{run_id} +GET /api/runs/{run_id}/results +GET /ui +``` + +Não há endpoints para iniciar, cancelar, apagar ou alterar runs. + +## Configuração de storage + +Os buckets por ambiente ficam em `configs/judge/storage.yaml`. Para usar outro arquivo, configure `STORAGE_CONFIG_PATH`. `JUDGE_GCS_BUCKET` sobrescreve o mapping quando necessário. diff --git a/deploy/env/evaluator.env.example b/deploy/env/evaluator.env.example new file mode 100644 index 0000000..4d80ed3 --- /dev/null +++ b/deploy/env/evaluator.env.example @@ -0,0 +1,22 @@ +# Copie para evaluator.env e não faça commit do arquivo real. +APP_ENV=dev +ADB_USER=admin +ADB_PASSWORD=change-me +ADB_DSN=oradb23ai_high +ADB_WALLET_PASSWORD=change-me + +LLM_PROVIDER=oci_sdk +OCI_AUTH_MODE=oke_workload_identity +OCI_GENAI_BASE_URL=https://.inference.generativeai.sa-saopaulo-1.oci.oraclecloud.com +OCI_GENAI_MODEL=ocid1.generativeaiendpoint... +OCI_GENAI_ENDPOINT_ID=ocid1.generativeaiendpoint... +OCI_COMPARTMENT_ID=ocid1.compartment... +OCI_REGION=sa-saopaulo-1 + +ENABLE_GCS_UPLOAD=true +JUDGE_GCS_BUCKET= +STORAGE_CONFIG_PATH=configs/judge/storage.yaml +GOOGLE_APPLICATION_CREDENTIALS=/var/secrets/google/GCP_ACCESS_KEY.json + +# Langfuse per-agent credentials are supplied by the mounted YAML file, not env names. +LANGFUSE_CREDENTIALS_CONFIG_PATH=/var/secrets/langfuse/langfuse_credentials.yaml diff --git a/deploy/k8s/.DS_Store b/deploy/k8s/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..707cc982b9c3b344c1d7e485fce508d9fc1062f3 GIT binary patch literal 6148 zcmeHKOHRWu5FM8Y1+kzzSbD{p?pT^CoS+wg&V$FGExgW9j{ScPi9w5-3}tn z@Lw6=XO~e)m$als`uvVx`o6j;in3}-*z(ug_VfMl?JV`rH~QO~{=4yvRJ>FTJ<@{i zsKWkFv`qKkyX93&H!pQOTI5?qcLC% z7y~N?`1|0Yj6tyzjGqn+u>}BTFh{|hdkN0*ib1gxga_gz6)34rTMQ@ZuzQUQik+aO zlhfwI>CR3&6c=~L`aXt}3xYNp1I9qgz>Zvwx&EL3-v6gbc4Z7016##_%Zgbs#Vu)V x9o!t(S`WQ|vT$4{xJtn!mSV(mDL#Tmf!*^27!*4}SRnQz5NWW%82D2Lz5$YCO@ROa literal 0 HcmV?d00001 diff --git a/deploy/k8s/base/configmap.yaml b/deploy/k8s/base/configmap.yaml new file mode 100644 index 0000000..22edd21 --- /dev/null +++ b/deploy/k8s/base/configmap.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: evaluator-config + namespace: agent-evaluator +data: + ENABLE_LANGFUSE: "true" + LANGFUSE_CREDENTIALS_CONFIG_PATH: "/var/secrets/langfuse/langfuse_credentials.yaml" + PUBLISH_LANGFUSE_SCORES: "false" + LLM_PROVIDER: "oci_sdk" + LLM_PROFILE: "judge" + LLM_PROFILES_PATH: "configs/llm_profiles/llm_profiles.yaml" + LLM_TEMPERATURE: "0" + LLM_MAX_TOKENS: "900" + LLM_TIMEOUT_SECONDS: "120" + AGENTS_CONFIG_PATH: "configs/judge/agents.yaml" + TRACE_PROMPT_PATH: "configs/judge/trace_metrics.yaml" + SESSION_PROMPT_PATH: "configs/judge/session_metrics.yaml" + OUTPUT_DIR: "/app/output" + BATCH_SIZE: "50" + MAX_ATTEMPTS: "3" + ENABLE_GCS_UPLOAD: "true" + STORAGE_CONFIG_PATH: "configs/judge/storage.yaml" + ADB_TABLE_PREFIX: "AGENTFW" + ADB_WALLET_LOCATION: "/opt/oracle/wallet" diff --git a/deploy/k8s/base/cronjob.yaml b/deploy/k8s/base/cronjob.yaml new file mode 100644 index 0000000..44e3caf --- /dev/null +++ b/deploy/k8s/base/cronjob.yaml @@ -0,0 +1,94 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: evaluator-batch + namespace: agent-evaluator + labels: + app.kubernetes.io/name: evaluator-batch + app.kubernetes.io/component: batch + app.kubernetes.io/part-of: agent-framework +spec: + schedule: "0 2 * * *" + timeZone: "America/Sao_Paulo" + suspend: false + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 5 + startingDeadlineSeconds: 1800 + jobTemplate: + spec: + backoffLimit: 2 + activeDeadlineSeconds: 21600 + ttlSecondsAfterFinished: 86400 + template: + metadata: + labels: + app.kubernetes.io/name: evaluator-batch + app.kubernetes.io/component: batch + spec: + restartPolicy: Never + serviceAccountName: evaluator + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: evaluator + image: agent-framework-evaluator:latest + imagePullPolicy: IfNotPresent + command: ["python", "-m", "evaluator.cli", "run-agents"] + args: ["--source", "langfuse"] + envFrom: + - configMapRef: + name: evaluator-config + - secretRef: + name: evaluator-secrets + env: + - name: TNS_ADMIN + value: /opt/oracle/wallet + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "2" + memory: 2Gi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: wallet + mountPath: /opt/oracle/wallet + readOnly: true + - name: output + mountPath: /app/output + - name: tmp + mountPath: /tmp + - name: gcp-credentials + mountPath: /var/secrets/google + readOnly: true + - name: langfuse-credentials + mountPath: /var/secrets/langfuse + readOnly: true + volumes: + - name: wallet + secret: + secretName: evaluator-wallet + - name: output + emptyDir: {} + - name: tmp + emptyDir: {} + - name: gcp-credentials + secret: + secretName: evaluator-gcp + optional: true + - name: langfuse-credentials + secret: + secretName: evaluator-langfuse + optional: false diff --git a/deploy/k8s/base/init-db-job.yaml b/deploy/k8s/base/init-db-job.yaml new file mode 100644 index 0000000..98a8255 --- /dev/null +++ b/deploy/k8s/base/init-db-job.yaml @@ -0,0 +1,63 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: evaluator-init-db + namespace: agent-evaluator + labels: + app.kubernetes.io/name: evaluator-init-db + app.kubernetes.io/component: database-migration +spec: + backoffLimit: 3 + ttlSecondsAfterFinished: 3600 + template: + metadata: + labels: + app.kubernetes.io/name: evaluator-init-db + spec: + restartPolicy: OnFailure + serviceAccountName: evaluator + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: init-db + image: agent-framework-evaluator:latest + imagePullPolicy: IfNotPresent + command: ["python", "-m", "evaluator.cli", "init-db"] + envFrom: + - configMapRef: + name: evaluator-config + - secretRef: + name: evaluator-secrets + env: + - name: TNS_ADMIN + value: /opt/oracle/wallet + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: wallet + mountPath: /opt/oracle/wallet + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: wallet + secret: + secretName: evaluator-wallet + - name: tmp + emptyDir: {} diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml new file mode 100644 index 0000000..342631b --- /dev/null +++ b/deploy/k8s/base/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: agent-evaluator +resources: + - namespace.yaml + - serviceaccount.yaml + - configmap.yaml + - cronjob.yaml + - monitor-deployment.yaml + - monitor-service.yaml +commonLabels: + app.kubernetes.io/managed-by: kustomize diff --git a/deploy/k8s/base/monitor-deployment.yaml b/deploy/k8s/base/monitor-deployment.yaml new file mode 100644 index 0000000..046a97f --- /dev/null +++ b/deploy/k8s/base/monitor-deployment.yaml @@ -0,0 +1,82 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: evaluator-monitor + namespace: agent-evaluator + labels: + app.kubernetes.io/name: evaluator-monitor + app.kubernetes.io/component: monitoring + app.kubernetes.io/part-of: agent-framework +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: evaluator-monitor + template: + metadata: + labels: + app.kubernetes.io/name: evaluator-monitor + app.kubernetes.io/component: monitoring + spec: + serviceAccountName: evaluator + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: monitor + image: agent-framework-evaluator:latest + imagePullPolicy: IfNotPresent + command: ["python", "-m", "uvicorn"] + args: ["evaluator_monitor.app:app", "--host", "0.0.0.0", "--port", "8080"] + ports: + - name: http + containerPort: 8080 + envFrom: + - configMapRef: + name: evaluator-config + - secretRef: + name: evaluator-secrets + env: + - name: TNS_ADMIN + value: /opt/oracle/wallet + readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 15 + periodSeconds: 20 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: wallet + mountPath: /opt/oracle/wallet + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: wallet + secret: + secretName: evaluator-wallet + - name: tmp + emptyDir: {} diff --git a/deploy/k8s/base/monitor-service.yaml b/deploy/k8s/base/monitor-service.yaml new file mode 100644 index 0000000..138e88e --- /dev/null +++ b/deploy/k8s/base/monitor-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: evaluator-monitor + namespace: agent-evaluator + labels: + app.kubernetes.io/name: evaluator-monitor + app.kubernetes.io/component: monitoring +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: evaluator-monitor + ports: + - name: http + port: 8080 + targetPort: http diff --git a/deploy/k8s/base/namespace.yaml b/deploy/k8s/base/namespace.yaml new file mode 100644 index 0000000..dd2b322 --- /dev/null +++ b/deploy/k8s/base/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: agent-evaluator + labels: + app.kubernetes.io/part-of: agent-framework diff --git a/deploy/k8s/base/secret-template.yaml b/deploy/k8s/base/secret-template.yaml new file mode 100644 index 0000000..93e3bf9 --- /dev/null +++ b/deploy/k8s/base/secret-template.yaml @@ -0,0 +1,35 @@ +# TEMPLATE ONLY. Do not apply or commit real values. +apiVersion: v1 +kind: Secret +metadata: + name: evaluator-secrets + namespace: agent-evaluator +type: Opaque +stringData: + ADB_USER: "REPLACE_ME" + ADB_PASSWORD: "REPLACE_ME" + ADB_DSN: "REPLACE_ME" + ADB_WALLET_PASSWORD: "REPLACE_ME" + OCI_GENAI_BASE_URL: "https://private-endpoint.inference.generativeai.region.oci.oraclecloud.com" + OCI_GENAI_MODEL: "ocid1.generativeaiendpoint.REPLACE_ME" + OCI_GENAI_ENDPOINT_ID: "ocid1.generativeaiendpoint.REPLACE_ME" + OCI_COMPARTMENT_ID: "ocid1.compartment.REPLACE_ME" + OCI_AUTH_MODE: "oke_workload_identity" + GOOGLE_APPLICATION_CREDENTIALS: "/var/secrets/google/GCP_ACCESS_KEY.json" +--- +# Langfuse credentials are one data-driven YAML, not one env namespace per agent. +# Add as many entries as required; no Python or manifest change is needed. +apiVersion: v1 +kind: Secret +metadata: + name: evaluator-langfuse + namespace: agent-evaluator +type: Opaque +stringData: + langfuse_credentials.yaml: | + langfuse: + agents: + - agentId: REPLACE_WITH_AGENT_ID + host: https://langfuse.example + publicKey: pk-REPLACE_ME + secretKey: sk-REPLACE_ME diff --git a/deploy/k8s/base/serviceaccount.yaml b/deploy/k8s/base/serviceaccount.yaml new file mode 100644 index 0000000..d0ec71e --- /dev/null +++ b/deploy/k8s/base/serviceaccount.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: evaluator + namespace: agent-evaluator +automountServiceAccountToken: false diff --git a/deploy/k8s/overlays/dev/kustomization.yaml b/deploy/k8s/overlays/dev/kustomization.yaml new file mode 100644 index 0000000..888b487 --- /dev/null +++ b/deploy/k8s/overlays/dev/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: agent-evaluator-dev +resources: + - ../../base +images: + - name: agent-framework-evaluator + newName: REPLACE_REGISTRY/agent-framework-evaluator + newTag: dev +patches: + - target: + kind: Namespace + name: agent-evaluator + patch: |- + - op: replace + path: /metadata/name + value: agent-evaluator-dev + - target: + kind: CronJob + name: evaluator-batch + patch: |- + - op: replace + path: /spec/suspend + value: false diff --git a/deploy/k8s/overlays/hml/kustomization.yaml b/deploy/k8s/overlays/hml/kustomization.yaml new file mode 100644 index 0000000..8821b3b --- /dev/null +++ b/deploy/k8s/overlays/hml/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: agent-evaluator-hml +resources: + - ../../base +images: + - name: agent-framework-evaluator + newName: REPLACE_REGISTRY/agent-framework-evaluator + newTag: hml +patches: + - target: + kind: Namespace + name: agent-evaluator + patch: |- + - op: replace + path: /metadata/name + value: agent-evaluator-hml + - target: + kind: CronJob + name: evaluator-batch + patch: |- + - op: replace + path: /spec/suspend + value: false diff --git a/deploy/k8s/overlays/prod/kustomization.yaml b/deploy/k8s/overlays/prod/kustomization.yaml new file mode 100644 index 0000000..5949df6 --- /dev/null +++ b/deploy/k8s/overlays/prod/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: agent-evaluator-prod +resources: + - ../../base +images: + - name: agent-framework-evaluator + newName: REPLACE_REGISTRY/agent-framework-evaluator + newTag: stable +patches: + - target: + kind: Namespace + name: agent-evaluator + patch: |- + - op: replace + path: /metadata/name + value: agent-evaluator-prod + - target: + kind: CronJob + name: evaluator-batch + patch: |- + - op: replace + path: /spec/suspend + value: false diff --git a/deploy/langfuse/langfuse_credentials.example.yaml b/deploy/langfuse/langfuse_credentials.example.yaml new file mode 100644 index 0000000..3b061d9 --- /dev/null +++ b/deploy/langfuse/langfuse_credentials.example.yaml @@ -0,0 +1,22 @@ +# Copy/mount this file as configs/langfuse/langfuse_credentials.yaml, or point +# LANGFUSE_CREDENTIALS_CONFIG_PATH to another location. +# +# Production recommendation: mount the complete file from a Kubernetes Secret. +# The evaluator loops over all entries; adding an agent requires no Python change. +langfuse: + agents: + - agentId: billing + host: https://langfuse.example.internal + publicKey: pk-example-billing + secretKey: sk-example-billing + + - agentId: sales + host: https://langfuse.example.internal + publicKey: pk-example-sales + secretKey: sk-example-sales + +# Alternative: keep only references in Git and source values from env or files: +# - agentId: collection +# hostEnv: COLLECTION_LANGFUSE_HOST +# publicKeyEnv: COLLECTION_LANGFUSE_PUBLIC_KEY +# secretKeyFile: /var/secrets/langfuse/collection-secret-key diff --git a/deploy/scripts/build-push.sh b/deploy/scripts/build-push.sh new file mode 100644 index 0000000..8e46159 --- /dev/null +++ b/deploy/scripts/build-push.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +source "$(dirname "$0")/common.sh" +require_command docker +validate_environment + +REGISTRY="${REGISTRY:?Defina REGISTRY, por exemplo: tim.azurecr.io}" +IMAGE_NAME="${IMAGE_NAME:-agent-framework-evaluator}" +IMAGE_TAG="${IMAGE_TAG:-$(git -C "${ROOT_DIR}" rev-parse --short HEAD 2>/dev/null || date +%Y%m%d%H%M%S)}" +IMAGE="${REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}" +PLATFORM="${PLATFORM:-linux/amd64}" + +echo "Construindo ${IMAGE} para ${PLATFORM}..." +docker build --platform "${PLATFORM}" -t "${IMAGE}" "${ROOT_DIR}" + +echo "Enviando ${IMAGE}..." +docker push "${IMAGE}" +echo "IMAGE=${IMAGE}" diff --git a/deploy/scripts/common.sh b/deploy/scripts/common.sh new file mode 100644 index 0000000..ad4f834 --- /dev/null +++ b/deploy/scripts/common.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +ENVIRONMENT="${ENVIRONMENT:-${1:-dev}}" +NAMESPACE="${NAMESPACE:-agent-evaluator-${ENVIRONMENT}}" +OVERLAY_DIR="${ROOT_DIR}/deploy/k8s/overlays/${ENVIRONMENT}" + +require_command() { + command -v "$1" >/dev/null 2>&1 || { echo "Erro: comando '$1' não encontrado." >&2; exit 1; } +} + +validate_environment() { + case "${ENVIRONMENT}" in dev|hml|prod) ;; *) echo "Ambiente inválido: ${ENVIRONMENT}. Use dev, hml ou prod." >&2; exit 1;; esac + [[ -d "${OVERLAY_DIR}" ]] || { echo "Overlay não encontrado: ${OVERLAY_DIR}" >&2; exit 1; } +} diff --git a/deploy/scripts/create-secrets.sh b/deploy/scripts/create-secrets.sh new file mode 100644 index 0000000..decb726 --- /dev/null +++ b/deploy/scripts/create-secrets.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +source "$(dirname "$0")/common.sh" +require_command kubectl +validate_environment + +ENV_FILE="${ENV_FILE:-${ROOT_DIR}/deploy/env/evaluator.env}" +WALLET_DIR="${WALLET_DIR:-${ROOT_DIR}/deploy/wallet}" + +[[ -f "${ENV_FILE}" ]] || { echo "Arquivo de variáveis não encontrado: ${ENV_FILE}" >&2; exit 1; } +[[ -d "${WALLET_DIR}" ]] || { echo "Diretório do wallet não encontrado: ${WALLET_DIR}" >&2; exit 1; } + +kubectl create namespace "${NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f - +kubectl -n "${NAMESPACE}" create secret generic evaluator-secrets \ + --from-env-file="${ENV_FILE}" \ + --dry-run=client -o yaml | kubectl apply -f - + +wallet_args=() +while IFS= read -r -d '' file; do + wallet_args+=("--from-file=$(basename "${file}")=${file}") +done < <(find "${WALLET_DIR}" -maxdepth 1 -type f -print0) + +((${#wallet_args[@]} > 0)) || { echo "Nenhum arquivo encontrado em ${WALLET_DIR}" >&2; exit 1; } +kubectl -n "${NAMESPACE}" create secret generic evaluator-wallet \ + "${wallet_args[@]}" \ + --dry-run=client -o yaml | kubectl apply -f - + + + +LANGFUSE_CREDENTIALS_FILE="${LANGFUSE_CREDENTIALS_FILE:-${ROOT_DIR}/deploy/langfuse/langfuse_credentials.yaml}" +if [[ -f "${LANGFUSE_CREDENTIALS_FILE}" ]]; then + kubectl -n "${NAMESPACE}" create secret generic evaluator-langfuse \ + --from-file=langfuse_credentials.yaml="${LANGFUSE_CREDENTIALS_FILE}" \ + --dry-run=client -o yaml | kubectl apply -f - + echo "Credenciais Langfuse atualizadas no secret evaluator-langfuse." +else + echo "Arquivo de credenciais Langfuse não encontrado: ${LANGFUSE_CREDENTIALS_FILE}" >&2 + exit 1 +fi + +GCP_CREDENTIALS_FILE="${GCP_CREDENTIALS_FILE:-${ROOT_DIR}/deploy/gcp/GCP_ACCESS_KEY.json}" +if [[ -f "${GCP_CREDENTIALS_FILE}" ]]; then + kubectl -n "${NAMESPACE}" create secret generic evaluator-gcp \ + --from-file=GCP_ACCESS_KEY.json="${GCP_CREDENTIALS_FILE}" \ + --dry-run=client -o yaml | kubectl apply -f - + echo "Credencial GCP atualizada no secret evaluator-gcp." +else + echo "Aviso: credencial GCP não encontrada em ${GCP_CREDENTIALS_FILE}; secret evaluator-gcp não foi alterado." >&2 +fi + +echo "Secrets atualizados no namespace ${NAMESPACE}." diff --git a/deploy/scripts/deploy.sh b/deploy/scripts/deploy.sh new file mode 100644 index 0000000..07bf9f5 --- /dev/null +++ b/deploy/scripts/deploy.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +source "$(dirname "$0")/common.sh" +require_command kubectl +validate_environment + +REGISTRY="${REGISTRY:?Defina REGISTRY, por exemplo: tim.azurecr.io}" +IMAGE_NAME="${IMAGE_NAME:-agent-framework-evaluator}" +IMAGE_TAG="${IMAGE_TAG:?Defina IMAGE_TAG com a versão já publicada}" +IMAGE="${REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}" +RUN_INIT_DB="${RUN_INIT_DB:-true}" +TIMEOUT="${TIMEOUT:-300s}" + +kubectl kustomize "${OVERLAY_DIR}" \ + | sed "s#REPLACE_REGISTRY/agent-framework-evaluator:[^[:space:]]*#${IMAGE}#g" \ + | kubectl apply -f - + +if [[ "${RUN_INIT_DB}" == "true" ]]; then + job_name="evaluator-init-db-${ENVIRONMENT}-$(date +%s)" + sed -e "s#agent-evaluator#${NAMESPACE}#g" \ + -e "s#name: evaluator-init-db#name: ${job_name}#" \ + -e "s#agent-framework-evaluator:latest#${IMAGE}#" \ + "${ROOT_DIR}/deploy/k8s/base/init-db-job.yaml" | kubectl apply -f - + kubectl -n "${NAMESPACE}" wait --for=condition=complete "job/${job_name}" --timeout="${TIMEOUT}" || { + kubectl -n "${NAMESPACE}" logs "job/${job_name}" --all-containers=true || true + exit 1 + } +fi + +kubectl -n "${NAMESPACE}" get cronjob,pod,jobs diff --git a/deploy/scripts/run-now.sh b/deploy/scripts/run-now.sh new file mode 100644 index 0000000..36c0c60 --- /dev/null +++ b/deploy/scripts/run-now.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +source "$(dirname "$0")/common.sh" +require_command kubectl +validate_environment + +CRONJOB="${CRONJOB:-evaluator-batch}" +JOB_NAME="${JOB_NAME:-evaluator-manual-${ENVIRONMENT}-$(date +%Y%m%d%H%M%S)}" +kubectl -n "${NAMESPACE}" create job --from="cronjob/${CRONJOB}" "${JOB_NAME}" +echo "Job criado: ${JOB_NAME}" +echo "Logs: kubectl -n ${NAMESPACE} logs -f job/${JOB_NAME}" diff --git a/deploy/scripts/status.sh b/deploy/scripts/status.sh new file mode 100644 index 0000000..fa3bba8 --- /dev/null +++ b/deploy/scripts/status.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +source "$(dirname "$0")/common.sh" +require_command kubectl +validate_environment +kubectl -n "${NAMESPACE}" get cronjob,jobs,pods,deploy,svc +printf '\nEventos recentes:\n' +kubectl -n "${NAMESPACE}" get events --sort-by=.lastTimestamp | tail -30 diff --git a/deploy/scripts/undeploy.sh b/deploy/scripts/undeploy.sh new file mode 100644 index 0000000..5a70183 --- /dev/null +++ b/deploy/scripts/undeploy.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +source "$(dirname "$0")/common.sh" +require_command kubectl +validate_environment + +kubectl delete -k "${OVERLAY_DIR}" --ignore-not-found +if [[ "${DELETE_NAMESPACE:-false}" == "true" ]]; then + kubectl delete namespace "${NAMESPACE}" --ignore-not-found +fi diff --git a/evaluator/.DS_Store b/evaluator/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..fe416d7402a51081a7f92d332af8a4d2a441ea5f GIT binary patch literal 8196 zcmeHMzi-n(6n;me5E>ydAhk;+OGuSqN(LC1LJ5Kmq_C&tM^hkWuV1|a zUV*h#fbS0hCfkO#v9j{kfkn0euq7Op1E0|c$e2LehPJV?G8B8N?m_5Kp<4{0!r>1X z4%^T+R#t_RP~jwWW}!P2A+sYckZ=+WW&P?E@CxJ=;NHE8ZoN%onp^#QdpM}Kx=}b* zLE*6*8Ai=!XBc&Ws62f2?fciQPq&t#dw+o*ADS#)V_Uj_w+ti9AsA2TF<2kjT(=lj zfHHZ0IDNn1S%^$IE=St@SWemmqX_JVcn{JA!-ysshGOT23ukh?`ZoNbwJh8-j8c}X zuoh1dGtaW*Iidq-IItCLUCH`QUcbK77TQ^Rn0#CqiwBAhqa=AusBd$){L=K9`CmLu z@+Yxo?jpaPH{iS#oOch`Xe6s>N^?8^#gih8$>aHMeW62cFO$zQTmi2#J)s$x4Ztl^ z+1KBJljS%0RBtD`WH4!+xG*-42dvMAtl`jB?)e+zyp27~Yv97zJmh&-!JyCIS=7f7_;7`EGu0^qhB}ZYx08E` zE>3i+(`vNESJYUy^WZyyb|LIETQQ~kd4$Pt_j7Y`kIC#~s^>;oeAJJ&O?d@7s2>P{wB#AY&>+I)f&o-%5reCLr%KNa~iier{fgeoA6V zqJC;wVoqsdNq$j$T4sfQVtQ&`NqkySVs2`Aeo=OOesX3!Tm?d{etdjpUS>&ryk0@& gEe@O9{FKt1RJ$TppxGe1i$RQ!%#4hTMa)1J0ODdT9{>OV literal 0 HcmV?d00001 diff --git a/evaluator/__pycache__/cli.cpython-313.pyc b/evaluator/__pycache__/cli.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..84070a375f2a63886218a31ca58133f24c9b3a23 GIT binary patch literal 8541 zcmb_BTW}lKb$1_l^C61h6C|~yo|MgpEXlTBv?!61_z)-&fGiuLRv@q>A%Y9)E+Cq0 zWYKXQ({5)>$8jxZGGnEanbI@;P=8wGtClCz(I*f{LwX}6ooS{s?iW?rRh;yr=iJ3E zK$>j&(aYi8d(XMgbIv_4JS{DC6DSjtH?JPwO31%q$1I}J;GXvIgnUc_B)|=jevWby zw)p{`^6(V~gnoev{UR0fa2vG&TpY0XJE()fYy-}I7j?0=eZbxCp`QK{TGC%iOZ&@c zS${b#XE?{emi`J_!P?G&t^Jj>lC@m}-u^0D#oF$H>i!y9!`hyK+WtCPCy`6G;REKe z3*QzJC<&Av;;1iBR!?ZdCExIo0@iE=$`5izJFumm9L$4h(bfq=g zTQpkQWOS7^x+$1idQ-7M9rJbs+ro%&N0ywFI^Y>etQMzR4_K z(X1VThIO-cT-q6EJbaOmh?9^@z7~s`5*g>m`PQ9ZBXwIj-4jvOWGa%1Csobg%InTp zB&DR{Q;P1wI;JF25g6utJ*g^v>R1xmwrLtyQ>`}Ld5$K>siJ9mX<$08stP@sOr)n& z-E|^jHr+wxjZ_|75rh?c6On06i5Va~Kq485$&qnIO=-~G(tACUNVD_ws^f800hqS{ z8&;;1T0E7cvjBH$N-7ms$8lcSC{4++$gDgQS7XVUxwdXxQ94Bx zHlav^gJ-4wbZlIa^6MrkEFexOiRoxw{+Q_Ds}l&}o}PlnWq5ZXa)1kPKxwDSh1n3S zhT%=FFD)d*6=-g9Y6ofM{n6sFF6exOTLfX_F77gaS-322ClWq#E3ENLA%lC$rK;thdp_8(I;RF*aMD$^N=tl zhWL;W5)TLg7b6h~+JFqiT)hue)dbw&I`^^Qx0Mi;gB44QmdZDSBCKc+xC5R;pvFR+ zhN=zz4(7=$ksZWI{LRhed9o)zCk8Yc<0Y~UCAB!uAm=@MjAbuk^<{AxXG&wb$P*O_lMu!1N!2?O$n5#8< z(+Z7(Xu*`jUHjWR$8PM@-HIi&Un9FfGE)(Xq6sg`#-{5ww#ym;gQsb5|5q@nOECBz zBEz`Ex)R__Iv(S!TLTXz5_AMOWF~%h!*`SUd_XWMsB)W(5sSv~=DOx?k-s9OAc4sD zL)@?>MTO&<^O#id-{ZPakk62?;Z0o|+!S~ZX2!iK#-WOTc3sb=v#oViThKmiy@H}U zIWP8xxH2$eA-;_47eF3MMG`1O2_1(&kVlnCTe+WB!8BSAl}>cW6evd!lvAIBVvD3x zNja{@Q?eGFP^KcZ1_lAbqg)4Zk`wW%cuMy$VUgo0WlEzc@wz((n%4l#@rR|a-vD5% zKYFoFbr%}bXx^x@c*f^^&rH$*NrbecX zl||fDcUn_XdQ*xn;G*{pPtqE=&0VhX6yE&iDWdA(7M{; zKwu{d{pe^ltfu9l`oq^`o~*R&{=3$nw=SK@wH#b-Idiw_?i;_{wrE?N_^^DXw(&F9 zr>@((ms)bQZHu;*YTqaE597C8(64S=6jmy0Z`FSC%A@9kU!=3mXP2AhY_puXqGZO@ z?3HBZ#+w=FHs4(K-CXQ>RNr_j_SwXz6Sr^Vnhq{E9ll$Ut3R?_fBf#X#okAC4Yz`y zUHtUo?L@9||8nEYcdq5?4lVYqcpE+$`f%uW&rF+UAe6cAS|;>*razpybR{!# zCDR{S9Qw6)bk#+?+rRSN{7$Qcd%fNL{7H$s*TJ1^67O|(cR)vXj)5d-6G{wNxf8}P zsvAsnP}B<)iA+)BQuGUH2~@f>5{t>m0@@1QRy%zGp>{CA=|qaQ0#rohNF9d;7T)RrY%aYM}k4#|`i=t2k`exKHeKeA(;(2&iyJM4MHk@uV&Q)mnM_ z5-dYsM%v&Ly&@STEnP!OdK6F=#895%7_wB zf{-~UFi}ah#YWfo2dH1cW^zw|3XP9R!50AUfIq~ksO|H15mzzGMta$_>0nbgo@4n9 z<|?GfX$)IhF(3rQfUVu~H&6q%+Wn_!6U@{3p@F#rmRltmnof$*lOk$NqDqu1 zpi5$smX1b2KaQmniP`qgdEJgWCauv%Sg6%zXfe<`Nvh}`Tp_PF^^!GO6pxMW8%56$ zEoYt^20`W6C`I>e2mEV?p_(VZ_iVdUmG!(d-?Jh*-|c^^e_5>0iuDh~2CMJOioOS8 z<0H|v7|iUtly#3h6fe_bFn!K}8hleW;d}f(o1|jHxiR^IjC}MJ+@d;9HLw!VNhQ{i zrx!gANDw~M?i6qRIblaKgpZvIZ`%i$=ipzfg$iyf)-LV%wb+4o>B($Al64<_C>}HJ z;?X9OGi}kNIu;*aBX(slm5JRV5VZI{1qUE3!WdSD^G>4K$8UgZ1Jz|3+@i6l+SZjK zEQ^5GqKmMo0@sxyES`B?DZ*}9hk|Yq7S-IkQiS!cLqWF)i{@)x38QglrDzF3@m{s-y4f36n$$l8kZr0ip1s7h^EMCn$XJ& zu&E>jhI&H*5Sfn4lgg}Y2veFWF})NV5QypWpw@CGv$~+AC`E4QC9v~VB?>-DGMUmH z-90Ddo`7Gk#Kx=LgM+<6d2r}tcd&2Bub0C~MiYv413CmN&~sRw$F;T&o$QlO_xij0 zyBPG3z^&t2z9uHsRv<=%_#?z0 z5RFjUd<3d(s55NYvSCZTnDYo}Aq5}bv}PMwx*!QcY#9^|=GQ3fP_SYln+au43c_;h zfUqHIG&nT(6fH)VSgZj;=ugPgf7i}u04uP~ix89d;`Xe2rbN*uKn zMN)#HV@{mS@c-QqUB;6!Sy5x~K#VqvCBZr-(-3%SNO}qU?z|u8Jer71jm9EfdKLI4 z1;5$^zXZcI2`cbF9_&4Qe>l7M-16RFc5g5fie#cwnPfUM^MeO_Z!qq{Y0zzl5O_R#FDWZ(vo?@oa;2R3Thp9w{W4Cr!gT)4kv%V)Dwl z@s0<$kP|X#BBqJ1a;iNHZ_Ownd%y;6tpi-tDic=th8)8+g`U~ExFci>dV=nd04WLk z`hh+)c76R>nr#CN zv${{~ZoizXdw$XW+se9I@g--rY2Td-xuzp4dpj5VZcSvpyB~S$ZVfKgEqOBg{r9im zpU7Mo$<$wd;FST9Y3RGZ>wazK{N+r&{Lp)4)j_r$U3~F7rsUr4_H>tVGz$Ac9A8k1 zWmFkcr~;RRlr-d8N+@1Y7u4hoMFoZ8l~O1^G>GZZO+vDuOz18YJn*@qlN4VU)4#es z0^t)_1eZmO))L)eg#8*M>XVrGh6K27V~f!gu7oKY00}UWQUur)mQbhHa9w!vh6ZQH z)=+N&3s^aB85a==t=C$+YHL~01(URUP<_X;16#H(I37WYWbns>zo^aZJ(g?k%6VT| z_V#4GJvr~G1=pi0>2F3BdVdRPgx=5mpZb?3at$wjwWW5Yao6A1-L?I5`7g_}t!J{0 zXBPaA>UJ-cT4nl zMB4$?hNL)~tv-8S$yEn)?$AT=faQW z`=cAuz0waPKV)+0Btl1^GS~1!_OLJ><$2Bxp_w@z7ohLsc5RC5xlP|p#o9(9E zDawO_yOB?ZvOH)_kr`syi*~~kG&E`BN*a$sACr#9r0p^3eoWe-{jYYHmnCRDXT+I`n=eiJEyNazZNalR)!$8|dq+px--t9BC| z%5jz7@-}WOV!f-_S}ik?mvUVFw|o`14UxW8Y^}DNNIcZHu$pEb3Y&$6FXT8cqF-g` cD%|%~M2jU0b2*{zE79>zFQ|U8o)PkY03^@`q5uE@ literal 0 HcmV?d00001 diff --git a/evaluator/__pycache__/engine.cpython-313.pyc b/evaluator/__pycache__/engine.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..388cdb7c9995449c80032862b890818f282018d1 GIT binary patch literal 17441 zcmb7sYj7LKncxgCcn}0g5Fo)fIeZJG1d4i44^p%sJ|w?1YYA14EmErsPd7ySG*OaaEF> zvh%vC`??1MKp^N`TEsWsbocc0>#x7>>#y~R#bP2L+`jzcrOi%)_$TzD21-H5Pr6Bh zc!gjHh8!hENS>q-t{K(v8vNAqTKuGV3O{wc4nOt09-i7!!-$bLju?0Y{*D?g7%}mt zkwU(3#LSy4a2bU>qn0qG>q7cv-QGqrk;t;mEuIO|)fK>g7UFepW8dt z`P}64#l^6&z@e^HswpGUi|*ywa4b8~v3zAQ8k`Mq=~4r8VHg!<=oP3{9^@xq0^}9q zJi(I;0W?bjy%sUWjJArnL|r0*YOPEOL#5tyjE%7~y40JV(K7}Y$&@li9HZe4Oc~=~ z3R3Atrktr@OsTg5rjjW{IB#OA7&FES84GU)N~o5tnIRaA%!eb~FCEY@Pbk$++*-8e z;GO*BCT=rJ9wT?*1cI0?APAc9!&+1VLo%8pBugx&V|?^tiuMS?545R-gQ+LvOs!~0 zEzFauSL6J}Dwg%FQH`4?oLbKiOprCW9E%3Rk#H;^gci7^ptFFtLoV6Gr)O9$xxz>1 zc}@@lAt2|A!O&&C1m)3PLIlH?BHXr^BI3Xmb^iG>lu&RfdNB}0a}Vo-61c@Vz6Rp? zS^xqXHkzj8;m)NVeuroysQ67llT4xZgg20@*32%l4g2mJ>mHW1B+A<0-Sn{W zaH8?(Ds|sj|FCjjqVm8h_3w&4fsct$dM@Avjr>>9tk=L?Ho+{vLa>xKJ+COg5@5~= zmA}vgOL)`st>z#IFJe zM*`)^dVV<)2+ukxz75KgbwUg#S~e^JjbUV#iv`1rf>SSRuSBn4f&o(HG5-pv)QCw) zHiRR>6=0=uVOR*NjMr)48&JW;YTkfSA&;8NI}vEd6?0dCzGRXY(0~9XqQE!eqyV=R zj>(SgM1XL=UQegx9C9S2BwKZ;SU0Y6A_pG`f7CWTk#*c~{*Se(QmUwYX z(oz1Z;bp@kEn#haq$A9Yk99=BS@N-oD5?03fLi*}z7uBh`{W5D^>^+zY)C^(%;r>AK)lb+#V zOq#kl(y8~zMwMRx6EZ5RJUlBG=d5>`T7tc9Wv>uj=0hC4v@FEviyWmJ4z+9A<>$cIrQ|X9= zg%bdP5(-OhoJ$ne-`{m)^Kg9EnAk7|QDRwZqOkRT=TM^al(_2@$~2uQtp4`+wQ}5uZ9h|(stfq3!b6P&;-ah}Jqj?w98nK^Hezy{=FQ4;j+`ToY3qqO5_2+(2lh1c z6^;)_X9K{)_?T=;y>XFQg~e#1msfVHVi1itLV7klOM@gyhmgp|FVXOg5Ef!wB*Y!1 zW!sqBb8?7f1_L7#CjxGEV0dc9gtyu~u$S&xG15>KNN>kLzBA(jQQZ)szQI<#ycnAT z>1$g-fg$(s*xgGndNrgrc3C^m#bh%G zkx1ypD~jKTG87D63vpMFbS!Iy81F2^%Zvw=gpiArcE(*OE-$mf8HTxbBnwp}|2#up?b?8$)QChX;_)gU~tJVjkhOT%+*F#66 z*m7F*`r@bkV$=DAV@9;jByEmY+h1 z0y&`h6+w(?CQuQRz@9+@8PiOqif3{0tmZKk&uad$xIAesduXrt-J=31_V>aJ48=?2 ziD$@vRp&lT{1?)FfcloZ5YRg=(%ntn>7)?8ALY1nfJAsVhKEfjdbM{ojVBIi?>3Vs z+Vpo@EC}DHgPeB{=}>B~7VtEef{KVD_BqZbK1YiDfPf4e+LlEBl{Ri#p zU~0F?Q&-;p1K&5Y`p+8!7&~pmIO;oq(1^tx0m^Hf*qEbv!hkr9wR(GU>mdl0M`OCl zzY6LZdZ+f#1g!buK{EeZ%Qxa|3De-x4w4a@9|=b|ur_ta5qT&KZoDj@cRb*_Jj0)Tx~B%{~OTbviy|q?N{?< zo~*$?puTyBR4nt%8u>DHY@eD#;DJx+2|IDDW~PNL%UM0?5}DQ%ooz4ekJ;JVd`-+Q z=uu{UW@>OADZpzWrv|1uKR?J@M!Sm=G+6T~bC)$E4G5k&nooyEF7&dVK)-j;Jq7xN z-Hi-&G+0^a5YK@@9O)5SEavl|seuB(b8{SzG@Jmonj2EtK-oZ?PswEJpBNh(?4KI; zoLp&1nY_V}O$%TbLPmEw7@0*%Q`(F!$mCHcC0i6TKyWb}gsfNx3qf-r7Fis*Sy{Ij zM#h|iRQZDgzcrfu7@|OY}+!Ed-aYfGU)(7a9pB5`y;wSgBWYU*y0z zz^R#(kD#@(fpL#dA|*>V3hEHOVpg*VbSJ%HNPP~Ya~73(4uEV;)fiBFDi;k;4URK` zVGlFtMFU(g91%Dk#+Cl9o({*jB|$b0`X(oo#6fJZheR+P6h@b0SC+wO3fkPtB$)SU zWFHI9g+thKPp2#k0)4Wdp%oK64IM#qw+D}-&;t#Q(SsiMiLt=|whV2m zD(>qfkGO!WS0aX>9;|v|QbZ|WK*9y%2wV@2ZB|T?g+YaSkyTM?1*|thQIpLP?zuo( z&s7xTun>h2UJAzeNt6$5E3zppfaNk4gcX5pRdR94s1!)if^109Go}++yU0akLn@DK zP36iEj3o%g0fhuo#DUDDXh3-zs&W(oq?pMiTm&6R1?EFSK$$Oo7{wcc!u3cf9F+^; zKp?gNi+Gkt61QwZIj_L-4PTS>%aO%!Io{|gTA#WR)~~!SvG@>f#a_NyQ`TU`9-OGU%Y(Z?cS|vi4DcsP@;TRDw$o?Co9{e z%D(kpsiQaE(VOV#TP;df*1lGEt4^vs5U)ItsO(-XdZam25GEh#341xVIZUt3h~2|m zEg!nxaf!oc#RKO=?vlg_aZY$d5YLnS$R04ISwsc_^Z;YQlqQlg7DTbJAc}1fQ4KOW zbiCYgbM7|xgUB0^&9HcSM&z&E=zxOk!#L|O&N{3~R@6up?eU8C_3j_^z0sGbII%jc z+AOBm&WrocC2W4l>=(`cq@()P$jcG2tYz)=+6z+KGx4@(5^Zj=^@M2eUmbW@TzS)Z zzolC$J`^uLlpGkB24>;|Gs#kVWAN|C|NFQ&?U$zK;?r~D(z9To86+=|V4@i$ZD6B$ z#um5s+%Sk8Jun_HwwJzkDOp-|b9DXa_d^?Pf4g}5!sfC2r>8`gOPrn)C+EfaYohb| zW39H__H#$;V;wv{BeW&9#|S(!5e@AU-4&<1ZtoV)1*CKH@pJQHG$ztrBE2kCE#ELC zckh*UyW_jvn{CplKR)UgXD&)Jm*X>+#icJKc7HK$bwZ}57ST#49aXPhdim0tW4(IA z^w$2(=kE1xy2Qz-*nK78cviALD_WmTjabs^NLs<1SxrmT9r5apWLaghq3e-_u-85!40f-mV+CWn};{sw}!++Ou{xLnWseaRI-MCZT!~wdTXMlOLBCro_tv4 zT91AA#n)fl)PGp?PEn%j%<5>mkhOESk8Joi#>K9Y`#!&Tem>z_xX~`!!jd^Gn!^w6 z)slS={2@bG({jRbRkU7xNVmbjenfZO-*;-OLp*a{Ix`3rTaiWUh;w>+YNBbW6mRBVx}|!WNOt5z!n;mR-MT5X-Kwv){e& z`i1xU#Yul+?7X;i?W3~mt3$ss5w#6#j{h{aI{r`e^&94-wN|n=#H|f$yKhrJwH|z| zB^p-9pQ9D4}M`zDhpP6D2(PL<5ua#7b0*k+OM8c69y19Db}Ps>9?1^TPi!M;@0F zmKR9X@)ssi|2kjplx+Lrwtcs|-#jAqjK+IL-($qnQ}6l`wrR;c9XC&>hhfbpcJ*)e zZyppohcU7H5~t6iu{$T3&&AE>P{!<4y&z7W7lmuErcY>2Axk1MA;Q~~<_uyElNz5! zS@CD_ia(1jiDxy*y4KsRKaSjq+z)_Zb1o5>kJ}gi!MuPCi(&Hj!YbHTUutKZTNZNCL~RxI0sfAeoU~BySQ-)DNlsd+clOo+{*xkd z(oX%PxB}sAa<+<1OUrUHW)yE8riQNpG?KBgt~?7@A#Ms#aF3 zDo@zRYET=Kd9pfWE`3V1(yFQzK7i5ZJ1+9sSPG1y+HEZ{fC`wu7C2tYI7nm+Dd@oI z4;W%HjZI_u44=L;I~jwpdY#9-n>k?5?FUkxJ%0ngm*BSuzajXwz^@3@UaQ&+PepT*+T;X(+GNTa)tP&0ZgSMf9Y@;$=3FCId-IeCMpeZ$nNu2L zfcE9fZ*pl<3aHU-#0n@pa}>}*4?hjPS*hq(ghiw#KBXBtNkzCk54m`B00QS`DM3;; zOuD^O!|t&abc#i*N&z2+T%ZYL1b-~^Td>T}UqWc_qAr!6MwB)JCom!ABv;D6W=OY)yUWd%WQDDP) zWlfPineQ}EuYxamW zQ<7t9^<-uP^4}hLQ`kK8!!K?`woZxrSY?kAl|2e?Wu0+b=WTt$c3d(a7tP0!R9kzi zcI|4S(k0nlqS^Jhn6UXt*cL4l4XC(8PxcN+`RcCMS-Vio8aqj?<1(# z^Z4%`8Bis^7jDBCzSJ<-Mt+OzFQL9o0rq-Ze=TvxO7`2SJ0;}^-$nM9QFoju{w~Cq zQ+G=m5x$@7ucYoCr~v$)ne4Bo?pd@5w`!o6dv+4xWf+!YSczdZhP9@_X6?PULxc6& zx9Z4&2K`(0t$@F+BL|K8xAk^}SCWwD?J6CV_IAAv;mumWL#cxqXgVDF!4Gr$A>f7= zEDFExBqEwW#yx-~{YZ_)A)gD;PW&#SWwmJ|FSP0;Iq~0sQy?nN3CP)m(PZ)E^nL++ zS^K$sS!;)KPXVzQ|X(W~a;D zV|Ls(>N8)fn6J!tZKvHI?{2i3GxN-Q&~!~5!Q9s1QZgT>vAoAFWuH@u#)~*;PRNV` zt|vbwqCHE84*(2ab%L|XVGUm7h>#Olqc?rnkx7EsJO>{TyEl_Z#UK_Lg>nMyk&5B= zDQV~=IS$(iGS~z5FF<=5wV}Z^c*{=}c@8buv_nqrJb(+0$%{;1IYFgoI9zZO#4@bv zB}>HUM|L(GSjz#Dv4C@|k=5AKha#Dy27L6WX3lzgp|sq>rMQ}Q01wViCnPH}l+rI!_L4wn> z4Id<%^B>HB8+6HDk~I&4GanqRXgo;_{|dtwF)RW|dj1cF?_&s(D8#ry_B^203bJPq zI4*JmP@}S*-a}b0Db!cyKcdF61!~{^fOMIEe&JYIS*iI-WX$=cOMHM);4V0+9S3Jk zIK%)vBSYdIehI^U7#4uE{Q!O&@cYl$xvSJ8TTZyA`cDOzDG(RId|f!Hm{gIh$-{7f z(G(e5!Ji5I^@7g@E(zfJL?O$mV!jXbPoRSU^l9-hWCQ^W>Oi z9KEn3J(w#)p=I!xx~{xh!Gkdrj$kXk5>vqAmmu6UC`9K4s&|(e*QUs|G!|<^q&gSy z1Rg`|N4nt!K>A2kJA}2~ND$GId^0?yL2jP2zz4=*SA*kAz;eS?;uRfDj5jN*b)G<@ z_&Ny}CHTL9tk_Nt?s6K}UcL+7!AC4Sn{vDSQp&j63x`N>IKcPGySEuvReQFpcM%;T z2z{<}?!SWKgeCyLMl!HSU-;JmCq9umdLtv*yc`svnLgRtnPRH|PLKoly%?9nuqul?G>vay(EHoa17l zg#h#=WsX-&@1~1E6fJ~TI7Nv-`h#4i*v>H}rEK>26loy{S4Cnxe*-e{NS&06mZETG z5`gOxVSI*0VS>}4OB4c+Jd))?>?WujKgc#H;3}6oi%{I@c(mEe_LL`IDoH@MHssQ* zW~93%>w{OWAXmn(p+?|o{0|6w1%dyFz^e%0Gk}t;0u<82%8y*6v?+bEr9(XxWD1QA z)Qa1vE`U@kGdk276c5gWid`SZ4IME!ixhPVZT$Fd0y^dZBffe}@Ld#7p<4KVk5DbF zP2^9AF2CS&0IX%9Yck5IqH==s5@hAS0YE^WuXWz)l&X5-RXrO+Z=I3`&cp}KB&vK;sc&`gAMF*lS~B`eV51N; z6y3d%&EBo6V)wM7;)v!m4=ZYKP*B#*>$f_=+6;E)=*v;b(Gz#{Y@AFu1|;i%XdQT@ zag@xFDBDWBs{Ka)!}7YD!rG8j-VrbFxV`@ehu=8-w@25RhqawjZQn<=eQ(D8e&v6x zY&HGx#YFAIjiF>y+jowB^QhExB;IsnqczdwmTKJ)t#+X|Ue&uXy)~DpnvqIp#L^i^ zk*d1lRb5imV7zK@6H-{IlodybV4khNS3xprMu&$yVs}hmmWa5 z+HbWdE9#OJyOLGT^|tj&v0{I+q3t{M->gqIIi;op@umYAq;BJKXM&{SQau5{_pAZ^_T(T)twK-Q!OQqBC(&^M3txd1bY%Fda+H#7$Q}@pY#6Jlo z&d=U#7ppj_6#a>%-0vRgP=UXFVk9a?uqFH)n7`*97)Kx2`W_gEeQ3t(!0+10F`NEwDTVM#w`;6f^Ok`eE7QMaY(jVkIaZ;6Yj-2y zZx@kc)%v%KG2TUv)#=~fhd=l+O^!9_f80PprXRPUQnzfR$3|`0-Bx&cr-}5G>ECJY z0Q@J#q^E-Vi4`^IU6S-vQ}1dT5#CICYN>ZyY5;!^;_IpRG=&I<`~aaifMq09@LmOm z)fm=c*nnZP$y2O*&$Y{A(!K8>$19ERm$xGP80jI5@Av5d|3GViay~HWAm<0gsLLOe zp^`tSz_1#_S`6zEe9FFC2;2sE4GB}IT;otRgINkLIKsaJn&#{a9i*L@)(M&}LI8RF z6tK7`n{N*fYsNvxMH+{oiKMS=fHfDc+#P)ru8BaMtUh9x_$s<0aui%UPz+ca3==Q; z(-z^(m4M7eA%9x7W5|pne}0$BeBZO0Jy3pHj!elPs^ks231Ztgs>G6ZO6j5D=AP=fg56x; zCLY|H0{KSW25Goo1r}1ccopGhUB5&EBacRzKLfyN25)`16UxnEDL~djP1yViSnwKL z(}9C_nF?{?Mc8S%I8NFTv;VF?Oi|p^uBH7Xz@t^DL^N8-XbVuw+=MiUS}Z zqHS7fsH_V`;m!eU=ip;wR%5{dB9`}l0570=8@9`lN=wp?6W@bu!UF(bBL1b=k+hfH z;BR!Vjz4tNym2y7+xxwf>(9Qn_>ILkPR46{6OLo6g~`IQn#v*EgWoUxLFM-<;W+mTiJsA|OYt6GqU&s;{am8dzdHEP zR`KedukBeoyw<+%UXMtwp}1=(;W{Pm85Zf0gl%*ccJA7nq1P_my0l)MsOwFXA6q^3 zu)ON#;lFr()%>8kF;U$A^+Pv;-&y?T;_bzF>xoy%TX2%zzv&Z;$3^4#Z=RG9b;o}5 zq?E7@kOHz-y;9cSZM;`%?}vl94n07pj;A3W%Z*qp#MkMt0PECvJm3IR1Ia7ZnE}k) z`K7-|d!i8zf3Z>yS`oe-+Ml`{no0#%q_n#ZvVtpj?=a>>uNeC}VBph(T*Q;i-_Og*mGTGx?<53dZunE;Q;kk{@Y5oLuHsW0r5h(7nF&r4hVyHSuJY5m9$lFHbcHnQ zJmjxxoVtCVO_N1{D`ldRoV!t9=9YoFpf5aZV+8*k0(%hnO9cKW0-Fe6T0ny*>toli zfPTj7Q8ZZF2_9*bawVtSVvq8!l!H=~7hD&?9}!coT$?vty~wC|fHmjMgT0M1SD^8o z#vQ6dS09z@j^)`!q5l`ag^Lj|c|ZL%PF4TQ07X$P%l?|@Tv+cEuo8-ow&bcz2CUr) zuEoWr07jLo;dCa0;*RJ_{bvG-?;qSOv!PMT%9eEp&Xu`)u3SdX8kqF$a;Q?hagG&O zLAi(CmZ>;j*7Rj`<#Kv`-rUIHO=&0gfaB3R#*0w4=JGmBll*;@p#_0P)GK4JvO@a! z{|&EbBNY&xKhlyU`7zP>GonKxIub<3&xj_#<3z{DL=XNu^)a#kW1{n8!u2!anU9G> z9}{l;ck*Lm{A1$8&+XMWr`DR*gtc9_E`Za}dU%6+J-SK7T?1mxK-@kk64uZ3mo%h@ Ld_v#{Zq)w+nDb4A literal 0 HcmV?d00001 diff --git a/evaluator/__pycache__/time_utils.cpython-313.pyc b/evaluator/__pycache__/time_utils.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..57df6f4db98015b78de790506e977755177fb220 GIT binary patch literal 1875 zcmb7FO>7%Q6rTO@+VMI`6T)B8O4FoBu}YJaG>0Zo)Izw5E4=)sOTXQmxvPwjvVazkZRRRAdpaRNC|h|8{0{fRtUzLw=?g( zd3x`AKi+O{PZC%^>K|SCK_ldMytEeFaVgs#&|AbJmNH8+3Rfte)mb&8aV?{AHKTJq z6XP+-*Jh1OoX077Ki*F)eM~_cgz$uwv|_#Fiq6|C(=y~KX{D^VJeeR($W!WaTe|%* zK1nN)>G*yi9N`9jnARhG#Stt@AQf(bMY^O3q~*Y8@E~ykzB7J4D0iJLFz)7@Vao~Z z1*hl*xuz>Q|P~UN{G1>|!q)tem@|!yT3iShCa6Grns8ez{qt_g-$U-UtI=Pq~x~oKn zD7z4MTIVJ>5XCuL^PC)uG>}J0Cv1TPssJ z7j~XI1y&09RXfPJ_O8aB?l2m*i^BE7p|$Ig;e_@jE_gQ>Z-X5U0jZE*e0-x8 z?|abM^Tqn7>vyJZu2(w;tBJt}=AkXKuV(grHB~ohRj1qVf;>A}pM&HUS>8?L5@`YZ z?@3xT(B}TN`x4zeu*RDAMa1&JvJ^`lQyWZ3r#4?Uj*pCtkBmghg|s09=N%yCSP+p` z6uGF)4TC%;Q1BE=I)Q`|e%^&j2@wE-IlIhXg6koaz6PX19-3WWzVg`{TjoH`9Juq& zx97e&S2s^?nG-d0qHY3zx@J!Qv|3%T>Sng8XE_w?e&{ebS?6EF0iXlgSc}aNT6g^R zLme;2S{w)zeq+t*U*3h#Qcr8X^1m!;mJYwzShH&m*izr?KBWSb#ym%-IF6*pFCv)$ z;=VcANuaFJuPB}i3YT1;arB0_14(O92WNT4nV(~}91yWkI9x;;^H)Ojo>JPtxdSQ_ zRIb}4*Ix-roPw|ib~p_L&`$Mir4HXu9lrg+z3j%(jq{tz_ZNR2t)oHr9QJB+Yl!?ns`5W-3vQJQXzk|3X;}L`kh{dhBZ%A^%FAgdB_qLZzEL( z%(o)<<>g5M9{`m|3k3)65mZNrB)9ysD6V;Ixit@8-JOgV)9q2xw(~_W$!wcrTBYU2 zV_6Nd-xByofP;`HPz5Wb5h!++Ot=i`E|DHL+wb*4uu}Bc89oR~*q7l+Ab)9!qC6t~ sf0EHU8GS^K{AP4iJNxU#Kt>P{wB#AY&>+I)f&o-%5reCLr%KNa~h+YFT1VX<|u! zk$z%cVoqgAW^%E9e0*kJW=VX!UP0w84x8Nkl+v73yCPPgdXRa=AjU^#Mn=XWW*`dy DTHG6z literal 0 HcmV?d00001 diff --git a/evaluator/analytics/__pycache__/vloop.cpython-313.pyc b/evaluator/analytics/__pycache__/vloop.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5edba9e5f34c903e4a8e2086358f8d55dc49638b GIT binary patch literal 3369 zcma)9O>7&-6`tiTmlT&2LsFD%QIZ!n{XEsO0|6fb`H1`V%HuACz9tn%SX*HpUIyhIltL+H4$U%{5sB`a zbIr1tr8+yu&APFBmd8BQeJQRVt#H_r5>jrec~kxrPqnU8Al04n5`BfizEm*PMcajC zPvW_6NWcVR39_aY4cSnOnog=YWl1jQjhvb_5-y8Jo~h6Y#sk%tHj9s;ioA$XZshI2x6)+*jm$S<%Ue zqBWq9nrsyDxU9+fYlfQD$FJmz#nM;_=9ZE53~|E&u=c=D-w#CvHG76?<=eTfp5vQ6 z$Ey?fxRV=UC>}y5dSV^KhpuA{=Jw3%m_R7)CSwti&;l7{m!z19)}Aq+J6>tPjV%%&B*jj&aMQN#i1381YY$u3&bX3m&lPaaxXxE&zthL<*g9X0>+oUs zjPvwPxjTHgXY}DEW=~7rnaK|r2Zy-?KnqFkXSNSfd;b61YaYo%u(iylh4wjloahS# zp}PcNL`CyT-j3M;5+S$QI7)b~zcGE)z;f0Qjkgq$Xo{$6hN2l_Syyu6 zd5gE>Bs`d)4ioaCS!S`zieY)6kPQP{tPX`63bmBzvI1Di7I7|9#2FY9KL<{hPg8V5 z$F9=bvL6|Hc6bOHiHFX^&R5U_ zPiSrQcF&gQ$foB=GZL#Mw+2Qx2S(Rd%z-mzWU}gQMh9z!t)cPFq45p3Ids~LzPRdp z5Q)7vQ%`P8-ZjjDb7o|k+P$}2FKkG6&zVE7n9(0reGmLSYd^mA=FKGvn!ovg2zvFFY3 z3zfNT9(9Fo_*eaFi>7zzF>-YsZV1sEXIIbGj+w%t$1F757kY2Jy!vwOd!`WI_8_-! z+lRb?-~6f(j5LMrYWeq*TS9D8h?&A*L2`tuD9Kepf$N3Q z82GtTE{&B?1YH0eLYl$`+9I_RUvo?DcGB<&t|E;kp2-+Oi>b*)gk)%yfL=xc-auBI zJi{!K*^%wv*cl~@)zN!|=1u+ZPkCH7L>J>!;XFu@>}It(6J@H3g4ha?Q+tL*}1wQAAj5LB5*cEsqUH-SR-4M6x{DGRBZT zmbZkJD{8TRH9V&`BNl5Y{{rQ5u^16zXkh2XA9L3L-9B1-VN zznDw-Fb6|fTpnn;gKneHh^ zk^0C6XND)LQ(pmtodoRvajM4t;lieWaLXUx^vCO$KKGA&8zQI=ql?Usn7;)+3H&wq znXvem;D^I^4^^i=3yrOp&CtZB{)tb5p9+ijxW)gN!XjPQ$7B1ZeEbvKM|g;)NIxi@ zDI%fq_QNBNFiE8ZV{a;h$t%)+f!Z|SqfgGvQeIxhC!ueS^b>@^-U!FmN7oOS!kK&A z8JeRBj$UGzz>u(uQO4T!Zt_e`!|{%Y@T&eP*pL|129r1RbszMJzZB<-ngX5VRR0@r zp5AHVLd`4|IMAM~ZG;I_?GZXcfYH+-@M$nj_$^;1vs8vjDVYo=+hYmYVjkZ2^k0CD zMBcboQnh7!dV~S!)la#VZV?3t-HAJhuG!uu?@NVZuAEoS;_rhGVIG}i>^93V%sv8_d5BGR!0$>3_egq>%ss literal 0 HcmV?d00001 diff --git a/evaluator/analytics/vloop.py b/evaluator/analytics/vloop.py new file mode 100644 index 0000000..f2c05af --- /dev/null +++ b/evaluator/analytics/vloop.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from collections import defaultdict +from typing import Any, Iterable + + +def _as_dict(value: Any) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def extract_intent(record_or_raw: Any) -> str | None: + """Extract the trace intent used by the TIM specification for the deterministic loop metric.""" + if isinstance(record_or_raw, dict): + metadata = _as_dict(record_or_raw.get("metadata")) + nested_raw = _as_dict(record_or_raw.get("raw")) + else: + metadata = _as_dict(getattr(record_or_raw, "metadata", None)) + nested_raw = _as_dict(getattr(record_or_raw, "raw", None)) + + trace = _as_dict(nested_raw.get("trace")) + detail = _as_dict(nested_raw.get("detail")) + trace_meta = _as_dict(trace.get("metadata")) + detail_meta = _as_dict(detail.get("metadata")) + + value = ( + metadata.get("intent") + or trace_meta.get("intent") + or detail_meta.get("intent") + ) + if value is None: + return None + text = str(value).strip() + return text or None + + +def max_consecutive_intent_streak(records: Iterable[Any]) -> int: + """Maximum consecutive streak of the same metadata.intent in trace order.""" + maximum = 0 + current = 0 + previous: str | None = None + + for record in records: + intent = extract_intent(record) + if not intent: + previous = None + current = 0 + continue + if intent == previous: + current += 1 + else: + previous = intent + current = 1 + maximum = max(maximum, current) + + return maximum + + +def session_loop_values(records: Iterable[Any], session_id_getter=None) -> dict[str, int]: + grouped: dict[str, list[Any]] = defaultdict(list) + for record in records: + if session_id_getter: + sid = session_id_getter(record) + elif isinstance(record, dict): + sid = record.get("session_id") or record.get("sessionId") + else: + sid = getattr(record, "session_id", None) + if sid: + grouped[str(sid)].append(record) + return {sid: max_consecutive_intent_streak(items) for sid, items in grouped.items()} + + +# Backward-compatible name used by older integrations. For a single trace the +# streak is either 0 (no intent) or 1 (intent present); the official metric is +# session-level and should use max_consecutive_intent_streak over all traces. +def vloop_flag(record_or_raw: Any) -> int: + return max_consecutive_intent_streak([record_or_raw]) diff --git a/evaluator/cli.py b/evaluator/cli.py new file mode 100644 index 0000000..8544e38 --- /dev/null +++ b/evaluator/cli.py @@ -0,0 +1,93 @@ +from __future__ import annotations +import asyncio +from datetime import datetime, timedelta +import typer +from zoneinfo import ZoneInfo +from rich import print +from rich.progress import Progress, SpinnerColumn, BarColumn, TextColumn, TimeElapsedColumn +from evaluator.config.agents import load_agents +from evaluator.engine import EvaluationEngine +from evaluator.persistence.repository import EvaluationRepository +from evaluator.config.settings import settings +from evaluator.time_utils import brt_day_window + +app = typer.Typer(help='Agent Framework TIM-style LLM Judge Evaluator') + + +def _run_progress(coro_factory): + async def runner(): + state={'run_id': None} + with Progress(SpinnerColumn(), TextColumn('[bold blue]{task.fields[stage]}'), BarColumn(), TextColumn('{task.completed}/{task.total}'), TextColumn('{task.percentage:>3.0f}%'), TimeElapsedColumn()) as progress: + task=progress.add_task('evaluation', total=1, stage='starting') + async def cb(event): + state['run_id'] = event.get('run_id') or state['run_id'] + stage = event.get('stage','') + msg = event.get('message','') + if state['run_id']: + snap = await EvaluationRepository(auto_init_schema=False).aget_run_progress(state['run_id'], event_limit=1) + total=int(snap.get('total_items') or 0) or 1 + done=int(snap.get('done_items') or 0) + progress.update(task,total=total,completed=done,stage=f'{stage}: {msg}'[:120]) + result = await coro_factory(cb) + progress.update(task, completed=1, total=1, stage='finished') + return result + return asyncio.run(runner()) + +@app.command("reset-db") +def reset_db(): + repo = EvaluationRepository(auto_init_schema=False) + repo.store.drop_schema() + repo.store._init_schema() + print({"status": "OK", "message": "Evaluator schema dropped and recreated successfully."}) + +@app.command('init-db') +def init_db(): + EvaluationRepository(auto_init_schema=True) + print({'status':'OK','message':'schema checked/created'}) + +@app.command('show-config') +def show_config(): + print({'env_path': str(settings.project_root / '.env'), 'adb_dsn': settings.ADB_DSN, 'wallet': settings.ADB_WALLET_LOCATION, 'langfuse': settings.enable_langfuse, 'publish_langfuse_scores': settings.publish_langfuse_scores, 'llm_provider': settings.llm_provider, 'llm_profile': settings.llm_profile, 'oci_genai_base_url': settings.OCI_GENAI_BASE_URL, 'oci_genai_model': settings.OCI_GENAI_MODEL, 'oci_genai_api_key_configured': bool(settings.OCI_GENAI_API_KEY), 'agents_config': settings.agents_config_path}) + +@app.command('run') +def run(period_start: datetime, period_end: datetime, source: str='langfuse', limit: int|None=None, show_progress: bool=True): + if show_progress: + result = _run_progress(lambda cb: EvaluationEngine(progress_callback=cb).run(period_start, period_end, source, limit)) + else: + result = asyncio.run(EvaluationEngine().run(period_start, period_end, source, limit)) + print(result) + +@app.command('run-agents') +def run_agents(source: str='langfuse', agent_id: str|None=None, limit: int|None=None): + async def main(): + results=[] + today_brt = datetime.now(ZoneInfo("America/Sao_Paulo")).date() + for agent in load_agents(): + if agent_id and agent.agent_id != agent_id: + continue + reference_date = today_brt - timedelta(days=agent.days_back) + start, end = brt_day_window(reference_date) + announced = {"run_id": None} + async def announce_run(event, _agent_id=agent.agent_id): + run_id = event.get("run_id") + if run_id and announced["run_id"] != run_id: + announced["run_id"] = run_id + print(f"[bold cyan][{_agent_id}][/bold cyan] RUN_ID=[bold]{run_id}[/bold]") + print(f" progress: python -m evaluator.cli progress {run_id} --events 100") + engine=EvaluationEngine(progress_callback=announce_run) + results.append(await engine.run_agent( + agent, start, end, source=source, limit=limit, reference_date=reference_date + )) + return results + print(asyncio.run(main())) + +@app.command('progress') +def progress(run_id: str, events: int=20): + print(asyncio.run(EvaluationRepository(auto_init_schema=False).aget_run_progress(run_id, event_limit=events))) + +@app.command('runs') +def runs(limit: int=20): + print(asyncio.run(EvaluationRepository(auto_init_schema=False).alist_runs(limit))) + +if __name__ == '__main__': + app() diff --git a/evaluator/collectors/__pycache__/agent_framework.cpython-313.pyc b/evaluator/collectors/__pycache__/agent_framework.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4b8bb9aec71c30b808ddf08dd0718b3b8d2809b3 GIT binary patch literal 4791 zcmc&2TWl29_0D6T_IlS|FUH2iV;6}GHHLsl%Cp({ZR0g~yx@8rro-;oUb8!E?#$xF zC2EyQ5&5VBiV&ev8&s+U)Q_e#A4ILx@>BnC`KWKmPyN=|JzO(XOKSzPo^%k&(?TTg6u*nJ zWzoQlPxRl%fykIOil*fZ#YpEg9fsbNVyH$ss{$P9&S|Gqs#}ZQxlBe)8aXPmW<&MD zn3~Mdlo_s`8&P##nNb0?W|S()j4A=4YSzeedQOuwIVEMu(pAGqYcqP%R?Ke8S~3s2 zV0?j05z0t}vJyi%iESa2mpqbJ;##7OLJHcm2?nHeH-u|BX(NXZ*HIjyA)Sx?TY zSw-Y1ilBJZrc1~D0vfNYnQ0wa3+86^l#mg`>|G!UkAt_X48-{=955xgD6^}q*&w+2EXX^> zT;O7^TCxe$jPb8=(~QW*?x%o*$Wz>)=>vNkvs6`5V4$)AO%7a0bkTZ5sGV17I+v1l zL!pN0FOO6$Wd>{`E19&StGemRq_b&5M-2r0Xakaa9G-12GI|-I7O?fb(A)>BFGrd$ zPM)8<6A?=h@#>~uZ@sqlMr%>}{v9cOJ2kVEN*7Pe-#M{RG!~X3g*(AQ(OW153r}qOfbx2O#d6R;LwkX2Bbw`*pO<5MmT2y{d#u_IfGvIwhUSTT z#wurKuVE`OFOY^l_sl1;yU@qst0NM(lS|atBSOGKc!}T5?J8d(fq=Atv%1<>wDHEh zVbaeGj1ZD{6EgqP`QOif9_Jh%Q$W*+D|xJUqD7AG}Sb-Lj*yzy2Lk`C^>6+rMHLcR-&`(^72P^yW!hV8M9lEj<#C8-C z!^E%R>*7uE(3(~Wyud1f|6!HDs#UfVDYV<~>_Py~964lL2GTS9IW_>lc5NcXoFy(U?5g z(7k)IAW5#d;5R3f-fz@!pAXc}NLKwXN7$rGOHA180t^qik#_nbTQqwXw&PUE@qxWb`GL!;jz(!y=6nj z#=pjfxFktnD>#8Q`8{LLGWO!pwUO(*D9q$^N?5}{&vAi*!@o*m5|hVt zDe}`&46md^e187Q1LXtB&wl6(Wg9B96`n*JF!-n0>G&iAg*-~lB-{O3D^nkhZMY-Ck5W4nda!DnlALyzOZ3b4l01f8vqHy$j&()KE z==-d%*mtC~@5q(r%k>{Of7E=v=gC8eIOOecCvyq@|8M5uOrHW?0KN<-$0}*lWmPjV z`U+@AcR*8Z9E!o6z6uS-q1A1+XRaUngt>L<=BeW06Q%wWSH#QhAB!J}*N;7+uL0)$ z|G;;j2T1f&w26R7f>qr_!WG-nGOdf9w?Zh$MsRf|S>NEPP!@_maXJkY&YY-}5 zH2Pg+^9DBE(CF)-abnK1y;Y4;Oxs^m{+CNpEGcxnbWn(f8qRu_1NHAszB_p*&{hhx z-43*09$yYMUaUD^^M2rL_i}CUz2SFQqyM_#h7`Hvi*^duivraudqJH zFGrfr@qZgQSZeG!*Ym;V%ip=0xUprqas9>N^TW4=oi`f)xNz&O;@J3YB~?`Cmz0x_ z2ziScV4<60lk5t{pbU0dR%ajMq`^M0JD40~598z@dju!31G+(W0tpt`H#q3Jxn2$) zPj4}O9H6xKb6*h>8Q>mpWRMwQ{<)#8Bpm%s&ysMo_`Sr^hN*M@yRAa0J@e_rjnt>d zmfAB#ZGNfs(+rEYDFK75GKO6n=nY^=J488(@Ejna53}u482f2{oOqd%P21aH8_?-#R{g ze%9C7ALed_{rw#tk+Ci_Jk(;M*S=&JK&+caEeH$2#UQ-V5^!tZ6wU~-oTgfLZxNz~ zQJxUJ_~g~#$u7&LUzW4ER6c`ntt_9+E1B|2P?o3DR5vndP1SO;OtCnoSS#AMB_iX~ z&qyaBtr;|Z9HNOX%hw3Sg4w=RxC4p}Ms^+P-y#2E8CN&+m78&$X1;W9a5dTZh!qp- zP@n0%k*A;!#jI#;%xtc_<-5vPF-P^TUFAi!IspFZUrSfjCHIcz(WJfwIzBSwX5va^zO6mc!18r0u-Y} p-vrGH$1u!2(s7S$yhrwaK?c7d{r5=MBff)aD!#PyO9H*c<-el%H4*>- literal 0 HcmV?d00001 diff --git a/evaluator/collectors/__pycache__/base.cpython-313.pyc b/evaluator/collectors/__pycache__/base.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6727b81aaed92a7ff54aef9b2c4a7b50e9726bb1 GIT binary patch literal 915 zcmZuw&2AGh5FUGfl7{|3OHfs)h}zRisJ&2eK}e8WBJF`aww2{W4M1G07$uP!7g-cp)ox3MkFHfcZdSxD0EnKw_U7NUegYH^%zNUmew6}5GdEbePT;M&o+RAcW zFPIR8o??E!DKn)HZ}X`x=h*7nWGv?s!I)_=HYtWv>F_dRZ>L<=J8O)MGE`b-xyTE~ z(20Q?)|fVP3hFFnTx-k*Q!SJYaQsXA0j@i)Ia+lmk~DA>FVII_8tlQ=-b zFjce4GE+DVOiaTrrn_r!w!4OOH?5Cu_M7|6qM79}^;`O^qP5Q^+OlPe#3JM| z_uKm%qJyPb`kj5nVsT%ISi;_|{jNT@=w@+Se`#NtSk~teJ?y=x-`iI%mb19M-`7_m zRv=yza8z?AO=4xBHsEBjsz6O}kv4~QZpEwv9nGnM$=03z)NK6l^)r*%DQxWQn zoDPjog)q$EX!L9+x)nMIw<`v$Is z6S%=F=43yxp8&57e>b0B!jCdBK5v@%m0Jj*1%Yp2t+X2k6x*ptB199`ZdB~icr+GI zgkmER#T1T?BoyOVBtZZ*F?>=nQO-DZ*zXr<5|pB#R#>dVdj_XnZjnqI=S2i!IjR`J zpZ-`7vl2XPLgXb(j)yx~%3;U?4?@6qVlZ1wI{^VSeTQ8%z_`JWX>@RYZj7U0a+mO< zSO_K}amB3u=8we*FcIU4Ap0UVqoI2I#aAPEhWoIzB5BSz+_T$3_1^=OIuO#m{MzfjyA77Am?TUKs}fQmgsG$(4z zrp2}xmxEKt!Er+XLiNeVo+Y(1@=bYeE8oO{NDmfgE6Dp9!Yr!~k?U%CQJ1d^23S7X3Jc&ydgLaSeJJ=(1UVgqt2y!|#R zvY9}C2R=3foU`DMNnDEYS`(+am&|_da2b~c{_Vz5UIelbopc9BCSnO9gdn?#$XP;d zL&pY%KfM&KcY#<&&Q6MZl;-6nI(CgujDSh(>CiJ_Ub%P`f=_`d(yG39d#^lxWA*kc ze0p`aw7UD<-Y?<=w$D~p^X*o}MC~fpY)2I9DV1l4fHY1qM`M#y3B@uom7rhy$i%71 zad5SX7)qR&h>k?!A|Y+NLnPLq7~=_1F-Btv#XLGb5lSeg;faZH#WWs`BYtus8dEHz z(Qzr>YHjLY-< z^!e#meRHPNs=jNz@~S?$s$X{9E7|T{lJP^&P;y_!RrY63E>x_ZH@{hQspxX^cS_`n z?K7rdc`9bMWNI2-=>M#U^FG2a+Br|<#r6yBvxnreRY@z6;IUT|ptUJS%U@M2c$()9 z%=f(BCwn?2XD1E8QIec`u3Iue8p7C?fL#$10H^%T1w{T}%0*O1XJ7eLr#>$|N5KYl z3WmnAPZKCgzz{Iz=wratWzvAWg=^$;7-{7~U_Ji)x(nc#RyJ;E09763b{iYGQNG=L z?!J8|LSkqHK(c;hVk(yKM;@Pwf(-yq{1c;o{I(_VJ<`E|v)z9tnm7Rf^^Z@0TaNhw zf>R@jDM(0umZ5_|=siF!ex1U1gRMFvu{gd5L-CPlRI$e+lV}fuNnEjpBcq|I@kG0Y z!LtZV2T-F$0B;;XZw^nLnv5$&y=O-vlY|ECJcCQy0z%~b+iw2{)0k1D^x!C(#y+M~70LM`YJA$#(3Y7aZQ3 z9IxitJTvv@-OoM!zO8=Ih@>y$L|y)}xyR03v)A{mHC|i8BOD<-)gCZ!6`XcqVF;Y^ z*fu_qOwzfW)#>GLV-w9FxCP}SjRED8{tz(_YM(tv%L0ZjLnq&IdwV(%-qzkPz?O_) zOWQ$n^O*uS2tZ%{2u2OihDb(a&+-}>J8DP2bS8S#NcaLKmK}ADO$AH>TgcEDdSTAS zLnxaE;)VJV(2-@ite9h!G1CyO9eXgN1xAx&huWwL$))3Go0j zv7Cw}J~1Uiy0>Y{O}jDoRU`qv-JVB^%ph*jNs$0UWuU}6Mc@Iacp>|%O+u@hIF*%~ zF`b-Z0f&#o!tFMd$2zk_F338}m~A&zlk1fc{|n`TAH-6UI^n2NE4xPd1>|HSzxo_i{3`smQ%8}8=$+LU|0uTNjDl2`APeY?`W zy(!;b*|$G=;G^i|4e#ptBiBzzM~9`uBPnlK@^S?Oyd(m*)nj!3Ri$#z;woJP20Jel^jN#lp!s^tER)AfAc`Mw(%#(uft zt`D4de`e&$t3P&?CG882n%Q32(Uf+yr5tUtW6d-BKN>xG!`U$R$n_nP5S03kr<@_l zF)W!xX%fnrc)O@TbO?X^&HswXQaxA05K`0zb$W$t*{7gB+e55;h5QDs#ypw^I=W(h zofhZi2Yw~3H!XcFDBpKnG9ABtL5P#E1<4??bU_LZTGyQNCaLc$ zCaKUOKzd$rQVT5t=*Wus^+m|*59ZSk_NOgBmd7%n6p^_L=8993N)aqnuz!VnNNG@` zL8t>o_$nrjARAMzE*Y4Fr7jpR9;KDw5oFC7t{_{%V0=PM1W7SCj&Bjlgy~?*1%a6a z_t_fR)tYvln=gN}>QdF^opSqKc)PmwXIdg3EU5Qiur?NA>VxjZ_?4B^wD7I zXiz%-n0$1cQYJ;&5tmHy+t-I^7elGPPS`-DTS!mx&JX3fp!w}Cr(8CQD%W1jYdS$Y(jp``s)vXmqWeqlL70r z5<(g}$P(;m3T?vNjg)-xMynO$>aYXZ)iyw-p<}V}1VOqD(eMB$HIrOOze2nQKdK&; zPp8X*S}AoFP*^1BI)%TiS#z<3D|SDB;QWD^e%a|yJNxGLy>a061K{>8U3j|^c;}(F zAG&(vN5`J&{iv$`PdBUjb=%y2$<%f`R6#RQp~{DDEG5($p@WSlZ*2?7-Z`iP&m_{n z>lw!(KEs#f@}ljE%>wdy6>6+xd>Yf{kLf(5&9Z*&q4O{z#Q_f_V#CzrstvEUeW z7A6$_uJ>z&@3}cBTwb4p70O$*^;+S(Rr|g*YqtIU<`AZ9+kiNRX%a^fDDHgVCk~~! z@>BYe;2xfWRRYkj8t!0fb7)Xa`8C8qR@fZKXwOXDIqXn{e=?TAM8@!hxdVlNWY{faV#m5yj7tdZeEBQCc6`LiK7ZR{k)`T4eYkSJt zzTmFSIBU~Rf6D2ftCgJ{xwjhGxn6&h8aK<%EqSk-Wano6b*@f^wQNZ}H;b*M_N4i@ zZq8ZCB>z9}mYj7E;ZzhSnxZ{4TR>p`=I4m~L5$<87=aGXw}DZPSm=46lImn3XNy}> z5x3G^H+f5-aI9YbXkFg*KXX8tVxIYV@ zAmWS%WQjg_(EVa$gmxuIBHebse`AM#0Q~=SB*(QFcj(BLKNQy30SI>o3T*1|A0fM) zUjs?4C2aELROCFK7^hO3JN)U%D^vMUg09&9U%vwrSqlVBvfa|a1ergOsHspxpve<~V_ zo&wxzpb7w@Z43slC$QHr0Wk@mv*OD*7Mm=?v1Q$0A|287ndXA!H>YomFhgf_lXo zy%wjDzRnXFV?5*PP54 zGw1cyymDVdv3*tKc!$SA_wW3+j1V*Uwm&cR$W>toIs*=|v+lc8yB{)H72 zB%HXypB1|)UlRpSAczwYV%U&H2RL((sZo3y>HnEBq!2(t!Is`?_$*lcDXae%HBGXs z`y1P5Lf<}lx%935=~Z19`Ion+T-}nbd&$-sGd+Ij@&I$(6&D>B9JBjmw?Ap7Op#ZQ zq#aE#u>GQ;MfU9a#?)-%ckP!qz7I&R>wV|0&p<%EL_j^y?>xVA<}^Gc zASR(VEw7r?4aQ&8G{~;)-`IlQbkB$0@}`@&p*LGnuI-X-d&W`x%%LwnFX!AlzxdqA zd3S&DxrKA@h|>goqiVm&^g|1RcWqVsi!AT9S`hxEwrYQg=_g$lgd@nMKfq0T_?v%^ z@FngMPK1L6JhXUtXBps`WfaYU(m+|job_F@1Uvx?^H{Rh!(XD%e*tg6%5vBO>A59U7`?meQ{WUw|$Nx`!AL0VCp~z2C3b;b zMJ71y9Xzr=41N4GsuPav!!~3rJaHx~=;T$ym=J3)PkPWQiNkD}Ex8Wr`$%M+A3cgd zUuVBr0f=0-NHx!6Z5ro`hU0#+_p7|FGrtL4%sO)-!nzD;3eO5$5VnGGX3nAc3PIwE zC8+f+9sd0ha6RI<@dR;I$WrJv+yJuTV;uXX#9hZH#u(Gh>lszBwZq>ZiA?%YP~*qL zk@1NbrhSYgFtx6*wnUKmVCz5^CgJo33sWSwc;i?WV)>`A?$MFB|ICRfIsq9EeM8Tw zIv7b%?sRlwDsE@3_FvBW-wsr{xR?zqDIb&5il0cTLO!kU_eN9Q7@N0CERA+ z4iMpy2ey&rk4D>CFwB8_$u|dvY|OHSTmnwT6;o&`aUwbc`Qe;bn^%9&{fD4N0Qkaw zh4-H`bo-&Z)3l0ziU0~e+(4&D;SWa?=OA|fVOZ@I#R*aMY}t!UDDdwnuVrMMD3W5L zRTrP5Br`d8#uXbDeiFW`eW`7I($@a#8?PU^ zevdSGL=ujrh2fMiEFBq<8p3iJcE%zZSH<&Boqy`NrzKlGn5MUn{~``YDDK(%o=$#_ z?^$npy{8z_5#spTLvaRAzXhzQ@D$)<7VP?316<5j52Zvf_VP!M4{9ayeglRMLoXjI zD!gBGwCw0hnpNyJh_mNZ=A+-%q{^5>Z~)lY69#ix>=jZ4eFv^ROB^mPOLkrsi z4?-Wa9N-U+6Cpjo5A-q`V;}gGR1zZR43Evk#1cz9z6ecV9_MdfM(;Ei{6J^y3F_9UtmEUuS;t2c8@QRN*ou`g z=R;vt*QTAO0k+kI4MD<2@#_fk*_#M;hf}r|pmw8?#K?&tgbPLx;|1IyKc~8S5iobi zI`RDzk4q!rS4ZX=-)Mckb>8%?wdtl~mrH+G{=M?c)AGh1yj^|ZXTmk%`cAp`A-qXL z$I|tWO7)MTuwqet_pccuDpH({qXed1k>ywAEn6togP`5bc!xMc2^T0Jz9BwE z!3z}3Qt$u*rA(8Cg7N4Wgd}*qDGq`-bdUI(2xC4t&Tt5mi2WKZ4;z#?N7Z~20i)?m zcFYS@XE%3n*#am{cV48Asm?RJx+05i&Rv=`Wm;B2VcXgz*{U-oWtr;!*$1WS{`n*C z47@$?eszDcKU3>}ZQ$jB`E6HfE^8Ozl{NBMU$K)+E1Xp#+wQ7FLTbnOGA#dI% zuiZzTE>XKs%+)m9q2zZr&&R*Bx5)Gk#e2*;-(X&pWV>Kf1lrO`hodh<+_O9MIXptm$N8srpmxt4?uw)CTT;VT7vMDVAZwPBDcw12U z9}eDmV6XGRytEuDKv?YrGC-`gkg@JyBX~CqGb1vDQWLO1qWQBFitTzgR-v-&;~#2-*Vd>U7S@6kkTxneqi1M=)6Da)|swC{ZW#bxJ=Y&^I{rH}%n}5`wavof$ zZ&~ubXg2!nNjpev+q$IvL!Uo+5B#!=8#8r{>AFp+x=ok2NjnAUp<_~TQWDR|b!XER zXWw_9g-2pV<8Mnir!V6vCzgZLR-d`J%4z+uO7AjL${r))Baof- zhh`1${LMZLINy&Ux0Ms%mmLv8PI@#67*=)t2o2fARf~I8JIA|VBny1N*xj82!JK{- zFx=@dL7pz7ql>_=2UpN~plfKRoHjE^=L?rTIGCU#;b7^qk9ZY~dGfodP`g@7+V4^u zyw)Guei~RJp<^0U34I$Pc@jEQvx!m-KiQ@NR6K#t%ERa)ZF!7wgmuguB>GWad=Pud3+C1Pti6h#DXQYd zi_|;~20|(Pkwq9oagZkh{&*^M7N-o!9%UMdM8}!*saQl{?Zhd?GMN2U`-9xk7_9%g zdjtBV*e4_KZjL3$Rsq8mDz;3;qK{8SVCA4=5yY$M>?;f&%Qpo)!Q01K5YKPV=BKg$ z2?q2G_ep8RtHxB#j@O@%Yj(($JLJ-?q~#-5xr&*N^4Zeo?#g(ol7}-|92mO*S*qv7 z#Kq|sroZi(d+=K|^XF2HU2;XY?B0oe1W(n)-7oB(^U9u8Y0vtUXZ`%>TO(Imf9kpV z;7@AgZHHygK+?A0DxWzayBZ{012d_CKE8||>3C|w3&28ULmvTSvup@S>Ql&)rUMm9 z=3$$zyhZq&m?0Z#(ySW(P&M_jcIl7VR1LmRwZ2A(^F-HjzyLOVv`S(X{EM_)g>dPO@AEIR17 zGj7pd76cdj#E90c!#`Nc+Rf zp0Dhc8)ZnBTlRcqnNrwc%7jwPJ=`#39>Ft64nWJEg6B#=Wc=I;`Ewp$UctL${8)CO zY>=FgmOVqHp|eNuYO~j=A5I-?)V}6^1&@{wE&?HSK8KpM??TQpQ=Xe)4eDY(jJu*h zdS$>8s}kIpk54fl)p@knst!Y_7m9@n!ApY_e1mJX^0{B3SnHjiJNglgi?O%& zFs}7lE}>eeMZT;&a`YcrZIySb?eIUSFaj*;)N0E84sF!pf=}zg(O#{MSX~ZQggWhe ztR5ES1_mwP`!^0P6@{TipQ${U2ocN{3-vn8&f&eGAcpH001SsrPZSe5H)|`E`yC?7 z6?+2CE*`lo!-OXd0d8n}_5&BN2o?=SO;|UqonL4Groa1pt=;d3HFubY+c6GSaBi?m z>tpU0Fn&i26iN!tdhrIGZWf{@LUEwD%N%PI8V4a+FMo8*ZNdCC4uT>rfAk#Sn#i$d z`FD=fa}Z4fW(fX*tJ|nS2Uho|>gr+_1pjHPbjYxjSNpMZGI;mx?G5eI(jL+|XkMCb zbI(hw)Y^WiDlb*X?7TF$mNPG{p+H)xmKG@KD2g>bG_Zdok6M@3Vq2S>thOVof3Qa0yOBa~*O7RL6A-fK9eGLvzt4qS18Wm##Rk(33 zMC}3YYoUuB6fZn7#-g+;SpcG%lu{L5^OMSNbAn`T{+imvE*B2ohtNQoS}-33&~N5| z5@t2)FHl*5xZU-EM9%Cu`W{-<5_hQo5%faDxlaIc#`^5(Hi$CxgeX;_zT*!q_72WF{Zt6aA0 z{f4cxr9W!CxG~kRHEET~c75WhOnX}1_q5DM-aL8f5@^58GlZkDSb{7*aPLO1SuKpG6lcO8*-9JykYcN}?T+wA6VY`e50*)LT; zxX{=#HzYT1#FodRldJWoYu2Y~pvGS9lWQJKmp_=iC)3!PZtQ~Adu5e$|HE?QaJptV zd2eRdp5(nBRy5BTGrp>eV;9C|qq46p?YnD!-lue)o;_$wT~-JJ3I)84fy z@7nL~yKMZ+9= z@ZDeN*!Ued7&K;DS4qyM-&8fsKrgiuL76%T{$Cfxd$IU@o8s?_rKDauI`*4lzkg8kk*aq)~%`5t(QlxY?52| z;P~=_4_A9gojtN|FKVf%&Un|%hf>~7$-4&ieB8cHayQO3r`)SD?v}K>E#+>T_uO!= z&v<-)>uFzD-F2l^>Iq1~QF-;(X8Kd!mW;PO?OmVpuAd+Iz`N^%`s!ZP}K*@56tIu`jGJcHio>Z*9uAc77k5 z3ygJkWWGkO--2;w+zn~>DvWY&>I3)s&&sLSm0WT5{KmI~SgVwCD?HLuE`O$J*EcZk_SGlYQwtyK5eIc+b(0WGgWo3HN4!A@m4YLX~E>p1?HP% z@A{0l;o^N4?wi{;UnYCkEqLo@BQj>TZSA6it8QQ9EY)Q*7Hp+?tAQ@~f8LUA>&E|A zM&FruXX5RNjJM|Ez=eUiZ6A2od{%)Xzk8bJy!Y@xr^Sa2T+`||YG1EisPoU&$aU)% zT2@PK-Ezy$g|!`0=YDx@Z)Vl51%LBgzwFd>3Mel+S994Jp6@H{ov;%obMrCsTuxVdA51WTgzatEnIL5v7(>N)7VlCla0Q$Sjl*&O z77~^*Kv>*tAjs`jbpzsa%5jqdmY1WVW1xRrb?M zMC^M}kcd5;@f(`c^_x=ln=&{~1I^7HX;r4IVzHQO*u2PD8|=U1tX1~&#YyW7 zwq$AJyCPZgSt-)KVE5NOTe+J52H$FW#j$#+%M?s+szLvXZK*;35Af!Ime6@AT_6pxl^Vt!s8FUv!7#h#pleO{hyD=};SVM0y#hIx>nLoKZvEn`J_>boIl zKqQkOPBazvSJ6+R#Oy2N$eFnuxocendq7t+^HLoFt2SP;W5BJ844#mwp#VAyZ7SrI zEiUL!qxUb@o;C|ml>YDP)JK%-ZLZG4UsIJdD9$Ch4G-O<<`SG35u~b^TW}9zC6_-) z^({~1xGjcQY2hzR)W{B~k)?u5HwZyk7F>nQor122VHOH2$ht}g#S=OZGb@lGRyRuu zy0Sm08`E+<*GF8IS3-B&`pOa>p-dkY)aRi_tyXfAr}@+B1{RUhg$L$Yk6xyBNf|iQ z=NUY8qjSMpJVUw}5>9}?MfK5I(OO_vMcSEro;R$s`8 z@5HL^*k>K%SLkz@HvSd9>l+Cxj44-t;W6dvFZ^AvUq1?7eaA8BkhDUd^-Yo$`kX6Y zxKFwAcUT7)O<}O5Csvcg8lh6Cf{79bd>Ay=Leerq*0K-jru7^R4fs0DFraT#YXLp5 z2v(tH*a2Dw0Y`m=8f}g4gB*$@bh$6pxAA@*o(0fG!HLa)x%pqc&@&OrsBj6*dBiU@b}y1A9!cX+zd$8@u^8Jl*_=zD7v=*D`_B55*9QiVq?pmZFOTBArJUeKb;<-t2J}=I<+3 zBK2?R3o$@82MuT6-`qPN`nm5t-_L8`tNojLY3NMq(3x))&GBy({bdn8ej!#-g{P<` zd-g^&n2E&4kN`b_(%B(hv6iw?s^XXuLqR&&gnUX7UiKk=D6F_}k10-%kI+4j>~aS? zE^mv)B5J;D9Qmx-s8W<|?@(B4sg4@)BY@2_dm3Cbn6|6Oh81h}gKB1Tsn#ylLWp}L zmEuV_Ku#utxPx*Gei+6xkzw2j_uc7Z4-~AX;4lJQxkVSp;+P6frs2MrJa^@0%JV0b z2bMT?b2+>IQY6>L_IgzvBLg2Bve-)fW+V6|rRhUcocU7*PikiHrs-20>in>)p<$V1 zm(yl1yko;2VQq&a;i*ZuoRH&54DKoMG4&v-K1W%zGY(F%%WbvhhDS8(I%019AXfHN zs|rH;!m4mY^DMSwAn|&{vZ~|?KTN|E|0i{^_NxfOH>o-tJArz6%guQ!vKQKU|NQB6 z#X1;8JnNITkDU#_F?u07XN6TS>Bv;nvMcl|<%%^Z z;#ocKxcnea&EB{xARU3@>M8lISjrPi+CKC&;8t9x_xbzI-*=;S!)4D8E5288r2@Y6 zSGUQveK(x_w|%UXH}1XKFW27xf%5^n!OxG|ayvHQlHAs|ZwX1$hn^bL=em#y@a3)AhTxsLxxI zw0%2<0DtqLRDR=e)xNBa{aIr z9+j)cl6@b!su!y3Qw?X|+HvLJjm@}M@T_$1NxAx|WdA3votGUyK9X+Tn`+&A;j!z6 zq+?8nMcy)t*P=>Y1NJ!O8Xx_ctkJu$QM;$0W+lA5>P@NAQ&&LnM%25YT7Q&PQX#P7!?m5XNl zVoe)&&UH)v?#q=3GSw{$)vM;tz47(ezkYdKUcK+?Bk5it)hkF34$HkGa&>sYzi$5F zHy^$9sI+T9UJp~_vGfC>)B_=DI3YhUCHqf*UW{rbTP17~WDVY!s@a&X=}OgfN!>6B z_v6bg3s=z$!^ZOi=LcqY%g*-s?l<>b+H+;Cy!IZLJkrkEl(TmBnCx6T|44fM?$r9- zS00ns_vsh4NR4!h%T24P!k%=%@at>RD%&MH+veB2*>R}@p8acjub%ta*ROqDdN3#- zIG(Hc5!tzB{@(PuU8!}ut_;cR4qe}s?jK6^4@r+i}g^-Kl$Dtb5d`xQFtWh@aCZR^LrMc8lu3J|i3dvFy|ptcmP|Ut-5D zp=F|~&vmQgWu4fUcX`bq+iyCy&aZcG9X5OOr}-G}PZ+?+SL&5`7j>OkC-0^hYp!yo zhOjz}k+PK#X?QeADobT)jIVZ48e^_3#wLgH;xYxthUw-^92>R=gQq6KQ`oi(D$Zc= zaoAO}UtB@;pJEutJyx*^e6|LI;fax8Q2Z9Pn55uE3NBIb76orpa18;NIQ^R_BVgMn zCc^3lDOqJj^2Jt}V0nF&1E=NWGTC8u$%oijfgE$3>!Fgjh~f{^5(T4 z`|20X6hT`+ax074xysdx4iL9xL6d%-|9tH9<;;0pZ_NhCI9w*qr+T* z2G@2j7KeGW{bN_%qM0J7ykhO5l|^k_nSZf}MRCV`-J*jbwa<&&$|V$ zIQN`Oy`R2QUKPr#ZiDi}keRW^eGKj9z?}7(0QvURY`Y}>sl;jd2=h|2HPE=lgQ!g% z4?_@`#i*mXCJD37)v)d@ogtlcHF1YW= z?p@%j`wsFe5y_aZv={t7Uf|yvQi(Cy@RwFf_ZHyVFZCZGQav-u^D7l29iT0d@wj?( zHPIfWCYM1Iu{$(vfnARPM*_Xyf4jwp0-J15-*5Sxe95i)9y4#3D%bxLM-kS~ F{{ykLJ}>|P literal 0 HcmV?d00001 diff --git a/evaluator/collectors/__pycache__/langfuse_credentials.cpython-313.pyc b/evaluator/collectors/__pycache__/langfuse_credentials.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a0d29f2b575df4ee6a57c77b8c19771fbcda126d GIT binary patch literal 7152 zcmb7JO>7&bO$|PH&s;j!HUcGwt>V5B3ziMa*AV~j|`0)OdR)oH#6+3yX0{7~3D13?}BynTt z21htd%dRmOalz9)=Dxubp1tG8JU6_=%gUZH-wlBXmA-!BhrZsiz>OdYVsw}9N0Ltx z&TyRwO8#yn1x~rF`A%66L`11c3U;IWJZX@cC6SdHr4}hHHPG^Xj)Zy<)Ll|zl`3ioOf3ZJ68h;R}DiEzYy0Hkx5m)r-Tg_U@x#W)<3B)(+j|9!|xUA03@`H^mrP#H8FOam z9xIH}7Enc`w?VdyUWCGHAFh75#s50=%g}Zxwk^b7K?8b%fE&{b>&+20A2M?D8D$FO zau`E)FCv&iBB9DzC6O=#i9}XQ+6Wz$9RYldH8D7*V3A_;I zi8ikgV~J39%(g^Ary7(@$c91E(>X)Y6A97@ZKQ){lygGQ6wByt=|hnH3wjY~_-M4; z6#nRzm5HH`#$I?DUb#5W(N}KdZDb{j2p#{%&hhYq?OZzuo9-N?NmB~yvuMiMg@vPf?cXU`%#>;XJk}^)i%3i&+1TRJOc+* zI(13nB-a^klImAovZ<)9xa$Jv&;nhhCT0*t+!KHD|efE4vK3RRxhl*z54c!*t&M}r#CnIisHT%jy9ZLJ-v2z^(+i5iw$dIt7BUoD`VSY zq$I`)Vyq}0TOKI~o7OI@U)kn6ncgI+n%Idho2ks_AZ~WpD~e%vpwF7krEob0#c3#s#<*}?TVR;q+Hddk`&u$DqLN zIUtx(3>}Z zJl0f@3Q=Yh0`>)H8rq(V(_@w`V4PE-;%jLm8kmT9P4}!~5D`uuIRu&M(sa`cuRui1 zW&mt?KuiR%+!8gzavwCaf}*wDXxwd`vs__`h?x4!q;gHms-&>JiD9xL=7 z+X=O-|J~1iTngbr2p2>B+d}_~Q0v;#>e6OMG1Rww{kbo+)6}xz|JeU%U~}%XBgLkn z70>fv(|UYk>fzMJy@&V8;m(b756^vaVOzv6n%Xw}5B-}%#ioJf(dS3cd>!2PNZRW9 z?EPZ!@HT(>nE{z)-fB&SA9!Zw?IakzPtg=qt+bc-0w{%QFawS>t{qWpHY-%@L&z|V zVnkiC(a1j5DL||#$`eZx;AI0##_28Me_ z5kNabrN^4~%6 zHbT(b6Xz$Xb5xgX8HUmI^9bc&Mf0;$ZFZ-)@6T83>6wG`-K%_#}Tz2{wUPFmS%X*=OSHmr;q zCYEZ}=&XE_x?^=oYPTg255$9vVn^KDh)%er&NKWEenN-b{GZBx3jnNL`On8SIfVi0 zC@{{XXLAsvU^^ti;P~GMumea?DJrFuMT#C-JiUZ7+H5*0XDpN*O#zrrXjX+HEdYZ| z6g(%B6gZ(kat`n!if;o}&g9f2L&uD!&FffO0NDLtPBs);poTxK=m9!-9!xdG%L>NU z;c7IHKkD#VbqM|pe=&r+cm{wkbr$D`qX2CN9=|wo?P^>aNnD;BxiT_wd-USCl(>9r z;_B$N#5)&nU#FlZTWATaGgzdc5+zha$N*%fU`KVPm-!jG>0{5)lqs-6BAv<)kI2cn zsx+%yvUM)uY)&^Ue*wH2>aC2XDCFAX2@7<6G{zlK7fgW`t^yyX-wJrgAl?<7BJI-P z)U19M2D5%nk@a(Gv|s+vD1-_XhhR^c&e&YxJWux{e>I9P+xmi?Nh_ES^Hsbga1~%A zs7#t*m^Vget8pHpCKofgfe74gP{BH3HGz|fBCRYivBC~)i@P?6PFO0NpuP~H=gSl) z!NEyql@UTTxE)xMvl-K`%QH$M18Pnpbe0(H7)XPg2vID!FeaZ$Cn3y98%kEE_Xh@O znw-lRu(R{?ikdP5|3CvCM$__JsMlaNI4se9VWBRB;#ROET3iQf7=&ryrp4rmrWq!9 zGfdCTfU1$KoJr@E6ahmFVHjn@6e?ays!lLEh0TgXQ{&YP($ys#eyVzws2&=f+=a1p z4*h?@OfI9>A_~8?apdD8TYL$>`|m<&=-tB5yTz8t<*{h!lfnDVG2VMt|*t9nM)8WnUm%3wx?%1=A?e5ri_r+rH zQYm=-i|OSnsSetdk_2~`>)xbYpWA0*LT7loBS^VTdrbwVC6~~X1}$1 zYjbpK@oVvDS!^$fJq596GxeY1z)K(6AKevDxM#)pB6e)~dbz1%qha&-#|v`r&DyWBhvYmy1sW#e?UH;qxoP^H9f5%f9WNd&QRbx5f8& z!tLvepIlfG{yPMdY+Zdk`ZQk*T`Gm{emPgVdbe=(F3@fp*t+ug`%nM*iw}!!@2v#t z>%R+i{ag1=OK-X5{IehYX8bGutEsPUf?$8hy+`HS{2rBWxDn*r&7j{q`qa zrRKgubKllLvH5%Djz0Qp?_0mO)wgwh>+E(o`qJ;|4Xg-7q5I_l)ZDv!9Eq*Vx0vq; zmsM?G=(P_Lvl(q#&Ln1_HZ3O~IM*x`J^`#Or?Lz+h?zMWQO6+;8%N*>I+rf;4*|w# zf0k@FhPh(klGdl{m5l0+yKN|x!BL32A;xp<6@^YwPFMmrt}>JTtO0SL0hb?caL1~y zCm%-)RKe*2X!X{DYTUUtk$(g$_)dq^;S)|yO;2j3>XOs*8uWCWo0>7;R@SFU$IYv@ zNRDGxGj3!YKN_!qb7I~cQsrxgr$Z>eY`$V{;XMErd% zt(tICO$>m58?apKIxg&3a|>Nr#9Mu7;WvG0U9?JcAVeA{;$b*K?m^iM)|h|OLstd> zSKR{L1D$GMWS<=yJ9F0vgUmwb1^Cy$h73Yw5y2&%Z?KGq9$(qUN4JHZ-$BsZezw?t zekE9LdTS%}FtjBUn<6amg>ZLeyxi8cap&QkM+;jIiu;cj+fJ0)-hR3WwvhtNxo6jZ za~>?;b*{hMe|T%(<9B!6Tm#$-+bHvg0L=KS&Qjk6_L@jNJ+Fyb!myAxw?rPZm ziKWd{J?g#zn8JDMo5L_sp_STjeF-wySC0Dz^?!@}-=My4Q0F(O{aY0LEsDPMB3E#I zu)uZv(Is$4cM)XpVZhb7ORKNj{an}U&UWq!_lM)BxYk|5p?!GQy`S^$deNciOCOZV Xjh*(JeFt~x8vuWA*B|{*VmthQz&B#P literal 0 HcmV?d00001 diff --git a/evaluator/collectors/__pycache__/mock.cpython-313.pyc b/evaluator/collectors/__pycache__/mock.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..40c1db8bcad4cd07d8f6c80dd1a1b7f696e5b8af GIT binary patch literal 1481 zcmYjR-EZ7P5Z|@ezK?uTI0-5~Od6qDYAy*WYDndY_T%~L(Dy6Np@of^PK0Dc6 z_u~{O0txX7Y9(HIqWm4a@E8f6RscSm(ke0Z-DSC>*|odWq4A@uY+02fFAX>Yg}%Mc-=MaU+t46<33C-B5l z<>hqA8N{zsPBU+Fl$$%;)Ds%3ysqSbTS(ZCRlu4vwijm9`e= znE^t!A)vT60w6;+8)3uNOMX$(watoKi7Q@+>t#2HYVj^?k4$Yd$@U8q8Tr82vAi%N zqh7tIEg&OcxkSh;%kV*#7^xIaRpAAPzGYgb+je|kswD45Kuj6itSp&z?Zlw6ND~3v ztd}Oe#8|KNYD^TA$G$G%1_}YoIv)?$R?lOH?fr+?e!0rFwRU#D+Lar*=pR9xkT1v| zS3iEt9&da<_iO9o(@VciU;ZWi#`}GG=4t2Xqfd{z<{2R$I3GHot~(p3gj1kecD4JQ zkVo!11M4QcgKC!@vcJ*!8N(P`_b}$gsCwRQA7ku}57y^ygzBz;fJ*!KQSJE;@R64u zqA}e^+35TKm5qOS&AV2kKatSsbi%rN6X+|3;;j)10QIM|G@II}+reSb$up_mMlX!d zR1TOzJ&H2XlTl=wQPj)3MT&SliVg~qR*|VF+D?== :start_at and CREATED_AT < :end_at + order by CREATED_AT desc + ) where rownum <= :max_rows + """, dict(start_at=period_start, end_at=period_end, max_rows=limit)) + sessions=cur.fetchall() + for session_id, agent_id, channel, ctx, meta, created_at in sessions: + if aliases and agent_id not in aliases: continue + cur.execute(f""" + select ROLE, CONTENT, METADATA_JSON, CREATED_AT, MESSAGE_ID + from {self.store.t('AGENT_MESSAGE')} + where SESSION_ID=:session_id order by CREATED_AT + """, dict(session_id=session_id)) + rows=cur.fetchall() + msgs=[] + for role, content, msg_meta, msg_created, message_id in rows: + msgs.append(ConversationMessage(role=role, content=content or '', created_at=str(msg_created), metadata=_json_loads(msg_meta.read() if hasattr(msg_meta,'read') else msg_meta,{}))) + input_text=next((m.content for m in msgs if m.role in ('user','human')), '') + output_text=next((m.content for m in reversed(msgs) if m.role in ('assistant','ai','agent')), '') + records.append(ConversationRecord(session_id=session_id, trace_id=session_id, message_id=rows[-1][4] if rows else None, agent_id=agent_id, channel=channel, input_text=input_text, output_text=output_text, messages=msgs, metadata=_json_loads(meta.read() if hasattr(meta,'read') else meta,{}), raw={})) + return records diff --git a/evaluator/collectors/base.py b/evaluator/collectors/base.py new file mode 100644 index 0000000..cc00c1f --- /dev/null +++ b/evaluator/collectors/base.py @@ -0,0 +1,8 @@ +from __future__ import annotations +from abc import ABC, abstractmethod +from datetime import datetime +from evaluator.core.models import ConversationRecord + +class ConversationCollector(ABC): + @abstractmethod + async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None = None, limit: int | None = None) -> list[ConversationRecord]: ... diff --git a/evaluator/collectors/langfuse.py b/evaluator/collectors/langfuse.py new file mode 100644 index 0000000..65c28e6 --- /dev/null +++ b/evaluator/collectors/langfuse.py @@ -0,0 +1,518 @@ +from __future__ import annotations + +import json +import math +import random +from datetime import datetime +from typing import Any, Iterable + +import httpx + +from evaluator.collectors.base import ConversationCollector +from evaluator.collectors.langfuse_credentials import LangfuseCredentials, resolve_langfuse_credentials +from evaluator.config.settings import settings +from evaluator.core.models import ConversationMessage, ConversationRecord +from evaluator.identity.resolver import IdentityResolver +from evaluator.time_utils import to_utc_z + + +def _iso_z(dt: datetime) -> str: + return to_utc_z(dt) + + +def _metadata(obj: dict[str, Any] | None) -> dict[str, Any]: + if not isinstance(obj, dict): + return {} + meta = obj.get("metadata") or {} + return meta if isinstance(meta, dict) else {} + + +def _first_value(*values: Any) -> Any: + for value in values: + if value not in (None, "", [], {}): + return value + return None + + +def _content_to_text(value: Any) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, (int, float, bool)): + return str(value) + if isinstance(value, list): + return "\n".join(filter(None, (_content_to_text(v) for v in value))) + if isinstance(value, dict): + for key in ("text", "content", "message", "value", "input", "output", "completion"): + if key in value: + text = _content_to_text(value.get(key)) + if text: + return text + choices = value.get("choices") + if isinstance(choices, list) and choices: + return _content_to_text(choices[0]) + return "" + return str(value) + + +def _raw_character_count(value: Any) -> int: + """Character-count equivalent of len(trace.input) without losing structured input.""" + if value is None: + return 0 + if isinstance(value, str): + return len(value) + try: + return len(json.dumps(value, ensure_ascii=False, separators=(",", ":"), default=str)) + except Exception: + return len(str(value)) + + +def _messages_from_value(value: Any, default_role: str) -> list[ConversationMessage]: + if value in (None, "", [], {}): + return [] + if isinstance(value, str): + text = value.strip() + return [ConversationMessage(role=default_role, content=text)] if text else [] + if isinstance(value, dict): + for key in ("messages", "conversation", "chat"): + if isinstance(value.get(key), list): + return _messages_from_value(value[key], default_role) + if "role" in value and "content" in value: + role = str(value.get("role") or default_role) + content = _content_to_text(value.get("content")).strip() + return [ConversationMessage(role=role, content=content, metadata={"source": "langfuse"})] if content else [] + text = _content_to_text(value).strip() + return [ConversationMessage(role=default_role, content=text, metadata={"source": "langfuse"})] if text else [] + if isinstance(value, list): + out: list[ConversationMessage] = [] + for item in value: + out.extend(_messages_from_value(item, default_role)) + return out + text = _content_to_text(value).strip() + return [ConversationMessage(role=default_role, content=text, metadata={"source": "langfuse"})] if text else [] + + +def _agent_id(trace: dict[str, Any]) -> str | None: + meta = _metadata(trace) + return meta.get("agent_id") or meta.get("agentId") or meta.get("agent") or trace.get("name") + + +def _trace_sort_key(trace: dict[str, Any]) -> str: + return str(trace.get("timestamp") or trace.get("createdAt") or trace.get("created_at") or trace.get("startTime") or "") + + +def _observation_sort_key(obs: dict[str, Any]) -> str: + return str(obs.get("startTime") or obs.get("start_time") or obs.get("createdAt") or obs.get("timestamp") or "") + + +def _session_entity_id(obj: dict[str, Any]) -> str | None: + """Return the identifier of a Langfuse Session entity. + + The generic ``id`` fallback is valid only for objects returned by + ``/api/public/sessions``. It must never be used for traces because a trace + id is not a conversation/session id. + """ + value = obj.get("sessionId") or obj.get("session_id") or obj.get("id") + return str(value) if value else None + + +def _session_id_from_metadata(obj: dict[str, Any] | None) -> str | None: + meta = _metadata(obj) + value = _first_value( + meta.get("session_id"), + meta.get("sessionId"), + meta.get("conversation_key"), + meta.get("conversationKey"), + ) + if value: + return str(value) + + business_context = meta.get("business_context") + if isinstance(business_context, dict): + value = _first_value( + business_context.get("session_id"), + business_context.get("sessionId"), + business_context.get("conversation_key"), + business_context.get("conversationKey"), + business_context.get("session_key"), + business_context.get("sessionKey"), + ) + if value: + return str(value) + return None + + +def _trace_session_id(trace: dict[str, Any], observations: list[dict[str, Any]] | None = None) -> str | None: + """Resolve the real conversation session id for a trace. + + Precedence: + 1. Native Langfuse trace.sessionId/session_id. + 2. Trace metadata session/conversation identifiers. + 3. Observation metadata session/conversation identifiers. + + Deliberately *never* falls back to trace.id. + """ + value = trace.get("sessionId") or trace.get("session_id") + if value: + return str(value) + + value = _session_id_from_metadata(trace) + if value: + return value + + for obs in observations or []: + value = _session_id_from_metadata(obs) + if value: + return value + return None + + +def _recovered_session(trace: dict[str, Any], observations: list[dict[str, Any]], session_id: str) -> dict[str, Any]: + """Build minimal session context for historical traces lacking native Langfuse sessions.""" + merged_meta: dict[str, Any] = {} + for obs in observations: + merged_meta.update(_metadata(obs)) + # Trace metadata has precedence over observation-derived fallback metadata. + merged_meta.update(_metadata(trace)) + merged_meta.setdefault("session_id", session_id) + return {"id": session_id, "sessionId": session_id, "metadata": merged_meta, "recovered": True} + + +def _items(payload: Any, keys: Iterable[str]) -> list[dict[str, Any]]: + if isinstance(payload, list): + return [x for x in payload if isinstance(x, dict)] + if not isinstance(payload, dict): + return [] + for key in keys: + value = payload.get(key) + if isinstance(value, list): + return [x for x in value if isinstance(x, dict)] + data = payload.get("data") + if isinstance(data, list): + return [x for x in data if isinstance(x, dict)] + return [] + + +class LangfuseCollector(ConversationCollector): + """TIM-compliant Langfuse v3 collector. + + Collection is session-oriented and bulk/paginated: + 1. Read every native Langfuse session in the BRT-derived UTC window. + 2. Read every trace and observation in the same window, once, page by page. + 3. Recover historical session ids from trace/observation metadata when the + native Langfuse session relation is absent. + 4. Deterministically sample the resulting logical session ids. + 5. Keep all traces belonging to sampled sessions. + + This preserves session-level semantics while eliminating the previous + trace-detail + observations N+1 pattern. + """ + + PAGE_SIZE = 100 + + def __init__(self, agent_id: str | None = None): + self.identity_resolver = IdentityResolver(settings.identity_config_path) + self.agent_id = agent_id + + async def _get_page(self, client: httpx.AsyncClient, path: str, params: dict[str, Any], creds: LangfuseCredentials) -> Any: + response = await client.get(path, params=params, auth=creds.auth) + if response.status_code >= 400: + raise RuntimeError(f"Langfuse API failed {response.status_code} {path}: {response.text}") + return response.json() + + async def _fetch_all_pages( + self, + client: httpx.AsyncClient, + path: str, + params: dict[str, Any], + keys: tuple[str, ...], + creds: LangfuseCredentials, + ) -> list[dict[str, Any]]: + page = 1 + out: list[dict[str, Any]] = [] + previous_signature: tuple[str, ...] | None = None + while True: + page_params = {**params, "page": page, "limit": self.PAGE_SIZE} + payload = await self._get_page(client, path, page_params, creds) + batch = _items(payload, keys) + if not batch: + break + + signature = tuple(str(x.get("id") or x.get("traceId") or x.get("sessionId") or "") for x in batch[:5]) + if page > 1 and signature == previous_signature: + # Protect against a legacy server ignoring the page parameter. + break + previous_signature = signature + out.extend(batch) + + meta = payload.get("meta") if isinstance(payload, dict) and isinstance(payload.get("meta"), dict) else {} + total_pages = meta.get("totalPages") or meta.get("total_pages") + if total_pages is not None and page >= int(total_pages): + break + total_items = meta.get("totalItems") or meta.get("total_items") + if total_items is not None and len(out) >= int(total_items): + break + if len(batch) < self.PAGE_SIZE: + break + page += 1 + return out + + async def _fetch_sessions(self, client: httpx.AsyncClient, start: datetime, end: datetime, creds: LangfuseCredentials) -> list[dict[str, Any]]: + params = {"fromTimestamp": _iso_z(start), "toTimestamp": _iso_z(end)} + return await self._fetch_all_pages(client, "/api/public/sessions", params, ("sessions",), creds) + + async def _fetch_traces(self, client: httpx.AsyncClient, start: datetime, end: datetime, creds: LangfuseCredentials) -> list[dict[str, Any]]: + params = {"fromTimestamp": _iso_z(start), "toTimestamp": _iso_z(end)} + return await self._fetch_all_pages(client, "/api/public/traces", params, ("traces",), creds) + + async def _fetch_observations(self, client: httpx.AsyncClient, start: datetime, end: datetime, creds: LangfuseCredentials) -> list[dict[str, Any]]: + # Legacy v3 deployments differ in accepted time-filter names. Try the two + # known public-API conventions, but never fall back to one request per trace. + candidates = [ + {"fromStartTime": _iso_z(start), "toStartTime": _iso_z(end)}, + {"fromTimestamp": _iso_z(start), "toTimestamp": _iso_z(end)}, + ] + last_error: Exception | None = None + for params in candidates: + try: + return await self._fetch_all_pages(client, "/api/public/observations", params, ("observations",), creds) + except Exception as exc: + last_error = exc + # Traces themselves normally carry input/output. Observations enrich context; + # a legacy deployment that cannot bulk-filter observations must not regress to N+1. + if last_error: + return [] + return [] + + @staticmethod + def _sample_session_ids(session_ids: list[str], percentage: float, seed: str) -> list[str]: + unique = sorted(set(x for x in session_ids if x)) + if not unique: + return [] + if percentage >= 1: + return unique + if percentage <= 0: + return [] + count = max(1, math.ceil(len(unique) * percentage)) + selected = random.Random(seed).sample(unique, min(count, len(unique))) + return sorted(selected) + + async def collect_sampled( + self, + period_start: datetime, + period_end: datetime, + *, + agent_aliases: set[str] | None, + percentage: float, + seed: str, + limit: int | None = None, + ) -> tuple[list[ConversationRecord], list[str], dict[str, Any]]: + if not settings.enable_langfuse: + raise RuntimeError("Langfuse disabled. Set ENABLE_LANGFUSE=true.") + creds = resolve_langfuse_credentials(self.agent_id) + aliases = {a for a in (agent_aliases or set()) if a} + + async with httpx.AsyncClient(base_url=creds.host, timeout=90) as client: + sessions = await self._fetch_sessions(client, period_start, period_end, creds) + session_map = {sid: s for s in sessions if (sid := _session_entity_id(s))} + native_session_ids = set(session_map) + + # Bulk-read traces and observations once for the full day. Observations + # are needed before sampling because historical/legacy traces may carry + # the real session id only in metadata instead of native trace.sessionId. + # This remains O(pages), never O(traces): no per-trace API calls. + traces = await self._fetch_traces(client, period_start, period_end, creds) + observations = await self._fetch_observations(client, period_start, period_end, creds) + obs_by_trace: dict[str, list[dict[str, Any]]] = {} + for obs in observations: + tid = obs.get("traceId") or obs.get("trace_id") + if tid: + obs_by_trace.setdefault(str(tid), []).append(obs) + + use_alias_filter = bool(aliases) + eligible_session_ids: set[str] = set() + trace_session_ids: dict[str, str] = {} + recovered_session_ids: set[str] = set() + traces_without_session = 0 + + for trace in traces: + if str(trace.get("name") or "").startswith("judge-job-"): + continue + aid = _agent_id(trace) + if use_alias_filter and (not aid or aid not in aliases): + continue + + tid = str(trace.get("id") or "") + trace_observations = obs_by_trace.get(tid, []) + sid = _trace_session_id(trace, trace_observations) + if not sid: + traces_without_session += 1 + continue + + trace_session_ids[tid] = sid + eligible_session_ids.add(sid) + if sid not in session_map: + session_map[sid] = _recovered_session(trace, trace_observations, sid) + recovered_session_ids.add(sid) + + sampled_ids = self._sample_session_ids(sorted(eligible_session_ids), percentage, seed) + sampled_set = set(sampled_ids) + selected_traces: list[dict[str, Any]] = [] + for trace in traces: + if str(trace.get("name") or "").startswith("judge-job-"): + continue + aid = _agent_id(trace) + if use_alias_filter and (not aid or aid not in aliases): + continue + tid = str(trace.get("id") or "") + sid = trace_session_ids.get(tid) + if not sid or sid not in sampled_set: + continue + selected_traces.append(trace) + + selected_traces.sort(key=_trace_sort_key) + if limit is not None: + selected_traces = selected_traces[: max(0, limit)] + + records: list[ConversationRecord] = [] + for trace in selected_traces: + tid = str(trace.get("id") or "") + sid = trace_session_ids.get(tid) or "" + record = self._to_record( + trace, + obs_by_trace.get(tid, []), + session_map.get(sid, {}), + _agent_id(trace), + resolved_session_id=sid, + ) + if record.input_text or record.output_text or record.messages: + records.append(record) + + stats = { + "credential_source": creds.source, + "sessions_read": len(session_map), + "native_sessions_read": len(native_session_ids), + "metadata_sessions_recovered": len(recovered_session_ids), + "traces_without_session": traces_without_session, + "sampled_sessions": len(sampled_ids), + "traces_read": len(traces), + "selected_traces": len(selected_traces), + "observations_read": len(observations), + } + return records, sampled_ids, stats + + async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None = None, limit: int | None = None) -> list[ConversationRecord]: + # Generic/manual command: keep all sessions, still using the same full-day + # paginated path and no N+1 behavior. + records, _, _ = await self.collect_sampled( + period_start, + period_end, + agent_aliases=agent_aliases, + percentage=1.0, + seed="manual", + limit=limit, + ) + return records + + def _to_record( + self, + trace: dict[str, Any], + observations: list[dict[str, Any]], + session: dict[str, Any], + agent_id: str | None, + resolved_session_id: str | None = None, + ) -> ConversationRecord: + trace_id = str(trace.get("id") or "") + session_id = ( + resolved_session_id + or _trace_session_id(trace, observations) + or _session_entity_id(session) + or "" + ) + session_meta = _metadata(session) + trace_meta = _metadata(trace) + observations = sorted(observations, key=_observation_sort_key) + + input_value = trace.get("input") + output_value = trace.get("output") + messages = _messages_from_value(input_value, "user") + _messages_from_value(output_value, "assistant") + for obs in observations: + messages.extend(_messages_from_value(obs.get("input"), "user")) + messages.extend(_messages_from_value(obs.get("output"), "assistant")) + + # Stable de-duplication while preserving chronological order. + deduped: list[ConversationMessage] = [] + seen: set[tuple[str, str]] = set() + for msg in messages: + key = (msg.role, msg.content) + if msg.content and key not in seen: + seen.add(key) + deduped.append(msg) + messages = deduped + + input_text = _content_to_text(input_value).strip() + output_text = _content_to_text(output_value).strip() + if not input_text: + input_text = next((m.content for m in messages if m.role.lower() in {"user", "human"}), "") + if not output_text: + output_text = next((m.content for m in reversed(messages) if m.role.lower() in {"assistant", "agent", "ai"}), "") + + trace_input = input_value if isinstance(input_value, dict) else {} + session_bc = session_meta.get("business_context") if isinstance(session_meta.get("business_context"), dict) else {} + trace_bc = trace_input.get("business_context") if isinstance(trace_input.get("business_context"), dict) else {} + identity_payload = { + **session_meta, + **trace_meta, + "business_context": {**session_bc, **trace_bc}, + "session_id": session_id, + "message_id": _first_value(trace_meta.get("message_id"), session_meta.get("message_id")), + "conversation_key": _first_value(trace_meta.get("conversation_key"), session_meta.get("conversation_key")), + } + business_context = self.identity_resolver.resolve(identity_payload) + + # Requirement: trace metadata first, session metadata as fallback. + ura_call_id = _first_value( + trace_meta.get("ura_call_id"), trace_meta.get("uraCallId"), + session_meta.get("ura_call_id"), session_meta.get("uraCallId"), + business_context.get("interaction_key"), + ) + channel_id = _first_value( + trace_meta.get("channel_id"), trace_meta.get("channelId"), trace_meta.get("channel"), + session_meta.get("channel_id"), session_meta.get("channelId"), session_meta.get("channel"), + ) + message_id = _first_value( + trace_meta.get("message_id"), trace_meta.get("messageId"), + session_meta.get("message_id"), session_meta.get("messageId"), + business_context.get("interaction_key"), trace_id, + ) + + metadata = { + **session_meta, + **trace_meta, + "business_context": business_context, + "ura_call_id": ura_call_id, + "channel_id": channel_id, + "session_metadata": session_meta, + } + raw = { + "trace": trace, + "session": session, + "observations": observations, + "trace_input_raw": input_value, + "prompt_length": _raw_character_count(input_value), + } + return ConversationRecord( + trace_id=trace_id, + # Preserve the resolved Langfuse/conversation session id. Business-context + # session_key may be a shortened/raw key and must not replace sessionId. + session_id=session_id, + message_id=str(message_id) if message_id is not None else None, + agent_id=agent_id, + channel=str(channel_id or "web"), + input_text=input_text, + output_text=output_text, + messages=messages, + metadata=metadata, + raw=raw, + ) diff --git a/evaluator/collectors/langfuse_credentials.py b/evaluator/collectors/langfuse_credentials.py new file mode 100644 index 0000000..963059e --- /dev/null +++ b/evaluator/collectors/langfuse_credentials.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from evaluator.config.settings import settings + + +@dataclass(frozen=True) +class LangfuseCredentials: + host: str + public_key: str + secret_key: str + source: str + + @property + def auth(self) -> tuple[str, str]: + return self.public_key, self.secret_key + + +def _read_secret_file(path: str | None) -> str | None: + if not path: + return None + p = Path(path).expanduser() + if not p.is_absolute(): + p = settings.path(p) + if not p.exists(): + return None + value = p.read_text(encoding="utf-8").strip() + return value or None + + +def _resolve_value(entry: dict[str, Any], direct_key: str, env_key: str, file_key: str) -> str | None: + """Resolve a credential field without imposing any agent-specific naming convention. + + Resolution order is intentionally declarative: + 1. direct value in the credentials definition file + 2. environment variable whose *name* is declared in that file + 3. file path whose *path* is declared in that file + + This lets Kubernetes mount the complete credentials YAML as a Secret, or lets + teams keep only references to environment variables/secret files in Git. + """ + direct = entry.get(direct_key) + if direct is not None and str(direct).strip(): + return str(direct).strip() + + env_name = entry.get(env_key) + if env_name: + value = os.getenv(str(env_name)) + if value and value.strip(): + return value.strip() + + file_name = entry.get(file_key) + return _read_secret_file(str(file_name)) if file_name else None + + +def load_langfuse_credential_definitions(path: str | None = None) -> dict[str, LangfuseCredentials]: + """Load all configured Langfuse projects and index them by logical agentId. + + No agent names are hardcoded here. The function loops over whatever entries + are present in the definition file. + """ + config_path = settings.path(path or settings.langfuse_credentials_config_path) + if not config_path.exists(): + raise RuntimeError( + f"Langfuse credentials definition file not found: {config_path}. " + "Set LANGFUSE_CREDENTIALS_CONFIG_PATH to a mounted YAML file." + ) + + data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + root = data.get("langfuse", data) + entries = root.get("agents", []) if isinstance(root, dict) else [] + + # Also accept mapping form for operational convenience: + # agents:\n my-agent: {host: ..., publicKey: ..., secretKey: ...} + if isinstance(entries, dict): + normalized = [] + for agent_id, item in entries.items(): + item = dict(item or {}) + item.setdefault("agentId", agent_id) + normalized.append(item) + entries = normalized + + resolved: dict[str, LangfuseCredentials] = {} + for item in entries or []: + if not isinstance(item, dict): + continue + agent_id = item.get("agentId", item.get("agent_id")) + if not agent_id: + raise ValueError("Each Langfuse credential entry must define agentId or agent_id") + agent_id = str(agent_id) + + host = _resolve_value(item, "host", "hostEnv", "hostFile") or settings.langfuse_host + public_key = _resolve_value(item, "publicKey", "publicKeyEnv", "publicKeyFile") + secret_key = _resolve_value(item, "secretKey", "secretKeyEnv", "secretKeyFile") + + if not public_key or not secret_key: + raise RuntimeError( + f"Incomplete Langfuse credentials for agent {agent_id!r} in {config_path}. " + "Configure publicKey/secretKey directly or their Env/File references." + ) + + resolved[agent_id] = LangfuseCredentials( + host=str(host).rstrip("/"), + public_key=public_key, + secret_key=secret_key, + source=f"{config_path}#{agent_id}", + ) + + return resolved + + +def resolve_langfuse_credentials( + agent_id: str | None = None, + *, + allow_global_fallback: bool | None = None, +) -> LangfuseCredentials: + """Resolve credentials from a declarative file, never from hardcoded agent names.""" + if allow_global_fallback is None: + allow_global_fallback = agent_id is None + + if agent_id: + definitions = load_langfuse_credential_definitions() + creds = definitions.get(agent_id) + if creds: + return creds + configured = ", ".join(sorted(definitions)) or "" + raise RuntimeError( + f"No Langfuse credentials configured for agent {agent_id!r}. " + f"Configured agents: {configured}." + ) + + if allow_global_fallback and settings.langfuse_public_key and settings.langfuse_secret_key: + return LangfuseCredentials( + settings.langfuse_host.rstrip("/"), + settings.langfuse_public_key, + settings.langfuse_secret_key, + "LANGFUSE_* (manual/global fallback)", + ) + + raise RuntimeError( + "Langfuse credentials missing. For run-agents configure the declarative credentials file; " + "for the generic manual run, LANGFUSE_PUBLIC_KEY/LANGFUSE_SECRET_KEY may be used." + ) diff --git a/evaluator/collectors/mock.py b/evaluator/collectors/mock.py new file mode 100644 index 0000000..3303001 --- /dev/null +++ b/evaluator/collectors/mock.py @@ -0,0 +1,9 @@ +from __future__ import annotations +from datetime import datetime +from evaluator.collectors.base import ConversationCollector +from evaluator.core.models import ConversationRecord, ConversationMessage + +class MockCollector(ConversationCollector): + async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None=None, limit: int | None=None): + agent = next(iter(agent_aliases), 'mock-agent') if agent_aliases else 'mock-agent' + return [ConversationRecord(trace_id='mock-trace-1', session_id='mock-session-1', message_id='mock-message-1', agent_id=agent, channel='web', input_text='quero minha fatura', output_text='Sua fatura está em aberto no valor de R$ 120.', messages=[ConversationMessage(role='user', content='quero minha fatura'), ConversationMessage(role='assistant', content='Sua fatura está em aberto no valor de R$ 120.')], metadata={'mock': True})] diff --git a/evaluator/config/__pycache__/agents.cpython-313.pyc b/evaluator/config/__pycache__/agents.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..78872dc9d552a870fdd856055356c59b0cd17ab7 GIT binary patch literal 2686 zcmai0O-vhC5Ps`j+v`7U3>afbx*=(?6$MBdDhMPIrD*~blCZdnWTMq#uN^1Ln!a6E z0adEi9!hc{QH5gEN^ne0rKjfDV~<2D6?f~jNIkTXT8W55j_tf%uc3%2@9_4`%zHEQ zX6AjfyMcfg!T5XT^V_3Ng#I9eJ%A>I+Ld8mK|JEAaWqC@iX&^sxMPgQ^caH~lcvX= z*a>`Q+%@LLZja#uwdwCD<<=Ggz&D^H2FGnYQ zFNiEb89q@_3Yw_p3yPX>7@mx%iD_9>Rl}LhOLArs_Au8(?G}7lRnoM)l2g+)#pZ{x z2x|8WSgfEKgee|j2Tx)8ZMeh?!_!?L;Tdq=36#sE+}OplKzU5ci`~2rC_kZQ*iJO# z;R6E{INkuFpWI4et;7nHdtuW01{3#u?Hrtka-JQzC{ArNvw)C}*ugwvn^ zv_LF^87V6kWlhM6X{~@47Cmxa)ly*RrV)`vC6_I#l3-aB<-Dj$s(}n|F0BglSjy(_ zrioE|kc82s5YED41trmxE!p=yO##1_+I?(vhdP0tP?HG<_5&M7U?NFVB{{1SBb*VF zJ{9Gn2!8gY&8YO4j;Y=A3x?bFu4#BLYTY$v2#%{Um`Z3n)ch#$Ft8aMT4%Pn!Drkc zEH~MqU67(D2+SKUENMloERs4gBasLitRN`joFoW_R}kh3nW9YAenI%8D9SZSP!O_t ztZH&zk(2_6fw8YF1pzmJ4cthkW-^g?V=yG(=52A5(Cj=eK#H^le1woKf}oPRP79)@ z;rwh-lT^^efegPzrVcWB?1_|#dL5>(&~`)PmlKudXzB9Ow9Z5;Es@gblBzS2O0>N+ z_Q0t#?Z3oNJk8Q67D&hxu8ay6JM%NjBCzm$_JnJNDZ zQo4dtKw9Il6n8>NoTuWo9T`XqxBo=#3ECRSx35-S3YBu%{3+I6?R`NrHRao*?Untk zY(4v|gTEZ=MT(;bDRj~?6|m3jeR-+}*(lV6uKbIT!X%kPcRF6fr)|r&9>tk-)*}EO zr-HUUp0W2zREkuIIZ{pbD(ST4sg^qLaBrNv%eGfP)B1)p?$_LawP#z`+4csV zUfx$fTgzms{S9aQdoi!u*^9y9_YLq32kr2|0hi)Rx~9kjIWX$+hx<(YgHV#5B5&!y zNV;tM9J)+Rfd4+Im?V7yB|?*i%fcQLtN?uo=!|9-doPLUTbxO8z`+aLTv63HKv@6< zi$9v-3fNi#h?|%&ZGwWq64ZVlKpT^qurMM_Rs;)8bigEu373VC2hgOxDG|bEw_xU> z-x*=cPr*V=!Q?Ru;+$-FR52?F&=fL;2TNi`(4@PXLFXiB5+4K5N|!J$U<+Hw)57G1 zV1lKXP~F!=YPe(b$W=XjZPS0P+R#>QiI$^Z->f!A%gt-i)#!TS@wDD{MsI$%8fhz^Sc|X5 z*9SM4AK35Nr?DTe>%Eut*cCl8T5atpPp{ory|FHDjOa)D_11yv@h;2W=IPO&&gk(; z{rJ^tYpi^Bt$($D{iBUey}eIweeY$!9rP~IJ0au`E)Omb-W#m=!^;!*Cd$XR{6}B5 zqsFeC4&)D)CM*mljV!^KpVga4(U$Nr}k$Sxf-eXsh?!ZmWX2?Y}3p$gJuhNjuPz40gzBlNazs zkU}z%O4RSr6h%Epoxh{_ugLctMW3VYZOZ$~;ib;+Aef-zID9*Vd=(2*Ve`uW0e99f A%>V!Z literal 0 HcmV?d00001 diff --git a/evaluator/config/__pycache__/settings.cpython-313.pyc b/evaluator/config/__pycache__/settings.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..deb131dd4809d9c7adb4dd6833b7ba6f79a80504 GIT binary patch literal 9539 zcmc(l&2t+^cEAU~kc0?Q;HUUaLi{Erl9KgdOSWhM1V{n|0SXQ3lhSwy3@OMU0M86% zi?(d89A=gWid}JKRhx@$cFCul@?H-x7?AL0 zt5m6-vYXStdHwqJ>+aVd(5u?oY7U+a;j=q!-5mEv%oLsq-^Ov@aD*eaIWA~xoA^8x?SI$+3T(ZldFp6`Ccxm)yiT=7P7nB^T(cBZo-!IlG5z<$5{dYUfDJlJkti zC?nS|)%S7*t#ZRsLu~M%=HWaN+c~bh#F9N$5V8~}wL_3wwXfPo4!=Y`VxulGlgX;0 zn#yLB2;}(%QN0a+E2dLYIyqveH9k?1h@`5i%&I~g3U5={%t~t2pGv5xB%KwLLNcpL znY%CTpm>8Hhu1iKa%S&JvYbjv39PX2R2na~SN{Y)`i5KLWEjRp;#TA>v%BCWo+5j8_vE<1WT)EWDQS8paP>xSR1>#_LR8 zPZ}6MOd3IdlZ7`k?q>Xm$y-P(<8@3=o5>xdo$-3o0eU)3UQfChZ(w@5Exd>EM#eoR zKT3KTZ({oUOzt54j5jm=1129NLyR9`^1~({A)}19Fn-MB$H^Gut&ESGe1e={yp8db zCa))NFy7AWJZ0gh8Sh~HjLFZEHyQ6_dfqa5JvqmC7n474;cqkE&GYXyVp9C21WqjJ?Gh~+WKE{J4uO}hK`{F=$HlN*eWF#Ss=ze#Q}KFYXY^7n|y_%X(p zO`af0#*Z^DnS6z;GCs!Yy>0Roxx@H4J0fiveC}n;mr_vmTZxak;rc@ogtsJ z7lY?pkiT!qpM(6Tmi&3he`d+Q4f#z={vF8w#FD=N`LD_6CACj7_E#nBUBaIi`N$D=F@-8e0fyiZ8O{$X8inyLuY295hol3HGM}Rj`p$rZvbGyo6kd#{9Gfc}`T653_O-YJqQl5&<26lS(E+8xdXchQool z5ROiH&xj>LU>=gXOSd=fY26ZgK}Fi<&^{s2xVh5!FC18 zMB#(^B7iALDWj&uv@(8ATuW1rsTEB3gBye1FcGGrk?G(Jba8x^dJ0;sg$1Ju=yvrs z<=3)__i0TyJP&S@~uHz@eLBp@ut!n7fqNC~UZ%PC;?M&edEDU-})QyF#Ra5jep$5(Ue*y-_;)Lqm+AN2>q)Lm4*mQ6|}nwf+U^jkGE zAt9Bd^+iqIg&-R_^+jD`F2zRJk)jkhI24$Q3qbtzNKq7AeMd^D0+1waF3K0$g$6Vi z#S2YC8(jImWC5fWi+UI1vjW;pYq5y9uHF{V$k#)jUbbi4bi_gOVG+mX*uH{5oU4!uoW^U9f;+P_4a}c+( zV<%}fDvSkYAePW-)F(@;uzOxLwzLo*-8fX@f%%0%%o|^f1!%rJy--lTG@*|A9jZANFJ`i1u3ozrO z5kG;zfJ#$qQg&U1xC<^wDjRKvCzJ^``X@vfeQJK%HFz;(l}@wQr85?d zvaWgp$4f;@uP9i=Lpvqd1FVbCmi)AyQ1Zvj@!UOHQwS__HmlO+!WLxQiV5gZsvz0q z{@dO1!2JsAP2d5v@L-=a4#*K`_G!Xt>&-h9@w5YHUDb<>-k?rzm7imKy#k?m)33XlTLV%KL^3`Jp%Uw z(C9@j27mWI1+#53O3C*jG5Up>CAUGI)|*!0{s#AA*t<#%B%9pxp1$AR|Mh*X_4sRB zfVU?HEWqm)OTVK9h27Xqv;v14SkX7M+jm;;JFRt|er-G8Ug@A6U1&#Li6>x}DWC~? zM|Ai8F8%AY)-w9q1{^JCKo1(wSz^GxJ*@~5Z2EyeM#pzYefp?R>+`*~Ieq2K>HSf2 z3f+^yVqar&U})mdhyQJ_R^a8whUNuZVa_FC&cU6W?N80mSpN186q*mQ7iM=qOw5Rk zbBWpnn4@@|Ml05Hun5yi8IA&}+<}odfHG|*4!7>@CHSnMh2ZVs4DPr?nltp5ZTbkd`Zwj8UY8Ivc%(F&Z|U#-pV9}KI;=Mh?>2=V$9Km<`dDbk zJ*PS6{<2LyiLLx?xu$ktdw%4^M*Gc<{u5HU0}eQ$H=O_nY{z%cg!D5%>Hr+dY?xn! zmp@~+7Y28kDNtCh?D7TBCBKWwzVR%-1CVx@Eh?TxeEGAO3IYnQY@6I4+>N{LLESyL z>-IkG-yQYpqh8JFJ=lr=FKw7Y^SzktZ^K1^w6#nd*l7;W0S6l}i4B<4M<+Gs0PzD>`O&KOGFIi!j=0asFF?$(1G;-aa}K<=F*D_inMPxJ${1rkqHcmv zev7~?8r9vSn)CHn=(FW4!mXlZtCGJeVD}MU?6bq8yFHrI^V*)l{p^8mx{P)d2hYB9 z=!1Qx^y%(C&Drr!t4= zt$pSU>Fy!TIrQ4*yj{+m^Jq?SD&@xt-cA_TO25QHk1_cik|!KonpVO!9vnQ``C^2Z ze+qmfe~XE~*2({d(mW(1cH4H$k;k}3g6CRat;&b=i;)!;wp= zEwgddEC>qzc$>gW3^}z7XHEs&`mZQ}!^b$w!LrK9nBb8~`5H#bm)u_Cz!PQrt$)7% zY*xFutT!e$!!H_JA2mO1(;KHYXC9Cpe(FV2+oRs?4ZX>?Ir|{7!~6DHyB;k+ozz=H zo8hg#9X_-NMH;qKdgDc-$i=wdKI^i#bxVl*kT*V6gu%G3LL%lzit zR^tvozt`2P9V6PxJ-ur~{em}}3v^p>d6m}t5Bk90hWK1=II z$YyBEvBQ%WUA>P_Y2KLLML^9WkiGVv#|~{WqPIskBU@8Dd~~m;|1r^~uIfF|gDnri zPuE^__i4vxwCfqYJBv;MRaxe{dF?$#?^5%=Q=zCfc1cU#(|f?5tn9`f*f9I_6a7fo zupx}V_q0<#GHrlq9D3r@alJKY^iy!JyYI1AyBOEI7xT3(A_{2!D|+`;tW4eEukQ5? zZ#QYP*Y&;|IMVLy@HbffL2c;+y+_X1FYlc^`&88~-_=iksPVmzz~LW0_q?g`tq=Ql z_&5JhQTxhnt7v`Y;3^M)hi^Yr+AGd}=Y;GYf1Kgx{PE`%wR`;OeC{;MjplQsEa%JT zd}VS%{+p1mY|h32);4GRt`cfO6T(Fr{=cZNDC;ZAzH6#|*VH;_vO$v_Gubhd9W~ie zll7ad-(-Cz>oeJ~$%aj4)QjyJG@=#8D0(O!A%fRGAB>DN<3hBuj+f;S)nF6LQF~UQ zRd{8VPA#)F4OiivJ+Loa)-YIz!Fyl zKVD_i@N*?-mhrBfUEIn4j^&t5PE!}VjH!ZOC{uDavoFw6=R3x#IVt}K6vd6Vf>HQ8 zhs|dDmh1f^cj#NL^;@p}zqrY7x$YM(_m=ak$z9h`-F0-w)wk<9rn`fChhajplm57pmu{XbOB+HHdmKKX&ezu(7suBPd$^C0VR?K*nkzvtb9yWQvY Z?(>_zFGD{GJ^1+de8+bVjyJZO{|ngmIFSGV literal 0 HcmV?d00001 diff --git a/evaluator/config/agents.py b/evaluator/config/agents.py new file mode 100644 index 0000000..1c15171 --- /dev/null +++ b/evaluator/config/agents.py @@ -0,0 +1,39 @@ +from __future__ import annotations +from dataclasses import dataclass, field +import yaml +from pathlib import Path +from evaluator.config.settings import settings + +@dataclass +class AgentConfig: + agent_id: str + enabled: bool = True + days_back: int = 1 + percentage: float = 1.0 + langfuse_agent_aliases: list[str] = field(default_factory=list) + gcs_prefix: str = "" + + @property + def aliases(self) -> set[str]: + return {self.agent_id, *self.langfuse_agent_aliases} + + +def load_agents(path: str | None = None) -> list[AgentConfig]: + p = settings.path(path or settings.agents_config_path) + data = yaml.safe_load(p.read_text()) or {} + agents = [] + for item in data.get("agents", []): + agent_id = item.get("agent_id", item.get("agentId")) + if not agent_id: + raise ValueError("Each agent entry must define agentId or agent_id") + cfg = AgentConfig( + agent_id=str(agent_id), + enabled=bool(item.get("enabled", True)), + days_back=int(item.get("days_back", item.get("daysBack", 1))), + percentage=float(item.get("percentage", 1.0)), + langfuse_agent_aliases=list(item.get("langfuse_agent_aliases", [])), + gcs_prefix=str(item.get("gcs_prefix", "")), + ) + if cfg.enabled: + agents.append(cfg) + return agents diff --git a/evaluator/config/settings.py b/evaluator/config/settings.py new file mode 100644 index 0000000..060e8d2 --- /dev/null +++ b/evaluator/config/settings.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import os +from pathlib import Path +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict +from dotenv import load_dotenv + +ROOT_DIR = Path(__file__).resolve().parents[2] +ENV_PATH = ROOT_DIR / ".env" +load_dotenv(ENV_PATH, override=True) + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file=str(ENV_PATH), extra="ignore", case_sensitive=False) + + app_env: str = Field(default="dev", validation_alias="APP_ENV") + + adb_user: str = Field(default="", validation_alias="ADB_USER") + adb_password: str = Field(default="", validation_alias="ADB_PASSWORD") + adb_dsn: str = Field(default="", validation_alias="ADB_DSN") + adb_wallet_location: str | None = Field(default=None, validation_alias="ADB_WALLET_LOCATION") + adb_wallet_password: str | None = Field(default=None, validation_alias="ADB_WALLET_PASSWORD") + adb_table_prefix: str = Field(default="AGENTFW", validation_alias="ADB_TABLE_PREFIX") + + enable_langfuse: bool = Field(default=False, validation_alias="ENABLE_LANGFUSE") + langfuse_public_key: str | None = Field(default=None, validation_alias="LANGFUSE_PUBLIC_KEY") + langfuse_secret_key: str | None = Field(default=None, validation_alias="LANGFUSE_SECRET_KEY") + langfuse_host: str = Field(default="http://localhost:3005", validation_alias="LANGFUSE_HOST") + publish_langfuse_scores: bool = Field(default=False, validation_alias="PUBLISH_LANGFUSE_SCORES") + langfuse_credentials_config_path: str = Field( + default="configs/langfuse/langfuse_credentials.yaml", + validation_alias="LANGFUSE_CREDENTIALS_CONFIG_PATH", + ) + + # llm_provider: str = Field(default="mock", validation_alias="LLM_PROVIDER") + # llm_profile: str = Field(default="judge", validation_alias="LLM_PROFILE") + # llm_profiles_path: str = Field(default="configs/llm_profiles/llm_profiles.yaml", validation_alias="LLM_PROFILES_PATH") + # oci_genai_endpoint: str | None = Field(default=None, validation_alias="OCI_GENAI_ENDPOINT") + # oci_genai_model_id: str | None = Field(default=None, validation_alias="OCI_GENAI_MODEL_ID") + # oci_genai_compartment_id: str | None = Field(default=None, validation_alias="OCI_GENAI_COMPARTMENT_ID") + # oci_genai_auth_type: str = Field(default="api_key", validation_alias="OCI_GENAI_AUTH_TYPE") + # oci_config_path: str | None = Field(default=None, validation_alias="OCI_CONFIG_PATH") + # oci_config_profile: str = Field(default="DEFAULT", validation_alias="OCI_CONFIG_PROFILE") + # llm_temperature: float = Field(default=0.0, validation_alias="LLM_TEMPERATURE") + # llm_max_tokens: int = Field(default=900, validation_alias="LLM_MAX_TOKENS") + + # LLM / OCI GenAI OpenAI-compatible, mesmo padrão do Agent Framework + llm_provider: str = Field(default="mock", validation_alias="LLM_PROVIDER") + llm_profile: str = Field(default="judge", validation_alias="LLM_PROFILE") + llm_profiles_path: str = Field(default="configs/llm_profiles/llm_profiles.yaml", validation_alias="LLM_PROFILES_PATH") + + oci_genai_base_url: str | None = Field(default=None, validation_alias="OCI_GENAI_BASE_URL") + oci_genai_endpoint: str | None = Field(default=None, validation_alias="OCI_GENAI_ENDPOINT") # compatibilidade + oci_genai_model: str = Field(default="openai.gpt-4.1", validation_alias="OCI_GENAI_MODEL") + oci_genai_model_id: str | None = Field(default=None, validation_alias="OCI_GENAI_MODEL_ID") # compatibilidade + oci_genai_api_key: str | None = Field(default=None, validation_alias="OCI_GENAI_API_KEY") + oci_genai_project_ocid: str | None = Field(default=None, validation_alias="OCI_GENAI_PROJECT_OCID") + oci_genai_endpoint_id: str | None = Field(default=None, validation_alias="OCI_GENAI_ENDPOINT_ID") + oci_auth_mode: str = Field(default="config_file", validation_alias="OCI_AUTH_MODE") + oci_config_file: str = Field(default="~/.oci/config", validation_alias="OCI_CONFIG_FILE") + oci_profile: str = Field(default="DEFAULT", validation_alias="OCI_PROFILE") + oci_compartment_id: str | None = Field(default=None, validation_alias="OCI_COMPARTMENT_ID") + oci_region: str = Field(default="sa-saopaulo-1", validation_alias="OCI_REGION") + + llm_temperature: float = Field(default=0.0, validation_alias="LLM_TEMPERATURE") + llm_max_tokens: int = Field(default=900, validation_alias="LLM_MAX_TOKENS") + llm_timeout_seconds: int = Field(default=120, validation_alias="LLM_TIMEOUT_SECONDS") + + agents_config_path: str = Field(default="configs/judge/agents.yaml", validation_alias="AGENTS_CONFIG_PATH") + trace_prompt_path: str = Field(default="configs/judge/trace_metrics.yaml", validation_alias="TRACE_PROMPT_PATH") + session_prompt_path: str = Field(default="configs/judge/session_metrics.yaml", validation_alias="SESSION_PROMPT_PATH") + output_dir: str = Field(default="output", validation_alias="OUTPUT_DIR") + batch_size: int = Field(default=50, validation_alias="BATCH_SIZE") + max_attempts: int = Field(default=3, validation_alias="MAX_ATTEMPTS") + context_turn_limit: int = Field(default=6, validation_alias="CONTEXT_TURN_LIMIT") + enable_gcs_upload: bool = Field(default=False, validation_alias="ENABLE_GCS_UPLOAD") + judge_gcs_bucket: str | None = Field(default=None, validation_alias="JUDGE_GCS_BUCKET") + storage_config_path: str = Field(default="configs/judge/storage.yaml", validation_alias="STORAGE_CONFIG_PATH") + google_application_credentials: str | None = Field(default=None, validation_alias="GOOGLE_APPLICATION_CREDENTIALS") + identity_config_path: str = "configs/identity.yaml" + + @property + def project_root(self) -> Path: + return ROOT_DIR + + def path(self, value: str | Path) -> Path: + p = Path(value) + return p if p.is_absolute() else ROOT_DIR / p + + @property + def ADB_USER(self): return self.adb_user + @property + def ADB_PASSWORD(self): return self.adb_password + @property + def ADB_DSN(self): return self.adb_dsn + @property + def ADB_WALLET_LOCATION(self): return self.adb_wallet_location + @property + def ADB_WALLET_PASSWORD(self): return self.adb_wallet_password + @property + def ADB_TABLE_PREFIX(self): return (self.adb_table_prefix or "AGENTFW").upper().rstrip("_") + + @property + def has_langfuse_credentials(self) -> bool: + return bool(self.langfuse_public_key and self.langfuse_secret_key) + + @property + def can_use_langfuse(self) -> bool: + return bool(self.enable_langfuse and self.has_langfuse_credentials) + + @property + def can_publish_langfuse_scores(self) -> bool: + return bool(self.publish_langfuse_scores and self.can_use_langfuse) + + @property + def OCI_GENAI_BASE_URL(self) -> str | None: + return self.oci_genai_base_url or self.oci_genai_endpoint + + @property + def OCI_GENAI_MODEL(self) -> str: + return self.oci_genai_model_id or self.oci_genai_model + + @property + def OCI_GENAI_API_KEY(self) -> str | None: + return self.oci_genai_api_key + + @property + def OCI_GENAI_PROJECT_OCID(self) -> str | None: + return self.oci_genai_project_ocid + + @property + def LLM_PROVIDER(self) -> str: + return self.llm_provider + + @property + def LLM_TEMPERATURE(self) -> float: + return self.llm_temperature + + @property + def LLM_MAX_TOKENS(self) -> int: + return self.llm_max_tokens + + @property + def LLM_TIMEOUT_SECONDS(self) -> int: + return self.llm_timeout_seconds + + @property + def LLM_PROFILES_PATH(self) -> str: + return self.llm_profiles_path + +settings = Settings() +if settings.ADB_WALLET_LOCATION: + os.environ["TNS_ADMIN"] = settings.ADB_WALLET_LOCATION diff --git a/evaluator/core/__pycache__/models.cpython-313.pyc b/evaluator/core/__pycache__/models.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..605f654d074e495e03b977a2eaed1f5a67e72b3d GIT binary patch literal 3518 zcma)8&2JmW72hR?yUXRLM2q^WL`jXT(v5A@j?&sr)5;Ppk+Nw;tMrgCSZv4#Q2?7Q+fKi}56uJcvd&#Nq%~CSyuuT@=kMCo4 zX6McCy*Fo(h%UkNx5A$vwG>JE7b@ou==jKI{{;L>G9*LZlyWkY=TP=-`f@(z6S9BP zpCgP2nQR7f3R7|_Q-vPb)N(-0HDK3(hdC0Qunrpe0 zn(ct9l`NOKl|2eF$=LP1w2uexS`K|sE72+s+^f)P34Hz)d%x&?r-yRy@9Z5AzLE+O zlMRXa44L^2A0vjJ1q{LzBfwNcVOm*DYdpAJw+&dk?wqRd7FV}F&gZlFbsk)O_+aaP zW+$`8)vcB7o$SheuH0M6-p{OkBjwXUu9>E7?NQU@x@qp!O7$wrVbgq4x2io)%rti^ z%yFv~o7y$gWMNo?MG&G0Ft6I0*8oOSs zF?t;qqgChH<9&u?5`f?CBmw>+b)w^q^-g@ck@cj=*ug!9E-`R;zZ@s2ZEut*y+O*i$~Ovl0B9QG^7-C4^~& zpCE|%;EQYqVGIGI;q77^rO9)^u@1g^zVVTnM$X%PWUO%yLU6F|y@#(1?7o)~@>w1{ zgS%gZd@BH9mcSzj%L=0Vw2|0q4u(_Fr0Q?3^og%l_OAHI=!tVDOMdet>| zts+oh|GQ|ZQglCs@$b$-27Jb~J?dJJGnNR_@aS@2Vqca4y%yn9n2O&+a6XnseORbd z=s>`acks0@rH&ps$aW&}gZrJC`9`j(x5@lT>Yd}AR_bOE=+8VHg(46|9BC-*b!4<0K8WNo#7kv3fZBiY$BV!*3X1-$jf4D|p^! zUx7fw^cjAG7=c^ZE0luLOZ7nybPw9r_`fN0%LjxBzyXVbC1?S?!60B)p*tvEcNnl^ z*HC(eDhy={Zde!e(1mtRkC4#LOHdf^PJGnxib_9SuE8g=Un6`7koK|PqPzmY1A-nr zUa@y6W3;sDRJ_CYf%g<|PTXIyU9L0g)T(v7i}5k&P8Tg~i_u~QtJoX5h!f zNpkw=$=4sXl1tv9FP+R^el~wBx8`rWB$G!s+vG++HO@zMk|~cGDUTYs&Hy$1V1P$H z!`0uWM&AX32x^dCFH@OPCUEnF3&P&#a4#(NpP(qYg7q6PXa4Er_fNT(;%o;7czLG| z%{!58V$iV6`W?QFfGaw93xgURnr@TnPWozN{bl6xD{|G_H;&3fd4g;1Zr!ajxaKn4 zI>QSTClqez46}r5kM~QK4HsF4b=K<~3Y3S6U1l0uup4@JR&jq5Hw)1$#E!*bitie{ z>~ZY_?|S_qE5HQERi_H@tzVYq*V4jkY38*w_77?CwUq1v=1x}vp&Jg!6U{`k z)RjQ!Mu~jqD2<*w-LNjde>B;Z;IBKO$g|D;t^|MGxFXM?9sas;pM32o{T~VbL?r$T D^Iaxu literal 0 HcmV?d00001 diff --git a/evaluator/core/models.py b/evaluator/core/models.py new file mode 100644 index 0000000..3191415 --- /dev/null +++ b/evaluator/core/models.py @@ -0,0 +1,56 @@ +from __future__ import annotations +from datetime import datetime +from enum import Enum +from pydantic import BaseModel, Field +from typing import Any + +class RunStatus(str, Enum): + RUNNING = "RUNNING" + COMPLETED = "COMPLETED" + PARTIAL = "PARTIAL" + FAILED = "FAILED" + +class ItemStatus(str, Enum): + PENDING = "PENDING" + PROCESSING = "PROCESSING" + COMPLETED = "COMPLETED" + FAILED = "FAILED" + SKIPPED = "SKIPPED" + +class ConversationMessage(BaseModel): + role: str + content: str + created_at: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + +class ConversationRecord(BaseModel): + trace_id: str | None = None + session_id: str + message_id: str | None = None + agent_id: str | None = None + channel: str | None = None + input_text: str = "" + output_text: str = "" + messages: list[ConversationMessage] = Field(default_factory=list) + metadata: dict[str, Any] = Field(default_factory=dict) + raw: dict[str, Any] = Field(default_factory=dict) + +class TraceJudgeResult(BaseModel): + judgeScore: float | None = Field(default=None, ge=0.0, le=1.0) + accuracyScore: float | None = Field(default=None, ge=0.0, le=1.0) + alucinationScore: float | None = Field(default=None, ge=0.0, le=1.0) + rationale: str = "" + judge_name: str = "trace_metrics" + judge_type: str = "trace" + +class SessionJudgeResult(BaseModel): + inferredCsiScore: float = Field(ge=0.0, le=1.0) + resolution: int = Field(ge=0, le=1) + conversationPrecision: int = Field(ge=0, le=1) + rationale: str = "" + judge_name: str = "session_metrics" + judge_type: str = "session" + +class CombinedJudgeResult(BaseModel): + trace: TraceJudgeResult + session: SessionJudgeResult | None = None diff --git a/evaluator/engine.py b/evaluator/engine.py new file mode 100644 index 0000000..46e2c15 --- /dev/null +++ b/evaluator/engine.py @@ -0,0 +1,192 @@ +from __future__ import annotations +import asyncio, inspect, json, math, random +from datetime import date, datetime +from typing import Any, Awaitable, Callable +from evaluator.collectors.base import ConversationCollector +from evaluator.collectors.langfuse import LangfuseCollector +from evaluator.collectors.agent_framework import AgentFrameworkCollector +from evaluator.collectors.mock import MockCollector +from evaluator.config.agents import AgentConfig +from evaluator.config.settings import settings +from evaluator.core.models import ConversationRecord, RunStatus +from evaluator.judges.llm_judge import TIMStyleLLMJudge +from evaluator.output.legacy_exporter import export_legacy_txt_gz +from evaluator.persistence.repository import EvaluationRepository +from evaluator.publishers.langfuse_scores import LangfuseScorePublisher +from evaluator.publishers.langfuse_job_audit import LangfuseJobAuditPublisher +from evaluator.publishers.gcs_export import upload_evaluation_file + +ProgressCallback = Callable[[dict[str, Any]], Awaitable[None] | None] + + +class EvaluationEngine: + def __init__(self, repository: EvaluationRepository | None = None, progress_callback: ProgressCallback | None = None): + self.repository = repository or EvaluationRepository(auto_init_schema=False) + self.progress_callback = progress_callback + self.judge = TIMStyleLLMJudge() + self.langfuse_publisher = LangfuseScorePublisher() + self.job_audit = LangfuseJobAuditPublisher() + + async def _emit(self, progress_run_id: str, stage: str, message: str = "", **details): + details.pop("run_id", None) + await self.repository.arecord_progress(progress_run_id, stage, message, details) + event = {"run_id": progress_run_id, "stage": stage, "message": message, "details": details} + if self.progress_callback: + r = self.progress_callback(event) + if inspect.isawaitable(r): + await r + + def collector_for(self, source: str, agent_id: str | None = None) -> ConversationCollector: + if source == 'langfuse': return LangfuseCollector(agent_id=agent_id) + if source == 'agent_framework': return AgentFrameworkCollector() + if source == 'mock': return MockCollector() + raise ValueError('source must be langfuse, agent_framework or mock') + + async def _audit(self, progress_run_id: str, audit_trace_id: str, agent_id: str, status: str, period_start: datetime, period_end: datetime, *, required: bool = True, **details) -> bool: + try: + result = await self.job_audit.publish( + audit_trace_id, agent_id, status, period_start=period_start, period_end=period_end, details=details + ) + if not result or not result.get("ok"): + raise RuntimeError(f"Langfuse audit did not confirm persistence: {result!r}") + await self._emit(progress_run_id, 'LANGFUSE_JOB_AUDIT', f'judge-job-{agent_id} status={status}', audit_status=status, audit_result=result) + return True + except Exception as exc: + await self._emit(progress_run_id, 'LANGFUSE_JOB_AUDIT_FAILED', f'Langfuse job audit failed: {exc}', audit_status=status, error=str(exc)) + if required: + raise + return False + + async def run_agent(self, agent: AgentConfig, period_start: datetime, period_end: datetime, source: str = 'langfuse', limit: int | None = None, reference_date: date | None = None) -> dict: + run_id = await self.repository.acreate_run(period_start, period_end, source, agent.agent_id) + reference_date = reference_date or period_start.date() + audit_trace_id = self.job_audit.new_trace_id() + try: + # Emit RUN_CREATED immediately after persistence so CLI/monitor can expose + # the RUN_ID before any external Langfuse audit/network call is performed. + await self._emit(run_id, 'RUN_CREATED', f'Agent run created: {agent.agent_id}', agent_id=agent.agent_id, source=source, reference_date=reference_date.isoformat()) + await self._audit(run_id, audit_trace_id, agent.agent_id, 'created', period_start, period_end, run_id=run_id, source=source) + collector = self.collector_for(source, agent.agent_id) + await self._emit(run_id, 'COLLECTING', 'Collecting sessions and conversations') + seed = f'{agent.agent_id}:{reference_date.isoformat()}' + if isinstance(collector, LangfuseCollector): + records, sampled_sessions, collection_stats = await collector.collect_sampled( + period_start, period_end, agent_aliases=agent.aliases, percentage=agent.percentage, seed=seed, limit=limit + ) + await self._emit( + run_id, 'COLLECTED', f'Collected {len(records)} traces from {len(sampled_sessions)} sampled sessions', + **collection_stats + ) + else: + records = await collector.collect(period_start, period_end, agent_aliases=agent.aliases, limit=limit) + records, sampled_sessions = self._sample_sessions(records, agent.percentage, agent.agent_id, reference_date) + collection_stats = {} + await self._emit(run_id, 'COLLECTED', f'Collected {len(records)} traces before session sampling') + await self._emit( + run_id, + 'SAMPLED', + f'Kept {len(sampled_sessions)} sessions / {len(records)} traces', + percentage=agent.percentage, + seed=seed, + sampled_sessions=sampled_sessions, + ) + inserted = await self.repository.ainsert_items(run_id, records) + await self._emit(run_id, 'ITEMS_INSERTED', f'Inserted {inserted} items') + summary = await self._process(run_id) + output_path = export_legacy_txt_gz(self.repository, run_id, agent.agent_id) + await self._emit(run_id, 'EXPORTED', f'Exported {output_path}', output_file=str(output_path)) + if not settings.enable_gcs_upload: + raise RuntimeError("TIM specification requires GCS upload for run-agents; set ENABLE_GCS_UPLOAD=true") + uploaded_to = await asyncio.to_thread(upload_evaluation_file, output_path, agent.gcs_prefix) + await self._emit(run_id, 'GCS_UPLOADED', f'Uploaded {uploaded_to}', uploaded_to=uploaded_to) + output_path.unlink() + await self._emit(run_id, 'LOCAL_FILE_DELETED', 'Local export deleted after successful GCS upload') + result = {**summary, 'agent_id': agent.agent_id, 'output_file': str(output_path), 'uploaded_to': uploaded_to} + # _audit() receives status as an explicit positional argument. The run + # summary also contains a "status" key (e.g. COMPLETED), so forwarding + # **result unchanged would pass status twice and raise: + # TypeError: _audit() got multiple values for argument 'status' + audit_details = dict(result) + # Never forward names that are explicit _audit() parameters. + for reserved_key in ('progress_run_id', 'audit_trace_id', 'agent_id', 'status', 'period_start', 'period_end'): + audit_details.pop(reserved_key, None) + await self._audit(run_id, audit_trace_id, agent.agent_id, 'completed', period_start, period_end, **audit_details) + return result + except Exception as exc: + await self.repository.amark_run_status(run_id, RunStatus.FAILED, str(exc)) + await self._emit(run_id, 'FAILED', f'Agent run failed: {exc}', error=str(exc)) + await self._audit(run_id, audit_trace_id, agent.agent_id, 'failed', period_start, period_end, required=False, run_id=run_id, error=str(exc)) + return {'status': 'FAILED', 'run_id': run_id, 'agent_id': agent.agent_id, 'error': str(exc)} + + async def run(self, period_start: datetime, period_end: datetime, source: str = 'langfuse', limit: int | None = None) -> dict: + run_id = await self.repository.acreate_run(period_start, period_end, source, None) + try: + collector = self.collector_for(source) + await self._emit(run_id, 'COLLECTING', 'Collecting conversations') + records = await collector.collect(period_start, period_end, limit=limit) + await self._emit(run_id, 'COLLECTED', f'Collected {len(records)} records') + await self.repository.ainsert_items(run_id, records) + return await self._process(run_id) + except Exception as exc: + await self.repository.amark_run_status(run_id, RunStatus.PARTIAL, str(exc)) + await self._emit(run_id, 'PARTIAL', f'Run failed: {exc}', error=str(exc)) + return {'status': 'PARTIAL', 'run_id': run_id, 'error': str(exc)} + + async def _process(self, run_id: str) -> dict: + processed_records: list[ConversationRecord] = [] + while True: + items = await self.repository.afetch_next_items(run_id, settings.batch_size) + if not items: break + await self._emit(run_id, 'BATCH_STARTED', f'Processing {len(items)} items') + for item in items: + item_id = item['item_id'] + await self.repository.amark_item_processing(item_id) + try: + raw = item['raw_json'] + if hasattr(raw, 'read'): raw = raw.read() + record = ConversationRecord.model_validate(json.loads(raw)) + result = await self.judge.judge_trace(record) + await self.repository.asave_trace_result(run_id, item_id, record, result) + await self.langfuse_publisher.publish_trace_score(record, result) + await self.repository.amark_item_completed(run_id, item_id) + processed_records.append(record) + await self._emit( + run_id, + "ITEM_COMPLETED", + f"Item completed {item_id}", + trace_id=record.trace_id, + session_id=record.session_id, + judgeScore=result.judgeScore, + accuracyScore=result.accuracyScore, + alucinationScore=result.alucinationScore, + rationale=result.rationale, + ) + except Exception as exc: + await self.repository.amark_item_failed(run_id, item_id, str(exc)) + await self._emit(run_id, 'ITEM_FAILED', f'Item failed {item_id}', error=str(exc)) + if processed_records: + sessions = await self.judge.judge_sessions(processed_records) + for sid, result in sessions.items(): + agent_id = next((r.agent_id for r in processed_records if r.session_id == sid), None) + await self.repository.asave_session_result(run_id, sid, agent_id, result) + await self._emit(run_id, 'SESSION_JUDGE_COMPLETED', f'Evaluated {len(sessions)} sessions') + await self.repository.amark_run_status(run_id, RunStatus.COMPLETED) + summary = await self.repository.asummarize_run(run_id) + await self._emit(run_id, 'COMPLETED', 'Run completed', **summary) + return {'status': 'COMPLETED', **summary} + + def _sample_sessions(self, records: list[ConversationRecord], percentage: float, agent_id: str, reference_date: date) -> tuple[list[ConversationRecord], list[str]]: + """Deterministic session sampling; all traces from selected sessions are retained.""" + if not records: + return [], [] + session_ids = sorted({r.session_id for r in records}) + if percentage >= 1: + selected = session_ids + elif percentage <= 0: + selected = [] + else: + count = min(len(session_ids), max(1, math.ceil(len(session_ids) * percentage))) + rng = random.Random(f"{agent_id}:{reference_date.isoformat()}") + selected = sorted(rng.sample(session_ids, count)) + selected_set = set(selected) + return [r for r in records if r.session_id in selected_set], selected diff --git a/evaluator/identity/__pycache__/resolver.cpython-313.pyc b/evaluator/identity/__pycache__/resolver.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7c642c17117313658f349a389b8ae06bbc6fa94a GIT binary patch literal 2312 zcmZ`)O>7fK6rTO{UlJQ^f^FKwgt)K)LXr|SQ30t%mB3L#>#ZeFTCFzr+AMX}?(7)V z9NL^J1FZ@zDT!Js;RL;L?}cNJL{W(u1*sD1fm>**BK6ca>s^Ng`lNmH_vX#pdEfhH zv$HdZKz=?iuL&kJ|kek=?fELM#7TQ7bj#a ztA15VqDvz7#ZezpBam=(*0q4=RrEp33UFEq zl~9h)`aJusHYa#qEq?MSLLsot@%>2Qrzkh5h*YwxWs}gLX`7Z!bSrOAv0&y26-owy zb+Momor+cXKPu#Fm~}C+SvV;W+GAYS%Qc-;@G-MsSi~gtW7x1O<;w;htk!8jD;P#q z17~;_>;y_cptBeOJ0# zx>2}3drf%mL%UKNe$)}3%WQ%JlxNBl_o}$fBQP_MK0%mM5$0763#zR8RGy{v5{E@d zvEZawQbjD6Bqcy2XWhWFZrsmygOhDplwrcjVh`kK9!B9vYk7m z4^WVO86)m`mR-&cQ8|?#Z=Z3J+n)6y@Kk09a|FCPRpE_WZMJj!n2l6nsEs4=(8djT z=z7)qwI0N69{k>}5>lk8D^iUVkB&h-_zf#xDVSD?`aN_|@iU`tD*?mxaMZwUX0X2u#WATO951EGeO<=t{dE5HlTK@R%g*zA4(qk*>u?GhqrpF$p zGppSbYuz9HhQTb;d@#=9ESyE2SwqdYkt#ZfOX@(u$ zKq~TdJK5*EJt~3ylFGje@h~!(1iKVE$2(Nv&g*df(`|U>B#%_iL6x09a1f51%(&$8 zdJw)Yx;8bvw^7_)W@YmBX^U=!Rz9Rn(u^{`oARj)fZ6)H6#O29Lq zv1;V0#JJF)vKte-YDSVEEpYl=W~A{~u<5QaaND;)0Y7#{8p0omH$_53a^XZb;2&d^z}gbl`C;0gSnC1Q_$J z!}kYP(5E_}Q;f)}QL>oZ|%!-KLg!r~<(W${Hg}bt+RjPLTAsA$Jt?-TkR$vqC z9f0QFF80Dq!`&W9!GrlJm9ar;*uo>s#>Z;d(q|2*;h?6?RtmK;>vw3{=QX|Tu7owM zXkwd`P0O$P(z^T9!-OnjF@>%Cj^@_HDcJB_6u^tcM^eP%$O-wo;HKr*XSQ! zltx~3M5JiTkS#;NY!NeFed#v>EU;n?Vurvlt5b#vaVAqsf6Fo#ezAKm7yS!?ay dict[str, Any]: + out = {} + + for key, spec in self.keys.items(): + value = None + for source in spec.get("sources", []): + value = _deep_get(payload, source) + if value not in (None, ""): + break + out[key] = str(value) if value not in (None, "") else None + + out["metadata"] = { + "identity_version": self.identity.get("version"), + "identity_source": str(self.path), + } + + return out \ No newline at end of file diff --git a/evaluator/judges/__pycache__/llm_judge.cpython-313.pyc b/evaluator/judges/__pycache__/llm_judge.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..214f163df624269f766d77668deec61d99ade281 GIT binary patch literal 8392 zcmb7JdrVtbnm^aqkLzoL0Yk6}j~hY&li)xeZJjHPhK$ z?d*5X^|eh3lgqtZ* za~4fAm=Q?caMpkg^q0jf{IX|_(DNv-ar< zTrpjVD{0ws)-hd$s~~R|xdwE>h^xg4u~M{>`~?QrxKJw+`35A~P8+mYVdtX39&1gJ!QN{`-_RFC-4y#qO zSPIG0sqmugmlf!z)`+sA0JpL_k3qGbJ9mCG8j)j3wK{}lDJch|(dA%>R*_+&aVZ>3 z;P`SP8Dg}F($oSeixBhhw~)Dxn$di*(O+0lu>vx!D1c5E`nRD&Xufb12w9Lci;#fk z%{slG({rU?*eoVcPD2m5)J}4up^uF*0j9H19ftAb$IpxLSM26Pd9nIz^3IR?IK{wYTfV@4+sa7NXLkg2}o)7S(t`iN{nE**QhEj$aDN338080n4gc zktG~jQW;S-E#i180cT)L%9oNVv#gN*f}l3Z7o})QO2+Z=3*-Qmn>;G3h8mrSp?uQ2|C| z`r)=pCWPjCyWpA{p@(SjmycXvFR~3Nz&g?CLRmk#m~p02G{Ss}GFO^UB@#{0Qk1>& zmXpSqac)E#vkiI7K6=0^!|%EXgy~6g5uSt0VOSDbx)V*dAQZ3OYYSjVSk_*bq!OKE zFQV%v0i8cYj#NO4%#$n#n_>4Bj4m00u6|4z7p1Y8_aV zWAdd09#Rhmmm&&C(q->JG#-+o%8<9%c#!mifzX%e3d*>~J~*=>y)*Il#Oj%~6Km4V z$c@O_{3q_-56=8SA&b1);bB(HDMiNp!V1SrFh;Up@EBEY8IT2v3P^rD5>xp|ERjl5 z*;H-uR8lW^OsW}c@WdxmI2MdXmLo~kx-4A^hL$9(U@{q8N2UeG4Oam^ZAF~E3`|6Y zQbZtg1#LO%-gy1m>v>1pTI-d`&n=G6ZFQ?%Yp?vw`$c`rnr+RVsqaotZozH1eC=}9 zHTLr}IoDXuF?P-PMRonEIq&jhU7k#R*Scq8B7bTkduk$c@|lcdGHu+lIo{;ov}J6K z8Drx^7z~9-qJIVXV;%x{!I8g+Fe4&R9}K7ttH={sq(Evl4z>n zvYf<`kfJ&iO-<<)9@q?^R$0J6Oi&J!G1Y3Vm!_D42`Rax+RLj{r_QIGq;hv9xloE6 zUBErSa@P&wg26~Ek_-k__oHXv)tg#Lr$Ic4No8T)uN8 z+d1;#Os;V@ZQSJC+s&SLJM&!=*{+Gq@kD@a;RDTS@E3PouO<3Y@2Q`TfcD|Q?!&?OtvQ(X;b8vO94z~A z;6$rFfBhGv-m0%tY z%~hb6Ly;IYv^0w9vYG;s71MSJfZ7Tc73gZP(^PZ|c}!GQt43nOsXSP41)EM~7eV^T z34t}oV(C>Nk&=?i$KrTdibhuCutLte2#0k+un;0ZBZq^U)KmKQZ_4UkbRED-c?*&& z=r@k$RVC**oHlOPxmVkM@&^tj=qAw~=fnB=41p7SYQ7djt+ zZf!msf*G0iosmhuB4kFy3>BFeJIqoux_n+>am36bBeDT=nG&F8R)7WD!cgpO9yXWr z0e(=k!05)U#=lgiHi&U$d?qFfnn(&#GAV;gARHCqP>Df&FLy6RVo6~kj)h2!ik&bm z&iIaNz%CGz1uj<7>q{MiqvU!2_^ki=?#U_NY(GB?9F#;{NlJnQmJN;oli2-#iVKPR z{~DGBFgbuH)F)sdDJ%iA4llpbkN}HF z4lYQMDBN;s0hS-6`%G*us!^A@OSTIrZs##Nec!CtH3lXd&gYu=2 zYNsp;4jsWKIJh7|9DpM@qA(#+2oeF-kWVwe4ir#^Iv%e^l1ti5)Q)cgjs{1M5S{{t z%D+QGEX^-!T|b?CYcgMZBwKrAZ6R0Nov-Xpvpa@L>nyY7tOre+b#||Za?TSQV;Sd} zjQz|56LNUJ2EeFtKSXvyCbX|!+Ck<{29ic&`rJ+>YB~JQ(%Vb<=HuDs;~Vx|^I+aJ zm~joJr*;f%1$3!Dn6K~6*7t6Ba`gju!kPM!PpU?~-gbg<$7t=DmcBb(`Tp;GWX<&X zGfiTqUd&gCA6JS0v%?aS|J$i2ga5Dc8Z6~%%g6w8gXuT^qY*MUYX;EI4UqW-KhnqE zH6I>1#oqNW!^g~byG$hCQ!~OD?im@E*sH^qTI~I7W_{?j2%CzSRhn_u45< z2c?De+nW2uy)xEB_Dh*{|CHj1M~o(+shw^ z79+Im;e|tsX})L!`pN7b`5+d+_=&JS@`>j88of%i^Z{gv+(|xW0D!I2>r1~Spaw8S z<=~!zeMp=Nup#I*PaGopK|qH%UdBhEkyz_}#y0RA{cfGM^b5ATy<}DFLj!@G$I>{U zFLAgY`NS$6+#k!KdLIro- z;`XJ8aPa)(lsJ3;C;q9?nSS2G;C84?k(dV015S&>&y1fJ#%6?o7`Xp8{wbhYIY6|B z`0R{0J4{-D0^l6#?WD)-@YwL|a6dl**1cVhDJd)mB_$MzsFs4cg&Tmm>e>%q0bTom z06#?%V&CJZ3B6`B24m8)jJ;(Q$;$~@HBs2p`t_48wDSPTQ4T{SN7&%MAjvREMj-K2 z;u9oKyjsn%siD68C2*gM9x^^mTH8oMHcT~zQ_Bg3xZOkGpq^%Qv0L!XML@LH*G36te>@T%M`PGBp|GgsNJ-%9&-H8B_~Z zb+o_vhYeR7(i3mYT$@=x@(XAk%%Gti#5~RHxGQRmY4c7aI^2o%D7+)+S{&MuFQ#%oM&{C8!OajTwUv9wD)R! zR_6f`sa7o_)q}v|HTa9}RbeprA)-b;CwhjNge#VUnIxVQ89(vbANfQBgzC)S4AqGc zcZUF<1t7nuLP}k!(y>6wKxmEwnmv?@zq&-P=K`F*jz?8R%sNkYoPGhTP8H(={9YPUCyzN6{5xK|GSfOtb9QEls-~kDD zfP^!pn*gC>kd!_{A)*R$K_jXL%9oAY0kH&aL&k>p6^g(f18#<*%HZ-~5C*>rsVDZq z!a_tz*(1IhhkgnVr4zcdx1s`YLrvbFukKc*UilILgr z!#;7;KXw0y_kS=Wj13FoxF|xlpT{J;kB^h&c}P@CVQ94~5?cT;m&2nVfiVa1;vldup!=Y$P)=Mj^&bJP7*0 z6pVybvy@24v9M~2fSagbzz&3o6F~hIe2(OZhA*iqOhCyiB^0omF{ILop?UBZ;J@+$Km9RZ9eMsQaQNniEr52=r*PRMeKooM;4?C{qOm zxBcYEzn#iGIhnS6TJ1^?Z`qwcJ@?kRto=~l-kG&`u2=uce)Iv0YTRE`R;AgkL!CF9 zZZzF&$=aLJ(_8MQcieBgv#q}WFy&f(Ik)el)JGRH?&qMXp*i2soo(pOxV@nFj#E`= zzhVb1SHbL9j0cSAi5(sR1MY233s$eU=Ulz{n%?y2wzGBZWX{=<9^0~45iPlTF4OhY zrhTATT=#6+PZdg;M(>94eeOLj(|7)(S8_ey*|hshgl)GwZ*^`Ad{~|98r`((3^R=< zH)h|Tdv7i?^jv1{`P}IjHtq9zr?Z+0+&Fp%=R6~u_R;O;o^11(jZ=`MM}Iu?par$| z?g*&LndTojk@G}ByN$TVyP10jnen6MdyS)2Q1GytTu=!i_K=5x#|~PzQdNAQ6kZd2 z5NrVRj2`q~0v&lzAa)WSC7q0#7?1{h5t5bWWD4Blc@l0bbOz@>i3|CcpaZpx2%gXm zS_saF6Qr|?2=)AE*&XrIw~LoF-c5w-Ja9WEh?+!AmmXVR^4L@eAro5>LdbkDxEv3s zq9kt*246`@(L#$O7^JRwG!m0z@n8@WL{se$2T4XkVBRgo!+67bR@5h)rlT+hFn7~F62MiZ?R@-lbFOq7qDp|vgp+xzcrJ&uDl(~ zUf66os$hDo7Q&Zci&|Tvf~N$p_QH*c9iu%kxJvC>>_qXB7hC{Vf+iYVQ8S%x>+Y>q zxdH8jhw$G5F<=H#$WcPrfMGsEt)C*>r^xylI{X>({u;fIMKAms4St4tzA{^w_TSZ7 SnWo>nUSb&MZxKo9#{WN3#UJYc literal 0 HcmV?d00001 diff --git a/evaluator/judges/llm_judge.py b/evaluator/judges/llm_judge.py new file mode 100644 index 0000000..2cb37f5 --- /dev/null +++ b/evaluator/judges/llm_judge.py @@ -0,0 +1,126 @@ +from __future__ import annotations +import json +import re +from collections import defaultdict +from evaluator.config.settings import settings +from evaluator.core.models import ConversationRecord, TraceJudgeResult, SessionJudgeResult +from evaluator.llm.client import LLMClient, create_llm_client +from evaluator.prompts.loader import load_prompt + + +def _json_from_text(text: str) -> dict: + try: + return json.loads(text) + except Exception: + m = re.search(r"\{.*\}", text, flags=re.S) + if not m: + raise + return json.loads(m.group(0)) + + +def _history(record: ConversationRecord, turn_limit: int | None = None, max_chars: int = 6000) -> str: + messages = record.messages or [] + if messages: + if turn_limit and turn_limit > 0: + # A turn is represented by one conversational message in the normalized record. + messages = messages[-turn_limit:] + text = "\n".join(f"{m.role}: {m.content}" for m in messages) + else: + text = f"user: {record.input_text}\nagent: {record.output_text}" + return text[-max_chars:] + + +class TIMStyleLLMJudge: + def __init__(self, llm: LLMClient | None = None): + self.llm = llm or create_llm_client() + self.trace_prompt = load_prompt(settings.trace_prompt_path, 'trace_metrics') + self.session_prompt = load_prompt(settings.session_prompt_path, 'session_metrics') + + @staticmethod + def _validated_trace_data(raw: str) -> dict: + data = _json_from_text(raw) + normalized = dict(data) + normalized.setdefault("judgeScore", normalized.get("judge_score")) + normalized.setdefault("accuracyScore", normalized.get("accuracy_score")) + normalized.setdefault("alucinationScore", normalized.get("alucination_score")) + normalized.setdefault("rationale", normalized.get("reasoning", "")) + # Pydantic validates the 0..1 contract here. Invalid metric output is treated + # like an invalid judge payload and receives the single specification retry. + TraceJudgeResult(**normalized) + return normalized + + async def _trace_json_with_retry(self, prompt: str) -> tuple[dict | None, str | None]: + """One normal attempt + one retry-hint for invalid JSON/metric contract.""" + raw = await self.llm.complete(prompt) + try: + return self._validated_trace_data(raw), None + except Exception as first_exc: + retry_prompt = f"""{prompt} + +RETRY-HINT: +A resposta anterior não pôde ser interpretada como JSON válido ou violou o contrato das métricas. +Retorne SOMENTE um objeto JSON válido, sem markdown, comentários ou texto adicional, +com judgeScore, accuracyScore e alucinationScore entre 0.0 e 1.0 e a chave rationale. +""" + retry_raw = await self.llm.complete(retry_prompt) + try: + return self._validated_trace_data(retry_raw), None + except Exception as second_exc: + return None, f"parse_failed_after_retry: {second_exc or first_exc}" + + async def judge_trace(self, record: ConversationRecord) -> TraceJudgeResult: + prompt = f"""{self.trace_prompt} + +HISTÓRICO: +{_history(record, turn_limit=settings.context_turn_limit)} + +MENSAGEM DO USUÁRIO: +{record.input_text} + +RESPOSTA DO AGENTE: +{record.output_text} + +METADATA: +{json.dumps(record.metadata, ensure_ascii=False, default=str)} +""" + data, parse_error = await self._trace_json_with_retry(prompt) + if data is None: + # Specification: after the second parse failure metrics become None; + # legacy exporter serializes those values as "0.00". + return TraceJudgeResult( + judgeScore=None, + accuracyScore=None, + alucinationScore=None, + rationale=parse_error or "parse_failed_after_retry", + ) + + data.setdefault("judge_name", "trace_metrics") + data.setdefault("judge_type", "trace") + data.setdefault("judgeScore", data.get("judge_score")) + data.setdefault("accuracyScore", data.get("accuracy_score")) + data.setdefault("alucinationScore", data.get("alucination_score")) + data.setdefault("rationale", data.get("reasoning", "")) + return TraceJudgeResult(**data) + + async def judge_sessions(self, records: list[ConversationRecord]) -> dict[str, SessionJudgeResult]: + grouped: dict[str, list[ConversationRecord]] = defaultdict(list) + for r in records: + grouped[r.session_id].append(r) + out = {} + for session_id, items in grouped.items(): + transcript = "\n".join(_history(r, max_chars=3000) for r in items)[-9000:] + prompt = f"""{self.session_prompt} + +TRANSCRIÇÃO DA SESSÃO: +{transcript} +""" + raw = await self.llm.complete(prompt) + data = _json_from_text(raw) + data.setdefault("judge_name", "session_metrics") + data.setdefault("judge_type", "session") + data.setdefault("inferredCsiScore", data.get("inferred_csi_score", 0)) + data.setdefault("resolution", data.get("resolution", 0)) + data.setdefault("conversationPrecision", data.get("conversation_precision", 0)) + data.setdefault("rationale", data.get("reasoning", "")) + out[session_id] = SessionJudgeResult(**data) + return out diff --git a/evaluator/llm/__pycache__/client.cpython-313.pyc b/evaluator/llm/__pycache__/client.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c40fe8e761cef72effe2fd413687fa6b5ce07a4c GIT binary patch literal 12040 zcmbVSeQX=&dB5ZLmqeN;iTbc4TC!x*AGBpla_snHswG*pOiAI1iBj1m81f|1rjE)T zi13YXP0(4!|qJP}=1{`};vS3Yz1=t^U?at`_ z+Vi}}BPohb(!CPjd++`Jc;4svJR z9*VWMQ>mzaNS+h&To*k<=2C70$#p}nrz*FRas=fEt*0* zKEWsh3-nzYr_4;{@{#40EVm3Dap^HZ$O(7N znnmQY3z}_3$StqrH4i*-<_SKz%!xQoFc{PV+|^`uHJQ%|hqBq_Ln$(s@X8I%k%CrP zF3*uw>vIenc@5B7bVKwB^*XiLxJz>E`kiQj_b=O>^j%88wcND{K6q$mF)s)^Aoag4 zNCjUBPUUz`vk6>&Rp5mHsi`>XF~HN5-+bNvp8XR^Cl7t89~fv zGd#!V5($C9?wb<{5l=6bNapiG=B3pls$qUCHOIN8WGM+#OQ0(6BBLic}PJd z1dZYpB+8+57Qe@FHbir&E0rU~Rc}2~JLHff{X>s}|N zgoe~AE?@)*B$NE%_kZ}8x!^J^FC4N6eUPuY1iS;QcyYAmRE~$+Ad+h|BXFsVh}lko zWF)iPIFPA_m0A^&sT=yM z56Co?;YoiaM+m_rLMF{}S%i_S>DA>GQD}rbgSPdPCq&Y+A{w01W>a8aE-58c+L-UpPmq4%+e@-=;H?dxmB z_ReBwuo&zuwjF?q9WJQY)&&)P-B8ilUAh?0o}vj+lDi?`jTYd*DLpTUDMF~oIRuPw z>&t}NVC{ZLL8KEZ7S|`#y--racTDXR@Ef2N)>7g=R@N%m7R@1t)-ruIau%*RZ0bu{ zu0pqD03-k-;7x!Z{pbKBfD4t&=1oSGp;BI)%9isMbxsg63rP2Z4I<4!s0L`|E-iQ= zL&SDLvUubI89tMTyWU-YW?`d3FSf+P2KLjs@119q*1>}1n+;@to^s>Hfj{XtE>6Xu zWo?MzB5Ok2bc)73#gGPW7Nq8p%aIVn<iqBy)&v7>@Q$anw^g-ot!WIe<@U~}w6|Ba$Ln7c%I#fo3+p&B2U4LMq(Y}r z%{bv0$8mXgYI^gKI ztxV}auk(1rNMJq289*KTQ|dHS>adNhchY~J;Q`0R-96OYV%)|0j+=Q;+!HgJ8=b>5 zwy)}?l#CtRo)@>WJK{9!51S6raYx+AGRG~otK}Qw4YBa{wrnGU)!52nU5?8nznmf-e zuW%p&LFj3&<>d85K6eFVpythImbu(&J|S`-H`AhKLBSfL2}}Z!=E|iqiQEduCo|d( z{h2`MrSs{v_BhDVm6ecPARCIcD?1JX6WEi`n?Wp9HR<3!mv)Qre z%vfw>HXa+(8Zjd}GMAX0o*bKEHHK7(M8~G%vk7+W?DW(qOQg1rNnOtaSm6i^*F3RR z9ydXMv+1WPjN-y9sikCIbC*Ch>K~ddC2&c=-1gK`E|cO!&0g9WVFFr&8i~jxxSG(4 z=HwO@xKus^^7I5$AiG)F{>p9&V^~T?wdoiiPJ$Cxhztcipej_srl(0^Lk;MMPQ`bC zw-Ef))OtI5E2=hy(O_gQIAf_ zM<_S2e9(Ad~{mz&lKz*xtW_+6?f-m zTmPSRt8D{v+rYhksck@Nn^l=v$unDYGg8NC#eI6yzf3{caj!U`TiYD~L5@7dkLwCq^ig1h5HE!ZC~xXA3BnY7s3mSOKFR_=yN> zzefEARIi~YwhrsiLkc1t)?%F${K}yO$mmBrWYxl20JWe_3)YJ^)~!QZLpTa{$g{Rn z<#z|`VQob5cS1=68Wt{F%B3z8SS%P8h-&U;U4m!P5%Lk{!H!PCO)n|vczf1P0=EdB z<9KB2aO;92ks#_Mkb=4gf*C%QT}^|zAeiRTnG{^*bdclIE4d7x7sDyID&>HygCG3} zFYf2Z)Q2fR*rlDJC<9?zDhOL=nvR5i%|&4tx~~G+p3 zns1wzg*UO*xT%q1ZAZL+_?S&MDzM46|n1)1%Rc)*8g* zK{1(INv>vd2M5-=m-6`)@x-A+M6c%f6c=7B(@HWEUTX@2@D1lct<7?&Y;HA;y6Bqg zkS-PuT^(2(uZW{B7y>*+OjJP>n#l|wtke!(TjF?x0A0Ci zp1Sr$kOipuAZ&yeU>zZ|W<>)Kr#TEVMbiwHZ&9N)Cc)=~Wv~RSafx!x-fdL0z0xF% zB4#0ao%*P$?e^rIn9B6aOuxeHRhh8Ngf~_dW=LgD%FN08tip^G9LO`9JJsgBa`Rrb z`Pq%>!g#TH@ZR(HCZv-y67!tod9LUV+<8WE_X0C5`sN;)O-$nh%GA*Kz()C63tqxd zQZQ&aApRq_?IJ)HFjj(LZj$1uivAiaQ%||gc3L*YP?4<(1}!5!vL*uBFILmgd|Cag z8twn7+8NZyn?Q-JqoBo*4cCOYscPIh%95dUu+$)!lVA;IvIY?D$ZKMC)TCG))hC9M zp(YqRQ|j09|LV7_{8T!x>DNlkXTR@!>!?uG{Z^?;wyOQBLnT|)sE~b%pkl;z{6c0C zY?c6c{TUvtl)w)Fx`K?ztR%CVA93xfP@M+=xfxW1#I>AoC7Vm86QE%Ldd%O@c5KVq zssUnLBoOGH3hmI$zzFm^-+(M3xtd=>!zw3C!#)z=EZ|x+FP2tM)x}+3N%CpL z>i|c|8Z_qu*pT&=xUqf-e`^-(RiJs$Z}xBg9Xf@=EIVDS{DHJ z#P-FD3B4cfD3;gVhB~cEjfCf6F`#-}$2EaE1q&4+xs{3ejfKMcjKO-g@vzcnRO%q3 z6n_iR>(s9t{*QgFs;^)6^{c*P8wXYIY1w;P@%}-JVw&Gydkrr&->?R{46 zeOBo@sdk-{yUr(&tUQ3Csc1r}58K7-MB{Zh$q1O<-WMyae<)H^y4_K(BEV7{9H+ zkF2BgFkRn(7pF zBJylHjS`Rx+Uo+=>VPXzCFc{#445%X{RsuAA$%30kWE+K5X|biIE3(Zh&0=kYe`{I z6cM-Rs+evNwU_VQYaC8HP8rTQp03!5Q8KdSl3FYlrvZzZ8ET>zOn2r=tGV7v z8{QJPl+hkgN)%+!Jj_B5@v@;%1UlPIxv5~4l{#($_s3I5KSTkwS&hDva6VvavLe0k zw#r=MHZY=>`v3!@{Uz{%Rh8Sp#9l6+_EeSFzyVV(;T`kXH3@OYJnkh4h7>N(1J=hn zV`U_2)Pr1CI~8{@Weja#0+8K~a%)7fmE9?$Sa26ra?3ay?gco5Ea_xEsWqTc0{QF; zKtK-gp(W2<&kL7ee9a9us}(R?ksQD{;M>Fq%ZPnK_%@~xA8*kS9FTDoHJvF)7$$f) zZ9ojwG*UG)Kxc8#Y!Slvt%=3 z9=une8it(2Ekbr&xQVrIK?KqjRw69pJ8rgSCGJnnyeuw~?G}+Zbanx;&M~}+t&1R@ zl&zZH)(S5}AEFG=>(s~ImYdhtTQ*u1?~#IKv$cEOvYxw_S6WZ1t)uT*3Jt$(8ZL}( zHg(*I-fL1IJ@)V3ro!rGp#9F!ch-LH+x5^+H6MHIpqTbM2RGX8feGZe>OXToRN>V`tgnQzqK6w**R(UMQJ`Mr52S7 zONDcr-nKh?-X2uFdu8w5VxaBzwOiNZuGyb}K%G?rvyyKXZ7MK@W8Zm2^7U?d{UC(B z=#eIjYpXK|De)^viT`aeD3Qn~_-uhU6Zr)BwV|HOs^b{KzDbC=j8x@m&@i}W(05Kk z+)~epQ_*j$xFjHb;AfQ)>L7n(gnAl)fopl|XO3RxqgA7~)r?+8hp4Izw+&3@*3UDa zZLNUgR=FX71IDc}yw^2>wHdwdEtMo;)VLK|*WXjSQNL#ijm3dTw~wTyBsLoz zo0?5TMuXsB!d%@41s<96V`pa*kTbf?p;XUqMVuF4c>XI`2!M14nJf&Yxo1Icoh`YI zXd?BY+5}XqcsW4-i_~WPHDqWWg5&u`q5*5362y6Vl@s%!Zs8l)=536SQ3{h7Aubax zV1)cfh+;H@(PfB0Ne3kufCM0YV{>qh@PSiO$cxu9pcMPD%GT{7-&@q#7-ESNR!*wd6u?p@3r?may6-YM&6&?7@=M@?N(5s6j8F&vO zun!!=AAN?`=~LWEsxK=0qKa?2VBKtKyA%4}!M6@>G$`%EYWtVabu}lSn1iHr@k>fe zLT%yX7EWndR9kY=m0ZF7p|}0cs@i=>?mi?Pj;h^La`#lRy}K~^k*66|DXD#*;@STM zjFN{oy#dv`PxkJ6VurVm@eQ3b5A2jP@INlf*F~(DsVXa`gKBD3nf)@eeU*!M$Kvv__|0LJ>64khjbVxDlb9p2|0tj>aFqF5RsU|;zkA~a$-i6j z4?$L-Qwme<)}Apqz0P?2lbk`!2O@-_P6jZ4Cb4dwbU>6z@UFaqyv;YI=_To#+Q*_;RE@GD!bqCmlIr z`AKIBq(K*0fC6%vK}BE2GzxbCaEyMPp$4H+yxLF$=}|zXtWu-O=haZp=cU;biF7Un z9~vHmcEDfACy2{jekqsMZ}L~MKXQri{yg11mcmkuM&T#E0MXZ|Us%R}V=-CxKX6bs z|09#r+WN>0sRx*P&;qH4rUvT_UCPAd<1UAF&x0mX-DmA7l|bsr?mlbyAq5eE!(=2J ztsE?nG9x7ZFGy5>U|R1B#413f!#p~M`&JXma1#U`Ya+&3fW-55qnc3;3M+gUTrNj# zhJ-EUy3rWh1Ei`gufSQ>I&TzH^=4jUqy;n32-F8cz=vkx;hR*D9$$bL;VeeiG24zj z9God@Z8cw9RNO7#5+$MnRa@P(+JmVh7~wwac1fCe=c~MM+78{(pTMsfbWfe|&rtmx z_=y>a!2O1H#y;8Cr}_?Uw5Z-;**mOwpLu}?QT(R%m^mZxUpyUW{x|*cGVf1#7%LC)m;d4^+gyM=wmIxsW*avbs@Cd+m z4By}`tm0QIK;J+cl<@Z$A-vM;@Da*VHuI8hJhtU;tblwXmq%+We@YBaEi`?HjZdd_ zJ9Jm299A7J*^r4z7r&&|J(xHH{Y#3<7UIU#c$GEBNk9hhmGEidU!atRpBRJaftjZ1 z52&7BQH{T(+#gV_A5bm7rotakr=HkMG<@v~(PLL5-F-9in1aXSo^E>R&Wn#Jcsw4Z aIhqdN%shsVtLgA>XUq;7K4%X?xBmyDP{~&S literal 0 HcmV?d00001 diff --git a/evaluator/llm/__pycache__/profile_resolver.cpython-313.pyc b/evaluator/llm/__pycache__/profile_resolver.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aabdea1cadd909e242387bd684ae0031174b8be6 GIT binary patch literal 5805 zcmcIoTWk~A8J@An*Ky*6#N-M%kN|ERhnB!Dn*a)lcS%An>xs9}l$}oONf>a(?wJXf zv``+Z)@-+hZG|?ptCC2S64VD)-K~nW51XedQu|mEA3PNGvl#I z;$7H>9@%Hkqy+N9QLiDC$#^F#}^H==CW`+@?5+qlc(L8qt1| zwhUR2MgLlnRsS-Gfv z9K=pr0TRWZnVu-@B}GN-n@6Jbo#3od z`cPpW5!Y6k(tMxWmt#X;W!4TeE0k{Ct5z!D9KnvbI3>&`R7`hZn#Xhq)BZW{{^LC@ zZwB@sKRVId@tw1QISbz&uwk2`A}NI(NK7S!n20S%#8zc?65A5VOCkzbu`GsGI3Evq0|*z$HxRYDN8XS!JGA@3sehT zc;K&GgXlVy_f+RRJ8zxIv29tlO=I_6@6WT|n>{yr=B@Lb#yfx2f+^h~_9j3JoZ6aTZ36S=1C0WaQm>eTRu2-#=#hXOI~7q?C3#xGHT;w$$4&K6AeR6|O95A4hbRk^2{Dc<`9xAE$zWGe9CVXW z#KaV8g5e69iO7~>C(lc=r1Cs&+Pdo@vyv#HG5`@MVZHzDihON-t~Qvh4QjQaH1oDA zU)S{Rm3OWzGnu*|bW*Z4 zYALEo7pRFf!kY6heWR8LSftItoPDu9>_OBrT2Wd9@MvL=qLvQm*Ub50oN`heL!`>q z*uJ#JzOja5%u!1Ha5l57)7AkifmK^;sFOXAVxo*We`2GCfy|;z#JhQ|F!w^y2I)Pz zboC2Lze1!Hi=S$xSjtaLQvt^a_5h!r7h@`ql1UX;gNpNGk#bbUNZS7gqU+R}cl#~%-K+0hU64PF|LNQx&V6uR z^Y*5hHDAMf`xaW4Iu<)J-|SzR)EWnJjiaj(%{P|wO=Nu&n(u7do%hwH-T!2LdC&Ho zXGhkvV|g$aJdq8a$OZee!Tvkv|Ju5${^c8)ZwuK$;h|?T@7%WLsk&ABv%w5-g(q0d z^kzN1#mvF1=OAP<)vcPRO-~%rJg+=v?RD-al-=b{yVolz->!83zq~bRb)LD)-eK=n z{H9{L^?vYP@P6p`A)u&VvQKOv^w>^S)f)WsEU!3W@jCy+0ad_1kTpU%_K6gk#FPT< zqa%(|&P;JMXE{Bgm;wX zgfhENXx=`6mbd2SwYRQ;^X1)>^9B}Bn#osgyLs-$xm$A!U0UV-v^8JjOFQ#TJMSL3 zbL6+(H-^&vYu<+Ww%^s}yn8k8-nHX>E8&lRaLf6y;lPJA?~i}bpfz-BRXw2b&HISP z*QdQ(ZEht@PS^PrL+2qj!k5`x&SF}{=UPxJ*b3;Igy+iIIQt9BIymPG%epxC3(K+` z^}@2AuOVCUHDoKlhOD;m?V_6_i_g-3b^_~Ft1 z@ToZ`DeeHf!j7j`7 z_^}cX_Ar%{WR;iVcpE8>C1+9sQfEZHEa27&;-VA>`)Qj=#>E6?0q!Xg2`X4c?4A)W z^J?;fC@a{jN;6_|R^=5D%w}A{&PhQL`B{{}4k0D+7sM+i4*|vkOZ@Ecm79jPAVP<_ zK@ZS6fhYtphPA~WQNG9*u9FJh1zWZ04ikD-;3a6N{1GDHhWcF#%u>Z-#qzZW7q$9V z*L=bTY%d1jP>)h?}RY;9-TLVRgvaVB$!6zm*(;$y4ak15vW)|q5v!+ngHK5#GT^*?W> zl9KVW0<9D%0i=t;bpF6yEPEd_jgasQ-613+Z3@pds#JvBqc??ejwn~0?N;+^8Zz_i zC(i8ubv9tu3zYmqv#~^q#=Nvtb9ce$1KsOJ|Ba}foZT=uAgAL|+ddE;t}+bs=`PMf z@=*o|#X5q>@^p;ouNLH0BH+^PC?XCL+DWSIvnk?)VkVY^*9>P|6jLOH{6wOgL~s`X zcigbKZ$cWolJNF{0K^Jr3-(?|qf4Bc0(V`y2u~}3PQw6`ITQ@AacPAo82$AGH~h!r zWA~2b+K*=2k3NvJ_LI5xk<}C0j?ssnv9A)nptTR?+Q(KWwH>cN^h7>s+;g|{PUqhn z+iwh}`|{pe&Fjy%?4959t4grRO~J=bs_nq7P}bR!_ts~e^*W>ijIn8CTXpBeARCKu zi6A~F+~n;@1g4YcSn^Wn|Dk^*DT~mM+yL}@2090+xd!6#On|Qw^p{fbrg-*jz@kfh zQMs`kkt+c7L(b<5H#97CtqUWGR08+%JkbGoE^zZau3d<^VTeD8NJJb@ zr3Ip&P){w4ZH#{Esj}5RZFSk|pKkZq+@CjB*czTt5E-pW8(0ki1UgS5vMcB~DfN)a zPt6i@0kb27guX|j!z3au5q6M=m4q~D+}rHxl@xfUi0nT)LL%}WqR$GYNXkbdGC}O@ zF*fBGQXxyQs{|l=Y^7=X5w-hMiv0)Wg5M*m{t>kuvQO;{-Sc_nak}BZ6hwMg{{wEP BlGXqK literal 0 HcmV?d00001 diff --git a/evaluator/llm/client.py b/evaluator/llm/client.py new file mode 100644 index 0000000..49fde57 --- /dev/null +++ b/evaluator/llm/client.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from typing import Any + +from evaluator.config.settings import settings +from evaluator.llm.profile_resolver import LLMProfileResolver + + +class LLMClient: + async def complete(self, prompt: str, profile_name: str | None = None) -> str: + raise NotImplementedError + + +class MockLLMClient(LLMClient): + async def complete(self, prompt: str, profile_name: str | None = None) -> str: + if "inferredCsiScore" in prompt: + return json.dumps({ + "inferredCsiScore": 0.5, + "resolution": 1, + "conversationPrecision": 1, + "rationale": "Avaliação mock." + }, ensure_ascii=False) + return json.dumps({ + "judgeScore": 0.7, + "accuracyScore": 0.7, + "alucinationScore": 0.1, + "rationale": "Avaliação mock." + }, ensure_ascii=False) + + +class OCICompatibleLLMClient(LLMClient): + def __init__(self): + self.resolver = LLMProfileResolver(settings) + + async def complete(self, prompt: str, profile_name: str | None = None) -> str: + effective = self.resolver.resolve(profile_name or settings.llm_profile) + provider = str(effective.get("provider") or settings.LLM_PROVIDER).lower() + model = str(effective.get("model") or settings.OCI_GENAI_MODEL) + base_url = effective.get("base_url") or settings.OCI_GENAI_BASE_URL + api_key = effective.get("api_key") or settings.OCI_GENAI_API_KEY + temperature = effective.get("temperature", settings.LLM_TEMPERATURE) + max_tokens = effective.get("max_tokens", settings.LLM_MAX_TOKENS) + timeout = effective.get("timeout_seconds", settings.LLM_TIMEOUT_SECONDS) + + if provider == "mock": + return await MockLLMClient().complete(prompt, profile_name=profile_name) + if provider not in ("oci_openai", "openai_compatible"): + raise ValueError(f"Unsupported LLM provider: {provider}") + if not base_url: + raise RuntimeError("OCI_GENAI_BASE_URL is required for oci_openai provider") + if not api_key: + raise RuntimeError("OCI_GENAI_API_KEY is required for oci_openai provider") + + from openai import AsyncOpenAI + client = AsyncOpenAI(base_url=base_url, api_key=api_key, timeout=timeout) + resp = await client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt}], + temperature=temperature, + max_tokens=max_tokens, + ) + return resp.choices[0].message.content or "" + + +class OCISDKLLMClient(LLMClient): + """OCI Generative AI native SDK, including dedicated endpoints.""" + + def __init__(self): + self.resolver = LLMProfileResolver(settings) + self._clients: dict[str, Any] = {} + + @staticmethod + def _normalize_endpoint(value: str | None) -> str: + endpoint = (value or "").strip().strip("'\"").rstrip("/") + if not endpoint: + region = settings.oci_region or "sa-saopaulo-1" + return f"https://inference.generativeai.{region}.oci.oraclecloud.com" + if "/openai/v1" in endpoint: + raise RuntimeError( + "OCI_GENAI_BASE_URL cannot contain /openai/v1 when LLM_PROVIDER=oci_sdk" + ) + return endpoint + + @staticmethod + def _config_and_signer(): + import oci + + mode = (settings.oci_auth_mode or "config_file").lower() + if mode == "config_file": + config_file = str(Path(settings.oci_config_file).expanduser()) + return oci.config.from_file(config_file, settings.oci_profile), None + if mode == "instance_principal": + return {}, oci.auth.signers.InstancePrincipalsSecurityTokenSigner() + if mode == "resource_principal": + return {}, oci.auth.signers.get_resource_principals_signer() + if mode in ("oke_workload_identity", "workload_identity"): + return {}, oci.auth.signers.get_oke_workload_identity_resource_principal_signer() + raise ValueError(f"Unsupported OCI_AUTH_MODE={settings.oci_auth_mode}") + + def _client(self, endpoint: str): + if endpoint not in self._clients: + from oci.generative_ai_inference import GenerativeAiInferenceClient + config, signer = self._config_and_signer() + kwargs: dict[str, Any] = {"config": config, "service_endpoint": endpoint} + if signer is not None: + kwargs["signer"] = signer + self._clients[endpoint] = GenerativeAiInferenceClient(**kwargs) + return self._clients[endpoint] + + @staticmethod + def _extract_answer(response: Any) -> str: + data = getattr(response, "data", response) + chat_response = getattr(data, "chat_response", None) or data + for attr in ("text", "message", "output_text"): + value = getattr(chat_response, attr, None) + if value: + return str(value) + choices = getattr(chat_response, "choices", None) or [] + if choices: + first = choices[0] + msg = getattr(first, "message", None) + content = getattr(msg, "content", None) if msg is not None else getattr(first, "text", None) + if isinstance(content, list): + return "".join(str(getattr(item, "text", item)) for item in content) + if content: + return str(content) + return str(chat_response) + + async def complete(self, prompt: str, profile_name: str | None = None) -> str: + from oci.generative_ai_inference import models + + effective = self.resolver.resolve(profile_name or settings.llm_profile) + model = str(effective.get("model") or settings.OCI_GENAI_MODEL).strip() + endpoint_id = str(effective.get("endpoint_id") or settings.oci_genai_endpoint_id or "").strip() or None + if not endpoint_id and model.startswith("ocid1.generativeaiendpoint."): + endpoint_id = model + compartment_id = ( + effective.get("compartment_id") + or settings.oci_compartment_id + or settings.OCI_GENAI_PROJECT_OCID + ) + if not compartment_id: + raise RuntimeError( + "OCI_COMPARTMENT_ID or OCI_GENAI_PROJECT_OCID is required for LLM_PROVIDER=oci_sdk" + ) + + endpoint = self._normalize_endpoint(effective.get("base_url") or settings.OCI_GENAI_BASE_URL) + if endpoint_id: + serving_mode = models.DedicatedServingMode(endpoint_id=endpoint_id) + else: + serving_mode = models.OnDemandServingMode(model_id=model) + + content = [models.TextContent(text=prompt)] if hasattr(models, "TextContent") else prompt + message = models.UserMessage(content=content) + chat_request = models.GenericChatRequest( + messages=[message], + temperature=effective.get("temperature", settings.LLM_TEMPERATURE), + max_tokens=effective.get("max_tokens", settings.LLM_MAX_TOKENS), + ) + details = models.ChatDetails( + compartment_id=compartment_id, + serving_mode=serving_mode, + chat_request=chat_request, + ) + response = await asyncio.to_thread(self._client(endpoint).chat, details) + return self._extract_answer(response) + + +def create_llm_client() -> LLMClient: + provider = (settings.LLM_PROVIDER or "mock").lower() + if provider in ("mock", "none"): + return MockLLMClient() + if provider in ("oci_openai", "openai_compatible", "oci"): + return OCICompatibleLLMClient() + if provider == "oci_sdk": + return OCISDKLLMClient() + raise ValueError(f"Unsupported LLM_PROVIDER={settings.LLM_PROVIDER}") diff --git a/evaluator/llm/profile_resolver.py b/evaluator/llm/profile_resolver.py new file mode 100644 index 0000000..e6b3ce6 --- /dev/null +++ b/evaluator/llm/profile_resolver.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import copy +import os +import re +from pathlib import Path +from typing import Any + +import yaml + + +def _canonical_profile_name(value: str | None) -> str: + name = (value or "default").strip() + name = name.replace("-", "_").replace(".", "_").replace(" ", "_") + name = re.sub(r"(? Path | None: + root = getattr(self.settings, "project_root", Path.cwd()) + configured = Path(getattr(self.settings, "LLM_PROFILES_PATH", "") or "").expanduser() + candidates = [] + if configured: + candidates.append(configured if configured.is_absolute() else Path(root) / configured) + candidates += [ + Path(root) / "llm_profiles.yaml", + Path(root) / "configs/llm_profiles/llm_profiles.yaml", + Path(root) / "config/llm_profiles.yaml", + Path("llm_profiles.yaml"), + Path("configs/llm_profiles/llm_profiles.yaml"), + ] + for path in candidates: + if path and path.exists() and path.is_file(): + return path + return None + + def _load_profiles(self, path: Path) -> dict[str, dict[str, Any]]: + # Expand ${VAR} placeholders, matching the way env-driven framework config is commonly used. + text = os.path.expandvars(path.read_text(encoding="utf-8")) + data = yaml.safe_load(text) or {} + raw = data.get("profiles", data) + profiles = {} + for name, value in raw.items(): + if isinstance(value, dict): + profiles[_canonical_profile_name(str(name))] = dict(value) + return profiles + + def env_defaults(self) -> dict[str, Any]: + return { + "provider": getattr(self.settings, "LLM_PROVIDER", "mock"), + "model": getattr(self.settings, "OCI_GENAI_MODEL", "mock-llm"), + "temperature": getattr(self.settings, "LLM_TEMPERATURE", 0.0), + "max_tokens": getattr(self.settings, "LLM_MAX_TOKENS", 900), + "timeout_seconds": getattr(self.settings, "LLM_TIMEOUT_SECONDS", 120), + "base_url": getattr(self.settings, "OCI_GENAI_BASE_URL", None), + "api_key": getattr(self.settings, "OCI_GENAI_API_KEY", None), + "project_ocid": getattr(self.settings, "OCI_GENAI_PROJECT_OCID", None), + "endpoint_id": getattr(self.settings, "oci_genai_endpoint_id", None), + "compartment_id": getattr(self.settings, "oci_compartment_id", None), + } + + def resolve(self, profile_name: str | None = None, **overrides) -> dict[str, Any]: + profile_key = _canonical_profile_name(profile_name) + effective = self.env_defaults() + + if self.enabled: + effective.update(copy.deepcopy(self._profiles.get("default") or {})) + effective.update(copy.deepcopy(self._profiles.get(profile_key) or {})) + + for key, value in overrides.items(): + if value is not None: + effective[key] = value + + effective["profile_name"] = profile_key + return effective diff --git a/evaluator/output/__pycache__/legacy_exporter.cpython-313.pyc b/evaluator/output/__pycache__/legacy_exporter.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d951230d0b397c596b66e8989a5c1a70e137ce65 GIT binary patch literal 11379 zcmb_CZBSd+mG|jediq3sf0%~>V;oz^*q9I7NifEk4+}>Rfo(NWWbwel@IVRCyJeg1k3IK2 zJt44M$8E38)BQO2oO{nb_uO;Nz4t8^6N2;)Uwe6MwhWwVq#agLN)2r*zW4)Bu_8NK&*dXO~y~Z9BHc5GXFWqCt zW+`vzwe(oARmvNCZ9R5um-41wM^7HkV^9NP>E|fyWSy+J6pd+cKAX>4q+9`OEk(GH zEnsa@xri-fi!k5RaoEoL23eF48J`xMY!Vx~|A?0Ci zBseu58xDtJP}H0a#xB9HVK^A$V&MsnUnf;jE*1;(qfyA`cU}pO zPe~ZP++-vgjzutmsh9{}3xt3S7n+KNuW*4dALIB~AR5D5@N&IbG>uISk8*4%f;rI= z427m}Fmzom+5uxI%u7h+8uKKMOiaf5IDRyCNz{x-B9o#$%#UywbHg3cunc9wTr@Hc zgO;D*RlY;r2rFwTtv&jQ@F$q0^d z*fn`wR0Uqb41gEHUvwXQX3!dKea#s^_(toZcezS%>`yrkCmn}Z*gG|XsI-Un#3ZPJ9UG-*3zUPtMPQZ$nfe9{#!y#M) zrBc%9fX@uNS5UZke)dR0Q@hj-A6x~s+bC+FLZY|?v|mTrOlB#Cd3ynfq#z8-hjOvH zNd`bX@{@&=k`MWHO8y~CFg(CyRNA+GS(UP+J8(&ggup$7t3kqL*}|dZwnpVc${dD4 zAl$%Bzg22jeBHPsUuY_J|tT&BI@6r(z=qTIx-r z{!$QjTnytqfIu|Cp5vpjARppH?ZxXcE-LDVxlm-7!=QbUs0C3EPl}qcDC|t_cqBL+ z6*Xy9q{hLk*&=1(BX}Y>9-ig`(1H&_ee#G$UGG?^er+nHuSn`ENd4H{SiDHERw+%;T^0l2;yQzgLsWt4rq9%^pf<8kW8Ad04ddFi=Ufk3A&XxkMPmooqiyezGZ~ z^w46+Vw+RfD82hBH{vN)<)e-Qj$jQ!yO5uZMM^#@C+&>TAE&Yf6;3I&f)_-VR~YA!ktP~DuZ6ft>Bvz-cS*48RWe!~ z&qZV0aFpO<7+EzI5PUGyz;=>t0OEqb=*#e#LBFo7U#|Xs-R-)S+P7Uls7X~mldODZ zmYpA(8(Q#v`-QvSnNw@}!uTFR&upmFM#s9daN)|$>BZ@#2BD}iRn)u^6r3#ybISuA zGTXi#T{v{Jd9gWuOmOZ^IbF*q1Y3hZyAvAs|31(nd&zx786Cec+c(tk`=z9;KiGZ3 ziaxTMPL!!XDy85X$`-TCarn0&N^mkzI~SG+UAFelDQ1675m!HfC{GiSdIWq%z_Vs_ zQG(BT(rld%S@1o70zMc1MhRY>lSjF>B8Vf5KR0|X{2+e}Xl!1SwIZ5Fs}=c7+6J9_ z?wM`GlIF<`Z_EMzuM&K2BO@35cmD*u zDF^;<{R#M7=6xM8wJ6K4~q7SQ(%LGOG|ulV#|m&rKUr!kWCe>WG)f&JQfU(i(1L}5pBe2 zC)Nu1#(~S+b&&@5IS>N>IS?Mk`CzKZrjbT6jWBU=B-2Q?f~*p1d<<$uQ`)VDexeyH zHsT;kBgDr^r&jWLhJ2odk4Q_Sfsg@MV4_(bQ@Ui8o0RUNE!~=_FvtKGL!vR=bTJeq z-W9Vg$<~-tvBt9J)d#?2K+)dbmxN5bo!??seS#) zef??Es%_P|C*d2K>APpJ&ztAWiGn)8uy+&bjMg>UIe%>KSfaRIp!aR6q2?ZKoo}9N zPB?1?de5d7YBq?SCfC&nbnT{|)ESVuI7OEw>9Rz*OQ0JzjilC;ReM07U7IwiH6yci z!-lj45Au+?Fj2VY9ZiZpkfaZ+l~p7R#b3doT(3T`Jo5dq+hcc%1XugVhf~$3lhvnZ zPtEtt^(>_E!pOS*guzMJP4&&Q`%3-%@a&(C<)-EDV{ ze9-oO+ubqY=yR*=P|AHS={~nGa&v5PY^ij$aDU>37iap{k34&~>_?R!RwkYc3GKtH zBV(x}my<^>&whRW<++y=l?PXCjft0{u-Gjp?jB2Zu*nWKF)$)@jIM?!QZ0P4g`cQ(zKp%7&|=V;BM8w%BU0Xs3%b=;iMKe>pmtCfRl8AC1!Dt?bPC2jIq1{+ zZ3?t6W3dHPXPh$DsG8UDTFD{JM0>y?t&()`6V1Y%iQW)b_RJzeiZdHkF-WE}LgFc3 z0E5vNgx~cBO#FU`b%r5IB85wI!O2OEAC_VtxS2F+!!d3miism$uZLJnt1;ug4;^3@l4Sdj-7LUvth)3#PGdo}Fh?~CCpETE`%uglF zPrY;I&R$`E$Is0te`7@Ue2~*_>MZr2mLFSwYFnkx{lxbEzB|>k9jgx4@|56c`pn$) zW7}u+xrFB2mXn$T?ed9~xq&#Y?*IE0eJ)xAV_e$P-m3brqWDCe?!(r)6E)h8swwzx z^I*x!OEUeMReI`p7;~*Z9<|#;39(R_oJvGqStYpUp@yz~~QY4}fIwS_sPIh$(tO%6BGZ$EXmm z^Xa?=JKG`JnXcmXKE0RldPiankDykv0}zUA#^Qp}nhvR1+DExFQC`^9dM}at9Z4Ah z>}tcHanLkKN44pl4YQqs)tQ`|eEZzoxwl!fOAWgjI7)N`Z{|(BkvH)4BSvodGe>TH zf_^q-En&v?L)s^sF>is3C-|p?0YREb;9W{X76mhJ_3u(@y^NCE`h{4smUXastg}@O z>umGcywzK+N{-F<)++g}U$&qW`AKM3O3sWlD74wa1JI(BU@nyckhpgTrzML)QDcK@9iCQPIF%14}3n zxjgM?pWxtfG1$NcBZ0GUeH{_??OmOofzE(P2m1QX1bTcYyE+5YmfG{R6Sc#E+S9dX zYT0SMYv9~~tLwt_m?_&8jB?~IfMGjXw!7aG=svlh>GquJ^m;o_20Ga80Nc^;?cC3J zJK6p|-vEH@XFB>lgPmTsost}yXNX+`-u8}8X@=O9JzwmSDW#{1UF{tmK7c)+p8Pg!Y2E=)pRc3aBTp^U(<_zR z`+#F%SI6mgkEgSbWX}M5VUZ-_VVAdkDA2>^X4!NZ1CJ)QrEzW~#*9V6JR{n&_}M+s zd1iW+2{REMD3FrOLagMsqmdbnT{&5;us~t9#HuY;OT&p^xVa>ENZ8fk?Q9?V8DWY?R~SVE}$wtdrjdz~z?oS|D~U z78sp&9UG5894Gp;D+Bm#f`hyp_yUalM2Y*Y!lm{%cgLT*RU5B;tMCu^iQW6n;!=2# zV3MLf(luE+x|4oNL83y$XcYD7is>W%9gh^AC--(zR|?fi#Uc~A|(wOw5Wy=<8!2}4n@P@d%|76W)hy=h&IKagmnk4 z9mOnxvPs_M#p{Z|@zU0_Akm1Wmp#F84)XwwPm<3N z>8DQxeBspoS$nS;nx8HK_7huY%6e% z9VIs_{;uN9YpKe{WM!k^I5g9%xg5(aaTMOsMUm1V1B~jBXl(h)f z)|sv~x*%S+w0Eg=`RK~kRC8Cdx$6^~(0nd&r4{@x5k{kr7=90CNid&^`l-??Tqdc*vStw~q z6t`|5b(?X+sE3hQGb6F-VI+2Bch29KyYcEbW;L7j$h7OhepIw~StAr4nCbsD%)aE6 zl7++Z{AK-TM)ykln!z})pVKdBZWrPrM1G^RAGIxuzq>p%0;2Db+&6QFMqaUt+af$cddlERk3tp zwr8!pV)o32%51qvtru1=orX~?sMi~h-C;i%dVlEd1>yMFRqtS`aVXh16hCyU`HkkK zw$(CM;`zXW;a+|@;r>#&knh?=>Vn;CMU^*iEZ$h^6^h)OYN)uEuILtuT$@^`04^5q z-Y_CZ*#k4GY+R{cxpucy*wwc{uQN?6A%SUIuskp!M{!*H9qYQiC~gq!Rpgfz>@{ol ziZy#>%3cji_q~o~(?9jEJSW(j*X%_%=|!4^knIiWLcC9~*Jm0I3HHNl_Odm5sr)}> z-xI&IQDVur&Ys*TM;7aR*Id`a$(!Aa-N}l^+XF&HqmbXYGLXz~O_|$%(@CKT>LqI9 z6orhYt;Kk=F+Td;qwlERIkX&myY+2LqVx!i=1xu0{`9Khnay4bnGSCBQOInc>HWj+ zA{2eg!;BAbhtMB#J$sT>E4!aqdFG|3czIAW`^Y${zg89{eK& zegNdl%{oA~YIH03zKoKf)kntmqBF{1=(;7Lj5 zy~p1JxoHO|bmj�xy|j@C&-NUL$Lv2580v1|NYBJe~7Ee2+Xmk@YgUBAKSi%O~jO zf%le@I-54h_@oM{GYN;l;}VIOm4N#J0<0@O_B`PGCc#IdN|M%=HTt(mk#v)(lQ3(2 znP>_GMyAM%m_Pu7+CumS`4D$mq|;8c#v5lFDWW0%Im7cuE9pJ9e+7$ZEtjK~L> zE_QtqY#mJEmSRcfnW76$hLP~7OL;9SpN`B{`y{;21e+dgCC)`o8~O35G=*$5K|VNs zJqAaI>k2tFq;MymA%kJar-po_gHAfxq%)4JzNDIQHOUbzFDr4wv5Cm=)HwGvCc#V! z{-Oi$0WqSe&(ZE*A^Mky{v1_*j*34=71IC0|3R+LQTykp str: + return '"' + str("" if v is None else v).replace('"', '""') + '"' + + +def _metric(v) -> str | float | int: + # TIM specification: trace metric None must be serialized as "0.00". + return "0.00" if v is None else v + + +def normalize_raw(raw): + if hasattr(raw, "read"): + raw = raw.read() + if isinstance(raw, bytes): + raw = raw.decode("utf-8") + if isinstance(raw, str): + raw = raw.strip() + if not raw: + return {} + raw = json.loads(raw) + if isinstance(raw, str): + raw = json.loads(raw) + return raw if isinstance(raw, dict) else {} + + +def extract_prompt_length(raw: dict) -> int: + """TIM metric: raw character count of trace.input captured by the collector.""" + nested = raw.get("raw") if isinstance(raw.get("raw"), dict) else {} + if "prompt_length" in nested: + try: + return int(nested.get("prompt_length") or 0) + except Exception: + pass + return len(str(raw.get("input_text") or "")) + + +def extract_ura_call_id(raw: dict, metadata: dict | None = None, message_id: str | None = None) -> str: + metadata = metadata or {} + raw_metadata = raw.get("metadata") if isinstance(raw.get("metadata"), dict) else {} + business_context = ( + metadata.get("business_context") + or metadata.get("businessContext") + or raw.get("business_context") + or raw.get("businessContext") + or raw_metadata.get("business_context") + or raw_metadata.get("businessContext") + or {} + ) + if not isinstance(business_context, dict): + business_context = {} + + nested = raw.get("raw") if isinstance(raw.get("raw"), dict) else {} + trace = nested.get("trace") if isinstance(nested.get("trace"), dict) else {} + detail = nested.get("detail") if isinstance(nested.get("detail"), dict) else {} + trace_input = trace.get("input") if isinstance(trace.get("input"), dict) else {} + detail_input = detail.get("input") if isinstance(detail.get("input"), dict) else {} + trace_metadata = trace.get("metadata") if isinstance(trace.get("metadata"), dict) else {} + detail_metadata = detail.get("metadata") if isinstance(detail.get("metadata"), dict) else {} + trace_bc = trace_input.get("business_context") if isinstance(trace_input.get("business_context"), dict) else {} + detail_bc = detail_input.get("business_context") if isinstance(detail_input.get("business_context"), dict) else {} + + return str( + business_context.get("interaction_key") + or business_context.get("ura_call_id") + or metadata.get("ura_call_id") + or metadata.get("uraCallId") + or metadata.get("interaction_key") + or trace_metadata.get("ura_call_id") + or detail_metadata.get("ura_call_id") + or trace_bc.get("interaction_key") + or detail_bc.get("interaction_key") + or message_id + or "" + ) + + +def _session_loop_map(rows) -> dict[str, int]: + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in rows: + session_id = row[1] + raw_json = row[8] + try: + grouped[str(session_id)].append(normalize_raw(raw_json)) + except Exception: + grouped[str(session_id)].append({}) + return {sid: max_consecutive_intent_streak(items) for sid, items in grouped.items()} + + +def export_legacy_txt_gz(repo: EvaluationRepository, run_id: str, agent_id: str) -> Path: + output_dir = settings.path(settings.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + now_brt = datetime.now(ZoneInfo("America/Sao_Paulo")) + path = output_dir / f"AGENTE_{agent_id}_LLM_JUDGE_{now_brt.strftime('%Y%m%d_%H%M%S')}.TXT.GZ" + + with repo.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select SESSION_ID, INFERRED_CSI_SCORE, RESOLUTION, CONVERSATION_PRECISION + from {repo.store.t('EVALUATION_RESULT')} + where RUN_ID = :run_id and JUDGE_TYPE = 'SESSION' + """, {"run_id": run_id}) + session_metrics = { + sid: {"inferredCsiScore": csi, "resolution": res, "conversationPrecision": prec} + for sid, csi, res, prec in cur.fetchall() + } + + cur.execute(f""" + select r.TRACE_ID, r.SESSION_ID, r.JUDGE_SCORE, r.ACCURACY_SCORE, + r.ALUCINATION_SCORE, r.RATIONALE, i.CHANNEL, i.MESSAGE_ID, i.RAW_JSON + from {repo.store.t('EVALUATION_RESULT')} r + left join {repo.store.t('EVALUATION_ITEM')} i on i.ITEM_ID = r.ITEM_ID + where r.RUN_ID = :run_id and r.JUDGE_TYPE = 'TRACE' + order by r.CREATED_AT + """, {"run_id": run_id}) + rows = cur.fetchall() + + loop_by_session = _session_loop_map(rows) + + with gzip.open(path, "wt", encoding="utf-8") as f: + f.write("|;".join(_q(v) for v in HEADER) + "\n") + for trace_id, session_id, judge, accuracy, alucination, rationale, channel, message_id, raw_json in rows: + session = session_metrics.get(session_id, {}) + raw = {} + ura_call_id = "" + channel_id = channel or "" + prompt_length = 0 + try: + raw = normalize_raw(raw_json) + metadata = raw.get("metadata") if isinstance(raw.get("metadata"), dict) else {} + channel_id = ( + metadata.get("channel_id") + or metadata.get("channelId") + or metadata.get("channel") + or channel_id + ) + ura_call_id = extract_ura_call_id(raw, metadata, message_id) + prompt_length = extract_prompt_length(raw) + except Exception as exc: + print(f"[legacy_exporter] metadata extraction failed trace_id={trace_id}: {exc}") + + vals = [ + _metric(judge), + _metric(accuracy), + _metric(alucination), + prompt_length, + loop_by_session.get(str(session_id), 0), + session.get("inferredCsiScore"), + session.get("resolution"), + session.get("conversationPrecision"), + ura_call_id, + channel_id, + session_id, + message_id or trace_id, + ] + f.write("|;".join(_q(v) for v in vals) + "\n") + f.write("|;".join([_q("TOTAL"), _q(len(rows))]) + "\n") + return path diff --git a/evaluator/persistence/__pycache__/oracle_store.cpython-313.pyc b/evaluator/persistence/__pycache__/oracle_store.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9b43ef7e42149522df5c9399655f343512cce826 GIT binary patch literal 15512 zcmd5@Yj7Lab-s(m0w4eq50Vneh~)Aik+5Zokt~^{L`e_{i7$Z^7Lce=5(EMZ5)uh8 zcR@WEd#XI9m0V{eHPP7VWNNDEjO9Ki{?X~k)6OVL-I+}CXAv|8XCt?nN&Cb9AXBMb zr$2h`E_S&f0f|b+ZC2vN-Fwe{o^!r)?qgRgE3F8ge>DElg}1v9`Zak`A4Z8#tA7rW zJBULZHHtnDHX9uIuf0W2!h8TS>T0FC+2y{Lc*m~ zk_Qsno4gvO3T=`f6a3K#QL94`xr3%in*0O`9aP`RZz~F+eyX#O6axwtb;HC>8C5FI z&`Y-?5A8n+U9j^>VG;A;h!9OABy)_9M;7PO9$GS95K>8ri7n1A2of!%N#~hMk-0@) zAbn${?L2vlq*Hj;0*{4+kmi$7epd>^@c6I*>3rA1Rmm166@}$`unS7s;7_QB;2L^l zbKLBEz3&$P=Eb-0d(1xx8QXwp99V@+6xEN7^j)5TzlO*i7$M0GQSE3-o2L+((n_ez zQD-3n?S@dGAWo!q_2HZhAgobnj_OUz_eOGRRW zRN=c4Soyi+$X6qwI_06T%m)_c*b+Dp^{p^vgW))G)Rw(>> zRi1{}=K|r^-_?3)^pLiD-Sk5j1z{8twFkID9%}U*gm=&xgeeYzT2h$a4k~)a$k82; zBgc$djFDp|%y4F)uoT`*@>_-cX2oW?j7@&CW6O-mQz=!A%hJX1>2xAFBYfNdRpTlM zB;%sMW681righ`KV-g(`lKv$t$SwOS>j%Y>9G4?=b9_2Hmx{^?swHS;%-&-@fAHuj z$)1i(gA6ZVKAyPpF7i}JmT)*3ndie{$r=vNr(%nc9hU6j@K+ZjbA=p7I2=!4Aw8E! z^2rorLxK(u!ePvk?zWJijr6%T3~B;>iH0NTG)_z}f=mEaJ0WnBfN&dsNe#$ggLS&Y=G?a+rl<4|_S0o@EsgVIz2EaUSXsx>)XNGCGT)Y0~f!HVQH%Kt6BfuPmelJ_e2A)Z)x6 z8xepqk?KmP!jQyAVx69y?0h60omIOwA4w$HxYjcveRYoSvPKIU1cOD000F#Ht4*Nq zchIDBPEIDlltve#0!A%ai>$|djsiLj&or_Pwn3&Vfp#0~n<79YKOu?(!@IZ8qoQ;$ z#3j>>9HXXhsF?{=sx`0us?ro7;I{7JOezj@=3Y|>&1wAylqX8F?gtzz&}rgUqVu~ zckoE~ly_{*7YvV$4|s#aX)dn$AlXp zhMP#xOahh!8cZ$iA|*qdFR+s(F||fEl8ng-gPfme1dI8vE`oZ+JXMlKC?K?qUXn$v zL9**>kj&%_ic97h9)K{7C1!B}oC(PUFp3ik5*>}ttm~a*4=g5$x8%b(g#m&S$wWFV zL}&T=2yt5kelCt*h7m~42w;XjTOb02eB{x#w&fwQYv4!0 z?6#5Omxh+*PI2eKA5~-kbB zyc0QVZeF=@<>qTQUt1o0XXx%wu6-~3zaM-s`TpdCGY`%@ayN<16Iu5O(SG9JY+qWj zJ9GBNjJ@#}_U1f|?Av7gkHSc_hs*=~QL7^mdA38j<~o)F6jg}Z(Yuu2W5fh=Bs!V8 zjLmXvF&$lR(gsO}LqF?4-Uo?7Edk?@eD(ELzd!W$*!RY=4v%Q@D1*>EJRs8s z5rh}w69(XS{ZqCr1^QE92+0leqB8GL{1FJl9nYxHax$Y@LS;B0T zp)6s7I8zrLG7M7ts3hHlLi8nchhpJ3K(t`pqlzU9%M_q^hIxR+iDrgiaRP12^=hVP zt@5>cd2QL!^2`?01U5e zIt9rZ=fPnD7kye#bdCjD*%V3Uf~&T!HpvuC&Ce&&xDF^K3kGL!IugAoG0{12&j5~y z2IAu+({%ANICg?$BsB>}NCI1+EUEAWA*j}?ZlO6ZWC*{3;A2qXUt6}WbnLut$yw?% zmim9Sw6D0TubWn2729*8CueJe|0``B@ATa5$+hjzwC%rs^wxolon7hN`9sUQmR#rI zOy}Y2N5$&)jD34?sge|qtvN^Y&mGO8A#TzdzIO7lV_z;F(PD zOfGmn1OFWt#n=T1uN(6Qb45KM&g&y!4sMR#82$F~ycJctZjRg-`S#dnPb!eJeia!i z+$8(>jpJ{Ouh?CA1`>Xow`g?#BfJ13|Hi=mXjX@egXY4R+HXbkl=- z&AJuNl)$aIt?T+QNSlfRu_Ve2Q_u`q_{+y#aYD|{4IsIsETYirlfg&BeLlxUZtUeeYu zDIEryaAGD2i#vFS4I@En(@%dHqDiPr7=qv$${S2XWF17J-=cqE-}c!fN8KtiR=6Lz zYi~ur(|LVh#nJFqH0x+to_cQ}yZx2>`+w4w-GBU%a|;OgZ}SyE`nf<>(%*2^pzm6} zCi*WLy%hZrQ6!`oNQe@KsSh!NGV}fcBH(3|WF4>o1%PV(v&K?@Y>jzqR4Vv#hAxA; zXT_o^V)Eq&TB}gwYrk44npC-AHKmCLNB4aTfyNL9|G+A9qY!|POjg0s6nN5|FL;Xb zSXiQ_=2^0x5{7klIGUPUoKFhHG?!<2aQKy765<50{cPXbTQMywFBFt&WLmPwHiK-P7QjAC^5Z0!fuLCFrK-Y!Q~_)# z62L-}if~3qX#!s0yIE{EZ8vOr#MIgWM;lw-KKi|*%YE5xUH8A1J2a6wG?6_N$W(Gq zY)nOndSXY;>RWr>JaGHKcMdMMWUAY9)t-A)*73r<6B$RBXz5biJiSC%pdDGx{t_ax zH(LTCCf%U;1vyilwO9{82USY?WU17*0X36nO)9|?Hh^Sm@}}gIP?a&D02l#ZSzB=z zR-YhK_Q)K-fg%O$62nGfF;=2jV8aEdVI8juV1bL71EwD!A$@g$Uym47;ljqW!V;;N zPepm8HQ@vVz$Zs_&e582v@Y*?Z#3sPoN*jh(PHOv`khO6FWtM4?dTPE^xfb7BU|q9 ziOk^>*~6T8C@2O);>mMjC@j{U|AjpQ3n!jK_nlHrWHjT}a;lUSArfS_za7%y`i45@5GAiicpTF_2yYa|9U5jU_Q*@8hdpqY@1%DuB=1s$13oS^7L?q&q@%9PVL|B@|+Qwyq`HozX95RJrmfIrxd$FM1v7TASFJi}{d@a5gG zDI&XsCa`Tn3hRWK_?e3NGb!fJBx9CL!TJTZtHauYy(ueZ=Shv=8bc2rO8V2#oI4_=cVfdzpQV*1>5zkvuN0(o`7yDuS-S~T7RdU1JZAw`&=Na@;h6OF!bNHdmD`pokoa0Y_uLR zm>+gHy#3~f`%EO|07c^c45a_qKtsxp8S9a1bI~ytSu4p9XM#y72XU%KfX>?#%Uf_K~oAI7Oag*9u7UFCcVxvWm?l96IvrH&1dz-zH(9wMH<9F&>D>r_9uU)^__}#kJ zs>_VOMn|)m@z<8AXN#_$tz~HRBdgnE+XQFJ$x*$owQ}>QAAQ5`HIHA?wYIIy*cx>- zo0-SW=}|e!Zql{ZT&72?j^??0#I=B>2P?7&=(F%Lv{1su2jnqKzP#6{fF4+KSh2{K zycNpAE{;+bJcV+ljYXxwzRRWCHr2AbsI1y@DIdbBf-?9J&{tyfkAowot}Hj<6}0=8 znzRLgx@madCykPNVXe9Cy7y`C)10^W|7Ct;Y$X<%z=xhYkWa34<0c?=2{ zVl6tXDk-M~N<{^GDd z2D@Ygyx2}V5zczq={sVzXBZ@o-EN3Bb!RHOErZ$ zcd~1P_uxe$L=V7FmBC(uB1I^=De!%MkqHL81HPh$cJJL+hIt$)I-G)UlDZ&To1}go zvoSE__4|EeYl04QQyO@lAH%&tnD~j{7mJ4h@2QPiq$PgQ8uUC^SbAFZI_Ok&P?lH_ zK*^~pgPO9mO;Z7dA;qeIi7GS`FQ<$fB5!9D$!-&f&H)Xu@Jfc^VDe=826v_1vsVCa1m%=n1JO|iv-8V$Kd`C zuzWM!7>7Rik8Orh?t^#iX)YOfRA9!TzNfft%06NVq*4@8_5;3Xu+l?~TZNiJ$B41_ zs201lZ0d_wuPaWpeW`DQr7!6$6p*3VTj=amlQ&dS>@G-d6%utlcWI!g)`HUn34c)D zvs89$|FDp-(5L4Hp+Y&bYyEV185}Lus;y#OGnELu9l#Ti&TL$YhbO8N7; za@!r`Pe6^6FC5jai;l(QMYu$m3}0A` z&F~%Cf^w}i@g$b)T4|6T$YW%?Et!;_K!Q!@C*a$pw+5-I*e6hvT>uKj_Fv7ZE@A7{ zJoiSsu4Y-+Y`B`VrL?_Pvz_a0vzk#E(A!}(wIZO}Tzo?rT7_m|)f5m4axNzeu#%=6 zh2bku0!K+8AHb(bj2wokC%bFsUo@QAN;I76t~=Dl{Xj0&=?}g1*K@xPxx(MWMjxEx z!cC=(hh4uSH(NV1_Rf3Nc|`4LT4`#1JN~`6xP5Q7X`k5mN*+0TD)Y2!m-7j7Rl4#! z;38A$0*A9rUa|2=9+?LyHeZEWw!dS#YZ5#9v+O~!`4H58nCe~W?SHW4{Vn3*iEQr) zu_us6aI#dHueOo;ZBT!muKxEb@5dg*-;ayFsqDcsV*gq3Y($)n=cZ>f)3bSmE>J$h zDtv)K9W}t)GtZV zD&NDNY*Vk;*q29J_p|vft%v)w>;bX4A9{GOL=P#BVprI9W$q7T*?zJ4AdnvFTj@La zV8{DAL~kJ5$BDf`nEDV*{VR=RFpV&n10|>aboSs^#QrI9DlDFl<<7@5=i}ncQuh4U z#PF*yiq|N=JeYBLFcX^?j0Fbc+~iJ>nYOP5-p5dbGv((|xTF|5=N5beH*OFVZC5 zMUnU}IlkLEhRkr0n%pmtuj|9-lJGquK13kGIGsw(0q~kIpI*dC`ARulD;XlRSfcmqDK#ZAhsADMz&{n63hg;zM!mv05F%O{8JN?mTeoy2#Lpo0W(9|K9|a5$EV z!gpDR$(vpIVr!mH&!%E{kWk5R2pa=PzA}KxwL%3a5_@8znBKeznQ9&zTueR5sL7Mq zCp#$S;1iRXaX)dJnO#rZ24ykVcC7 z2(|tiRs9m#K0@0+LM{J}IzB>MV5+D=>WRfdZOv~%2HU3w6V;kW5WuV%Y;dEHYW>uG OiKd3APmug0xAwn0>C#33 literal 0 HcmV?d00001 diff --git a/evaluator/persistence/__pycache__/repository.cpython-313.pyc b/evaluator/persistence/__pycache__/repository.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..038f9c1adb7ceaf2274b7647c6c9775258476c7f GIT binary patch literal 27646 zcmd6Q32+-%nqD{VlK>C!CJ3G)cz}|qlcHsV1SRtVDKw$6MNJTx1Z8R{P;P*-#hRL( z+1lEYE3<2Rj#yedD{ADd$l0xuoSnoA0i8YOy zS@WocwTxO>D=E{Dm5kb0+o+wjkCw8fB+oEbHd@Y>yQpfanPQB!6l2<};zz?)oKm^X zcQE0mIdeD~U5ba}3ro?MAJT@oa6A%UI1_=m+8aHOS+Phwz7RbXb8ES>p{3}#2pb~} z0+HDzc8)WR#3N@I=wms?83W5vA!?sy!?Tgm<+)RlKqR)j80X5ENGt|7lI6M8oOu%J zEk>C55*y*nq0_OYXlQQv%-NWjSX>Iv#b6x97ua|x8eTXTndP5^57qn+eZQ5)Nj^!j zG(*9}(yW?kW?C3kEp=LRnr1ajE7Qhkgp`)i)>5pFX=ipYIw4Qb=poO*bT9_YVU0L( ztch_mCXzNYW=LC@PR2sgR>lhH5~hnOA!!?9gS4ILX6z(g%9KL7jOk&@NV=TyFy$mx z!BjwrgXv`)Bwfi=Lb?h@xRa^E92jpkQw@1F%r2&e)Tw1^AzjDpX8M>qp-w$h4|xrc zw})vE@|=tl@*0`F%s!@3$a66+$ZKMn*ydAe_kONo_*{5#S$K(OmtqSr`RDIgVJ2sV zRYE3J3n5G|eZLbTKcr@0g86GksuiRe`k*REElN43D7Ok%l0Cc}UkWWm7viDV?2C~z zVYiyKKyy|H0jG`O`*y21O)RoF&sIQQ47+f-xRwY%xFxptEG)H!SUeJ)jr6h+aiE^F z=Q%@&?>fXaC_RfuEH1&0OCk6+wc&8RQ=J$XPdX+pX#T)-A2KQbEIy-nVP*2Xx5-jc zdftHWO^Z$Nx0%{2B=-t46x9Z)06u5l6)@&yKvHZQ)fH)DF5X5(MfE{_0AF0*CH5My zXX9cl zsKbAZ#$y!e6`^N9FSF^Owv}q4x~UGK(HzYByvjv2QZD$1rBA6Fsd?J1^?%V|a^+nM z(O86yyB4DHB^OsReB3)8^iGdV`a^-BKYR|lDrnIGC7}^tx66BU*gs8TQ^SFgNnePW z_6DZAT|#Qu4|SNyU|?t%ey6?DL8jX^Jvr?i4~vr{rPer0}J_;Y%!qQwQ77w#=C=rs8C{~LtEwi%`j1c;+@C3r_?(B9+|MkBZ zfv#SNgyW$w_BhKzG&090md_%>U?O*zZX>7TN5>h&!EqM(*f>*mXq=888T|6VW0BvJ zGi7hdnPhjga_U%|u z$0HmaXYo5?@$=>MupIm&{-d0hOsm_#;@W0i7&Kwfj6ov?P7IKEavJPAW`L-;9N8+& z7e*R_N9P(9DH;DLSOGuJ*iRt4Pv(v-9`ZBq1(hD<}#_LKd?z@pR#VMbuISI5~{8#Q{TGba<6r#T>CeiJJw7oXJ4k)wNcxWuI)F{D@TmwR^) z+0-A{OhZonEPQs-g%tTUL>(3)E$~JEkOF!o6=x#>33IY}%~)4~M)%S|`WU@mx04F0 zeDpqg1SrajKa*R!9#LGRVPtKCoMfEdCB0kMAulUj7@ zaMXF12pIq>+MqTd2S%Z$SRP?6g>?TtbQTqmZ5h89aq_`V6qxd~7Zf z4=*gn2t%Digmb&&sD;0S#o*QwVugGeH0Sg}Kb%G04ci6}>(-NxgYAMi+l@gF2Dpq_ zoNYqy+&Z2p5(4-P=6WIE^g?g!E=a|YLBgjIKOP%P1d7mEcIgxg&%w1Sen5G(a2&Be zfB-(BN2hwjsfHto`lFka`dJ#to(9OC3CUiKGgIGkd194a+q>pV)a}}Eb*&w}K5>Ih zx%xBC=F6uqoxU8s6iqojcg;GWc}9~!^U~I)Us{{Kh~YQ-mbZVVVFwVraor}pwXb9YFOgtHHAUKNLF5E7s6gtZ zUUflLIrZ#K3b=4571Sw{x*~U$h#Egq3>mdZ%z)R{^r>eg;x&Vk3AqAxF_!%;ls|$r zNCjot8e$A`C5~LoqbSH#;Lagei9go|YO^iH*bGuZas}{<&$|MyybMS}?=6M7;>-p$ zK@Fqzn6udu+ZF+=u^7mx!(>`Zcz(o)x)}|9s)kbo+98n0T(e8d(RfFv+cnQFopG)Fq3h)rBW%RQ zuOnzL2)0VSxD_WqD=lN|>~t3Pu#G#pRlKB!6XY)E*&~$t6nUf_Sb&@^>&9RL13w0n z7!a{>3S%fMa{4JyDuArXBA9ZvGfQ)kMWP)gtAaBQzcL#+i^^NKG0SjqdLFf7oPqyZ zBXfi)BU~4!2H_S&ZC-b)j=VA(L#Y<$nq0gbW8oWya1F({CPmPfS3%g0eFxSCFk$5t z7Yy5wN6RYT?0Tc?`#l$i{!4As>$*)JA2PIVNj(6hMR(`Z{^_!`dJ(=c?Ol#-0$t#nY_U?51{$%_9 zY(yw%abG)p*d-3|ZUt+LcsKCqTKsy5px zN5d-p#>DrYz1v0AHE-^wtVE245jIq3s_QSCE}1S{E?E+7dsEf>5>@**Db<0VPkMG; zKe2w|`pNZ^iT<%v&v>GHVw2MNfy316Yf!LmXnC*m-Ol$s?|M?Z2a}~k4{UntF#Xwn z6q=9Fl)d_b?H?XgQPoZNDHTjhRr8;J`Qo!(f^03c z`XYv0+PdJo1s>HJmM=uES#hoA)R}{higTxjlAVVVj)|LdA3G*)&4aQyZJS6KCX%)Z z1@07WKY;V}eILN)V+z~qpn5A|8+cdES1)kS;K<1pKuIO<3P}8Bwm{ZGfd^G=A#tFJ zT&D`OyfOhCo>GAeEvJraW2#;-4gja^ps1xFqeZ4YP?H^*7!$e2Db)n%*36WPit0d> zp$XK9HL|~AUFcsg^$)G0`WeJ6a?KzyCf9SPL~c#wy5Jr#0=!2|6@?!1dAL@{CuQ1s zF{YoVit}KSN=?{uSOqYlTx%Op z%lAgYaCA->3^1jj+dc~_YcNv+cQg-*kH}oEo+Gbe4v**>0L4l870^io+?|cblt%}) z&Is#z;k-=iUQp33FedEM%hBaCt^wdJUI@o$Ukt?-Rw6mIMTLuA7&fQlN6lH}BjHTh z=7ckM+ly;okYiw{F(BN6j<+Qcdm9^pY|cCviOsSLgkj*ci%Ty@Sk5pXf$@bG7dg$` z0_eWgD+^~ul{V4d%wvbBuVJ6ZKIl2rWiQ4!H9Vt&LfcOI3q>QZfO&y%56UWo{A#D5 z^Vla4D5`O?%tFRuLp?UYX+!;{fdUQo=%vxiq*x2r0e>Ub$uJPt#Z}%hN~m(+M9Ik%{V)FO^yS!*PG2$m*;x#k9$+){Xgzq zJ@$I|4d0uiZ;bxS@%7#dzC?L{X3zfj#@`+Pi9c07^l$uY%ny8TmtUNI%lXTSj(7dZ z@}bY%0Lw{h)93fvsft6Nb3)GFpk@d;!?y;@e5LB!J@r1l?gL{5#6M_o`YgH+4yhpi zq27dHsma&)gk$nqcoqdrE)D0uUanmkm-Fj=3py;0%}*ygpH4WA-D>^VaqOe^q~loH zb}V5ymb4ucuvu;qz|VnTvmXe~V`8%^sM-pfziM2TVeB?hxro*RwPtyjh)FZJ;P%J;Z#NOO2m$<)*t_yA)+Oywjt~ zaRNNzm<0JaXYgR+LCKehx-2MBD3Ggw>rj^Dh=gVmOt_+u#;d=1OA+aT*LA}}q+qzb zbX9RC%zlGVN7QxCqWU0uigPP|Axc%h1D(b2OHrhig@(RdNo8NEdViwo0LW4Yc|#wF zQa~eXdy}=j>Ds-?TA+n;1$U;d`L14rc0FYs_wK<;_3cX2VCxf3tIp$lEYxv3!b(%j zbt=9M#kyduq+$rUirVg>Wam)A5xfQRMDQb!CxU5PFkuKLZ9xTjLZVqKk4dwNYR9+M zbQh5^M4CYif(pJXMU8$++HJ`%g!d$wktm#!?4YFQ4yuDFGstKcP(R5R1?FG4o;wy) z@5K)euCIc=y`DoNkm|toNc8P1rc&(7@MLb&LljMfJQ;^3C#J@S!AXQ?3LPOfG7rYJ z7tx_0MAT~FW0hmBL$a6E7ZI+nz*JYB{bpvD&YWG0fJ+D0q4>!t)chXYA$Ac0pkEJn zTu|Ws@|7pG(eE>U{3fvSlS?;U|#R~^DUy|a1D?^r}`{|dac*B9fWFRD1a**4Kuv|76T*d&6A+vJo zCuxZXc(d8SGdgkn0pv6C zu7IRn&KAg8NSt4`RH+5n{Pl9$2cG2y|4Ky;ZOSn=PURE{Oj@3cEzlxG{i&j%k(Y1( zIKNx+c`GX`T{5jCP(2BD^SdatO3FQdA8C*)*YZ2x{`mMMp5MN3aNA>Gm8V4N9cVFX z6gi_bE9@Mk2~h^x+JkW-Y!hEHTvqJwV1S|(A;r)EA;74J%vxMy_^II|6XBXr>LSvc zBJT_CBvcGJQXsPi^K0f;&0yTCsmN4RqsH&e@i)d3O}kPRyA$PoVBFi|{>0sLy>`9! zdi{ERV&Bn}``JXt2xucmX*Z}NJ3R_z+ELoLX|p&io3&I`!$wU@x~3;t)03|0OV;#l zRJziYJCc<<(v>~Q%AQPZ>s_O^#Ppk*QlptgSDPH?PW0X(eAG|i%a!lQ0Dl?S_9F5C z1^k36s?F|ZfV%1}>S_Xl>;(6cE=dy9<@BS?_5+p^iK@XJbg~jdPKjPtV$3NqWJ?O3 zz!*gj9XVAM#-^l$6Gsa^T;`xz$uCQ+p~%q{yOZRPB~1&(m}L~z0zO;7bMV)UU5qJt zVk!D;ffKBvrA)148H2h?{#jbsKLL9}f`*_mXxiHLRb)OT`!)FNz@6b>XT%n#1iRtm+(1@SkmcnNh4AvDO z#olq~OBnZ~_zjaCd3i2Pfw&nyfrC|0sN-BzMqUX~EdoOW4*d8x!w4ZR7N(z%M%vn^3gNXJJ1}aqV(ZKOr)} zAPx{E54R;>C^><*1_vLU4pw>)_HE2Ih$DxMpV@DRjF#6^BWosp5BSR~tV?6zbCD3g zD)^3z*mDZ%{WbI#Ylp85_utiDJidDJ#-5bpKpJ*7OkOZ;R5o0kUtPT6OI03BSNd*# z4f0{DLkf22nKtZI7j>(awYikN?}B!ty!PV0)u*9m`N0bY(k{_{;AU&8^63lajf%R9 zM^?vfRHQ2Qr7H$*HeWD4hAa3}sfNRe`lmN3{WItaZfLPTD5IQB?^Il?{ZZ}J`c#7l zcD{fTDO2BkdGgX^#@U*7_9mUZQn~|di0C-p*qUkX%(U&yG_+?L+tQ7DlZ|^*jr()8 z>USIMAXb%D=edm|#M~|$^b8x-KQo$!tDbNv3dK_eDXJ`7un@Q%irT@#ikrA%ifYrJ zNp?Pya0G7c`PdQoXkXG1NZSGlLm+7j2wEq(o?G%ApMT=Gr2_s&2Rk_*hrcld4Uf#< z6uXlWebZ9I#ySE|Iq@GM`m>)gdXi z0RG1ErTH5|b2*y`oC0gZqlsSdbLJ^7YN-f|!u}D&!hf_~rbO;6t;m50+y!Ac3K4b0 ze+1NXux)gR84)=SU?C>QgSZbCb{4=U2pnh?ADT zN3|GNQ8*9*y!vNj3p`V>w785+0wm6Y)9V5rK!630%sFuOh3Gs0c4TfSw!jdkzyw$* zbk8|pgXcN%WP~a4PJqsD;Y%i-G{nm{kG)XCIN9q^g)>US;}^#&wrhg``nZDeA7OyY zeLHT&L99pk1DRtm?`KR#a23n5%mK11JOiNM3|vl#x%g@X{8u*iDAc?UUB}SpE1v_H zNDLs%=db`rZa$xOOn}p6%7N^`SBvX9g6n#~b$cPM19aE7B7i$PHmaN#&n0#oytyY; z^~{E|ZFPQa@un~3^lmh?t{zYH_-@Um8pgIp_ZlmpyRAY2-QVvT^!S?9A2yqOJD+fL z|8rawg*ws-9<=ioL=n4^!ip$?#lZeP_FbSUmFz@&pw46Z?4T`M&IQ1Y@DgdflD?p* z9UI4!cjf9{`@klxth)sC$!m$ptD4T2K9Y-1yee6)rOM+qqJI?bz9 z3hOh59wm?Ef+KAf&>rAjaL8W!%iWi{QGUqvzNvMALQt^4kX`k18}(b>wn3x%w$U{B zq)Q6_9$!KsEwS_V7TNSG8^`jixsZm8V4sAB$n6-Yfg?K71mXINe1}`;I6-Up5Whe0 zvQdAvbfm~AA4Y$QS;ZgL4sAa$P5GCSB*Exp8cviP@*6?=q3sNPX2uw#chXtz8fH+z znl4gl*!4imb~|bq6MnV@QjTTWf`BhLPAWGne-6MSNREJEvEACl*?vZa(@IJ{Vin-;2aU)&~*?Mp9j)iB8;VI8JwEzHG@@s%t#a z37ht(9JhUpv3{($&Kou9j8r zYWRxp+Q^#sYDr$@On1*(c-@!m9(X&*ck=ekt=79%eMO1r-Ui%*a5#l;_i&Z^XH}-* zwkKQ+`72n)3amp(IA=w0EkzWJ@}+eUmrYUS;y|)nbmnG;^KUc;WW?@P^&{o(IsEwNWdu+8G(0@QUkiq6>Z^MYm-Y@n8jC}~f%C~b60MS{1`R!~G z-0rzU1@|Ca0KW&3J&eQnzH*g^_<``<2%*BRIu~wJE9hBaYaQ;C!(C`Ob%o9NoIxz* zG z)N+I{;hIv^$eqRlUP#5pBxNv46&gpsD zcAnD`+=k70FeLdlyckEHVb5S!^ep>J_$8;R8EGgOw`E1_C; zy)pW__hQqgnQ}H}8Ua8%fAaFpmu|JE_m3p^kF4#wzHfcspB?zjDlKEUZm z5oZDOG}1Qn?*g`y1C9vdgR2z&<_NHQ%n=2~oDze)1Vl@t&<7EZv$zQ*%0jSK>Q!=7 zg1X8=uyS2xAvjxCLjHWgH|IHQ^5#$L zju45~jO_}cg?rWyrt13>b%)^m!2#H_&L0j_-f%IJC~ZZ&Yv1gqY8x)sU#frGdB4F{ zQF5=Is_l5)yIDgux>o5+a~W6DYSR_XM(d6>daY?)b9IE^+Zugk?y5WEY$SE~pL|Jj zN9WqEwPWjht{=Ry>*}|p94-cS#f2OeX>WduKo2_sj*MoE-LvjVI7V-R6U*o=aAFxv+eQ7}5my;Fn+;{1Uu^U!t0YUxGaF zYexbl!z3+JnuAGY9xGtdu!Ko)3{6o?fxV^c}N%A*=WFTV_UInct>F5(ti z>_|6xm2V;e`@hi1_Q9Aai>loT7}r3xw61!ufHx)RJmIIodGV!7(;3JF?P#VQ5bw%d#tkU8UXkhS zTJx?;H9fsI==Hfw6KGPeR3Jn)U(sSOgIC(*XjWGOXy$~Q-NoC=&`eCp(5!06s(ydB z*ED3%-5yXu;sb*gL#t`1?g>Y;f56pIXnM5@U?r>wWu<#g9duA*!`K@dUEMS>2e|1Y zpmIT!;xsQTEiLkg3!7LM*3)3nB*mIBMz}I$5xy|NVf?Ody8LK10EgxHG8-kO(BIQg z?gxVdahxJ8o{`#mr5IoYwPMC|i-c|jYX^UiuVHC5uIZT}nR8Zv9DVq87S}dQxI^AV z!Z(^1b4mFZCej@5@O2o~I2Ph)gsn_`lWeZMBPo}6Js_=wb0SGx{%qO0=)BG9F}R9> z3GXGZNr(it=TD0KKT=I3Q8`af*l1+9`TJwdoKcg<{|4sLcku1yH2%-n4(EoSB~ner z5;8+(jAm2$O{7A;2?cA#-GYCo1^0P`LYyfSg5xWf7cp)Lg#o)Ubd>ey5!C)PO zKgZzTVL<#5{sLovg~4BA@CgRL#^CQUxQhY#c>WP$oFha$L>6Z8oG^Cb1^B}du~2B8 zBD=H*jb%Y2Op(9}k=TzR_>@40Ag5TbU7w(=FqwB`pLF6hVytr;k*f%rbG_NmSaQ77Mj zg|_5gH?5t7+c9)d-ftCDsG@HotJn1KvgCk45o+=Eor~XCoeRfN6|h;0ElHEKpqK zEG}TKIa?p}uHxVR@hp))5e3r^TSWLjB7_snD_1MBP4Mqlg}~c$F~ str: + return json.dumps(value or {}, ensure_ascii=False, default=str) + + +def _json_loads(value: str | bytes | None, default: Any): + if value is None: + return default + if isinstance(value, bytes): + value = value.decode("utf-8") + try: + return json.loads(value) + except Exception: + return default + + +@dataclass +class OracleSettings: + user: str + password: str + dsn: str + wallet_location: str | None = None + wallet_password: str | None = None + table_prefix: str = "AGENTFW" + + +class OracleStore: + """Oracle Autonomous Database store following the Agent Framework pattern. + + Uses direct oracledb.connect() with wallet arguments. Synchronous DB calls are + exposed through asyncio.to_thread(), matching the main framework style. + """ + + def __init__(self, settings, auto_init_schema: bool = False): + self.settings = settings + self.cfg = OracleSettings( + user=settings.ADB_USER or "", + password=settings.ADB_PASSWORD or "", + dsn=settings.ADB_DSN or "", + wallet_location=getattr(settings, "ADB_WALLET_LOCATION", None), + wallet_password=getattr(settings, "ADB_WALLET_PASSWORD", None), + table_prefix=(getattr(settings, "ADB_TABLE_PREFIX", "AGENTFW") or "AGENTFW").upper().rstrip("_"), + ) + if not self.cfg.user or not self.cfg.password or not self.cfg.dsn: + raise RuntimeError("ADB_USER, ADB_PASSWORD and ADB_DSN are required") + if auto_init_schema: + self._init_schema() + + @staticmethod + def now() -> datetime: + return datetime.now(timezone.utc) + + def t(self, name: str) -> str: + return f"{self.cfg.table_prefix}_{name}".upper() + + @contextmanager + def connect(self): + import oracledb + oracledb.defaults.fetch_lobs = False + kwargs = {} + if self.cfg.wallet_location: + kwargs["config_dir"] = self.cfg.wallet_location + kwargs["wallet_location"] = self.cfg.wallet_location + if self.cfg.wallet_password: + kwargs["wallet_password"] = self.cfg.wallet_password + conn = oracledb.connect(user=self.cfg.user, password=self.cfg.password, dsn=self.cfg.dsn, **kwargs) + try: + yield conn + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + async def to_thread(self, func, *args, **kwargs): + return await asyncio.to_thread(func, *args, **kwargs) + + def _exec_ddl_ignore_exists(self, cur, ddl: str): + try: + cur.execute(ddl) + except Exception as exc: + msg = str(exc) + if "ORA-00955" in msg or "ORA-01408" in msg or "ORA-02275" in msg: + return + raise + + def _column_exists(self, cur, table_name: str, column_name: str) -> bool: + cur.execute(""" + select count(*) + from user_tab_columns + where table_name = :table_name + and column_name = :column_name + """, {"table_name": self.t(table_name), "column_name": column_name.upper()}) + return int(cur.fetchone()[0] or 0) > 0 + + def _ensure_column(self, cur, table_name: str, column_name: str, ddl_type: str): + if not self._column_exists(cur, table_name, column_name): + cur.execute(f"alter table {self.t(table_name)} add {column_name.upper()} {ddl_type}") + + def drop_schema(self): + tables = [ + "EVALUATION_RESULT", + "EVALUATION_FINDING", + "EVALUATION_METRIC", + "EVALUATION_ITEM", + "EVALUATION_PROGRESS_EVENT", + "EVALUATION_RUN", + ] + with self.connect() as conn: + cur = conn.cursor() + for table in tables: + try: + cur.execute(f"drop table {self.t(table)} cascade constraints purge") + except Exception as exc: + if "ORA-00942" in str(exc): + continue + raise + + def _init_schema(self): + with self.connect() as conn: + cur = conn.cursor() + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_RUN')} ( + RUN_ID varchar2(64) primary key, + AGENT_ID varchar2(128), + SOURCE varchar2(64), + PERIOD_START timestamp with time zone, + PERIOD_END timestamp with time zone, + STATUS varchar2(32) not null, + TOTAL_ITEMS number default 0 not null, + PROCESSED_ITEMS number default 0 not null, + FAILED_ITEMS number default 0 not null, + RETRY_COUNT number default 0 not null, + ERROR_MESSAGE clob, + LAST_HEARTBEAT_AT timestamp with time zone, + CREATED_AT timestamp with time zone not null, + UPDATED_AT timestamp with time zone not null + ) + """) + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_PROGRESS_EVENT')} ( + ID number generated always as identity primary key, + RUN_ID varchar2(64) not null, + STAGE varchar2(128) not null, + MESSAGE varchar2(1000), + DETAILS_JSON clob check (DETAILS_JSON is json), + CREATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_PROGRESS_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID) + ) + """) + self._exec_ddl_ignore_exists(cur, f"create index {self.t('IX_EVAL_PROGRESS_RUN')} on {self.t('EVALUATION_PROGRESS_EVENT')}(RUN_ID, CREATED_AT)") + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_ITEM')} ( + ITEM_ID varchar2(64) primary key, + RUN_ID varchar2(64) not null, + TRACE_ID varchar2(256), + SESSION_ID varchar2(256), + MESSAGE_ID varchar2(256), + AGENT_ID varchar2(128), + CHANNEL varchar2(64), + STATUS varchar2(32) not null, + ATTEMPT_COUNT number default 0 not null, + ERROR_MESSAGE clob, + RAW_JSON clob check (RAW_JSON is json), + CREATED_AT timestamp with time zone not null, + UPDATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_ITEM_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID) + ) + """) + self._exec_ddl_ignore_exists(cur, f"create index {self.t('IX_EVAL_ITEM_RUN')} on {self.t('EVALUATION_ITEM')}(RUN_ID, STATUS, CREATED_AT)") + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_RESULT')} ( + RESULT_ID varchar2(64) primary key, + RUN_ID varchar2(64) not null, + ITEM_ID varchar2(64), + TRACE_ID varchar2(256), + SESSION_ID varchar2(256), + AGENT_ID varchar2(128), + JUDGE_NAME varchar2(128) not null, + JUDGE_TYPE varchar2(32), + SCORE number, + JUDGE_SCORE number, + ACCURACY_SCORE number, + ALUCINATION_SCORE number, + HALLUCINATION_SCORE number, + INFERRED_CSI_SCORE number, + RESOLUTION number, + CONVERSATION_PRECISION number, + TOOL_USAGE_SCORE number, + ROUTING_SCORE number, + RATIONALE clob, + REASONING clob, + RESULT_JSON clob check (RESULT_JSON is json), + CREATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_RESULT_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID), + constraint {self.t('FK_EVAL_RESULT_ITEM')} + foreign key (ITEM_ID) references {self.t('EVALUATION_ITEM')}(ITEM_ID) + ) + """) + self._exec_ddl_ignore_exists(cur, f"create index {self.t('IX_EVAL_RESULT_RUN')} on {self.t('EVALUATION_RESULT')}(RUN_ID, ITEM_ID)") + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_METRIC')} ( + METRIC_ID varchar2(64) primary key, + RUN_ID varchar2(64) not null, + ITEM_ID varchar2(64), + METRIC_NAME varchar2(128) not null, + METRIC_VALUE number, + DIMENSIONS_JSON clob check (DIMENSIONS_JSON is json), + CREATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_METRIC_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID) + ) + """) + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_FINDING')} ( + FINDING_ID varchar2(64) primary key, + RUN_ID varchar2(64) not null, + ITEM_ID varchar2(64), + SEVERITY varchar2(32), + CATEGORY varchar2(128), + TITLE varchar2(512), + DESCRIPTION clob, + EVIDENCE_JSON clob check (EVIDENCE_JSON is json), + CREATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_FINDING_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID) + ) + """) + + # Non-destructive compatibility for older generated schemas. + for col, typ in [ + ("RETRY_COUNT", "number default 0"), + ("ERROR_MESSAGE", "clob"), + ("LAST_HEARTBEAT_AT", "timestamp with time zone"), + ("UPDATED_AT", "timestamp with time zone"), + ]: + self._ensure_column(cur, "EVALUATION_RUN", col, typ) + for col, typ in [ + ("ID", "number generated always as identity"), + ]: + # Identity column cannot always be added cleanly; ignore if table is old without ID. + try: + self._ensure_column(cur, "EVALUATION_PROGRESS_EVENT", col, typ) + except Exception: + pass + for col, typ in [ + ("JUDGE_NAME", "varchar2(128) default 'unknown_judge' not null"), + ("JUDGE_TYPE", "varchar2(32)"), + ("SCORE", "number"), + ("JUDGE_SCORE", "number"), + ("ACCURACY_SCORE", "number"), + ("ALUCINATION_SCORE", "number"), + ("HALLUCINATION_SCORE", "number"), + ("INFERRED_CSI_SCORE", "number"), + ("RESOLUTION", "number"), + ("CONVERSATION_PRECISION", "number"), + ("TOOL_USAGE_SCORE", "number"), + ("ROUTING_SCORE", "number"), + ("RATIONALE", "clob"), + ("REASONING", "clob"), + ("RESULT_JSON", "clob"), + ]: + self._ensure_column(cur, "EVALUATION_RESULT", col, typ) diff --git a/evaluator/persistence/repository.py b/evaluator/persistence/repository.py new file mode 100644 index 0000000..5fbacca --- /dev/null +++ b/evaluator/persistence/repository.py @@ -0,0 +1,411 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from evaluator.config.settings import settings +from evaluator.core.models import ConversationRecord, ItemStatus, RunStatus, TraceJudgeResult, SessionJudgeResult +from evaluator.persistence.oracle_store import OracleStore, _json_dumps, _json_loads +from evaluator.time_utils import brt_naive + + +class EvaluationRepository: + def __init__(self, auto_init_schema: bool = False): + self.store = OracleStore(settings, auto_init_schema=auto_init_schema) + + def create_run(self, period_start: datetime, period_end: datetime, source: str, agent_id: str | None = None) -> str: + run_id = str(uuid.uuid4()) + now = self.store.now() + with self.store.connect() as conn: + conn.cursor().execute(f""" + insert into {self.store.t('EVALUATION_RUN')} + (RUN_ID, AGENT_ID, PERIOD_START, PERIOD_END, SOURCE, STATUS, TOTAL_ITEMS, + PROCESSED_ITEMS, FAILED_ITEMS, RETRY_COUNT, LAST_HEARTBEAT_AT, CREATED_AT, UPDATED_AT) + values (:run_id, :agent_id, :period_start, :period_end, :source, :status, + 0, 0, 0, 0, :heartbeat_at, :created_at, :updated_at) + """, { + "run_id": run_id, + "agent_id": agent_id, + "period_start": brt_naive(period_start), + "period_end": brt_naive(period_end), + "source": source, + "status": RunStatus.RUNNING.value, + "heartbeat_at": now, + "created_at": now, + "updated_at": now, + }) + return run_id + + async def acreate_run(self, *args, **kwargs): + return await self.store.to_thread(self.create_run, *args, **kwargs) + + def record_progress(self, run_id: str, stage: str, message: str = "", details: dict | None = None): + with self.store.connect() as conn: + conn.cursor().execute(f""" + insert into {self.store.t('EVALUATION_PROGRESS_EVENT')} + (RUN_ID, STAGE, MESSAGE, DETAILS_JSON, CREATED_AT) + values (:run_id, :stage, :message, :details_json, :created_at) + """, { + "run_id": run_id, + "stage": stage, + "message": (message or "")[:1000], + "details_json": _json_dumps(details or {}), + "created_at": self.store.now(), + }) + + async def arecord_progress(self, *args, **kwargs): + return await self.store.to_thread(self.record_progress, *args, **kwargs) + + def insert_items(self, run_id: str, records: list[ConversationRecord]) -> int: + inserted = 0 + now = self.store.now() + with self.store.connect() as conn: + cur = conn.cursor() + for record in records: + try: + cur.execute(f""" + insert into {self.store.t('EVALUATION_ITEM')} + (ITEM_ID, RUN_ID, TRACE_ID, SESSION_ID, MESSAGE_ID, AGENT_ID, CHANNEL, + STATUS, ATTEMPT_COUNT, RAW_JSON, CREATED_AT, UPDATED_AT) + values (:item_id, :run_id, :trace_id, :session_id, :message_id, :agent_id, + :channel, :status, 0, :raw_json, :created_at, :updated_at) + """, { + "item_id": str(uuid.uuid4()), + "run_id": run_id, + "trace_id": record.trace_id, + "session_id": record.session_id, + "message_id": record.message_id, + "agent_id": record.agent_id, + "channel": record.channel, + "status": ItemStatus.PENDING.value, + "raw_json": record.model_dump_json(), + "created_at": now, + "updated_at": now, + }) + inserted += 1 + except Exception as exc: + if "ORA-00001" not in str(exc): + raise + cur.execute(f""" + update {self.store.t('EVALUATION_RUN')} + set TOTAL_ITEMS = ( + select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id + ), + UPDATED_AT = :updated_at + where RUN_ID = :run_id + """, {"run_id": run_id, "updated_at": self.store.now()}) + return inserted + + async def ainsert_items(self, *args, **kwargs): + return await self.store.to_thread(self.insert_items, *args, **kwargs) + + def fetch_next_items(self, run_id: str, batch_size: int) -> list[dict]: + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select * from ( + select ITEM_ID, RUN_ID, TRACE_ID, SESSION_ID, MESSAGE_ID, AGENT_ID, CHANNEL, + STATUS, ATTEMPT_COUNT, RAW_JSON + from {self.store.t('EVALUATION_ITEM')} + where RUN_ID = :run_id + and STATUS in (:pending, :failed) + and ATTEMPT_COUNT < :max_attempts + order by CREATED_AT + ) where rownum <= :batch_size + """, { + "run_id": run_id, + "pending": ItemStatus.PENDING.value, + "failed": ItemStatus.FAILED.value, + "max_attempts": settings.max_attempts, + "batch_size": batch_size, + }) + cols = [d[0].lower() for d in cur.description] + return [dict(zip(cols, row)) for row in cur.fetchall()] + + async def afetch_next_items(self, *args, **kwargs): + return await self.store.to_thread(self.fetch_next_items, *args, **kwargs) + + def mark_item_processing(self, item_id: str): + with self.store.connect() as conn: + conn.cursor().execute(f""" + update {self.store.t('EVALUATION_ITEM')} + set STATUS = :status, + ATTEMPT_COUNT = ATTEMPT_COUNT + 1, + UPDATED_AT = :updated_at + where ITEM_ID = :item_id + """, { + "status": ItemStatus.PROCESSING.value, + "updated_at": self.store.now(), + "item_id": item_id, + }) + + async def amark_item_processing(self, *args, **kwargs): + return await self.store.to_thread(self.mark_item_processing, *args, **kwargs) + + def mark_item_completed(self, run_id: str, item_id: str): + now = self.store.now() + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + update {self.store.t('EVALUATION_ITEM')} + set STATUS = :status, + UPDATED_AT = :updated_at + where ITEM_ID = :item_id + """, { + "status": ItemStatus.COMPLETED.value, + "updated_at": now, + "item_id": item_id, + }) + self._refresh_run_counters(cur, run_id, now) + + async def amark_item_completed(self, *args, **kwargs): + return await self.store.to_thread(self.mark_item_completed, *args, **kwargs) + + def mark_item_failed(self, run_id: str, item_id: str, error: str): + now = self.store.now() + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + update {self.store.t('EVALUATION_ITEM')} + set STATUS = :status, + ERROR_MESSAGE = :error, + UPDATED_AT = :updated_at + where ITEM_ID = :item_id + """, { + "status": ItemStatus.FAILED.value, + "error": (error or "")[:4000], + "updated_at": now, + "item_id": item_id, + }) + self._refresh_run_counters(cur, run_id, now) + + async def amark_item_failed(self, *args, **kwargs): + return await self.store.to_thread(self.mark_item_failed, *args, **kwargs) + + def _refresh_run_counters(self, cur, run_id: str, updated_at): + cur.execute(f""" + update {self.store.t('EVALUATION_RUN')} + set PROCESSED_ITEMS = ( + select count(*) from {self.store.t('EVALUATION_ITEM')} + where RUN_ID = :run_id and STATUS = :completed + ), + FAILED_ITEMS = ( + select count(*) from {self.store.t('EVALUATION_ITEM')} + where RUN_ID = :run_id and STATUS = :failed + ), + UPDATED_AT = :updated_at + where RUN_ID = :run_id + """, { + "run_id": run_id, + "completed": ItemStatus.COMPLETED.value, + "failed": ItemStatus.FAILED.value, + "updated_at": updated_at, + }) + + def save_trace_result(self, run_id: str, item_id: str, record: ConversationRecord, result: TraceJudgeResult): + judge_name = getattr(result, "judge_name", None) or "trace_metrics" + judge_type = (getattr(result, "judge_type", None) or "TRACE").upper() + score = getattr(result, "judgeScore", None) + accuracy = getattr(result, "accuracyScore", None) + alucination = getattr(result, "alucinationScore", None) + rationale = getattr(result, "rationale", None) or "" + with self.store.connect() as conn: + conn.cursor().execute(f""" + insert into {self.store.t('EVALUATION_RESULT')} + (RESULT_ID, RUN_ID, ITEM_ID, TRACE_ID, SESSION_ID, AGENT_ID, JUDGE_NAME, + JUDGE_TYPE, SCORE, JUDGE_SCORE, ACCURACY_SCORE, ALUCINATION_SCORE, + RATIONALE, RESULT_JSON, CREATED_AT) + values (:result_id, :run_id, :item_id, :trace_id, :session_id, :agent_id, + :judge_name, :judge_type, :score, :judge_score, :accuracy_score, + :alucination_score, :rationale, :result_json, :created_at) + """, { + "result_id": str(uuid.uuid4()), + "run_id": run_id, + "item_id": item_id, + "trace_id": record.trace_id, + "session_id": record.session_id, + "agent_id": record.agent_id, + "judge_name": judge_name, + "judge_type": judge_type, + "score": score, + "judge_score": score, + "accuracy_score": accuracy, + "alucination_score": alucination, + "rationale": rationale, + "result_json": result.model_dump_json(), + "created_at": self.store.now(), + }) + + async def asave_trace_result(self, *args, **kwargs): + return await self.store.to_thread(self.save_trace_result, *args, **kwargs) + + def save_session_result(self, run_id: str, session_id: str, agent_id: str | None, result: SessionJudgeResult): + judge_name = getattr(result, "judge_name", None) or "session_metrics" + judge_type = (getattr(result, "judge_type", None) or "SESSION").upper() + rationale = getattr(result, "rationale", None) or "" + with self.store.connect() as conn: + conn.cursor().execute(f""" + insert into {self.store.t('EVALUATION_RESULT')} + (RESULT_ID, RUN_ID, SESSION_ID, AGENT_ID, JUDGE_NAME, JUDGE_TYPE, + INFERRED_CSI_SCORE, RESOLUTION, CONVERSATION_PRECISION, RATIONALE, + RESULT_JSON, CREATED_AT) + values (:result_id, :run_id, :session_id, :agent_id, :judge_name, :judge_type, + :csi, :resolution, :precision, :rationale, :result_json, :created_at) + """, { + "result_id": str(uuid.uuid4()), + "run_id": run_id, + "session_id": session_id, + "agent_id": agent_id, + "judge_name": judge_name, + "judge_type": judge_type, + "csi": getattr(result, "inferredCsiScore", None), + "resolution": getattr(result, "resolution", None), + "precision": getattr(result, "conversationPrecision", None), + "rationale": rationale, + "result_json": result.model_dump_json(), + "created_at": self.store.now(), + }) + + async def asave_session_result(self, *args, **kwargs): + return await self.store.to_thread(self.save_session_result, *args, **kwargs) + + def mark_run_status(self, run_id: str, status: RunStatus, error: str | None = None): + with self.store.connect() as conn: + conn.cursor().execute(f""" + update {self.store.t('EVALUATION_RUN')} + set STATUS = :status, + ERROR_MESSAGE = :error, + UPDATED_AT = :updated_at + where RUN_ID = :run_id + """, { + "status": status.value, + "error": error, + "updated_at": self.store.now(), + "run_id": run_id, + }) + + async def amark_run_status(self, *args, **kwargs): + return await self.store.to_thread(self.mark_run_status, *args, **kwargs) + + def summarize_run(self, run_id: str) -> dict: + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select + (select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id), + (select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id and STATUS = 'COMPLETED'), + (select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id and STATUS = 'FAILED'), + (select count(*) from {self.store.t('EVALUATION_RESULT')} where RUN_ID = :run_id and JUDGE_TYPE = 'TRACE'), + (select avg(JUDGE_SCORE) from {self.store.t('EVALUATION_RESULT')} where RUN_ID = :run_id and JUDGE_TYPE = 'TRACE') + from dual + """, {"run_id": run_id}) + r = cur.fetchone() + return { + "run_id": run_id, + "total_items": int(r[0] or 0), + "completed_items": int(r[1] or 0), + "failed_items": int(r[2] or 0), + "evaluations": int(r[3] or 0), + "avg_score": float(r[4]) if r[4] is not None else None, + } + + async def asummarize_run(self, *args, **kwargs): + return await self.store.to_thread(self.summarize_run, *args, **kwargs) + + def get_run_progress(self, run_id: str, event_limit: int = 20) -> dict: + summary = self.summarize_run(run_id) + total = summary["total_items"] or 0 + done = summary["completed_items"] + summary["failed_items"] + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select * from ( + select STAGE, MESSAGE, DETAILS_JSON, CREATED_AT + from {self.store.t('EVALUATION_PROGRESS_EVENT')} + where RUN_ID = :run_id + order by CREATED_AT desc + ) where rownum <= :max_rows + """, {"run_id": run_id, "max_rows": event_limit}) + events = [ + { + "stage": s, + "message": m, + "details": _json_loads(d.read() if hasattr(d, "read") else d, {}), + "created_at": str(c), + } + for s, m, d, c in cur.fetchall() + ] + return { + **summary, + "done_items": done, + "percent_complete": round((done / total) * 100, 2) if total else 0.0, + "events": events, + } + + async def aget_run_progress(self, *args, **kwargs): + return await self.store.to_thread(self.get_run_progress, *args, **kwargs) + + def list_runs(self, limit: int = 20): + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select * from ( + select RUN_ID, AGENT_ID, PERIOD_START, PERIOD_END, SOURCE, STATUS, + TOTAL_ITEMS, PROCESSED_ITEMS, FAILED_ITEMS, CREATED_AT, UPDATED_AT + from {self.store.t('EVALUATION_RUN')} + order by CREATED_AT desc + ) where rownum <= :max_rows + """, {"max_rows": limit}) + return [ + { + "run_id": r[0], + "agent_id": r[1], + "period_start": str(r[2]), + "period_end": str(r[3]), + "source": r[4], + "status": r[5], + "total_items": int(r[6] or 0), + "processed_items": int(r[7] or 0), + "failed_items": int(r[8] or 0), + "created_at": str(r[9]), + "updated_at": str(r[10]), + } + for r in cur.fetchall() + ] + + async def alist_runs(self, *args, **kwargs): + return await self.store.to_thread(self.list_runs, *args, **kwargs) + + def list_results(self, run_id: str, limit: int = 100) -> list[dict]: + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select JUDGE_NAME, JUDGE_TYPE, SCORE, JUDGE_SCORE, ACCURACY_SCORE, + ALUCINATION_SCORE, INFERRED_CSI_SCORE, RESOLUTION, + CONVERSATION_PRECISION, RATIONALE, TRACE_ID, SESSION_ID, CREATED_AT + from {self.store.t('EVALUATION_RESULT')} + where RUN_ID = :run_id + order by CREATED_AT desc + """, {"run_id": run_id}) + return [ + { + "judge_name": r[0], + "judge_type": r[1], + "score": r[2], + "judge_score": r[3], + "accuracy_score": r[4], + "alucination_score": r[5], + "inferred_csi_score": r[6], + "resolution": r[7], + "conversation_precision": r[8], + "rationale": r[9], + "trace_id": r[10], + "session_id": r[11], + "created_at": str(r[12]), + } + for r in cur.fetchall()[:limit] + ] + + async def alist_results(self, *args, **kwargs): + return await self.store.to_thread(self.list_results, *args, **kwargs) diff --git a/evaluator/prompts/__pycache__/loader.cpython-313.pyc b/evaluator/prompts/__pycache__/loader.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9ae4608eb889856a806f6598a355ac5bb02cbcb0 GIT binary patch literal 774 zcmZ8fL2DCH5T3WYNj6!NRs)qbmu>OTrMOk3C)))@5gRCy5GdrZEZvthHQ5bsU&Qp% zfETTYdMQ*qsJHqLioYU|gYryIp1cXcyYn_#kq+$aH{YB2-pueO`Fsx0zG_DYPYJ+J z86>7m0h!ncPN51_Qig59h$*pBR+xf+wXAMyOhYbJ)t2BuW2vTMrGLp{iwMIDf{=T> z6$a4`#(G3KZw1Y0ZzeDEAdR02Kqf;3r%;z%Y?c4BX50$YU;~?#7ASRXiU2B7<%(gh zQpsK+G2EC@iOv>ToiRS%Xu=%`)ZOBJyv4f#P&X2VDyfl8rGVqt(BIRNdP?f~Bxm<- z3Sn|`3oLc#(0InePM4d{Y2Q37n79(N5%vP#Yz3y3VR8ncbv?c>w7%DAi)`dIsM`)b zUt}5ee3#RgTUEZUL>VVG*O~2ymrsyAzSapmq+VT zk21U47lwrHlxOo;B9)>Fx&e&!`IG!{{+&P4OM?evWA4N`cFvYRuDxITaC>B|elzUz zhl5AoGr3ns str: + p = settings.path(path) + data = yaml.safe_load(p.read_text()) or {} + if key not in data: + raise KeyError(f"Prompt key {key!r} not found in {p}") + return str(data[key]) diff --git a/evaluator/publishers/__pycache__/gcs_export.cpython-313.pyc b/evaluator/publishers/__pycache__/gcs_export.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f277e17b90f6e867364f937bbf3c15925a483cec GIT binary patch literal 3626 zcmb7HU2GHC6~5Oq{)s2HL&D!C&F&Bk#NA-i4L=J6XowR+LY%AzBT&SSCid7&aAwS% zF+Z-hL{%S1sg$%5+EmL!R@#T9sd9!F!CV4ljZ zan~3_7)xfxS;E2?JMJFy5D$#`1a~KzWrXU6Z`@vc&R*#NlT82w0=uBh+vbG zR8=!1L(x;I*ZUT zY>%lZg_00}t?gKSptt_agbcKjYYu@w63Cs*!AU^fsS8bRV#>wF!*C zp2UuoGq|xIoQX$Akcy{wIBQaPy4{hF)z9qcOE8nd$z9I;=2LJVL5}t$28*sHi>|j= zbZu+Vv&ACQWRdyr7Bj&&V(6tY>~}e5mvM?o?%DL)k(@8lfv&Ju+&Xgv>8=hm`U8ZD z5QD;S%I(A;1Md@bk9VB4Db}%`>~h97pN{1weYc}8x+eM&;@}(TjOtd|Fzo8cSrqXl zN(Ti)BT`Q0)0&!9az$e0F`rfPG7nkJXSJf5ImMUaQC^S@UbsA&9K0|r4oxOTE?y9) z1}~5DhQ>?$wZVx?_9`%BNQw%H&+E4B=u9zvQ#SNy$z3$Ehd-pXA**RE13IR+u4H=a zTBZFL)!R~D$!uND&q;*>HLIw6BuLy40+TI3v6&rp8=`#((Hiu~cNN{x&DLa5HIzAd zm=KMa>^*5NZ?@=CRu=P`lrdX~EM-JPzH69%MOReakkqtnvKb|9m`qMKOt)glbGpgs z1~J`v?T$<$9+PX{ip@xdM10^7p~Yq5n!N(Opb;ZlxM%LP{1NRai1o9Q?|~6&Q~wuq^JuNT^V7q>Jp6^P#t&5Zfol7i`SDe5 zca7_=aNRZTy$biQQh<^!1Jb}i)|8#OLm;lkidx%-FDx}F8f zU5RRNvh15&jUE5$lP912=Kf1A{_Fdn8!NY-@2j@<{CsE$KO9-Sv~cNR;_=W5{%qt$ zYtNJWFSvM_jel3=;yT5nKb;B0PkNpQ4?+J1tkP_-V6_t<@oli`l$z8AWd;lFwjBV6 zo9DLy;H^+6;Qs}~Y7_W6Wt-%j{WqVfO<>2VO)z~IybU;7f~(2%Z9E%r0kkvC(4N9! z6k!u4Dji5@{DmQbw{xvg`f>)!o$!NGQ{r&q%9+x5lpm_2g8U!F2t~LZ7{Gdh<*jEkguLTZO0*Ah2s)4@wk-u^~ta9rA?P%?2yaK=e3s6p@ z_^MS^SDUJ89jdB*eqsYQM(H}OQ5iKdGx2|9W(Kliny$2((4bZs3od~fXU167#iQv) z?l{?`0ud)0IpVBGh~8(ewk3g9wzi`{L(Xc4HdHp{N;U|4bA_X)x>N2X{YYC{s&SD} zJt@z2q`OIFsyF4`jwqXGtWgDmNARA6^9sHc16j?cSfE@Wvq(##^Z`xeVu4~gPvl!g zSvMe$@7$mSHk=qdf5}e6)YPTP!T6c1l-FfKKTiZ(fNkPf8oo=T(Mkf!#2Y^aVLuS( zBe?PaQ6EQ0+u`7bMbQv@xT-X-ts4;cTPVQi$wx97VMB!X(n$M zuL({kW#I9Hne^4g5Kvwr9-(dMk< zQbn>06e2!CueGM9K;*1)*Z%NaooOv|oHjEnrw!4zo!+!fAQ_NOsokb>2Yq&LA7OG5 zG!M`vzk&`Bh=R=l_)#M4nydz=%D$;rZ20f(;cELq;O_QapZ5Nu_a8Lys|Viy>e`cQ zmGFrT23|LitHU!t4!9EBwfOPE$JIdh{K#scWAXaJ_2tvmz#(|!b}XJ=IKA|vDz|?= zzRLP)Y#cUDG$)ZRY&@jF99+p>T z>_rrN%y$|Wp{RX0rxg$5*YEmq`#(EFIA)pnH+?bk0{{R3 literal 0 HcmV?d00001 diff --git a/evaluator/publishers/__pycache__/langfuse_job_audit.cpython-313.pyc b/evaluator/publishers/__pycache__/langfuse_job_audit.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..71848dfaf818a354de9a2c8b2e15d5ada7b0d725 GIT binary patch literal 3393 zcmai0Uu@gP89#~?_1BhdS+*?47UMX!mDsY8I!B*#a zq|#W)ux^O!VZl;INy4F)@T z#{4TWqWItUS2iqQsBEXo zXEm}gXB?BRiut0bt9ZO%TdIvWf&$hEB5^~!ux0oa=`BIWVp^g1B#V)(BY$o(x!>Ln)`#LMZ7tSr35XO0jlt0#4Rfp8=FIPaf${ z_`f=ekUdW#@HJ!gu0;EbV?4z>?DM3!lT;G+aMIC=UUDp-;F-81$)2Se+QT;B&;**J zcxRkZJV~Z80v<^^?UpzNt;U!LAxJX+Pp{YD6dm}Q;GN%u=ULFr!@GvTq$JI|XP`D$ zI;bzi3^>G-q$eoAdI^^IrGmB>QeMGdwy}wb zEj8-_lyoHyHQTntxAK8A4hmfYo7Z9YT%~T&OHMyhd`Vv-YNMJp-?n@w!8O~q#J8WJ zF9EN9X%2tVpBS(a&6@9+{tI~LC_*WCt%nbOJ+>9`oiH+F3C8w>1lp6Iy>l)B zvmew1ydGK(f(Ps^(VPbHU2$d*xzW;fcvJ9(52I0J&H(YU{NYA^WPeJ|OxD4nD8!OO zOJis0D9Tc&=qwfMUMjK2CnlyRCJgG$lDEl#K>sY9Wxc2wt~wdU7{dXy;UT#IhBgZZ zqlhFqY{CJDR3XM3I0R~9vK0)ctQ3m+Il_Aen*{=p;M26aS;$Mel))t62CEiz02xk` zf=hmrn#LfHK;DTlm;~Qo^erHhK2uy%Gh2`QK_tm(O@y3N#CgL-6oQ#bp>eSwTP&C! zhtvaFLMj6+v!SVq!H7kDU49pwQR=d(k2ATEv-IOij%n@vf?>V^wTg*@=EL2Ek3 zBr)Muf^)9xdf}SkJFjgi=~wbHV8V`ow$hMs*mm-Z`K1maJfi&s0`eHWjh=O)ws18t4FCOL_x-lfyUTlR zQ`^q_k%9L|e?9uawL3R{e`7B)S&f{lM9%F+Uam$iRw5U7!)5+Edy(&M2Of4r_IpP5 zySUro+um~L@%)_WV({u7n{6QA`@?lMoDbYJWCg?GDezFLE`)KLzN?)gUd`v$9h(|5yvo+^(_ z!)W;C(nEjnxBLg4cOrj?{PECUU;Lxiaxzot(#j#dMmvJuZPzmvb@o()gP#Tm?@E8# z_;BOiKs7k`X>e}0YriM@e($e(%SUJLovOZ`s=S_hueaQ@S`DslU)T=>tu%Z4rMMIT5el??n0gO6ig3FclX|P zmsh@DU6CsA4`j>k>=O?e{yez)oUY>lS`NyC}M8IEeM391i|nM!ls%j!buejzaaduDCX-U0YS*hSkv>eA}J~i zy96PlrUe0akeObA!UzfEYNN|(LDY3DuN8Gk1Hc>skR)jCeM~MGWkBvQPA8_WVhaI7slL$2y1-oi4Q*@*D&YH&*{iu7W)?$jSC^T3L zm|`317_OP$!-U@G^Pt&_NNFiTbcnSObF)S8H<*hdOCpP82rXHR!~(AZM`Chk!sH@q z#2Porcv{Wp;c}~DE#A0EHVIHODbDCQh*z?5E^c4>O=7g1@nr`Y?pZ4a=P(aT!pTP( op_m#?QPcx8@Cf<-j)otgu1CoK50rR-;?LO;Do6bbkwZrd$eI(@P{YTJlQ>K3Jz(i91nw)8Fu`pM=f*k0!OD;GV*(LenlLVo z`G{}KPyB7D1+^nt+<|1@TbyT!1ZKI2e}!!#nAWI*7S!^f4zjtH*T9$=RA`)q^D$cx^ZAFQfMn!7_pXG2vxV=60Yt z2vX*WI4eYav}r;$X48(P%V1hR>CEJ^Rz@c)K2X;9O?Cr-e~u7d>ji2DY6H{L6?UxT zv_b&nwbDl2j!*)}N;TX{B83r}-E56Hf8p8Wi$(In)5973);?sM;9~B{KCc4Q;PVu= zlewtyA9i@q*!FLYm$oB-M%~NU>E;a~AxP+5(c-$1#YyNmL~bTz5w0oHpz+*k6a;>~=Kkfv3%%&lxQ2dHL0!nT z4!7B>NdN^aH=W(%H0bVuGk_^jvUZr?gVOjQpT?2UB$do`I+rzcWsbG4VzI7^2-I)0 zmL5$}!BFRQDxOtyj!ylWIX|x(Hf<_vshZ}1O!`Af%UuA?MA_-jWB1K1{GAt&SaM$o8bsgIr9wlSuma) ziwZMlYLPU9)p#UCSWDBerW%TC$I2+V(lGI2uxP+dOUb6FRAG5Ol{5O{CZeTlM7LdH zwObi?9MLM`rljU{T3b%t5OBnTOo_zu27A!W9+DZ@I ztFyCHZIgJ_Wz*E^uEgP$I=TSKmy71r$Ys!d6I+hs=oYem@M@k_#!mfcrFS3>zWhVxU&;?zuGY9?o;lNllHye^%M^b77h&FjsJ4`=hKC7ve1xRuR#ql4CV^b)Pr!xI$zqZ2GRD$rPQSh z`9S!uXCc&(zz?LUKbn*4VoAc&*(~?BwFH&_?4ADMR{YaI)lj!^tIEe3KM34q2@m!8mK#Kdw{nagu3`AAc&J}k-n)(A zy_nJaFvI&@IN%#@^55a`uo%3<2N@o!2JUq<84*YXA|vdxK=v`fzN7Gs4=4(oDvCm@ z6=mK`IXQ+073CvG&6PMIMM-CgWwY62n2JL7gD!1U6xJ+RO;K%|WM>>(w}AI1@Ydi( zq?-YD|6RcRS?peZf!4V?aVP03b0SsDciOQX0=@}+QV?keKwB&A64gv2ot=%AfBbmk zrExQ#=urq?&skoM%$?_)1>`wn0nxyKm*v}J0JhK~$O;2k dict[str, str]: + config_path = settings.path(settings.storage_config_path) + if not config_path.exists(): + raise RuntimeError( + f"Storage configuration file not found: {config_path}. " + "Set STORAGE_CONFIG_PATH to a YAML file containing storage.buckets." + ) + data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + storage = data.get("storage", {}) if isinstance(data, dict) else {} + buckets = storage.get("buckets", {}) if isinstance(storage, dict) else {} + if not isinstance(buckets, dict): + raise RuntimeError(f"Invalid storage.buckets mapping in {config_path}") + return {str(k).lower(): str(v) for k, v in buckets.items() if v} + + +def resolve_gcs_bucket() -> str: + if settings.judge_gcs_bucket: + return settings.judge_gcs_bucket + env = (settings.app_env or "").lower() + bucket = _load_bucket_mapping().get(env) + if not bucket: + raise RuntimeError( + f"No GCS bucket configured for APP_ENV={settings.app_env!r}. " + "Configure storage.buckets in STORAGE_CONFIG_PATH or set JUDGE_GCS_BUCKET." + ) + return bucket + + +def upload_evaluation_file(path: Path, prefix: str = "") -> str: + if not settings.enable_gcs_upload: + raise RuntimeError("GCS upload requested while ENABLE_GCS_UPLOAD=false") + + from google.cloud import storage + + bucket_name = resolve_gcs_bucket() + object_name = "/".join( + part.strip("/") for part in (prefix, path.name) if part and part.strip("/") + ) + + client = storage.Client() + bucket = client.bucket(bucket_name) + blob = bucket.blob(object_name) + blob.upload_from_filename(str(path), content_type="application/gzip") + return f"gs://{bucket_name}/{object_name}" diff --git a/evaluator/publishers/langfuse_job_audit.py b/evaluator/publishers/langfuse_job_audit.py new file mode 100644 index 0000000..a2b492a --- /dev/null +++ b/evaluator/publishers/langfuse_job_audit.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import Any + +import httpx + +from evaluator.collectors.langfuse_credentials import resolve_langfuse_credentials +from evaluator.config.settings import settings + + +class LangfuseJobAuditPublisher: + """Audit each agent batch as judge-job-{agentId} in that agent's Langfuse project.""" + + def new_trace_id(self) -> str: + return uuid.uuid4().hex + + async def publish( + self, + trace_id: str, + agent_id: str, + status: str, + *, + period_start: datetime | None = None, + period_end: datetime | None = None, + details: dict[str, Any] | None = None, + ) -> dict[str, Any] | None: + if not settings.enable_langfuse: + return None + creds = resolve_langfuse_credentials(agent_id) + event_timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + body: dict[str, Any] = { + "id": trace_id, + "timestamp": event_timestamp, + "name": f"judge-job-{agent_id}", + "metadata": {"status": status, "agentId": agent_id, **(details or {})}, + } + if period_start or period_end: + body["input"] = { + "period_start": period_start.isoformat() if period_start else None, + "period_end": period_end.isoformat() if period_end else None, + } + if status in {"completed", "failed"}: + body["output"] = details or {"status": status} + payload = { + "batch": [{"id": str(uuid.uuid4()), "type": "trace-create", "timestamp": event_timestamp, "body": body}] + } + async with httpx.AsyncClient(base_url=creds.host, timeout=30) as client: + resp = await client.post("/api/public/ingestion", json=payload, auth=creds.auth) + if resp.status_code >= 400: + raise RuntimeError(f"Langfuse audit failed {resp.status_code}: {resp.text}") + return {"ok": True, "trace_id": trace_id, "status": status, "credential_source": creds.source} diff --git a/evaluator/publishers/langfuse_scores.py b/evaluator/publishers/langfuse_scores.py new file mode 100644 index 0000000..0df91f7 --- /dev/null +++ b/evaluator/publishers/langfuse_scores.py @@ -0,0 +1,23 @@ +from __future__ import annotations +import httpx +from evaluator.config.settings import settings +from evaluator.core.models import ConversationRecord, TraceJudgeResult + +class LangfuseScorePublisher: + async def publish_trace_score(self, record: ConversationRecord, result: TraceJudgeResult): + if not settings.can_publish_langfuse_scores or not record.trace_id: + return None + auth = (settings.langfuse_public_key, settings.langfuse_secret_key) + payloads = [ + {'traceId': record.trace_id, 'name': 'offline_judge_score', 'value': result.judgeScore, 'comment': result.rationale}, + {'traceId': record.trace_id, 'name': 'offline_accuracy_score', 'value': result.accuracyScore, 'comment': result.rationale}, + {'traceId': record.trace_id, 'name': 'offline_alucination_score', 'value': result.alucinationScore, 'comment': result.rationale}, + ] + payloads = [payload for payload in payloads if payload['value'] is not None] + async with httpx.AsyncClient(base_url=settings.langfuse_host, timeout=30) as client: + for payload in payloads: + resp = await client.post('/api/public/scores', json=payload, auth=auth) + if resp.status_code >= 400: + # Don't fail the run because score publishing is supplementary. + return {'ok': False, 'status': resp.status_code, 'body': resp.text} + return {'ok': True} diff --git a/evaluator/time_utils.py b/evaluator/time_utils.py new file mode 100644 index 0000000..18a0023 --- /dev/null +++ b/evaluator/time_utils.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from datetime import date, datetime, time, timezone +from zoneinfo import ZoneInfo + +BRT = ZoneInfo("America/Sao_Paulo") + + +def as_brt(value: datetime) -> datetime: + """Interpret naive datetimes as America/Sao_Paulo and normalize aware values to BRT.""" + if value.tzinfo is None: + return value.replace(tzinfo=BRT) + return value.astimezone(BRT) + + +def to_utc_z(value: datetime) -> str: + return as_brt(value).astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def brt_day_window(reference_date: date) -> tuple[datetime, datetime]: + start = datetime.combine(reference_date, time(0, 0, 0), tzinfo=BRT) + end = datetime.combine(reference_date, time(23, 59, 59, 999999), tzinfo=BRT) + return start, end + + +def brt_naive(value: datetime) -> datetime: + """Oracle TIMESTAMP storage helper: persist the wall clock in BRT without tzinfo.""" + return as_brt(value).replace(tzinfo=None) diff --git a/evaluator_monitor/__init__.py b/evaluator_monitor/__init__.py new file mode 100644 index 0000000..abeae50 --- /dev/null +++ b/evaluator_monitor/__init__.py @@ -0,0 +1 @@ +"""Read-only web monitor for Agent Framework OCI Evaluator runs.""" diff --git a/evaluator_monitor/__pycache__/__init__.cpython-313.pyc b/evaluator_monitor/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a036299dded2923d03d753584962c9734f2ab17d GIT binary patch literal 202 zcmey&%ge<81gY8eSuQ~OF^B^Lj8MjB4j^MHLoh=TLpq}-QJ9{d)mL=wtCW2HHmF5-e`DrrVVvmnc z$xn`tU&-(pWXCPpRG6B0sA>A~@tHtlnJyl1|b_lmZzg}(WI-ajAaVe#I*Fff0JA5x@S z-4UtSQ|vAF75is`{BUuk7(OH24)N!19VS}#|B3?{*=Z|Ai-!m@lteBvpvFWu?~^x2tU@qAI~kkrXrr;6vYZ%LARN^-P#nxDRP zdTD&Wv?U1=ixh{7!^M|pf{L^nSPjHSzap@>+zhLRVcM##n})TB_ikpD)jEp@n>|-m z%f5Jhu6gM7<>l*F?v+`cpqgE8HW=TC2bu$~FTZhZiCJ|FWz8c?Ow&0l+g{7Za`X6= zyK1c=kT0>iY3a7fH#W~*TxEu>T;=K-TQ~U~#f37t@`h;;%w|_QIhGvT++1R+mN1Rl zhO*}L=*Fr7no>9BlnUUPgC9@4VOU9JxvE=A*<1rjGzDIC!^RWnR%^gwvAW7tn<*7; zt|@kvDJ!a7uD0R1#7$$~TuDLzUAAu5ShMFY;}$U<)*u_M3n`S_r5wUbP?F@w$MNui zbkj9%s6{RYX5?Wh9$dT%*&b0Fwpr4_V#z93*_z79Wj~f46SC%MpP7=&Y)Y-yll6^e zSBXP+3ZPGnOd`}Ptf)wqU|I=h9rgDA5;~C z#ZWRhEKLO~a=dG?8MtwMbI>Os;S#V6DRf(~j)v9j^Tpci<4{Y9)WQdW)wvJ}{1`6H zZifth1~1R!-=fy9K=HqvJhL6o{o~|L`1*F}`ZpHI?Y*8pd9x3k>M2s;S7g3lkpT5y z;R-7Ytg>dH#>&1>(Fb8mf~o6xaF}Wd-Sep>g4*}HA$k5g2cM||;;#HbaItGZ8kTyc z#Sn?(V|g(i5W;Tut2N!SOGFsd*HzQkblW0A6o=D3znc%C^(FjUbRlN{eemQ@PklBx zv@v zKw%j%+y7A{{}HLPA7nV3ztm=M-^Tk}a#5ZM?C0AE+z1HXyU$5CNg?9bBdI@-BqLZ1 z+;nq?B;5#lvRmr&;97TY=2|rg2id{FP!&0$(B3SI13ET*> z0E6C);6>=EBISc;rHU*}xR+vAnB9f1mxP)&dkj<2R9oHbLwrN9Xc(IQhY;FkXsKZ+ zoBfL43?mCWMqVnL8f$iI%vSYUyxTE?L3Nc08z|MexyqRZUuZ?cpk;GdhPlo-?*`%V zP_x@nAwnsUV>LrWMl;Cnm4(GC_&Me?K!6@nPL@wrWc?5qvY@?CSJ&Y$Pn`Mb?|yb7 zxpN}<(cSG6$?X&KpFKDKerT^>8aVRq!q;Kx$b$UeUxtrAmV#Y}KMx=Oli<%T{c-Py zhyUpC?&;WWIQG%P-%UKS{&xEB3x8ACon6=+UHJ0o*^e$g8h&*5QT5Z?+tY9E99`Oe z>*n^r&Ch!e{m)(x(EP8J1`Y4YuLb2FpNUF;D!&~1Fcg*kEP#eTm!r~O_P-_t|LS7@ z#o6FL%y!}V=TZ<|n`fT1;}itesM%JB6_cF_EBX;oaTA%xB@ub||wG&V0j%0oRmM6x|1P zqg-of3<2H^?`3rQKcDCCp=p22@)Cux<>2CPf~kfMl)(nC#br+Jwm-;*xM|v)yln#i z))8Dp)Cs@vuxlqgvK<<6<7@wb%xBOGD?bblMY1lHYpP}ODKvM`NFot>p$O&Bxa#;< zV6icYc%D0_nPq#U&XlUXRx9-9=%&KUPMuZxW8Vdc2l3RllDdnolEoSE<4ZD(< zjwq>Cmx1h`kKEPSI#Qb*LDm+TIv-irHM^SE*j>HM65?ePSzNbuwU$6_Qe*kFhe#yo z!V~2~F6DF(8J4|KV>GLxWJzSoeauudIZz=3v#e6nSF1K61-~H;~@2ql) zJ=vkLmGsy++QjtPA*NNvD^uP`5*yantAgj&Wy8(0jSSTV*+sju($nJ;lQX`uDi!E>#pKXYHVr;3v!-jx z(8T2A#09@6!BtIfSlKC(fu~1OV;Vz%s=+OjXX_@Fq@c8&ifq+HmAQ|G@$reVjOGC* zofHL_{o;2G%F~X6H@V6|VM})O3fbYBfV?11n2?mZe@ zabsm*)@s1+&+iyB$)(CD#$>0)?p8f=Y;~o^04?Gh%z5l(pBreUrq(T%^&UDL4nFMT zsE*nsGmrimGBx2Cj>HZkdTdAZwDJ`5gO1IU$hJWGghMD!*MZKEo*BnpInR15Wl_v&fbG>Y#-;1N) z77MV_WU3vT9EE$j?k>FDsBX+)B+X zd#PCDBFcn*9H$)97Q>THiH+fHK~X5en|;hldmUtpK}5CV+vEXR(yANPCnqu&rV%VX z6i~JT-|}e4Qb-4X9XxC44DsyB#hD;!>uY2I|3-ja#WbO#Tny;Ts2*I<{k{j zrVBYuzpIF1F&{xG5Gk}aWUvGC^S^#_<3!=u*U$}1QCpiW&K`@ue~?Q7Yr&xn)b!_Q zfG7;xW^=PYN=ho-aw{TPD<(>8)I0e|%~Z7|EX!kYqN#$ywwg;hK<)d_b(uPfC_d5pBUaQMe;q5DGPN6f~tp$QeXti83#vH&WJrNh-ti2 z4IRL>xS4p^;<6$=ltg-hJe)|3-X(; zV*^%i&G|GPI;&x_A+j2c;td&XJ~H!df`@=@*n*2xgcQ6Stp8lfaq672V_N91G25-Q z89A8NpbMS<0)gD$!^>iG);x*kY-hBe_B(`gUq{ zQ~_{7iQSKmM6=Nmb*(-d9gXJbxn|R2fgV@IV}u@muVKdlXdeH!9P3*m;v#zxcQ@$*r5cT~xOs@ZoJ}$IIJkHEH!NA=VLqxM=!&pm{t-?z6TG+;^)y^!CqZ^+b9Brt=94o zC*6HuhgsdjwJ@uId)6am<^)U^i>qKRl*e@^5aH|#i7sE5xRPQ1y(n&`?;-UB0@X!kfX|d1wL}bbAk1UI($ZsgzushNgTHst{m2ehM?vQX?aa@F~zUU^8SVgT_OxW3$ zQl?{^7=!=)cIGVn4=#P4p%%3)sDga$32$quTGuc~8)bP~R=0=)t*A-bTYPeD-T<`+ zd$JEC+}a7x%chwZW5^pQcC^hJcdeLYwcq!(gB?Ew9u3;xzOgiS$vYwIkn#EGt4=xH z9vz%`QZL*2#m3qS<1sB+XPn9tY?fmmf!T~45x#a+zsIy#`dNUf!S;O+#8m?;?*Gs47?04aL18D_)*~*!>)?p;tt#wX`V6Eudaw% z2ypA=HVU13iRg4|5U;3r*DF}#_zojnw18z^Y3_2~aZ$~WG;Yz;#<8PAa$V@kd3o^Q z=G$=*Tv@)*A#BkPQMZR0gpyU}xjv(Pr&UHSFX~!UQd9}MQHfSKEy&I&C%{REZjACh zjBru2bA6{3IS>Hbb?(-F2ozq4hvfzg&r&v+jfg=wqAo5s48cu0J*Yi!pkAVr2)HM_ z<~xOqgKYI665(11|1Ad3M_uX;3?^+9#^U7M&&t`~#dW=N7B|0-XWab$)NqI;6G@}P z58HDv<;RX3cAM(e60M)S_)nFivkY=f)Ui!3Sy)c>i^LZHZUby8k2}F#(?K01Cb!Ba zXH*zs3&?v!#(@n`VcMQj-D(;KG46JwSxa;G%BCOWUZd;?+w;zKawKeu5K!~p+u%J)5ivw{;bl^7vfrictIAof}zeNYW z&Vgr99yscrTLnULv$s^DWkjj8`Mm#ag73lHB)O~@z37`CG=H9|xtt@jda_>V9EYKJmr5k%#;v^^;3G=ca!x$?|!E zJV}t}eZcL$7kA~Cz8If+6#P{FRNEO}__fp{A0f!IdjUM{9d~Ek?t6Jx9{FM{^GW%U z`swAJu{pwdfgq!V^Ma4_{bLWi-aWM~k9>{hs1Fcr0i5{RgGUVlx$-!$EXzj;=Bm6W up7)G^6dHJcb2s?>KZTBd9t!_t;k||JBj-Pfe>%Jy`u<**6gne5 EvaluationRepository: + return EvaluationRepository(auto_init_schema=False) + + +@app.get("/health") +async def health() -> dict: + return {"status": "UP", "component": "evaluator-monitor", "read_only": True} + + +@app.get("/api/runs") +async def list_runs(limit: int = Query(50, ge=1, le=500)) -> list[dict]: + return await _repo().alist_runs(limit) + + +@app.get("/api/runs/{run_id}") +async def run_progress(run_id: str, events: int = Query(100, ge=1, le=1000)) -> dict: + try: + data = await _repo().aget_run_progress(run_id, event_limit=events) + except Exception as exc: + if "no data" in str(exc).lower() or "not found" in str(exc).lower(): + raise HTTPException(status_code=404, detail=f"Run {run_id} not found") from exc + raise + return data + + +@app.get("/api/runs/{run_id}/results") +async def run_results(run_id: str, limit: int = Query(100, ge=1, le=1000)) -> list[dict]: + return await _repo().alist_results(run_id, limit) + + +@app.get("/", include_in_schema=False) +async def root(): + return RedirectResponse(url="/ui") + + +@app.get("/ui", response_class=HTMLResponse, include_in_schema=False) +async def ui() -> str: + return """ + + + + + Evaluator Monitor + + + +

Agent Framework Evaluator Monitor

Read-only UI · evaluator CronJob remains a separate batch component
+
+
+ + + Loading… +
+
+

Recent runs

CreatedAgentStatusProgressRun ID
+

Selected run

Select a run to inspect its progress.
+
+

Results

TypeSessionTraceJudgeAccuracyHallucinationCSIResolutionPrecision
Select a run.
+
+ +""" + + +if __name__ == "__main__": + import uvicorn + uvicorn.run("evaluator_monitor.app:app", host="0.0.0.0", port=8080) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..13de2b8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "agent-framework-evaluator" +version = "0.2.0" +description = "Offline LLM-as-a-Judge evaluator for Agent Framework conversations" +requires-python = ">=3.11" +dependencies = [ + "python-dotenv>=1.0.1", + "pydantic>=2.7.0", + "pydantic-settings>=2.2.1", + "oracledb>=2.4.0", + "httpx>=0.27.0", + "openai>=1.40.0", + "oci>=2.111.0", + "google-cloud-storage>=2.18.0", + "pyyaml>=6.0.1", + "typer>=0.12.3", + "click>=8.1.7", + "rich>=13.7.0", + "fastapi>=0.115.0", + "uvicorn>=0.30.0" +] + +[project.scripts] +af-evaluator = "evaluator.cli:app" + +[tool.setuptools.packages.find] +where = ["."] +include = ["evaluator*", "evaluator_monitor*"] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__pycache__/__init__.cpython-313.pyc b/tests/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d36d8397ab357bf4d4f21d128efdf4ba8ccea18d GIT binary patch literal 168 zcmey&%ge<81iM-4vq1D?5CH>>P{wB#AY&>+I)f&o-%5reCLr%KNa~imer{fgeoA6V zqJC;wVoqsdNq$j$T4sfQVtQ&`NqkySVs2`Aeo=OOesX3!T!nr~YH>-ietdjpUS>&r kyk0@&Ee@O9{FKt1RJ$Tppt&G>i$RQ!%#4hTMa)1J01x3Sd;kCd literal 0 HcmV?d00001 diff --git a/tests/__pycache__/test_full_tim_compliance.cpython-313-pytest-9.0.2.pyc b/tests/__pycache__/test_full_tim_compliance.cpython-313-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..be28409bf97b707f828cc1c9582c7dc54218b31b GIT binary patch literal 24116 zcmeHvd30RYdEa}pz8MTK*a?6DIRqDQNMf*a0XK0K36KzlN02n3Xgru1fI|*up!)_~ zAiL3Vd>qJ@BRZ{)pmUsKq;+D@r%mCuInhaSTFG`=)oGg<0u zC++XMZ<&R|MT(_=>Wjhm?!J8Y-tT_*`@Z+?m6o~%xb4cXpMNJH2>*owhO=uP@!lp; z5MC2xK^FUkK9Pw&@>}{X%*vm(KD%P^34IRc)KerT@e~(x@u!=)`Ll$T;A!pm^m&+v z=e6~h_LZ?R?zi`Q`^s54_dEJ4`YKr^_dENm`l?wq_e=dXeYLEX`(6Dj`s!GnPpA`` z1=-yo$R#}%t2e`#-s4tPJ~kD z@U%~aQepB`91A7JMkkf%fn+=$4X2U}k&P^>B;yyO!ML6n46|q?nn=Y$aRuom@nk3x z42>a%f;G|0DHaN+f)gw`K9LH>qlvN9xvqDJbEd~E(DX|SkTPTmWnDVg~QzhTbP*CuR@BNdOfXjyo6}Pevx=-*k^!h**YLgbvxpm!n;_Vo#YzmijMVH?7G;Y;B#n9!w&4;Z2No^OUV$rC)0ygS zi)?KZ+Qq{9)(P9JqoUt7pxOxSx2WT)jW(~}$u?8K5t^8YCL)N8g;T0EmH-%wrnJRH z)8e=DDWVEe=Tyf;2pe2cZK283IfkQ4PzY5Z493iLQ>`V&%%Lh;8p;;|Zqc+!o>$#3 zyO*T;j8s3<{oU%>(eK6QEemV9(hc23^T0c8F7h0%lU+{KJ z+irU6W~#r_@J7RTR=u%mu6oWhfALK=A77|Hu;4wImJWWbIMB#f_U+oYN5s0K(eZT! znm2&a`Y61HKr?~01o(EN4zqOx))U}+neVR_3KF1Yuap9OO}J~Z*y}!WSnY>IEhnXi z@ozePZHV^{BC3GWYa2|b^4M$+*$(KG9e|ST1a!#~pj&nUmdI}A87uL7)Y`)!nJ14P zKVTBflSEE$QLppp9$^3rTD;eXr3&5wON@Ajgop)dLmGF3l&iquB)7^o*)BVFSS>=r zGGr;X&}65vP<_}KgPe!SuP8yh3I2?Qh8u(#JJg)VXsk>EE%mN?AQcE z-3r{0eUJ`>0w=q2T76hKqVfe!c0WNmS`h^<%lBZE!XPc20w+u8DXpFYmu>INz8nk? z;pcA1YG7b+wGo$n${x9Nhv>s`zDC$0n8XPSZ`9%wRtrA-4gBeD>uOBQfwz>`H4Qda0FX;#qvYb4exD>pPc9O(?x>ZSm1oGixAv zIRvI5s%M;-9Er!m{n0DEzKILjTzaY!1>2V5sY-6@RFzp@w&^KPZVgkO+!9lbkLt@w zpv65NQxu>jn7rUGSL;o_ClF31Mq^_E18Zy#(5OnO==j8FEFM)W2f!mlP9m5jOdO^- z`z01jff*wvLG>Ak%w<8rIZ>@(;y{EYs<9-bC{dO=M9hWi*3c5X5WS+hH6(Dqjqs{U zTGpcpOj>f1g`_Bcy4abqNn$9ai`um zm0o{f(Q^OhC}?ehJWgBIaO9qt2roj6L4ZSp4g-D=h#=pzJPMp_HwY9b@Fo$@W=mKK4}w*~u*n6THaVw+6{OST zd9vk)a4({Uiya;~hV-6u!uzPw#4Hv#94}8=nppY*mwU(zR~d;z=6D8cjJWJmcFQG? zCD>ODf}O23i={`^Y7%MB0Naa3*;4?A**?IjFJ?(^4&fb%#j(}LIF*cqt|tcgRzL}qRs?I&>N62vmcy1F}lF{Z5nNosVpZ< z4sCo2@itkqw1txrEh$JRR4^0{M<-HBFp>;Uj#F!4(AGGh5>*d8iXJ^-CA-nS@~;72 z5*|T?($z06dcL$=Rjtz>JEuPmhpX*ojc>Mkp=QJM;Ve}-HM{SPFDx~+Wt!UN>lZyc zA4qA>G_}vKS@d*2kkXN9>X`3Z^lbZiQ3}K99#NPNDuvRD2T9O(NP`IfjSwp;f{uhhk|H6ibzG#3=~LQQ1m`PgsrvHzk4(DG5iL zAXh$XUd}ktD#F!fH$HglB7CaRo9wlb*FiMYPuO47uK>h42fNH@uwWK=m->JG&kg~;3V2jW_zMYb7YyPBz+~ZWJqdC(Q7L0N>_pl{;WfKYIF1v8rfp2*H)qRK z^@D?GOAYY5A|z#&K)XhX#GoM|)1*hPU7%{w%M)ym`rM-k46fy|1ea9?Su_+;f)I;< zuLDklmtv`N$;nhO5uAuaLYGWL1H0l$$W4?z0kif~=s9Nv=ysiMJoA%XZ#&A5Ic@w(b*<N>IS=|e&J=-EU3a;y4tnN?i|ssxI$o$Nf7cZQTJ ziEu2bTG?a*GMdpSB&peMYed}&8K@>;&^gn?=yp`Kh7wnB7pLXOhNv<}m`j$ln(ZWI zg=~Q(FX4J|S)uL{Rfp4%+Ntq~p9E#<(O(+&>F`o+0f5MRnr31Po{iJu`?V{k#aBvh zR@5zbZhg}>-}sj8t;m~SO1B?NpAM&^qv=y)nU-^U+I-nNjp@^8(nr3KX*qkxZf$Vi z6Redb(~b`%!8IuUw0!Ms-@I);^5$XUI_)>TwXeST^%rM4mR7cAR<_QKrnmOJvp;=e zIQ?rQ3oFA5-pI5K0+l7HE+f@l>t3jD|9f5h(9$b_TnHj!aHFJuX z*M<5`bNd#hKu$uY@yt70|6p{>m7WrvTXadqyK$j(REL#?E*TebPA2Uf393`?j_S}VP@USd^MRZB zK~5=osh}}7;^D(kV<|zsnruoa5iui2NieT*;lq!joqxy5q2?2mm7pAX)U>j8_T>Db zw@xl^?0#!+dgO&n9Rr=La8KKBOG3k{H}|B^#BSqaBlcVF2*Q?Q1gnl)rk%?bm9IB` zt8peeJ2YuaH_xS87TtbSwlTvMig+nbGx(&5|$5+<)+ly>CAE!A(zfJi>G zDDBFLCDHTjGnxAC`Ou=Y{r$#O)As4}kYX=aR=vLETU%x(XGi8*=65Vq?nZA6sywq| zsotNd_s_Lv>bJicSd@u#8qAa_G1Q!ZPxI~Y6BKA^f2Nj4se$EKnY_FKq(>qP*lYutRJ z1W7T-ew#;WtCqGm_&9!&Q+%%WM(yCM58O|Z6(~jdJu=FaAXa!#TS4LEB$N(E9<|gy zNlh21GekF`Y-sY&?|th)y7Ne;?C7-B#Qb7?U`|=6@0yoyNIM?HF+h;psGLay6kN{c3e*NK-CWR{SpBq$7HB^SBY zoOa3eZTZ)zV!0$-%1X>j8AhG_lR*-e6*yecfCv4GRw6e%LCtP?C81w#H1NvL&6Nce zQ;wz~x4eq-W^9hiNzg-u`?{=1zoeCn3J~+>)NV?IR%>E=7DGoE$tc43 zH=Rx^#CsbM^>|GcbB;Nn3?6&bMg;*0DbQ)hQ;-SAlwdS5nuI(l!tXMgC?+c{&atV; zUTh8+W*E0p=vCNU?FeA)2!Ee$a->nwZgR9YLH)2RNrYf(JNcP>-=%0uh1O8kJHjt+ z*EF3b`w}Wr?a;F`Xg#3?WnV@_fkIRD>Ngzy&{Wh*pA!ZG1r1M9=V zyC{6$$pvDqp(*FNXgr?ug{GuXLb()WzL3skv6qoglkt6>eBS_=GOHP3zei!cnh~R# z5wjX?g4pjPSK&mI>iFQthN4b>&eNbZdzm^$;sTn4+YVv%rsehR%Rc||noY~iTke!N zYHFckC>C4ct*g8(cq=OPS*obYpCuyIe3pn#YqQiQXiJ*gPVkjm6fspgVv-}!0bSG# zpVSvWXY}MYP7|*Xhrn0riu$XrkMG$K{EF2s@?i{Ilb8^fh0wd7;=&k>hr zie!$D?)}=n)`W}KECYx6>b|DwHB>2jIUGz~nSh)OK8R6AxbQA3|8miwFLN8Y&-f53 zSdW-Yyy#;NuKSEC02MWo`we4MV*Cv)W zeh8r3xBWTQ_D=w&vgQK1&8Bg1872daZU!t)c{jC<)1@OCX)7k?{y8$s`N|9+0&LNZfrnkV^-CfwTJ-M>lc%w+M~5==Aw3 z{KMVeJ5{N7w70Xpb8Bn+_STNBYgVcr*lyYRAqOkv0}$l?{; znRc{s+5D7bj9jhAGH5}~Qovuvhza4+T6UOxWG0}=HN4S8XaqKn21XfO1K33ZmjG0+ zKH#9j4LBl<4g_5Ts9gkO5q1J`evfvmJWuJoej?$FW*YJx18baBQ{6C30Fy}#8?q1u zQWC8N&e^3PNoL$QlD3iIw4-De{hS&*Fx3{yD+LPsVI&%&zV1KSE$!(yHP z0|C#EIvuX2I|6}np=K3i(A68KU630udjcdlUiNIiXNpT|S1r}FW@=ie-M1~}uFmC_ z&1BZF?q)@8dWD=G8C$40$Nfj%X}eKz_U5LJf4cemo9AOoTc62nedcC$9yUNrk?p17Ck!f`RDh^c%Z?7mlxZC>nx_!=rTkJpFEFtj2u1!ezQB%#KGU-R1 zUGV>Si~C@&>&IK2Nd56{_n{K!pE*5<|FbIT(8_#mC|y=?NfGZkK=X6O;=^x$VNFl1h%prZ~IKkDRH-!aQM?Oq@@SXsIoQspCnQ z%O_c@7EfjkT1HP^u@w)?R&6BEr6C3ze6p07azm-7otz+~lZa2U#8lRv4VLV@(E!Yb zd+|S2rc5eiK-}wt+*pZDHPMR}XQ8X?^)>GskFr=e)Fy|L!4sj$c(U0qF=C4}N+Xf6nP*fM`K{bIR9S4SK+dwh}lQnxhiOZ&GlgyIIlqy0kOhZK`2W#@}1=uFrVaFL+y)ylok8+q{_Zc0#vTT9K~ao+;hF99Zxi znij!Ar&si3%6pc)Pi4GMEqM1MU)73q!+}iIfu+iUOy$5rYW4zozd{I*4Dry1NZ zXIqq7&0x=b>V~xIK_=fMjWamfK7Y~(c{kl1;Gp@3%A6ZG7u&wjSMCr8ix3!tkZ^mo zBE}=KBX{G2Xco)ke1bfA#O^q>BqX5(sdjNv{ptElM4XuD0tfSCQ$ox&a&C=SsV;L} zPuVu%-JB`=WNNf^J4`O5Xaa8v!FX({EJik~DQtVl&n9AK*q|AnQM?Rgrq-D?8k-De z6%(l1@8T4J{WtWD-6HS{0M*XM;>i*A-znmM68IkkID4qe?*11=(5$d$0n{>M?`oWC zAqnW_BtY*g?Qd>Dn;_mdY3_BSYp}lvUjPHG_CIRxf5OWC1ie(~O7pl)UsFHRpQ&kC zst#nT19KPOt%j-Ua!vi~&wuOrxzkslPuKJ;)$GjF>|ChXontPRZfc#oxaiqxa!e0I z%sfAL>RnIw9hXqE?Y3K3xjMaOXQpB2QvKmf{o(Y9(`mr^GfVZ)XX>9{sQ==0nZBt( zLJ~Km9!;9E9s?MFC5aEuLg5XWqOc%=mk7L0;CBf8oWNZI9}`$dfGX0etR-Ks0Vq*v z1)vLGt-bATyBnac)P7dHyQ;$elz4Z8#ol|@z0=-#cS`iwcYoL@*|%ynQ6+S=5G4)< z^MpR8PUp9=w{u>Mz}_--I^tt=I+h|j9qW(6on06X|qppOa4Ka)MqteWXr@bToBPpKSZJb1MQ^3-SOLD>^uaZ{}dD=xDDji;Hw31U7ZHoej{+1eb z8P6!*D%~wk{PA#$k@>XKNQ2hRm;A+08OQ$rHAeWf7{QLnX{m^&jN{)6{I5Cl2RZm} z{zUkP@G_^BqWB-)VD#GPv(d{CW;&;|&&Zd3HVY4y#(Xr(Yd=vhpJbJo7&UwMB&)N1|%dP=ylrv7%wDQlO| z%Av|ZW9Zol5U{>l9$a6Mujb9%qgL0(+|~8iaaS6Bd~D87gG+zF`WfM}ayO{p1NhvO zgGk*AeAfS+@VV*#FMPr}U^87a5|y%;s2Zvqs^XOH`^Hj@K(v;#Rvxwz-Jo+@sC1Un z<&^S0I8UI%Preqbj8!YQ>eRnl4#;go)%tEN{{*|W+^EZV>bsS%FkrwXx9j-u{R$r( zA05BK2dGbWeRTe6zvSVg>sRX|i;pv*?7ies_DWCN2mbnT7E4b%2J}~nVX2f;uW6~& z{TWy)o#l6a)B4pgdL@(Di&1tZc?~u_}6nf?p@_4S=bV^OKRWXlrOP5=%|hX#vR0TMKH{ zERK#w@D5?I`qoo@zYkC#IO4`A>bcK;b~F|2b?U*>0J?S2%9iCd>z5n5me;@{cz4>j zXSu0)dDX@{rOt{VwCeTt2llv3M=R8L+mAx;dRRFlC2vPiqIWLWX)Dii}an@>9L}2=3LlS8GRS7qcPb zlzm(oYwduAK3+lRX7{kf=QGZ2-0{*D&4cRX13wzkEa)FP4P*|aLSv8D%|DY3ZETWU zE2f{LO@$b9V=|hTS8slQL*Mec?&S@wcbv90wRfBn@NIPyzTGCiJ!QHv1p5~#T`21M zLyG%b0HPsT+Xh!HESFqmAFeBLm1*KE%9tm_0#?a=LW9jO{?f5lV5o8H4muUqJE!#! z@FB)Z?2nOF#5f6GJoqKgZ9y+hdn6J>@?8#lBu_ac2*dX1 z0~!Vi6}cf!)r#rYwCgoppB*O zWA<+m;N2TwKQX*H;^IH2I4_nQExnvQQ~~=SR~_twT=iORo{^KI%IqgRidY0y(af#Q zCut|yzoTaUJpnp2S(-ow;8q#=76>d7xB)O#pIcsohhK<#uFH~IzYtVgL^IF@kwYHs;e029=t)Z2GVZ4*q z8pwMiYUb6mCiqZE=R>+om1K{IylT{HdN&py;?vXd-UyV#2L4dc7+^PsYvE1Ou)6aG zLxZO-s4EMP4yukbgVBgun%9I{s~yPZOSpO}`zax?_Mv7_>}P~!I$*P=PwXZIX@{xq zrzWm&8WCb@h4$7Oi%u~55FWJOst3uzkx7`(Lp`lJw3mI=x}bvhfMMuFS%ah(3&r`j zrWblrLA*NylPqp!pe=#-s5d1vB{iBMSH6uE z`;U}PhVbk^5qO`#UlRDw0BQ+!QCpXtxq%KXpf%c^8`zqgttU5-mu*{anH{;=+H(V4 zx!HP@8uW?i)&ow{{6GCmBFJnT@I{1331J>#nq@owo~`o(+Zub>V)gnD1Ux@FEIRD< zu+0J}7b+Uk4VxCqH@|fBr=HbmUpLT#lj;=CTX0>=Tp0(3Y&qQGzRd$*z zv&y0(K7iBsl$dUKDlI)VeF;yP6)Z{m?>fbqAXWzN_1Ea~L5MHL63+GcTBVOb?=$ zy|q_gczG%koW zMvhS<>oVSTY3W#c<1r0pyvLR#{dXPl-gR1VQKIZTWg#b;{2Q|+%%V4>W6R#^bj{|B zcS~C8m=^)k-YsN9(y=7zzw3IO76E$>v38H-ikyH&A&iJ%zUN?Sy1cMAdFqo(x$lzfV6l0Tt&vaX-U$5*Y!9p zVv&b5{{j^;^O+uG;azRbcmrvvYkmzt+8dZ(gRU=0`tQ0Pr$xX9201kU0u?dynI1Be zAp~s5cw5rambngqw6|rh18827^xt(oPK#LNAy1KsAWbaD6v$XJ2)+lo(~;O)_5Ipo;p`=Rvprex=k^_Y-6 zXv)vQ13mU5g7zUB=UDL z7*c=ZsZ0MwNbvB@(uo5wJ}l9sYOvd>&|kLA@>#}9_GJ9VzKN=l%M!!qj5THt43)-O z!c)Lnc519;OyP{_HgH!vh?~fAm7UCo2x7#xo*8_CUo9;dzB@9V%h# z32Y$HLSQ2SKLPqWHrot<7wf5g)=Geok;^V@W$k8+#+z){xGXKu(f+WHj2K)kNdXg4 z$P0t`Ml`5|5D)4qk0 z{qVmP(LhPxl1=|zryTpupys*3i~O0R>Uk}dfv75@vD%34Sk4s zbP{hP!0wWP9Kcq_;59jh)%u{uodz`i4!k7ZZqPnP=rW_oVw1;d^K+5vu_P7#kuO!6 zgmozKTpGIzo6gj^cx*&7`YbgP1LH~Hj5Eo5fHeO8fW}eN9Z+47yEgdO6S?$I;ol5X zec4p);|+m$bSxCULX0l#(V~perI>jLkQIa~okV{Qz~ojlGV)QJjZB0%Q^%*SKt!WT zyR;8B?_np}P`OXMz{M_k^bRgu3^H)&Eu4`JOQN zp3wH5u>T{6LnKpuvFW2qr+7%bBM|(cI%pBM&YW3l3S^oB9|#n>eb6r0JkwJP*7{|e c=Qodk<@i^hx$8J0ih+BlEEcg}`_ko=B6$Z%Vj|7qWjrGm>q?MpF|u5{O(tqw+fyYGMj2;(z(6EK!2$u! z0(2~NI?*iD6E`TNJI36ppJ3SZ! zKD_t#?c(7P*!}i9%H;+*IPPwLd^e9c?ptESM^DUgFzzL@@w47FP11N5HiY?Y&d=_La)3BMx)?*9pQ zhmRX&z7*+er2Vl&PRPk+-sEItjjnUQfmZA6`6#CH501M7qlL#>I@Y?^vz(k-Pb~6s z`u50Tq7=pU^vQN(W9=-r&N7lScWB*6+uBeZ(Y>DKN|8Y_tGwEmPre%$YcMt!KW+RJ z>SSN9CuhWZz01oPIXjn_g{L)>@_;-@i{bCoORR^1Wpo@z8y$HG$C(C6wqk5Ub1bN2E9A7nMNY*lo# z>Sor~SCtzp*T9+3%>&*Yn?7IPSt#0~Zi+;`RJ4P%Z8dONJ4ggcXd~j%qr7U{8g^(* z31Yxma>e?Nvt^m$Y)xc0pM#r=b!=6!X4`Hd)|9ep7#nJN`Lf73T~JOgP6>R z!2s6OisESZonY`y7*w+zMa97=?WJI?j>}f9u3{}1P;7NmBc5OgB9)DXZaBILNgEd2 z!VXR<_LkK!DzukUHLMNQP)xO^1^Kt=Bj}d-dyFj{j4PWKuBncq-LD&}sXEY7Oksf} zB#^PgN1pFBE`~@9~@yQ{Kh z@a?CW<@Z_4-9m*!Vb()SpH2G0QU^t66D@hdY-qN7bQS>4&nB7W_gT!{LQ99j1rL=T z-1CJ}2SsNSl|11>XtsNF768r%_n77PSeZ z$3PM3_zB}{gd!H@9IZ2aQr7Y6oTqgjepzSoWrYWzwI8qZ2-KOUsQ7qwj^F2npY%SD zKKJ*TA@2CAd?r-PdI4G+eWCC37}PoOGwOWtC)IiUxz{-nf2qf+GwTI)9)F>Ao_JYj z^3}iSIs+eAaZj|_Skrsh!Nt%_9FOZCVZh z@gna|#;`o}#L(P5#n5655$Du;{R;eoOq8*k+t$s>6;TfJ2{KV)?y$2{*RF_f8x7mh z@QwE{*-95P_%)KsIU;6=fDJx}&k*qoA_^dELTpfID~fmu^xy(TqA^O&;@Qle60@*NUXZJz=f2Joja?2XK|1gRq|`@-_)fC862;nwcTH~BGd{Aw$I zt&_j*=dXM3-}XS}Kis`eTb5h-E1mp`pI`Z+>g89w{0F<&4}}vRn%ZCVg{cmT&L*1L zyBb>UuF9Ihx4+0Nzt3Xs7Mh}HSlnOn1+jyovx!84hKbegs;n7&`zy@y`z+>eA@NW+ z=b?oMGrq9ULDAVn3!ZQ;G}}Eo3jpVX8D{x?7IU}I!l7{5L(}_JUzqNo=xn0tz2Aja zyQ{Kh@av&th%}!oP%9 zqz`D54~J|Ik0#)~IX=*q(Kz>q?}0+%Z;#MrtxKwX-A6|~GpPnt8OezrYm79DJskH` zs_co>T9<|+9|<`g0|iee*m^+$P-V|OD@Mf~D=XFuruM8nzzDoSS$KW~o|*XLzsYnW zaxlRUt-2?FFe6GZ8^Uh6JGmEAZbsVDRKwYF2cT7#V0O0MF{Vm?3isH5c2ED%CjpVs zFdW-WN)4S(<}g1~+GM^0G;KYuith&*748|0G$UCuviu|24p@+_*fLmx1;d&cHj99OjSXnzz#_`SSQ^QL$1~oZ z5nvq>Dw|EnDVwA=>*SE#RHZ6Vx#W;T4toH6*saPQq}X1$I_8wa7B^UvqKZ>qk2JQ* zz_Qu4tGnmD*RQ)@_xJVFdLZCKP(G1}7=s)zsDehW_Sr0J?eSkzHGNWjO!Az9) z*-;j|?3NpGn`{(~crahr2w1Q+UhK77ANE1(8f_Z!W4~>~jRr=VakJfbk8T?Y;^0UN zZi%AH{$1#DXb@(kCDmJN8|%UPN2;$;TI-7*x@g?K4~hH?gF7N<7ZQaoBziCVQt{{Q z2VFH8j$G~(eTOe1B!>_J+XK(r?n2JYQB+_HY^>=aHHtB$N!E44k}cKH&2jkV&dJs^ zwC*|C%1*~v(k5w!oRg-C*%`%>X63m#RWFbbR!l>?rbvaXS=T+J`#vf_Mck&WDJJ2k zR86DSzO12XN|sJS0$8zRRg(;yQ?N;zEd?~jtduo$OEyU`uj;a%Ra9(9azW88(lRAy zXN-Ja%_@?t%V7VkK{QYkW$AXDQa=D@JqnT!&}D=f5n)zj;LzDFggMa#wOiz{r{Iox zNO;^hhmBk@Yn_sDE^FizGp(4G`LGklXY1Z<=U%8G?9BQ}mU>Pgoyph=T8Yd+6C8W} zST@8|!#bCu6`gvFkj`c}N1yUIz0GzPnqX6Ot}R7$iQEvAfmv#0V#BZK9%6M@gIB*} zv)O`ISHn8)z&g0hHb+0fHH;2A(#>`dc1(+Nb93-L_8LNs9#U`tx0Z|zQYw9+hZ6`* za4#B{YTxP!Jk&**4oCNW=-4Y!!4H$72ezd^B)hkW;B)iToQO0d(E}^_&hxgQ;a|MQ zX0+e!jE|zRzarFxpk1%E6DZa5ytfG^u@MOXBZ>**#H(DBwW4epxWx6x=jNZei&lQ% zjXMkx6dmp(U7y$)Y-FX;>HEdm?AFOv?y+|sSqz#0JN5J z!y+!z!i2jnYcs^BnyPME6kkM00^*?yL+IoRacjnP1(VR(lP6CnPrr3Wdh3TL#?#X2 z<0A0@>J>dlny{h)RRHWQgLnYiif#adn5KfQ6YmsdE#@Yisc3oP1;D}xif|5?WSVdm zRusbN6kL?&qO_2UiCNuBw^pbHc zj<1bz-pw(PZ@}NT|1O}NXwN25!vmIw2=&0uqiDSMh^q(XnXj?f`$~(US6LKL%_8O- zFZoU2HNbOxUY^xT2kKvkebHzwr!Z+Z8Z~7-H)Y&7T$c{yu`xRUNkLvM_%P*r9EPei zyukwTx#E+}$ zFiPXlPb-==V2X&C3-5-8uvU(^viX8Z^ACt8-bf?XC{{}NGmN@)>=Bs;S`Y6O3T}J|MX`= zD=ibN!M18JS_wv%cMdM?JoJxP`NZ4hp-aobOoL|s($2y1q0!~wnWt_P`VsTgg+iC? zT4tUGQ1k9`p#N!*YQW>Onii(9doswbJhQD<0{zcS5jOR^1IdKzo-dHx;l39RBm?}t z*A5*U_TKLnpnw0n0?6<07Qp8IurKKgEP8m5F9v+c?xw{s58A~YzT_V7VjmASiwR%y z2!98S$9N)05`E_-iTEUG7BW+f_WhDX-`Kiln*^ZmI~E01S>$gZ;FkR|C^r&8}$DWKwebIr@qf+4Ne_s|c^Dgy~Hq!cQoIF0Y@I6HL^1{Hb~Da0J;eox*-ZYh$+l(|Bj*l zG5p#vF-;cn_w32jFmY0bBk-<_-~zh_s3O!dxhV=#P?MB-U{D)o2l^qH@UA^ZwFgO wp(l|JW{g>wuI@-wb|ju4`t|5_#QATPmRy~yoc|YNKOKAT>?0TAy6hwSA4xr`*_y4rWuLx%`aaM596heD_Yr9S;`-YQy+K0$fG?HC+a2z42S>;?A`+1sA;TQy z!q|6=IH;37UBkTL2$Nwq^_0H|RA67c)C*7Nh;O)#)(!ipk9~KI_=oFhJ?rx$4Z{H% zV14&UaCigV!1|t%jl&@tVtrwxakz;#vA%a?({M9wW_{mC%Wx}gW&OI5w&8Z#KHNb& z!lZ>ni0E%4Vtv2Ez60INa1DS9yzLzws9dr3?01$&Zhbo}2KSvNMBYdUjD4*(rqYX$ z{&F`=GLDQRx?u_5qnuSIYnpD#W>(jXWYl4~(z2;o0zxx8slW)=3E4ak&&`wa6)6S7 zlvLi#URI>6W-6K~875Wai@21k%V|l@0MdZhjYGP2S)m47dsIp3G!59`k;}50XSgF; zCad8N!IQ_1i{`Ydy!qzw;e0xSxF+R_N#&F&<)}WHGbL5gGH?us)i6#?DJ@S+Q&}yo zPg#PgOL;RT&7>-q#V7EeJsyYQagDr9C?^u?5IK+lM;oCo(Fu?jUDTc7qi)M{RKAGY zFKvMFsWK061%gzDf9`QFbgq%J<&_B;0rxhQ+63RW*?0hN4c;Mi^w^$Z*(tGd`)Aao~X*S8Q_WRGP?Mi8JC#6I7m5rgVBy z(o`{aIe``qiF{&3dy zQ~zSqrdiiVzQ=%HON$SX78l@4CkX>R#>jqd6bG(qBG+Ay7gDSEF)~`EOBG2*%Q{=b z5YW^LWw(>v&IvB+OfDgT%Y>v&*vC^_&YYf$I_M_syKPA+(~veJPDp|9s0v7Q_39N2 z&M{sBumqQ16#O6iKJ@)uz8d(*w-^Y0{NB~~76My~fvq>WQXoq)9!USF!Jxq)T~OEq@6}z&>i2DhRx$OA>8?#pqT5R+9u81p1JY zWYeVC@jP%*l5P{)hC}cc6wI6rP&f@gfFF}b4hP@-o#5a*?SVLEeGNDOU!J7_pa*V| z2j~&q00q$l&?^c6eWI7vWqi>(%a1Aw@^lhKY^fbLtBO3lc;KHY$*vQ|?y(N!^!VGzAhZ)HU7xolQdj7IBf-zzaV=4=_&1uUq&rp_A(b5J|FB}fr z!u$g2w^T8IV2Q!HnuY}SPq_lDF_knLLaTQ;#U#BP=(hwq)#e)T9P5V4zoEc+tHI;}HeK#9bP9wrW_`l_NLo z&02-in&WI0H;SBI)gt8dHqk5k&Q|3{tP}mem@D<_wWj9>dWCCO1Uh06EjAMp4>UwXEYfSc*C@-G)Q+oJE7e+L)_Cn`Jw>V5L_H znH1JP)!9&Wwy={EAyl~_;5jOxMrFlS;{t3Pw{~1@=i~eBGiKnF_DLs5G?=s;MvukG z;4*!NVi*vzNZGXI%WAp2DVY#3Sll>_>9x2NJo+pyov}~&`-7HeJgb8B&Cpi>MY|Am z16U!W$lX44Z^O_zz*QU^Pyx)fzF|E#chZ4@R~qE+I5;GcTqf7}FgXsk8)=;3(0$!@ zP*kwwZI14N=WVCuh$SqJT23QdQStX}755H6^aR>W4_GJGs<9!1GlM%wSwpH2U|cpp z?`175rNPb7N$~UrIFs0bszZEX9Eep{jKkgr9^}vH@d}}V8~M3y|LDK%zw0c%{OV_e zpA6o}KYfha@Ik1PyW#ER{y&IA2Vq0{8iFAJWs7H%gT9Vvv`Y#6wCwB$ZcM@1yKZ;g z?JVxt&(1#elpcdoQ6EL8V7c}23t$s14|r!~3Op`m4OUQMG*l|&GnkF0G5E%K11wQY z0Qt$N*S5wGu3_YASZ)YF!FyXgbL3V7dsNr4+z=M)TG|TO&*}$y1lK+az`%2XF=y?N z&F=&=&;JX)TKoSZvDk*^z~K1L?~_0Jx7^x(x3B0wc>kHbhh9G9oAo^LlZ`E(ZTV!& zd^l0s&@=14A5JWU2fqmqe)VEOJXJU?&!;kl^J+oU3zv*i$}ENRMPc(Se?QW@5E=R= zGW6Ba!r1A;nbf?JEnLtFxl08jUs5iYB2z_S>#V!#WahTa3wx_?U-G5aUBBXQ^?cFw zdDoYGv31w4op+O^JtOnNo4*Zhn;ly82N(PuMSsWrU}!!xa6MlN4U~cd?CrqysZ!`b zDR|&Nd1rmYBPVH$JO$Xc+d%vOb$8R@CSiq?W5@w!fP0*S4rGulXi0Dh?t6Q#>Q~l_ z^BQCTqd?--triL`3upM6(PP1fB}^)&jFG?Pw(~1`5Eim{-8`?*CA3fwe=>Q+qn2Y| zdII_@0^c1>uvA^op|N;o=4S0)D^kxj5lkg9#sR!feqY}__wvnSh1OWXA7A_*US2P> zb`|{H_OSBH!#Xz#*~13nt+zvs`PN-`rb?{`zuIt57+M=oR2g*$4uEOMvq^5VFIJ5R zm&cJncs^OcQ%@RP*AG=ilt6~e{tW(udIV*K4wt6WH> z?+MCU`k|d?f<6bG<cgV>DQi;<2a-U=m9kc zQwQYMsx}76IA#P8>G82;QGM0tbKvvcNvlE5K#@VtV(Gz@4J%N2*C!zS1w^odNa^EL z)>1Nr+~q0b7xk#CtU!jWLelL^>8c8T0b+-l4dZg^qCPQ^O)1^71_cil1|wZ8!OpWW zxX7doC_yNYd^4>p2kCS-4)aXPxg7Z8m0|ierh07MtgM#nA%b{uYkB^x@rPhD z17i{h&-)Mk)49^dp{~D2>HCiG-a0gYp~IYkqWn>K(BoQR{*aeUD*gsK%cVlB7CT1K2CZrpnPI#x z$HvQthz>A$W$9sg+j?W1%g2CDx{@;;X{eRx)A5UYj5r25@f6jy3;K9$TAoyAHe-S6 zjIJsB!jZvAg=SN7T$FX`gq&CP$V`KghYE<6-WQ&LKu7`3q0P6vP_0Va(V}1{V#lvu}ms2tfZ|SYZ>`{V=*`}b^-t<2113#-cq1%A+Wy~*k1}9 zoONL~Re!a0U3*2ZN6h-Kr)EH8K01d5%%E+^d@Ct<76@Kc85) z?0(K=_j9uQxyD?7@%{MkIj8eE?iW~f4R{^n*SlYZx?uvltv?IW!qx3LW?=zwRjbX) zfB+@`4gO75#q%d;1W=#u3kDe9^I;^x23hqhP7hvb z)xo14I)f|lYz50r6(g^jP!v`tdo4FZ_F{1uvAuR!fofu>3+41sFj9&p)2wbCTWfgO zd8M384Mn1gNvFY1rBaY5Gv6~IXVpAaYI$L0skdMg!-XjUriJ%6wk`(S7K86Tqf|kfs-Ig^G>cvqu)&!wcXi;Z7}Vjukh@?l|ZDJO3DJn>%vz)k1r`6iUn63CnoaJh(FxS}VCUg2+&^=N+gx`C+6>X`{3nYJ2-$)g7C=3w3Y%^IBCA(K*nDX5yVgoMeItgCkH=tso8Nm zPH3(Et`dXbP@RUNiV?evi4jG;w3=(*Au^rFW@7d)Uv>o9wEQcm9sDk!sD@pd2-d#Y zH3hpi(N;q!!SUCns63&W(Da6 zyek?yOE?L4AA*pIv5(1wBVzDI**gt6;pudE9nCKWN9S@xx$2}mCe<$Sc$OjF- zC4t|O&IhFBPh{u;i9aAaACT?`WcLH|+V9BT|8je{&Tk2TCyhrrF8uR73!&}B(Do+; h-yTAY;_}bVl$@=LF8@!C|LFMNzxB{bTy0E7{u}m?^o#%i literal 0 HcmV?d00001 diff --git a/tests/test_full_tim_compliance.py b/tests/test_full_tim_compliance.py new file mode 100644 index 0000000..f951366 --- /dev/null +++ b/tests/test_full_tim_compliance.py @@ -0,0 +1,279 @@ +from __future__ import annotations + +import asyncio +import os +import tempfile +import unittest +from datetime import datetime +from pathlib import Path +from unittest.mock import patch + +from evaluator.collectors.langfuse import LangfuseCollector +from evaluator.collectors.langfuse_credentials import resolve_langfuse_credentials +from evaluator.config.agents import load_agents +from evaluator.output.legacy_exporter import extract_prompt_length + + +class _Response: + def __init__(self, payload, status_code=200): + self._payload = payload + self.status_code = status_code + self.text = "" + def json(self): + return self._payload + + +class _Client: + def __init__(self, pages): + self.pages = pages + self.calls = [] + async def get(self, path, params=None, auth=None): + self.calls.append((path, dict(params or {}))) + page = int((params or {}).get("page", 1)) + return _Response(self.pages.get((path, page), {"data": []})) + + +class FullTIMComplianceTests(unittest.TestCase): + def test_per_agent_langfuse_credentials_are_data_driven(self): + from evaluator.config.settings import settings + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + f.write( + "langfuse:\n" + " agents:\n" + " - agentId: arbitrary-new-agent\n" + " host: https://lf-arbitrary.example\n" + " publicKey: pk-arbitrary\n" + " secretKey: sk-arbitrary\n" + ) + path = f.name + old = settings.langfuse_credentials_config_path + try: + settings.langfuse_credentials_config_path = path + c = resolve_langfuse_credentials("arbitrary-new-agent") + self.assertEqual(c.public_key, "pk-arbitrary") + self.assertEqual(c.secret_key, "sk-arbitrary") + self.assertEqual(c.host, "https://lf-arbitrary.example") + self.assertIn("arbitrary-new-agent", c.source) + finally: + settings.langfuse_credentials_config_path = old + Path(path).unlink(missing_ok=True) + + def test_agents_yaml_accepts_document_camel_case(self): + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + f.write("agents:\n - agentId: billing\n daysBack: 2\n percentage: 0.5\n gcs_prefix: agnt_ai_contas\n") + path = f.name + try: + agents = load_agents(path) + self.assertEqual(agents[0].agent_id, "billing") + self.assertEqual(agents[0].days_back, 2) + self.assertEqual(agents[0].percentage, 0.5) + finally: + Path(path).unlink(missing_ok=True) + + def test_pagination_reads_all_pages_without_n_plus_one(self): + c = LangfuseCollector("billing") + # PAGE_SIZE is changed only inside this test to keep fixtures small. + c.PAGE_SIZE = 2 + client = _Client({ + ("/api/public/traces", 1): {"data": [{"id": "t1"}, {"id": "t2"}], "meta": {"totalPages": 2}}, + ("/api/public/traces", 2): {"data": [{"id": "t3"}], "meta": {"totalPages": 2}}, + }) + from evaluator.collectors.langfuse_credentials import LangfuseCredentials + creds = LangfuseCredentials("http://x", "pk", "sk", "test") + rows = asyncio.run(c._fetch_all_pages(client, "/api/public/traces", {}, ("traces",), creds)) + self.assertEqual([x["id"] for x in rows], ["t1", "t2", "t3"]) + self.assertEqual(len(client.calls), 2) + self.assertFalse(any("/api/public/traces/t" in path for path, _ in client.calls)) + + def test_trace_metadata_precedes_session_metadata_and_session_falls_back(self): + c = LangfuseCollector("billing") + trace = { + "id": "t1", "sessionId": "s1", "input": "abc", "output": "ok", + "metadata": {"ura_call_id": "trace-call", "channelId": "trace-channel", "message_id": "trace-message"}, + } + session = {"id": "s1", "metadata": {"ura_call_id": "session-call", "channelId": "session-channel", "message_id": "session-message"}} + record = c._to_record(trace, [], session, "billing") + self.assertEqual(record.metadata["ura_call_id"], "trace-call") + self.assertEqual(record.channel, "trace-channel") + self.assertEqual(record.message_id, "trace-message") + self.assertEqual(record.raw["prompt_length"], 3) + + trace["metadata"] = {} + record2 = c._to_record(trace, [], session, "billing") + self.assertEqual(record2.metadata["ura_call_id"], "session-call") + self.assertEqual(record2.channel, "session-channel") + self.assertEqual(record2.message_id, "session-message") + + def test_export_prompt_length_prefers_raw_trace_count(self): + raw = {"input_text": "normalized", "raw": {"prompt_length": 123}} + self.assertEqual(extract_prompt_length(raw), 123) + + + def test_alias_filter_is_enforced_with_yaml_per_agent_credentials(self): + from evaluator.collectors.langfuse_credentials import LangfuseCredentials + from evaluator.config.settings import settings + + class Collector(LangfuseCollector): + async def _fetch_sessions(self, client, start, end, creds): + return [{"id": "s-agent-a"}, {"id": "s-agent-b"}] + + async def _fetch_traces(self, client, start, end, creds): + return [ + { + "id": "t-agent-a", + "sessionId": "s-agent-a", + "name": "agent-a", + "input": "hello a", + "output": "answer a", + }, + { + "id": "t-agent-b", + "sessionId": "s-agent-b", + "name": "agent-b", + "input": "hello b", + "output": "answer b", + }, + ] + + async def _fetch_observations(self, client, start, end, creds): + return [] + + class NoopAsyncClient: + def __init__(self, *args, **kwargs): + pass + async def __aenter__(self): + return self + async def __aexit__(self, exc_type, exc, tb): + return False + + collector = Collector("agent-a") + creds = LangfuseCredentials( + "https://langfuse-agent-a.example", "pk", "sk", + "/mounted/langfuse_credentials.yaml#agent-a", + ) + old_enable = settings.enable_langfuse + try: + settings.enable_langfuse = True + with patch("evaluator.collectors.langfuse.resolve_langfuse_credentials", return_value=creds), \ + patch("evaluator.collectors.langfuse.httpx.AsyncClient", NoopAsyncClient): + records, sampled, stats = asyncio.run(collector.collect_sampled( + datetime(2026, 8, 13, 0, 0), + datetime(2026, 8, 13, 23, 59, 59), + agent_aliases={"agent-a"}, + percentage=1.0, + seed="agent-a:2026-08-13", + )) + self.assertEqual(sampled, ["s-agent-a"]) + self.assertEqual([r.trace_id for r in records], ["t-agent-a"]) + self.assertEqual(stats["selected_traces"], 1) + self.assertIn("#agent-a", stats["credential_source"]) + finally: + settings.enable_langfuse = old_enable + + def test_evaluator_remains_cronjob_only_and_monitor_is_separate(self): + root = Path(__file__).resolve().parents[1] + k = (root / "deploy/k8s/base/kustomization.yaml").read_text() + cron = (root / "deploy/k8s/base/cronjob.yaml").read_text() + monitor = (root / "deploy/k8s/base/monitor-deployment.yaml").read_text() + self.assertIn("cronjob.yaml", k) + self.assertIn("monitor-deployment.yaml", k) + self.assertIn("monitor-service.yaml", k) + self.assertIn("run-agents", cron) + self.assertNotIn("uvicorn", cron.lower()) + self.assertIn("evaluator_monitor.app:app", monitor) + self.assertIn("suspend: false", cron) + self.assertIn('timeZone: "America/Sao_Paulo"', cron) + + def test_fastapi_is_confined_to_separate_monitor_package(self): + root = Path(__file__).resolve().parents[1] + for path in (root / "evaluator").rglob("*.py"): + source = path.read_text(encoding="utf-8").lower() + self.assertNotIn("import fastapi", source) + self.assertNotIn("import uvicorn", source) + self.assertNotIn("from fastapi", source) + monitor_source = (root / "evaluator_monitor/app.py").read_text(encoding="utf-8").lower() + self.assertIn("from fastapi", monitor_source) + + +if __name__ == "__main__": + unittest.main() + + +def test_metadata_session_recovery_when_langfuse_sessions_endpoint_is_empty(): + from evaluator.collectors.langfuse_credentials import LangfuseCredentials + from evaluator.config.settings import settings + + class Collector(LangfuseCollector): + async def _fetch_sessions(self, client, start, end, creds): + return [] + + async def _fetch_traces(self, client, start, end, creds): + return [ + { + "id": "trace-real", + "sessionId": None, + "name": "agent.gateway_message", + "metadata": {"agent_id": "telecom_contas"}, + "input": "ola", + "output": "resposta", + }, + { + "id": "judge-audit", + "sessionId": None, + "name": "judge-job-telecom_contas", + "metadata": {"agentId": "telecom_contas"}, + }, + ] + + async def _fetch_observations(self, client, start, end, creds): + return [ + { + "id": "obs-1", + "traceId": "trace-real", + "metadata": { + "session_id": "default:telecom_contas:session-123", + "agent_id": "telecom_contas", + "message_id": "msg-1", + }, + } + ] + + class NoopAsyncClient: + def __init__(self, *args, **kwargs): + pass + async def __aenter__(self): + return self + async def __aexit__(self, exc_type, exc, tb): + return False + + collector = Collector("telecom_contas") + creds = LangfuseCredentials("https://lf.example", "pk", "sk", "test#telecom_contas") + old_enable = settings.enable_langfuse + try: + settings.enable_langfuse = True + with patch("evaluator.collectors.langfuse.resolve_langfuse_credentials", return_value=creds), \ + patch("evaluator.collectors.langfuse.httpx.AsyncClient", NoopAsyncClient): + records, sampled, stats = asyncio.run(collector.collect_sampled( + datetime(2026, 8, 14, 0, 0), + datetime(2026, 8, 14, 23, 59, 59), + agent_aliases={"telecom_contas"}, + percentage=1.0, + seed="telecom_contas:2026-08-14", + )) + + assert sampled == ["default:telecom_contas:session-123"] + assert len(records) == 1 + assert records[0].trace_id == "trace-real" + assert records[0].session_id == "default:telecom_contas:session-123" + assert stats["native_sessions_read"] == 0 + assert stats["metadata_sessions_recovered"] == 1 + assert stats["sessions_read"] == 1 + assert stats["traces_without_session"] == 0 + assert stats["selected_traces"] == 1 + finally: + settings.enable_langfuse = old_enable + + +def test_trace_id_is_never_used_as_session_id_fallback(): + from evaluator.collectors.langfuse import _trace_session_id + assert _trace_session_id({"id": "trace-only", "metadata": {}}) is None diff --git a/tests/test_monitor_and_run_id.py b/tests/test_monitor_and_run_id.py new file mode 100644 index 0000000..c0dcff5 --- /dev/null +++ b/tests/test_monitor_and_run_id.py @@ -0,0 +1,25 @@ +from pathlib import Path + + +def test_run_agents_announces_run_id_and_progress_command(): + source = Path("evaluator/cli.py").read_text(encoding="utf-8") + assert "RUN_ID=" in source + assert "python -m evaluator.cli progress" in source + assert "progress_callback=announce_run" in source + + +def test_monitor_is_separate_from_batch_cronjob(): + cron = Path("deploy/k8s/base/cronjob.yaml").read_text(encoding="utf-8") + monitor = Path("deploy/k8s/base/monitor-deployment.yaml").read_text(encoding="utf-8") + service = Path("deploy/k8s/base/monitor-service.yaml").read_text(encoding="utf-8") + assert "run-agents" in cron + assert "uvicorn" not in cron + assert "evaluator_monitor.app:app" in monitor + assert "kind: Service" in service + assert "type: ClusterIP" in service + + +def test_monitor_routes_present(): + source = Path("evaluator_monitor/app.py").read_text(encoding="utf-8") + for route in ("/health", "/api/runs", "/api/runs/{run_id}", "/api/runs/{run_id}/results", "/ui"): + assert route in source diff --git a/tests/test_no_product_hardcodes.py b/tests/test_no_product_hardcodes.py new file mode 100644 index 0000000..f3ee31c --- /dev/null +++ b/tests/test_no_product_hardcodes.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import ast +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from evaluator.publishers.gcs_export import _load_bucket_mapping, resolve_gcs_bucket + + +ROOT = Path(__file__).resolve().parents[1] +OFFICIAL_AGENT_IDS = { + "billing", + "sales", + "collection", + "backoffice-anatel", + "backoffice_anatel", + "telecom_contas", + "retail_orders", + "financeiro_agent", +} + + +class NoProductHardcodesTests(unittest.TestCase): + def test_agent_ids_are_not_python_string_literals_in_runtime(self): + offenders = [] + for path in (ROOT / "evaluator").rglob("*.py"): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Constant) and isinstance(node.value, str): + if node.value.lower() in OFFICIAL_AGENT_IDS: + offenders.append(f"{path.relative_to(ROOT)}:{node.lineno}: {node.value}") + self.assertEqual(offenders, []) + + def test_storage_buckets_are_loaded_from_configuration(self): + with tempfile.TemporaryDirectory() as td: + cfg = Path(td) / "storage.yaml" + cfg.write_text( + "storage:\n buckets:\n sandbox: bucket-from-config\n", + encoding="utf-8", + ) + with patch("evaluator.publishers.gcs_export.settings.storage_config_path", str(cfg)), \ + patch("evaluator.publishers.gcs_export.settings.judge_gcs_bucket", None), \ + patch("evaluator.publishers.gcs_export.settings.app_env", "sandbox"): + self.assertEqual(_load_bucket_mapping(), {"sandbox": "bucket-from-config"}) + self.assertEqual(resolve_gcs_bucket(), "bucket-from-config") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_tim_spec.py b/tests/test_tim_spec.py new file mode 100644 index 0000000..5dac84d --- /dev/null +++ b/tests/test_tim_spec.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import asyncio +import unittest +from datetime import date, datetime +from pathlib import Path + +from evaluator.analytics.vloop import max_consecutive_intent_streak +from evaluator.config.agents import load_agents +from evaluator.core.models import ConversationRecord +from evaluator.engine import EvaluationEngine +from evaluator.judges.llm_judge import TIMStyleLLMJudge +from evaluator.output.legacy_exporter import extract_prompt_length +from evaluator.time_utils import brt_day_window, to_utc_z + + +class FakeLLM: + def __init__(self, replies): + self.replies = list(replies) + self.calls = 0 + + async def complete(self, prompt): + self.calls += 1 + return self.replies.pop(0) + + +class TIMSpecTests(unittest.TestCase): + def test_brt_day_converts_to_expected_utc(self): + start, end = brt_day_window(date(2026, 8, 12)) + self.assertEqual(to_utc_z(start), "2026-08-12T03:00:00Z") + self.assertTrue(to_utc_z(end).startswith("2026-08-13T02:59:59.999999Z")) + + def test_sampling_is_session_based_and_deterministic(self): + records = [ + ConversationRecord(session_id="s1", input_text="a"), + ConversationRecord(session_id="s1", input_text="b"), + ConversationRecord(session_id="s2", input_text="c"), + ConversationRecord(session_id="s3", input_text="d"), + ] + engine = object.__new__(EvaluationEngine) + first, first_sessions = EvaluationEngine._sample_sessions(engine, records, 0.5, "billing", date(2026, 8, 12)) + second, second_sessions = EvaluationEngine._sample_sessions(engine, records, 0.5, "billing", date(2026, 8, 12)) + self.assertEqual(first_sessions, second_sessions) + self.assertEqual({r.session_id for r in first}, set(first_sessions)) + for sid in first_sessions: + self.assertEqual( + sum(1 for r in first if r.session_id == sid), + sum(1 for r in records if r.session_id == sid), + ) + + def test_loop_is_max_consecutive_intent_streak(self): + records = [ + {"metadata": {"intent": "billing"}}, + {"metadata": {"intent": "billing"}}, + {"metadata": {"intent": "billing"}}, + {"metadata": {"intent": "other"}}, + {"metadata": {"intent": "other"}}, + ] + self.assertEqual(max_consecutive_intent_streak(records), 3) + + def test_prompt_length_is_raw_input_character_count(self): + self.assertEqual(extract_prompt_length({"input_text": "12345", "output_text": "ignored"}), 5) + + def test_official_agents_and_prefixes(self): + mapping = {a.agent_id: a.gcs_prefix for a in load_agents()} + self.assertEqual(mapping, { + "billing": "agnt_ai_contas", + "sales": "agnt_ai_ofertas", + "collection": "agnt_ai_cobranca", + "backoffice-anatel": "agnt_ai_bko", + }) + + def test_cronjob_is_not_suspended(self): + text = Path("deploy/k8s/base/cronjob.yaml").read_text() + self.assertIn('timeZone: "America/Sao_Paulo"', text) + self.assertIn("suspend: false", text) + + def test_trace_parse_retry_success_and_failure(self): + async def scenario(): + llm = FakeLLM([ + "not-json", + '{"judgeScore":0.8,"accuracyScore":0.9,"alucinationScore":0.1,"rationale":"ok"}', + ]) + judge = TIMStyleLLMJudge(llm=llm) + result = await judge.judge_trace(ConversationRecord(session_id="s", input_text="x", output_text="y")) + self.assertEqual(llm.calls, 2) + self.assertEqual(result.judgeScore, 0.8) + + llm2 = FakeLLM(["bad", "still-bad"]) + judge2 = TIMStyleLLMJudge(llm=llm2) + result2 = await judge2.judge_trace(ConversationRecord(session_id="s", input_text="x", output_text="y")) + self.assertEqual(llm2.calls, 2) + self.assertIsNone(result2.judgeScore) + self.assertIsNone(result2.accuracyScore) + self.assertIsNone(result2.alucinationScore) + + asyncio.run(scenario()) + + +if __name__ == "__main__": + unittest.main()