first commit

evaluator/llm/client.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from evaluator.config.settings import settings
+from evaluator.llm.profile_resolver import LLMProfileResolver
+
+
+class LLMClient:
+    async def complete(self, prompt: str, profile_name: str | None = None) -> str:
+        raise NotImplementedError
+
+
+class MockLLMClient(LLMClient):
+    async def complete(self, prompt: str, profile_name: str | None = None) -> str:
+        if "inferredCsiScore" in prompt:
+            return json.dumps({
+                "inferredCsiScore": 0.5,
+                "resolution": 1,
+                "conversationPrecision": 1,
+                "rationale": "Avaliação mock."
+            }, ensure_ascii=False)
+
+        return json.dumps({
+            "judgeScore": 0.7,
+            "accuracyScore": 0.7,
+            "alucinationScore": 0.1,
+            "rationale": "Avaliação mock."
+        }, ensure_ascii=False)
+
+
+class OCICompatibleLLMClient(LLMClient):
+    """
+    Mesmo padrão do Agent Framework:
+    - LLM_PROVIDER=oci_openai
+    - OCI_GENAI_BASE_URL
+    - OCI_GENAI_API_KEY
+    - OCI_GENAI_MODEL
+    - llm_profiles.yaml opcional
+    """
+
+    def __init__(self):
+        self.resolver = LLMProfileResolver(settings)
+
+    async def complete(self, prompt: str, profile_name: str | None = None) -> str:
+        effective = self.resolver.resolve(profile_name or settings.llm_profile)
+
+        provider = str(effective.get("provider") or settings.LLM_PROVIDER)
+        model = str(effective.get("model") or settings.OCI_GENAI_MODEL)
+        base_url = effective.get("base_url") or settings.OCI_GENAI_BASE_URL
+        api_key = effective.get("api_key") or settings.OCI_GENAI_API_KEY
+        temperature = effective.get("temperature", settings.LLM_TEMPERATURE)
+        max_tokens = effective.get("max_tokens", settings.LLM_MAX_TOKENS)
+        timeout = effective.get("timeout_seconds", settings.LLM_TIMEOUT_SECONDS)
+
+        if provider == "mock":
+            return await MockLLMClient().complete(prompt, profile_name=profile_name)
+
+        if provider not in ("oci_openai", "openai_compatible"):
+            raise ValueError(f"Unsupported LLM provider: {provider}")
+
+        if not base_url:
+            raise RuntimeError("OCI_GENAI_BASE_URL is required for oci_openai provider")
+
+        if not api_key:
+            raise RuntimeError("OCI_GENAI_API_KEY is required for oci_openai provider")
+
+        from openai import AsyncOpenAI
+
+        client = AsyncOpenAI(
+            base_url=base_url,
+            api_key=api_key,
+            timeout=timeout,
+        )
+
+        resp = await client.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "user", "content": prompt}
+            ],
+            temperature=temperature,
+            max_tokens=max_tokens,
+        )
+
+        return resp.choices[0].message.content or ""
+
+
+def create_llm_client() -> LLMClient:
+    provider = (settings.LLM_PROVIDER or "mock").lower()
+
+    if provider in ("mock", "none"):
+        return MockLLMClient()
+
+    if provider in ("oci_openai", "openai_compatible", "oci"):
+        return OCICompatibleLLMClient()
+
+    raise ValueError(f"Unsupported LLM_PROVIDER={settings.LLM_PROVIDER}")