first commit

evaluator/llm/client.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from evaluator.config.settings import settings
+from evaluator.llm.profile_resolver import LLMProfileResolver
+
+
+class LLMClient:
+    async def complete(self, prompt: str, profile_name: str | None = None) -> str:
+        raise NotImplementedError
+
+
+class MockLLMClient(LLMClient):
+    async def complete(self, prompt: str, profile_name: str | None = None) -> str:
+        if "inferredCsiScore" in prompt:
+            return json.dumps({
+                "inferredCsiScore": 0.5,
+                "resolution": 1,
+                "conversationPrecision": 1,
+                "rationale": "Avaliação mock."
+            }, ensure_ascii=False)
+
+        return json.dumps({
+            "judgeScore": 0.7,
+            "accuracyScore": 0.7,
+            "alucinationScore": 0.1,
+            "rationale": "Avaliação mock."
+        }, ensure_ascii=False)
+
+
+class OCICompatibleLLMClient(LLMClient):
+    """
+    Mesmo padrão do Agent Framework:
+    - LLM_PROVIDER=oci_openai
+    - OCI_GENAI_BASE_URL
+    - OCI_GENAI_API_KEY
+    - OCI_GENAI_MODEL
+    - llm_profiles.yaml opcional
+    """
+
+    def __init__(self):
+        self.resolver = LLMProfileResolver(settings)
+
+    async def complete(self, prompt: str, profile_name: str | None = None) -> str:
+        effective = self.resolver.resolve(profile_name or settings.llm_profile)
+
+        provider = str(effective.get("provider") or settings.LLM_PROVIDER)
+        model = str(effective.get("model") or settings.OCI_GENAI_MODEL)
+        base_url = effective.get("base_url") or settings.OCI_GENAI_BASE_URL
+        api_key = effective.get("api_key") or settings.OCI_GENAI_API_KEY
+        temperature = effective.get("temperature", settings.LLM_TEMPERATURE)
+        max_tokens = effective.get("max_tokens", settings.LLM_MAX_TOKENS)
+        timeout = effective.get("timeout_seconds", settings.LLM_TIMEOUT_SECONDS)
+
+        if provider == "mock":
+            return await MockLLMClient().complete(prompt, profile_name=profile_name)
+
+        if provider not in ("oci_openai", "openai_compatible"):
+            raise ValueError(f"Unsupported LLM provider: {provider}")
+
+        if not base_url:
+            raise RuntimeError("OCI_GENAI_BASE_URL is required for oci_openai provider")
+
+        if not api_key:
+            raise RuntimeError("OCI_GENAI_API_KEY is required for oci_openai provider")
+
+        from openai import AsyncOpenAI
+
+        client = AsyncOpenAI(
+            base_url=base_url,
+            api_key=api_key,
+            timeout=timeout,
+        )
+
+        resp = await client.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "user", "content": prompt}
+            ],
+            temperature=temperature,
+            max_tokens=max_tokens,
+        )
+
+        return resp.choices[0].message.content or ""
+
+
+def create_llm_client() -> LLMClient:
+    provider = (settings.LLM_PROVIDER or "mock").lower()
+
+    if provider in ("mock", "none"):
+        return MockLLMClient()
+
+    if provider in ("oci_openai", "openai_compatible", "oci"):
+        return OCICompatibleLLMClient()
+
+    raise ValueError(f"Unsupported LLM_PROVIDER={settings.LLM_PROVIDER}")

evaluator/llm/profile_resolver.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import copy
+import os
+import re
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def _canonical_profile_name(value: str | None) -> str:
+    name = (value or "default").strip()
+    name = name.replace("-", "_").replace(".", "_").replace(" ", "_")
+    name = re.sub(r"(?<!^)(?=[A-Z])", "_", name).lower()
+    return re.sub(r"_+", "_", name).strip("_") or "default"
+
+
+class LLMProfileResolver:
+    def __init__(self, settings: Any):
+        self.settings = settings
+        self.path = self._find_profiles_file()
+        self.enabled = self.path is not None
+        self._profiles = self._load_profiles(self.path) if self.enabled else {}
+
+    def _find_profiles_file(self) -> Path | None:
+        root = getattr(self.settings, "project_root", Path.cwd())
+        configured = Path(getattr(self.settings, "LLM_PROFILES_PATH", "") or "").expanduser()
+        candidates = []
+        if configured:
+            candidates.append(configured if configured.is_absolute() else Path(root) / configured)
+        candidates += [
+            Path(root) / "llm_profiles.yaml",
+            Path(root) / "configs/llm_profiles/llm_profiles.yaml",
+            Path(root) / "config/llm_profiles.yaml",
+            Path("llm_profiles.yaml"),
+            Path("configs/llm_profiles/llm_profiles.yaml"),
+        ]
+        for path in candidates:
+            if path and path.exists() and path.is_file():
+                return path
+        return None
+
+    def _load_profiles(self, path: Path) -> dict[str, dict[str, Any]]:
+        # Expand ${VAR} placeholders, matching the way env-driven framework config is commonly used.
+        text = os.path.expandvars(path.read_text(encoding="utf-8"))
+        data = yaml.safe_load(text) or {}
+        raw = data.get("profiles", data)
+        profiles = {}
+        for name, value in raw.items():
+            if isinstance(value, dict):
+                profiles[_canonical_profile_name(str(name))] = dict(value)
+        return profiles
+
+    def env_defaults(self) -> dict[str, Any]:
+        return {
+            "provider": getattr(self.settings, "LLM_PROVIDER", "mock"),
+            "model": getattr(self.settings, "OCI_GENAI_MODEL", "mock-llm"),
+            "temperature": getattr(self.settings, "LLM_TEMPERATURE", 0.0),
+            "max_tokens": getattr(self.settings, "LLM_MAX_TOKENS", 900),
+            "timeout_seconds": getattr(self.settings, "LLM_TIMEOUT_SECONDS", 120),
+            "base_url": getattr(self.settings, "OCI_GENAI_BASE_URL", None),
+            "api_key": getattr(self.settings, "OCI_GENAI_API_KEY", None),
+            "project_ocid": getattr(self.settings, "OCI_GENAI_PROJECT_OCID", None),
+        }
+
+    def resolve(self, profile_name: str | None = None, **overrides) -> dict[str, Any]:
+        profile_key = _canonical_profile_name(profile_name)
+        effective = self.env_defaults()
+
+        if self.enabled:
+            effective.update(copy.deepcopy(self._profiles.get("default") or {}))
+            effective.update(copy.deepcopy(self._profiles.get(profile_key) or {}))
+
+        for key, value in overrides.items():
+            if value is not None:
+                effective[key] = value
+
+        effective["profile_name"] = profile_key
+        return effective