first commit

evaluator/llm/client.py

@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+from typing import Any
+
+from evaluator.config.settings import settings
+from evaluator.llm.profile_resolver import LLMProfileResolver
+
+
+class LLMClient:
+    async def complete(self, prompt: str, profile_name: str | None = None) -> str:
+        raise NotImplementedError
+
+
+class MockLLMClient(LLMClient):
+    async def complete(self, prompt: str, profile_name: str | None = None) -> str:
+        if "inferredCsiScore" in prompt:
+            return json.dumps({
+                "inferredCsiScore": 0.5,
+                "resolution": 1,
+                "conversationPrecision": 1,
+                "rationale": "Avaliação mock."
+            }, ensure_ascii=False)
+        return json.dumps({
+            "judgeScore": 0.7,
+            "accuracyScore": 0.7,
+            "alucinationScore": 0.1,
+            "rationale": "Avaliação mock."
+        }, ensure_ascii=False)
+
+
+class OCICompatibleLLMClient(LLMClient):
+    def __init__(self):
+        self.resolver = LLMProfileResolver(settings)
+
+    async def complete(self, prompt: str, profile_name: str | None = None) -> str:
+        effective = self.resolver.resolve(profile_name or settings.llm_profile)
+        provider = str(effective.get("provider") or settings.LLM_PROVIDER).lower()
+        model = str(effective.get("model") or settings.OCI_GENAI_MODEL)
+        base_url = effective.get("base_url") or settings.OCI_GENAI_BASE_URL
+        api_key = effective.get("api_key") or settings.OCI_GENAI_API_KEY
+        temperature = effective.get("temperature", settings.LLM_TEMPERATURE)
+        max_tokens = effective.get("max_tokens", settings.LLM_MAX_TOKENS)
+        timeout = effective.get("timeout_seconds", settings.LLM_TIMEOUT_SECONDS)
+
+        if provider == "mock":
+            return await MockLLMClient().complete(prompt, profile_name=profile_name)
+        if provider not in ("oci_openai", "openai_compatible"):
+            raise ValueError(f"Unsupported LLM provider: {provider}")
+        if not base_url:
+            raise RuntimeError("OCI_GENAI_BASE_URL is required for oci_openai provider")
+        if not api_key:
+            raise RuntimeError("OCI_GENAI_API_KEY is required for oci_openai provider")
+
+        from openai import AsyncOpenAI
+        client = AsyncOpenAI(base_url=base_url, api_key=api_key, timeout=timeout)
+        resp = await client.chat.completions.create(
+            model=model,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=temperature,
+            max_tokens=max_tokens,
+        )
+        return resp.choices[0].message.content or ""
+
+
+class OCISDKLLMClient(LLMClient):
+    """OCI Generative AI native SDK, including dedicated endpoints."""
+
+    def __init__(self):
+        self.resolver = LLMProfileResolver(settings)
+        self._clients: dict[str, Any] = {}
+
+    @staticmethod
+    def _normalize_endpoint(value: str | None) -> str:
+        endpoint = (value or "").strip().strip("'\"").rstrip("/")
+        if not endpoint:
+            region = settings.oci_region or "sa-saopaulo-1"
+            return f"https://inference.generativeai.{region}.oci.oraclecloud.com"
+        if "/openai/v1" in endpoint:
+            raise RuntimeError(
+                "OCI_GENAI_BASE_URL cannot contain /openai/v1 when LLM_PROVIDER=oci_sdk"
+            )
+        return endpoint
+
+    @staticmethod
+    def _config_and_signer():
+        import oci
+
+        mode = (settings.oci_auth_mode or "config_file").lower()
+        if mode == "config_file":
+            config_file = str(Path(settings.oci_config_file).expanduser())
+            return oci.config.from_file(config_file, settings.oci_profile), None
+        if mode == "instance_principal":
+            return {}, oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
+        if mode == "resource_principal":
+            return {}, oci.auth.signers.get_resource_principals_signer()
+        if mode in ("oke_workload_identity", "workload_identity"):
+            return {}, oci.auth.signers.get_oke_workload_identity_resource_principal_signer()
+        raise ValueError(f"Unsupported OCI_AUTH_MODE={settings.oci_auth_mode}")
+
+    def _client(self, endpoint: str):
+        if endpoint not in self._clients:
+            from oci.generative_ai_inference import GenerativeAiInferenceClient
+            config, signer = self._config_and_signer()
+            kwargs: dict[str, Any] = {"config": config, "service_endpoint": endpoint}
+            if signer is not None:
+                kwargs["signer"] = signer
+            self._clients[endpoint] = GenerativeAiInferenceClient(**kwargs)
+        return self._clients[endpoint]
+
+    @staticmethod
+    def _extract_answer(response: Any) -> str:
+        data = getattr(response, "data", response)
+        chat_response = getattr(data, "chat_response", None) or data
+        for attr in ("text", "message", "output_text"):
+            value = getattr(chat_response, attr, None)
+            if value:
+                return str(value)
+        choices = getattr(chat_response, "choices", None) or []
+        if choices:
+            first = choices[0]
+            msg = getattr(first, "message", None)
+            content = getattr(msg, "content", None) if msg is not None else getattr(first, "text", None)
+            if isinstance(content, list):
+                return "".join(str(getattr(item, "text", item)) for item in content)
+            if content:
+                return str(content)
+        return str(chat_response)
+
+    async def complete(self, prompt: str, profile_name: str | None = None) -> str:
+        from oci.generative_ai_inference import models
+
+        effective = self.resolver.resolve(profile_name or settings.llm_profile)
+        model = str(effective.get("model") or settings.OCI_GENAI_MODEL).strip()
+        endpoint_id = str(effective.get("endpoint_id") or settings.oci_genai_endpoint_id or "").strip() or None
+        if not endpoint_id and model.startswith("ocid1.generativeaiendpoint."):
+            endpoint_id = model
+        compartment_id = (
+            effective.get("compartment_id")
+            or settings.oci_compartment_id
+            or settings.OCI_GENAI_PROJECT_OCID
+        )
+        if not compartment_id:
+            raise RuntimeError(
+                "OCI_COMPARTMENT_ID or OCI_GENAI_PROJECT_OCID is required for LLM_PROVIDER=oci_sdk"
+            )
+
+        endpoint = self._normalize_endpoint(effective.get("base_url") or settings.OCI_GENAI_BASE_URL)
+        if endpoint_id:
+            serving_mode = models.DedicatedServingMode(endpoint_id=endpoint_id)
+        else:
+            serving_mode = models.OnDemandServingMode(model_id=model)
+
+        content = [models.TextContent(text=prompt)] if hasattr(models, "TextContent") else prompt
+        message = models.UserMessage(content=content)
+        chat_request = models.GenericChatRequest(
+            messages=[message],
+            temperature=effective.get("temperature", settings.LLM_TEMPERATURE),
+            max_tokens=effective.get("max_tokens", settings.LLM_MAX_TOKENS),
+        )
+        details = models.ChatDetails(
+            compartment_id=compartment_id,
+            serving_mode=serving_mode,
+            chat_request=chat_request,
+        )
+        response = await asyncio.to_thread(self._client(endpoint).chat, details)
+        return self._extract_answer(response)
+
+
+def create_llm_client() -> LLMClient:
+    provider = (settings.LLM_PROVIDER or "mock").lower()
+    if provider in ("mock", "none"):
+        return MockLLMClient()
+    if provider in ("oci_openai", "openai_compatible", "oci"):
+        return OCICompatibleLLMClient()
+    if provider == "oci_sdk":
+        return OCISDKLLMClient()
+    raise ValueError(f"Unsupported LLM_PROVIDER={settings.LLM_PROVIDER}")

evaluator/llm/profile_resolver.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import copy
+import os
+import re
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def _canonical_profile_name(value: str | None) -> str:
+    name = (value or "default").strip()
+    name = name.replace("-", "_").replace(".", "_").replace(" ", "_")
+    name = re.sub(r"(?<!^)(?=[A-Z])", "_", name).lower()
+    return re.sub(r"_+", "_", name).strip("_") or "default"
+
+
+class LLMProfileResolver:
+    def __init__(self, settings: Any):
+        self.settings = settings
+        self.path = self._find_profiles_file()
+        self.enabled = self.path is not None
+        self._profiles = self._load_profiles(self.path) if self.enabled else {}
+
+    def _find_profiles_file(self) -> Path | None:
+        root = getattr(self.settings, "project_root", Path.cwd())
+        configured = Path(getattr(self.settings, "LLM_PROFILES_PATH", "") or "").expanduser()
+        candidates = []
+        if configured:
+            candidates.append(configured if configured.is_absolute() else Path(root) / configured)
+        candidates += [
+            Path(root) / "llm_profiles.yaml",
+            Path(root) / "configs/llm_profiles/llm_profiles.yaml",
+            Path(root) / "config/llm_profiles.yaml",
+            Path("llm_profiles.yaml"),
+            Path("configs/llm_profiles/llm_profiles.yaml"),
+        ]
+        for path in candidates:
+            if path and path.exists() and path.is_file():
+                return path
+        return None
+
+    def _load_profiles(self, path: Path) -> dict[str, dict[str, Any]]:
+        # Expand ${VAR} placeholders, matching the way env-driven framework config is commonly used.
+        text = os.path.expandvars(path.read_text(encoding="utf-8"))
+        data = yaml.safe_load(text) or {}
+        raw = data.get("profiles", data)
+        profiles = {}
+        for name, value in raw.items():
+            if isinstance(value, dict):
+                profiles[_canonical_profile_name(str(name))] = dict(value)
+        return profiles
+
+    def env_defaults(self) -> dict[str, Any]:
+        return {
+            "provider": getattr(self.settings, "LLM_PROVIDER", "mock"),
+            "model": getattr(self.settings, "OCI_GENAI_MODEL", "mock-llm"),
+            "temperature": getattr(self.settings, "LLM_TEMPERATURE", 0.0),
+            "max_tokens": getattr(self.settings, "LLM_MAX_TOKENS", 900),
+            "timeout_seconds": getattr(self.settings, "LLM_TIMEOUT_SECONDS", 120),
+            "base_url": getattr(self.settings, "OCI_GENAI_BASE_URL", None),
+            "api_key": getattr(self.settings, "OCI_GENAI_API_KEY", None),
+            "project_ocid": getattr(self.settings, "OCI_GENAI_PROJECT_OCID", None),
+            "endpoint_id": getattr(self.settings, "oci_genai_endpoint_id", None),
+            "compartment_id": getattr(self.settings, "oci_compartment_id", None),
+        }
+
+    def resolve(self, profile_name: str | None = None, **overrides) -> dict[str, Any]:
+        profile_key = _canonical_profile_name(profile_name)
+        effective = self.env_defaults()
+
+        if self.enabled:
+            effective.update(copy.deepcopy(self._profiles.get("default") or {}))
+            effective.update(copy.deepcopy(self._profiles.get(profile_key) or {}))
+
+        for key, value in overrides.items():
+            if value is not None:
+                effective[key] = value
+
+        effective["profile_name"] = profile_key
+        return effective