Merge branch 'main' of https://git.tech-lad.com.br/alex.a.alves/litellm-oci

.dockerignore

@@ -0,0 +1,4 @@
+__pycache__/
+*.pyc
+.env
+.git/

.env.example

@@ -0,0 +1,21 @@
+# LiteLLM
+LITELLM_MASTER_KEY=dummy-key
+
+# OCI auth
+OCI_USER=ocid1.user.oc1..xxxxxxx
+OCI_FINGERPRINT=e6:17:xxxxxxx
+OCI_TENANCY=ocid1.tenancy.oc1..xxxxx
+OCI_REGION=us-chicago-1
+OCI_COMPARTMENT_ID=ocid1.compartment.oc1..xxxxxx
+
+# OCI key file path inside container
+OCI_KEY_FILE=/app/oci_api_key.pem
+
+
+# Optional params for embeddings
+OCI_SERVING_MODE=ON_DEMAND
+OCI_INPUT_TYPE=SEARCH_DOCUMENT
+OCI_EMBED_TRUNCATE=NONE
+
+# Required only if OCI_SERVING_MODE=DEDICATED
+OCI_ENDPOINT_ID=

.gitignore

@@ -0,0 +1,2 @@
+.secrets/oci_api_key.pem
+.env

.secrets/oci_api_key.txt

@@ -0,0 +1,3 @@
+salvar arquivo .pem aqui
+
+oci_api_key.pem

Dockerfile

@@ -0,0 +1,13 @@
+FROM docker.litellm.ai/berriai/litellm:main-latest
+
+USER root
+RUN pip install --no-cache-dir "oci>=2.150.0"
+
+WORKDIR /app
+COPY oci_embedding_handler.py /app/oci_embedding_handler.py
+COPY config.yaml /app/config.yaml
+
+ENV PYTHONPATH=/app
+
+# Base image entrypoint already launches litellm proxy.
+CMD ["--config", "/app/config.yaml", "--port", "4000"]

README.md

@@ -1,3 +1,217 @@
-# litellm-oci
+# LiteLLM OCI Proxy
 
-This repository contains a Docker Compose file for quickly launching a liteLLM container integrated with OCI GenAI and a custom model integration.
+A proxy server that provides **OpenAI API compatibility** for all models available through **OCI Generative AI**, including xAI Grok, Meta Llama, and Cohere models.
+
+This proxy enables you to use the standard OpenAI SDK and API format to interact with OCI Generative AI models, making it easy to integrate OCI models into applications that expect OpenAI-compatible APIs.
+
+## Setup
+
+1. **Clone the repository:**
+```bash
+git clone https://git.tech-lad.com.br/alex.a.alves/litellm-oci.git
+cd litellm-oci
+```
+
+2. **Install dependencies:**
+```bash
+uv sync
+```
+
+3. **Create your configuration file:**
+   - Copy the example configuration file:
+   ```bash
+   cp config.yaml.example config.yaml
+   ```
+
+4. **Configure OCI credentials:**
+   - Open `config.yaml` in your editor
+   - Replace the placeholder values with your OCI credentials:
+
+     **Values from OCI CLI config file (`~/.oci/config` section):**
+     - `oci_user`: Your OCI user OCID (from `user` field)
+     - `oci_fingerprint`: Your OCI API key fingerprint (from `fingerprint` field)
+     - `oci_tenancy`: Your OCI tenancy OCID (from `tenancy` field)
+     - `oci_region`: Your OCI region (from `region` field, e.g., `us-chicago-1`)
+     - `oci_key_file`: Absolute path to your OCI API private key file (from `key_file` field, e.g., `/Users/yourname/.oci/oci_api_key.pem`)
+
+     **Values from OCI Console:**
+     - `oci_compartment_id`: Your OCI compartment OCID (find in OCI Console under Identity & Security → Compartments, or use `oci iam compartment list` to find it)
+   
+   **Note:** Use absolute paths for `oci_key_file` (the `~` tilde is not expanded by LiteLLM).
+
+5. **Ensure your OCI API key is available:**
+   - Make sure your OCI API private key file exists at the path specified in `oci_key_file`
+   - The key file should have appropriate permissions (typically `600` or `400`)
+   
+   For help creating OCI API keys, see the [official Oracle tutorial](https://docs.oracle.com/en-us/iaas/Content/API/Concepts/apisigningkey.htm).
+
+## Running the LiteLLM Proxy
+
+1. **Activate the virtual environment:**
+```bash
+source .venv/bin/activate
+```
+
+2. **Start the proxy server using the configuration file:**
+```bash
+litellm --config config.yaml
+```
+
+The proxy will start on `http://localhost:4000` by default.
+
+**Note**: If you're using Python 3.14 and encounter `uvloop` compatibility issues, use the wrapper script:
+
+```bash
+python run_proxy.py --config config.yaml
+```
+
+This wrapper script patches LiteLLM to use the `asyncio` event loop instead of `uvloop`, which is not compatible with Python 3.14.
+
+## OpenAI API Compatibility
+
+This proxy provides **full OpenAI API compatibility** for all OCI Generative AI models. You can use:
+
+- **OpenAI Python SDK** - Drop-in replacement for OpenAI API calls
+- OpenAI-compatible HTTP clients in any language
+- Standard OpenAI API endpoints (`/v1/chat/completions`, `/v1/models`, etc.)
+- OpenAI response format (same structure as OpenAI responses)
+
+All OCI models are accessible through the standard OpenAI API interface, making it easy to switch between OpenAI and OCI models without changing your application code.
+
+## Using the Proxy
+
+### Python Example
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    api_key="sk-any-string",  # Required by client but not validated by LiteLLM
+    base_url="http://localhost:4000/v1"  # LiteLLM proxy endpoint
+)
+
+response = client.chat.completions.create(
+    model="xai.grok-3",
+    messages=[
+        {"role": "user", "content": "Hello, how are you?"}
+    ],
+)
+
+print(response.choices[0].message.content)
+```
+
+### cURL Example
+
+```bash
+curl http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-any-string" \
+  -d '{
+    "model": "xai.grok-3",
+    "messages": [
+      {"role": "user", "content": "Hello, how are you?"}
+    ]
+  }'
+```
+
+### Running the Example Script
+
+The `main.py` file demonstrates OpenAI API compatibility by using the OpenAI SDK to call the LiteLLM proxy:
+
+1. **First, activate the virtual environment and start the proxy** (in one terminal):
+```bash
+source .venv/bin/activate
+python run_proxy.py --config config.yaml
+```
+
+2. **Then activate the virtual environment and run the example** (in another terminal):
+```bash
+source .venv/bin/activate
+python main.py
+```
+
+The `main.py` script dynamically loads all available models from `config.yaml`, allows you to select a model interactively, and makes requests using the standard OpenAI SDK format. This demonstrates full OpenAI API compatibility for all OCI Generative AI models.
+
+## Configuration
+
+The `config.yaml` file (created from `config.yaml.example`) contains all supported OCI models with shared authentication credentials using YAML anchors. The configuration includes:
+
+- **OCI authentication credentials** (user, fingerprint, tenancy, region, key file, compartment ID)
+- **Region**: `us-chicago-1`
+- **Serving mode**: `ON_DEMAND`
+- **drop_params**: `true` (automatically filters unsupported parameters)
+
+### Available Models
+
+All supported OCI models are configured in `config.yaml`:
+
+**xAI Grok Models:**
+- `oci/xai.grok-4`
+- `oci/xai.grok-4-fast-reasoning` (Reasoning mode - for complex, multi-step problems)
+- `oci/xai.grok-4-fast-non-reasoning` (Non-Reasoning mode - for speed-critical queries)
+- `oci/xai.grok-3`
+- `oci/xai.grok-3-fast`
+- `oci/xai.grok-3-mini`
+- `oci/xai.grok-3-mini-fast`
+- `oci/xai.grok-code-fast-1`
+
+**Meta Llama Models:**
+- `oci/meta.llama-4-maverick-17b-128e-instruct-fp8`
+- `oci/meta.llama-4-scout-17b-16e-instruct`
+- `oci/meta.llama-3.3-70b-instruct`
+- `oci/meta.llama-3.2-90b-vision-instruct`
+- `oci/meta.llama-3.1-405b-instruct`
+
+**Cohere Models:**
+- `oci/cohere.command-latest`
+- `oci/cohere.command-a-03-2025`
+- `oci/cohere.command-plus-latest`
+
+To use a specific model, use its `model_name` when making requests (e.g., `oci/xai.grok-4`).
+
+## Security Note
+
+⚠️ **Important**: The `config.yaml` file contains sensitive credentials. Do not commit it to version control. Consider using environment variables or a secrets manager for production deployments.
+
+
+# LiteLLM + OCI Embeddings (Custom Provider)
+
+This folder packages LiteLLM with a custom OCI embedding handler and environment-driven OCI config.
+
+## 1) Prepare environment
+
+```bash
+cd /Users/alexalves/Documents/New\ project/litellm-oci-proxy
+cp .env.example .env
+# edit .env with real OCI values and key path
+```
+
+## 2) Build and run
+
+```bash
+docker compose up -d --build
+```
+
+## 3) Health check
+
+```bash
+curl -s http://localhost:4000/health
+```
+
+## 4) Test embedding model
+
+```bash
+curl -s http://localhost:4000/v1/embeddings \
+  -H "Authorization: Bearer ${LITELLM_MASTER_KEY}" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "cohere-embed-multilingual",
+    "input": ["teste de embedding"]
+  }'
+```
+
+## Notes
+
+- OCI credentials and region come from env vars through `config.yaml` + handler fallback.
+- You do **not** need to rebuild the image when only auth values/region/compartment change.
+- If you change handler code or dependencies, rebuild with `docker compose up -d --build`.

config.yaml

@@ -0,0 +1,129 @@
+# Common OCI authentication parameters (using YAML anchor)
+oci_auth: &oci_auth
+  drop_params: true
+  oci_user: os.environ/OCI_USER
+  oci_fingerprint: os.environ/OCI_FINGERPRINT
+  oci_tenancy: os.environ/OCI_TENANCY
+  oci_region: os.environ/OCI_REGION
+  oci_key_file: os.environ/OCI_KEY_FILE
+  oci_compartment_id: os.environ/OCI_COMPARTMENT_ID
+  oci_serving_mode: os.environ/OCI_SERVING_MODE
+
+general_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+  health_check_details: false
+
+model_list:
+  # xAI Grok Models
+  - model_name: oci/xai.grok-4
+    litellm_params:
+      <<: *oci_auth
+      model: oci/xai.grok-4
+
+  - model_name: oci/xai.grok-4-fast-reasoning
+    litellm_params:
+      <<: *oci_auth
+      model: oci/xai.grok-4-fast-reasoning
+
+  - model_name: oci/xai.grok-4-fast-non-reasoning
+    litellm_params:
+      <<: *oci_auth
+      model: oci/xai.grok-4-fast-non-reasoning
+
+  - model_name: oci/xai.grok-3
+    litellm_params:
+      <<: *oci_auth
+      model: oci/xai.grok-3
+
+  - model_name: oci/xai.grok-3-fast
+    litellm_params:
+      <<: *oci_auth
+      model: oci/xai.grok-3-fast
+
+  - model_name: oci/xai.grok-3-mini
+    litellm_params:
+      <<: *oci_auth
+      model: oci/xai.grok-3-mini
+
+  - model_name: oci/xai.grok-3-mini-fast
+    litellm_params:
+      <<: *oci_auth
+      model: oci/xai.grok-3-mini-fast
+
+  - model_name: oci/xai.grok-code-fast-1
+    litellm_params:
+      <<: *oci_auth
+      model: oci/xai.grok-code-fast-1
+
+  # Meta Llama Models
+  - model_name: oci/meta.llama-4-maverick-17b-128e-instruct-fp8
+    litellm_params:
+      <<: *oci_auth
+      model: oci/meta.llama-4-maverick-17b-128e-instruct-fp8
+
+  - model_name: oci/meta.llama-4-scout-17b-16e-instruct
+    litellm_params:
+      <<: *oci_auth
+      model: oci/meta.llama-4-scout-17b-16e-instruct
+
+  - model_name: oci/meta.llama-3.3-70b-instruct
+    litellm_params:
+      <<: *oci_auth
+      model: oci/meta.llama-3.3-70b-instruct
+
+  - model_name: oci/meta.llama-3.2-90b-vision-instruct
+    litellm_params:
+      <<: *oci_auth
+      model: oci/meta.llama-3.2-90b-vision-instruct
+
+  - model_name: oci/meta.llama-3.1-405b-instruct
+    litellm_params:
+      <<: *oci_auth
+      model: oci/meta.llama-3.1-405b-instruct
+
+  # Cohere Models
+  - model_name: oci/cohere.command-latest
+    litellm_params:
+      <<: *oci_auth
+      model: oci/cohere.command-latest
+
+  - model_name: oci/cohere.command-a-03-2025
+    litellm_params:
+      <<: *oci_auth
+      model: oci/cohere.command-a-03-2025
+
+  - model_name: oci/cohere.command-plus-latest
+    litellm_params:
+      <<: *oci_auth
+      model: oci/cohere.command-plus-latest
+
+  - model_name: cohere-embed-multilingual
+    litellm_params:
+      <<: *oci_auth
+      model: oci-embed/cohere.embed-multilingual-v3.0
+      input_type: os.environ/OCI_INPUT_TYPE
+      truncate: os.environ/OCI_EMBED_TRUNCATE
+    model_info:
+      mode: embedding
+
+  # Google Gemini Models
+  - model_name: oci/google.gemini-2.5-pro
+    litellm_params:
+      <<: *oci_auth
+      model: oci/google.gemini-2.5-pro
+
+  - model_name: oci/google.gemini-2.5-flash
+    litellm_params:
+      <<: *oci_auth
+      model: oci/google.gemini-2.5-flash
+
+  - model_name: oci/google.gemini-2.5-flash-lite
+    litellm_params:
+      <<: *oci_auth
+      model: oci/google.gemini-2.5-flash-lite
+
+litellm_settings:
+  # Embedding specific settings
+  custom_provider_map:
+    - provider: oci-embed
+      custom_handler: oci_embedding_handler.oci_embedding_llm

docker-compose.yaml

@@ -0,0 +1,16 @@
+services:
+  litellm:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: litellm-oci-proxy:latest
+    container_name: litellm-oci-proxy
+    restart: unless-stopped
+    env_file:
+      - .env
+    ports:
+      - "4000:4000"
+    volumes:
+      - ./secrets/oci_api_key.pem:/app/oci_api_key.pem:ro,Z
+
+    command: ["--config", "/app/config.yaml", "--port", "4000"]

oci_embedding_handler.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Optional
+
+import oci
+from litellm import CustomLLM
+from litellm.types.utils import Embedding, EmbeddingResponse, Usage
+
+
+class OCIEmbeddingLLM(CustomLLM):
+    def __init__(self):
+        super().__init__()
+        self._client = None
+        self._client_signature = None
+
+    @staticmethod
+    def _pick_value(optional_params: dict, param_name: str, env_name: str, default: Optional[str] = None):
+        value = optional_params.get(param_name)
+        if value is None:
+            value = os.getenv(env_name, default)
+        if isinstance(value, str):
+            value = value.strip()
+        return value or default
+
+    def _build_oci_config(self, optional_params: dict) -> dict:
+        # Build OCI SDK config from LiteLLM params first, then fallback to env vars.
+        oci_config = {
+            "user": self._pick_value(optional_params, "oci_user", "OCI_USER"),
+            "fingerprint": self._pick_value(optional_params, "oci_fingerprint", "OCI_FINGERPRINT"),
+            "tenancy": self._pick_value(optional_params, "oci_tenancy", "OCI_TENANCY"),
+            "region": self._pick_value(optional_params, "oci_region", "OCI_REGION"),
+            "key_file": self._pick_value(optional_params, "oci_key_file", "OCI_KEY_FILE"),
+            "pass_phrase": self._pick_value(optional_params, "oci_pass_phrase", "OCI_PASS_PHRASE"),
+        }
+
+        if oci_config.get("key_file"):
+            oci_config["key_file"] = str(Path(oci_config["key_file"]).expanduser())
+
+        # Remove optional empty values so OCI SDK handles defaults cleanly.
+        if not oci_config.get("pass_phrase"):
+            oci_config.pop("pass_phrase", None)
+
+        missing = [
+            key
+            for key in ("user", "fingerprint", "tenancy", "region", "key_file")
+            if not oci_config.get(key)
+        ]
+        if missing:
+            raise ValueError(
+                "Missing OCI config values for embedding provider: " + ", ".join(missing)
+            )
+
+        return oci_config
+
+    def _get_client(self, optional_params: dict):
+        """Lazy init OCI client and recreate only when OCI auth/region changes."""
+        config = self._build_oci_config(optional_params)
+        signature = tuple(sorted(config.items()))
+
+        if self._client is None or self._client_signature != signature:
+            self._client = oci.generative_ai_inference.GenerativeAiInferenceClient(config)
+            self._client_signature = signature
+
+        return self._client
+
+    def _build_serving_mode(self, model_id: str, optional_params: dict):
+        serving_mode = self._pick_value(
+            optional_params,
+            "oci_serving_mode",
+            "OCI_SERVING_MODE",
+            default="ON_DEMAND",
+        ).upper()
+
+        if serving_mode == "DEDICATED":
+            endpoint_id = self._pick_value(optional_params, "oci_endpoint_id", "OCI_ENDPOINT_ID")
+            if not endpoint_id:
+                raise ValueError(
+                    "oci_endpoint_id/OCI_ENDPOINT_ID is required when oci_serving_mode is DEDICATED"
+                )
+            return oci.generative_ai_inference.models.DedicatedServingMode(endpoint_id=endpoint_id)
+
+        return oci.generative_ai_inference.models.OnDemandServingMode(model_id=model_id)
+
+    def embedding(
+        self,
+        model: str,
+        input: list,
+        model_response: EmbeddingResponse,
+        optional_params: dict,
+        encoding=None,
+        api_key: Optional[str] = None,
+        **kwargs,
+    ) -> EmbeddingResponse:
+        del encoding, api_key, kwargs  # not used by OCI embedding endpoint
+
+        optional_params = optional_params or {}
+
+        # model arrives as "oci-embed/cohere.embed-multilingual-v3.0"
+        model_id = model.split("/", 1)[-1]
+
+        compartment_id = self._pick_value(
+            optional_params, "oci_compartment_id", "OCI_COMPARTMENT_ID"
+        )
+        if not compartment_id:
+            raise ValueError("oci_compartment_id/OCI_COMPARTMENT_ID is required")
+
+        input_type = self._pick_value(
+            optional_params,
+            "input_type",
+            "OCI_INPUT_TYPE",
+            default="SEARCH_DOCUMENT",
+        ).upper()
+
+        truncate = self._pick_value(
+            optional_params,
+            "truncate",
+            "OCI_EMBED_TRUNCATE",
+            default="NONE",
+        ).upper()
+
+        inputs = input if isinstance(input, list) else [input]
+
+        detail = oci.generative_ai_inference.models.EmbedTextDetails(
+            inputs=inputs,
+            serving_mode=self._build_serving_mode(model_id, optional_params),
+            compartment_id=compartment_id,
+            input_type=input_type,
+            truncate=truncate,
+        )
+
+        resp = self._get_client(optional_params).embed_text(detail)
+
+        model_response.model = model_id
+        model_response.data = [
+            Embedding(object="embedding", index=i, embedding=emb)
+            for i, emb in enumerate(resp.data.embeddings)
+        ]
+        model_response.usage = Usage(
+            prompt_tokens=resp.data.usage.prompt_tokens if resp.data.usage else 0,
+            completion_tokens=0,
+            total_tokens=resp.data.usage.total_tokens if resp.data.usage else 0,
+        )
+
+        return model_response
+
+    async def aembedding(self, *args, **kwargs) -> EmbeddingResponse:
+        # LiteLLM proxy is async; OCI SDK call is sync.
+        return self.embedding(*args, **kwargs)
+
+
+oci_embedding_llm = OCIEmbeddingLLM()