commit 029adfafc37107db6662c906db969966d4aa345f Author: Cristiano Hoshikawa Date: Sat Jun 27 09:32:54 2026 -0300 first commit diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..ec3d829 Binary files /dev/null and b/.DS_Store differ diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..0c670ad --- /dev/null +++ b/.env.example @@ -0,0 +1,23 @@ +# LiteLLM Proxy +LITELLM_BASE_URL=http://localhost:4000/v1 +LITELLM_MASTER_KEY=sk-master-poc +LITELLM_MODEL_ALIAS=oci-gemini-25-pro + +# Configuração OCI para LiteLLM provider OCI +OCI_REGION=us-chicago-1 +OCI_COMPARTMENT_ID=ocid1.compartment.oc1..replace_me +OCI_AUTH_MODE=api_key +OCI_USER=ocid1.user.oc1..replace_me +OCI_TENANCY=ocid1.tenancy.oc1..replace_me +OCI_FINGERPRINT=replace_me +OCI_KEY_FILE=/home/opc/.oci/oci_api_key.pem + +# OCI Usage API REST +OCI_USAGE_ENDPOINT=https://usageapi.us-ashburn-1.oci.oraclecloud.com +OCI_USAGE_API_VERSION=20200107 +OCI_USAGE_SERVICE_FILTER=generative_ai +OCI_USAGE_PRODUCT_FILTER=Google - Gemini 2.5 Pro + +# Ledger local da PoC +LEDGER_DB=./storage/ledger.sqlite +CONFIG_FILE=./config/apis.yaml diff --git a/Quick_Test_pt.md b/Quick_Test_pt.md new file mode 100644 index 0000000..d4df590 --- /dev/null +++ b/Quick_Test_pt.md @@ -0,0 +1,54 @@ +# PoC - Rateio de custo OCI Generative AI via LiteLLM + +Objetivo: simular várias APIs consumindo Gemini via OCI Generative AI por meio do LiteLLM Proxy, registrar uso por API e ratear o custo total mensal retornado pela OCI Usage API. + +## Fluxo + +1. Cada API consumidora chama o LiteLLM com uma virtual key própria. +2. O request também envia `metadata.api_id` e `metadata.api_name`. +3. Um callback do LiteLLM grava o uso em SQLite: input tokens, output tokens, total tokens, custo estimado LiteLLM e latência. +4. O job `allocate_costs.py` busca o custo mensal total na OCI Usage API via REST assinado. +5. O custo OCI é rateado proporcionalmente pelo consumo registrado no LiteLLM. + +## Rodar + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +cp .env.example .env +# edite .env com OCI_REGION, OCI_COMPARTMENT_ID e credenciais OCI + +docker compose up -d +python src/create_litellm_keys.py +PYTHONPATH=src uvicorn api_service:app --host 0.0.0.0 --port 8080 +``` + +Em outro terminal: + +```bash +python src/simulate_load.py --calls 30 --concurrency 5 +python src/allocate_costs.py --month 2026-06 --oci-cost 1234.56 +``` + +Para usar a OCI de verdade, remova `--oci-cost`: + +```bash +python src/allocate_costs.py --month 2026-06 +``` + +## Ajustes importantes + +- `config/apis.yaml`: cadastro das APIs, virtual keys, dono/squad e pesos de token. +- `config/litellm_config.yaml`: modelo OCI exposto como alias `oci-gemini-25-pro`. +- `.env`: credenciais OCI, endpoint da Usage API, filtros de custo e banco local. + +## Fórmula padrão + +```text +weighted_tokens = input_tokens * input_weight + output_tokens * output_weight +share_api = weighted_tokens_api / sum(weighted_tokens_todas_apis) +allocated_cost_api = oci_total_monthly_cost * share_api +``` + +Para PoC, `input_weight=1` e `output_weight=5`. Ajuste conforme a tabela de preço efetiva do produto/SKU Gemini no contrato/OCI. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0efd2a5 --- /dev/null +++ b/README.md @@ -0,0 +1,384 @@ + +# OCI Generative AI Cost Allocation per API using LiteLLM + + +## Overview + +Organizations commonly expose a single OCI Generative AI endpoint to dozens or hundreds of internal APIs and AI Agents. + +Although OCI provides consolidated billing information for Generative AI consumption, it does **not** provide an out-of-the-box cost breakdown by individual application or API. + +This project solves that problem. + +By introducing LiteLLM Proxy between the applications and OCI Generative AI, every request can be attributed to a specific API, while the official monthly OCI cost is collected through the OCI Usage REST API and proportionally allocated back to each consumer. + +The result is an accurate chargeback/showback model without changing the existing applications. + +--- + +## Business Use Case + +Typical enterprise scenario: + +- Multiple REST APIs +- AI Agents +- LangGraph applications +- MCP Servers +- RAG services + +all consume the same OCI Generative AI endpoint. + +Finance receives only the total monthly OCI invoice. + +Engineering needs answers such as: + +- Which API generated the highest cost? +- Which squad consumed the most tokens? +- Which application should optimize prompts? +- How much does each business unit owe? + +Thi material answers those questions. + +--- + +## Architecture + +Applications -> LiteLLM Proxy -> OCI Generative AI + +LiteLLM records detailed token usage per virtual key/API. + +OCI Usage API provides the official monthly cost. + +The allocation engine proportionally distributes the OCI invoice according to measured consumption. + +--- + +## Why LiteLLM? + +LiteLLM provides several capabilities particularly useful in OCI environments: + +- Unified OpenAI-compatible endpoint +- Multiple LLM providers behind a single API +- Virtual API Keys +- Callbacks +- Usage tracking +- Authentication abstraction +- Gateway functionality +- Rate limiting +- Budget enforcement +- Model routing +- Observability integration + +This makes LiteLLM an ideal component for enterprise cost attribution. + +--- + +## Cost Allocation Strategy + +The project intentionally **does not** trust LiteLLM estimated prices. + +Instead: + +1. LiteLLM measures usage. +2. OCI Usage API provides the official invoice amount. +3. The invoice is proportionally distributed. + +Formula: + +``` +weighted_tokens = +input_tokens × input_weight + +output_tokens × output_weight + +share = +weighted_tokens(API) +/ weighted_tokens(all APIs) + +allocated_cost = +share × official OCI monthly cost +``` + +--- + +## Integration with OCI Usage REST API + +The allocation job authenticates using OCI Request Signing and queries the Usage API. + +Returned information includes: + +- Monthly cost +- SKU +- Service +- Currency +- Usage period + +The value becomes the source of truth. + +--- + +## Components + +## LiteLLM Proxy + +Acts as enterprise LLM Gateway. + +Responsibilities: + +- forwards requests to OCI +- authenticates +- tracks usage +- executes callbacks +- stores request metadata + +## Custom Callback + +Captures every request and stores: + +- API identifier +- request count +- latency +- input tokens +- output tokens +- total tokens + +## SQLite Ledger + +Stores the local consumption ledger used for allocation. + +## Allocation Engine + +Reads: + +- Ledger +- OCI Usage API + +Produces: + +- proportional cost allocation + +## API Service + +Simple REST service used to simulate multiple APIs. + +## Load Simulator + +Generates concurrent traffic for validation. + +--- + +## Repository Structure + +``` +config/ + apis.yaml + litellm_config.yaml + genai.pem + +src/ + api_service.py + custom_callbacks.py + ledger.py + allocate_costs.py + simulate_load.py + create_litellm_keys.py + oci_usage_rest.py + settings.py + +storage/ +scripts/ +docker-compose.yml +requirements.txt +``` + +--- + +## Configuration Files + +## .env + +Contains environment configuration. + +Important parameters include: + +- OCI credentials +- LiteLLM endpoint +- Usage API endpoint +- database path + +## config/apis.yaml + +Defines: + +- monitored APIs +- owners +- virtual keys +- allocation metric +- model alias +- token weights + +## config/litellm_config.yaml + +LiteLLM Proxy configuration. + +Includes: + +- exposed models +- OCI provider +- callbacks +- authentication +- master key +- database + +--- + +## Source Files + +### allocate_costs.py + +Performs monthly allocation. + +### oci_usage_rest.py + +Consumes OCI Usage REST API using OCI request signing. + +### custom_callbacks.py + +Receives LiteLLM callback events. + +### ledger.py + +Persists local usage statistics. + +### api_service.py + +REST service representing client APIs. + +### simulate_load.py + +Generates concurrent traffic. + +### create_litellm_keys.py + +Creates LiteLLM virtual keys. + +### settings.py + +Centralizes application configuration. + +--- + +## Running the Project + +### 1. Create Python environment + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +``` + +### 2. Configure + +``` +cp .env.example .env +``` + +Fill OCI credentials. + +### 3. Start infrastructure + +``` +docker compose up -d +``` + +### 4. Create LiteLLM keys + +``` +python src/create_litellm_keys.py +``` + +### 5. Start API service + +``` +PYTHONPATH=src uvicorn api_service:app --host 0.0.0.0 --port 8080 +``` + +### 6. Generate traffic + +``` +python src/simulate_load.py + +or + +python src/simulate_load.py --calls 30 --concurrency 5 +``` + +### 7. Allocate costs + + +This command will calculate the total cost came from OCI Billing API: + +``` +python src/allocate_costs.py --month 2026-06 +``` + +But, if you have the total cost or want to simmulate, you can run: + +``` +python src/allocate_costs.py --month 2026-06 --oci-cost 1234.56 +``` + +--- + +## Expected Result + +The report shows, for every API: + +- Requests +- Input Tokens +- Output Tokens +- Weighted Tokens +- Consumption Percentage +- Allocated OCI Cost + +--- + +## Reference Documentation + +Oracle Cloud + +- OCI Generative AI +- OCI Usage API +- OCI Request Signing +- OCI SDK for Python + +LiteLLM + +- LiteLLM Documentation +- LiteLLM Proxy +- LiteLLM Callbacks +- LiteLLM Budget Management + +--- + +## Possible Future Improvements + +- PostgreSQL ledger +- Grafana dashboards +- Prometheus metrics +- Langfuse integration +- OpenTelemetry +- Multi-tenancy +- Department cost centers +- Daily allocation +- OCI Object Storage exports + +--- + +## Conclusion + +This material demonstrates a practical enterprise architecture for implementing cost visibility over OCI Generative AI. + +Instead of relying on estimated model pricing, it combines precise request-level telemetry from LiteLLM with the official OCI billing information, enabling transparent chargeback and showback across APIs, business units and AI platforms. + +The approach is lightweight, provider-independent inside OCI Generative AI, and can easily evolve into a production-ready FinOps solution. diff --git a/config/apis.yaml b/config/apis.yaml new file mode 100644 index 0000000..5850c40 --- /dev/null +++ b/config/apis.yaml @@ -0,0 +1,24 @@ +month: "2026-06" +allocation_metric: "weighted_tokens" # weighted_tokens | total_tokens | requests +currency: "USD" + +model: + alias: "oci-gemini-25-pro" + oci_model_name: "google.gemini-2.5-pro" + # Peso lógico para rateio interno. Para Gemini, ajuste conforme o pricing usado no contrato/OCI. + input_weight: 1.0 + output_weight: 5.0 + +apis: + - id: "api-billing" + name: "API Fatura" + virtual_key: "sk-ph3FgX6ph3FgX6ph3FgX6ph3FgX6ph3FgX6ph3FgX6ph3FgX6" + owner: "squad-cobranca" + - id: "api-offers" + name: "API Ofertas" + virtual_key: "sk-ph3FgX6ph3FgX6ph3FgX6ph3FgX6ph3FgX6ph3FgX6ph3FgX6" + owner: "squad-ofertas" + - id: "api-backoffice" + name: "API Backoffice" + virtual_key: "sk-ph3FgX6ph3FgX6ph3FgX6ph3FgX6ph3FgX6ph3FgX6ph3FgX6" + owner: "squad-operacoes" diff --git a/config/genai.pem b/config/genai.pem new file mode 100644 index 0000000..e0711f4 --- /dev/null +++ b/config/genai.pem @@ -0,0 +1,29 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +-----END PRIVATE KEY----- +OCI_API_KEY diff --git a/config/litellm_config.yaml b/config/litellm_config.yaml new file mode 100644 index 0000000..e71aaa1 --- /dev/null +++ b/config/litellm_config.yaml @@ -0,0 +1,23 @@ +model_list: + - model_name: oci-gemini-25-pro + litellm_params: + model: oci/google.gemini-2.5-pro + oci_region: os.environ/OCI_REGION + oci_compartment_id: os.environ/OCI_COMPARTMENT_ID + oci_auth_mode: os.environ/OCI_AUTH_MODE + oci_user: os.environ/OCI_USER + oci_tenancy: os.environ/OCI_TENANCY + oci_fingerprint: os.environ/OCI_FINGERPRINT + oci_key_file: os.environ/OCI_KEY_FILE + timeout: 120 + +litellm_settings: + callbacks: ["custom_callbacks.proxy_handler_instance"] + drop_params: true + set_verbose: false + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + database_url: os.environ/DATABASE_URL + store_model_in_db: true + always_include_stream_usage: true diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b0ca057 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,34 @@ +services: + postgres: + image: postgres:16-alpine + container_name: litellm-postgres + environment: + POSTGRES_DB: litellm + POSTGRES_USER: litellm + POSTGRES_PASSWORD: litellm + ports: + - "5432:5432" + volumes: + - pgdata:/var/lib/postgresql/data + + litellm: + image: litellm/litellm:main-latest + container_name: litellm-proxy + depends_on: + - postgres + ports: + - "4000:4000" + env_file: .env + environment: + DATABASE_URL: postgresql://litellm:litellm@postgres:5432/litellm + PYTHONPATH: /app + volumes: + - ./config/litellm_config.yaml:/app/config.yaml:ro + - ./config/genai.pem:/app/config/genai.pem:ro + - ./src/custom_callbacks.py:/app/custom_callbacks.py:ro + - ./src/ledger.py:/app/ledger.py:ro + - ./storage:/app/storage + command: ["--config", "/app/config.yaml", "--port", "4000", "--host", "0.0.0.0"] + +volumes: + pgdata: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8cab654 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +fastapi==0.115.13 +httpx==0.28.1 +pydantic==2.11.7 +pydantic-settings==2.10.1 +PyYAML==6.0.2 +python-dotenv==1.1.0 +requests==2.32.4 +oci==2.180.0 +pyyaml +uvicorn>=0.33.0,<0.35.0 +pandas>=2.2.0 +openai>=2.20.0,<3.0.0 \ No newline at end of file diff --git a/scripts/run_poc.sh b/scripts/run_poc.sh new file mode 100644 index 0000000..cfa37bf --- /dev/null +++ b/scripts/run_poc.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +cp -n .env.example .env || true +mkdir -p storage + +echo '1) Subindo Postgres + LiteLLM Proxy...' +docker compose up -d + +echo '2) Registrando virtual keys por API consumidora...' +python src/create_litellm_keys.py + +echo '3) Subindo simulador de APIs na porta 8080...' +uvicorn src.api_service:app --host 0.0.0.0 --port 8080 diff --git a/src/__pycache__/api_service.cpython-313.pyc b/src/__pycache__/api_service.cpython-313.pyc new file mode 100644 index 0000000..b52bc75 Binary files /dev/null and b/src/__pycache__/api_service.cpython-313.pyc differ diff --git a/src/__pycache__/ledger.cpython-313.pyc b/src/__pycache__/ledger.cpython-313.pyc new file mode 100644 index 0000000..cb75bfd Binary files /dev/null and b/src/__pycache__/ledger.cpython-313.pyc differ diff --git a/src/__pycache__/oci_usage_rest.cpython-313.pyc b/src/__pycache__/oci_usage_rest.cpython-313.pyc new file mode 100644 index 0000000..507755b Binary files /dev/null and b/src/__pycache__/oci_usage_rest.cpython-313.pyc differ diff --git a/src/__pycache__/settings.cpython-313.pyc b/src/__pycache__/settings.cpython-313.pyc new file mode 100644 index 0000000..56b5206 Binary files /dev/null and b/src/__pycache__/settings.cpython-313.pyc differ diff --git a/src/allocate_costs.py b/src/allocate_costs.py new file mode 100644 index 0000000..37490e1 --- /dev/null +++ b/src/allocate_costs.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import argparse +import csv +import json +from pathlib import Path +from typing import Any + +from ledger import Ledger +from oci_usage_rest import fetch_monthly_genai_cost_rest +from settings import Env, load_config + + +def compute_allocations(rows: list[dict[str, Any]], total_cost: float, metric: str, input_weight: float, output_weight: float) -> list[dict[str, Any]]: + enriched = [] + for r in rows: + weighted_tokens = (r['input_tokens'] or 0) * input_weight + (r['output_tokens'] or 0) * output_weight + r = dict(r) + r['weighted_tokens'] = weighted_tokens + enriched.append(r) + + denominator = sum(float(r[metric] or 0) for r in enriched) or 1.0 + for r in enriched: + share = float(r[metric] or 0) / denominator + r['allocation_metric'] = metric + r['share_pct'] = round(share * 100, 6) + r['allocated_cost'] = round(total_cost * share, 6) + return enriched + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--month', help='YYYY-MM; default vem de config/apis.yaml') + parser.add_argument('--oci-cost', type=float, help='Permite testar sem chamar OCI Usage API') + parser.add_argument('--out', default='./storage/allocation_report.csv') + args = parser.parse_args() + + env = Env() + cfg = load_config(env.config_file) + month = args.month or cfg.month + rows = Ledger(env.ledger_db).monthly_usage(month) + + if args.oci_cost is None: + cost_result = fetch_monthly_genai_cost_rest(month) + total_cost = cost_result.amount + currency = cost_result.currency or cfg.currency + Path('./storage/oci_usage_raw.json').write_text(json.dumps(cost_result.raw, indent=2, ensure_ascii=False), encoding='utf-8') + else: + total_cost = args.oci_cost + currency = cfg.currency + + allocations = compute_allocations( + rows=rows, + total_cost=total_cost, + metric=cfg.allocation_metric, + input_weight=cfg.model.input_weight, + output_weight=cfg.model.output_weight, + ) + + fieldnames = [ + 'month', 'api_id', 'api_name', 'requests', 'input_tokens', 'output_tokens', 'total_tokens', + 'weighted_tokens', 'litellm_estimated_cost', 'allocation_metric', 'share_pct', 'allocated_cost', + 'avg_latency_ms' + ] + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + with out.open('w', newline='', encoding='utf-8') as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + for r in allocations: + writer.writerow({k: r.get(k) for k in fieldnames}) + + print(json.dumps({ + 'month': month, + 'oci_total_cost': total_cost, + 'currency': currency, + 'allocation_metric': cfg.allocation_metric, + 'report': str(out), + 'allocations': allocations, + }, indent=2, ensure_ascii=False)) + + +if __name__ == '__main__': + main() diff --git a/src/api_service.py b/src/api_service.py new file mode 100644 index 0000000..6c16fe5 --- /dev/null +++ b/src/api_service.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import random +import time +from typing import Any + +from fastapi import FastAPI, Header, HTTPException +from openai import OpenAI +from pydantic import BaseModel + +from settings import Env, load_config + + +env = Env() +config = load_config(env.config_file) +api_by_id = {a.id: a for a in config.apis} + +app = FastAPI(title='LLM Consumer API Simulator') + + +class PromptRequest(BaseModel): + prompt: str | None = None + max_tokens: int = 256 + + +def client_for(api_id: str) -> OpenAI: + api_def = api_by_id.get(api_id) + if not api_def: + raise HTTPException(status_code=404, detail=f'API {api_id} não cadastrada em {env.config_file}') + return OpenAI(api_key=api_def.virtual_key, base_url=env.litellm_base_url) + + +@app.get('/apis') +def list_apis() -> list[dict[str, Any]]: + return [a.model_dump(exclude={'virtual_key'}) for a in config.apis] + + +@app.post('/apis/{api_id}/ask') +def ask(api_id: str, body: PromptRequest, x_request_id: str | None = Header(default=None)) -> dict[str, Any]: + api_def = api_by_id.get(api_id) + if not api_def: + raise HTTPException(status_code=404, detail='api_id inválido') + + prompt = body.prompt or f'Responda em uma frase: qual é uma boa métrica para FinOps de LLM? rand={random.randint(1, 9999)}' + t0 = time.perf_counter() + response = client_for(api_id).chat.completions.create( + model=config.model.alias, + messages=[ + {'role': 'system', 'content': 'Você é um assistente técnico objetivo.'}, + {'role': 'user', 'content': prompt}, + ], + max_tokens=body.max_tokens, + metadata={ + 'api_id': api_def.id, + 'api_name': api_def.name, + 'owner': api_def.owner, + 'external_request_id': x_request_id, + }, + ) + elapsed_ms = int((time.perf_counter() - t0) * 1000) + usage = response.usage.model_dump() if response.usage else {} + return { + 'api_id': api_id, + 'model': config.model.alias, + 'latency_ms': elapsed_ms, + 'usage': usage, + 'answer': response.choices[0].message.content, + } diff --git a/src/create_litellm_keys.py b/src/create_litellm_keys.py new file mode 100644 index 0000000..60cd39c --- /dev/null +++ b/src/create_litellm_keys.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import httpx + +from settings import Env, load_config + + +def main() -> None: + env = Env() + cfg = load_config(env.config_file) + headers = {'Authorization': f'Bearer {env.litellm_master_key}'} + + with httpx.Client(timeout=30) as client: + for api in cfg.apis: + payload = { + 'key': api.virtual_key, + 'aliases': {cfg.model.alias: cfg.model.alias}, + 'models': [cfg.model.alias], + 'metadata': {'api_id': api.id, 'api_name': api.name, 'owner': api.owner}, + 'team_id': api.owner or api.id, + 'user_id': api.id, + } + r = client.post(f'{env.litellm_base_url.replace("/v1", "")}/key/generate', headers=headers, json=payload) + if r.status_code in (200, 201): + print(f'OK key criada/registrada: {api.id}') + elif r.status_code == 400 and 'already' in r.text.lower(): + print(f'OK key já existe: {api.id}') + else: + print(f'ERRO {api.id}: {r.status_code} {r.text}') + + +if __name__ == '__main__': + main() diff --git a/src/custom_callbacks.py b/src/custom_callbacks.py new file mode 100644 index 0000000..59b42af --- /dev/null +++ b/src/custom_callbacks.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import json +import os +from datetime import datetime, timezone +from typing import Any + +from litellm.integrations.custom_logger import CustomLogger + +# Este callback roda dentro do LiteLLM Proxy. +# Ele persiste um ledger local por API consumidora usando metadata enviada na chamada. +try: + from ledger import Ledger, now_iso +except Exception: # pragma: no cover - fallback quando import path muda no container + from src.ledger import Ledger, now_iso + +def _extract_metadata(kwargs: dict[str, Any]) -> dict[str, Any]: + candidates = [] + + candidates.append(kwargs.get("metadata")) + + litellm_params = kwargs.get("litellm_params") or {} + candidates.append(litellm_params.get("metadata")) + + standard_logging_object = kwargs.get("standard_logging_object") or {} + candidates.append(standard_logging_object.get("metadata")) + + proxy_server_request = kwargs.get("proxy_server_request") or {} + candidates.append(proxy_server_request.get("metadata")) + + for candidate in candidates: + if isinstance(candidate, dict) and candidate: + return candidate + + return {} + +def _month_from_ts(dt: datetime) -> str: + return dt.strftime('%Y-%m') + +def _json_safe(value): + """Converte objetos não serializáveis do LiteLLM em tipos seguros para JSON.""" + if value is None: + return None + + if isinstance(value, (str, int, float, bool)): + return value + + if isinstance(value, dict): + return {str(k): _json_safe(v) for k, v in value.items()} + + if isinstance(value, (list, tuple, set)): + return [_json_safe(v) for v in value] + + # Pydantic / dataclass-like + if hasattr(value, "model_dump"): + try: + return _json_safe(value.model_dump()) + except Exception: + pass + + if hasattr(value, "dict"): + try: + return _json_safe(value.dict()) + except Exception: + pass + + # Último fallback: vira string + return str(value) + +def _usage_get(response_obj: Any, key: str, default: int = 0) -> int: + usage = getattr(response_obj, 'usage', None) + if usage is None and isinstance(response_obj, dict): + usage = response_obj.get('usage') + if usage is None: + return default + if isinstance(usage, dict): + return int(usage.get(key, default) or default) + return int(getattr(usage, key, default) or default) + + +def _response_id(response_obj: Any) -> str | None: + if isinstance(response_obj, dict): + return response_obj.get('id') + return getattr(response_obj, 'id', None) + + +class CostAllocationCallback(CustomLogger): + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + metadata = _extract_metadata(kwargs) + + api_id = ( + metadata.get("api_id") + or metadata.get("user_api_key_metadata", {}).get("api_id") + or kwargs.get("user") + or "unknown" + ) + + api_name = ( + metadata.get("api_name") + or metadata.get("user_api_key_metadata", {}).get("api_name") + or api_id + ) + + prompt_tokens = _usage_get(response_obj, 'prompt_tokens') + completion_tokens = _usage_get(response_obj, 'completion_tokens') + total_tokens = _usage_get(response_obj, 'total_tokens', prompt_tokens + completion_tokens) + latency_ms = int((end_time - start_time).total_seconds() * 1000) + + # LiteLLM calcula response_cost quando conhece o modelo/preço. Para rateio final, + # a fonte da verdade será o custo total da OCI Usage API. + response_cost = float(kwargs.get('response_cost') or 0) + + ts = end_time if isinstance(end_time, datetime) else datetime.now(timezone.utc) + ledger = Ledger(os.environ.get('LEDGER_DB', '/app/storage/ledger.sqlite')) + + model = ( + kwargs.get("model") + or kwargs.get("model_name") + or (kwargs.get("standard_logging_object") or {}).get("model") + or (kwargs.get("response_cost_information") or {}).get("model") + or "unknown" + ) + + safe_metadata = { + "api_id": api_id, + "api_name": api_name, + "owner": metadata.get("owner"), + "model": model, + } + + ledger.insert_usage({ + 'ts': ts.isoformat(), + 'month': _month_from_ts(ts), + 'api_id': api_id, + 'api_name': api_name, + 'virtual_key_hash': str(kwargs.get('litellm_api_key') or kwargs.get('api_key') or '')[:12], + 'model': kwargs.get('model'), + 'request_id': _response_id(response_obj), + 'prompt_tokens': prompt_tokens, + 'completion_tokens': completion_tokens, + 'total_tokens': total_tokens, + 'response_cost': response_cost, + 'latency_ms': latency_ms, + 'status': 'success', + 'metadata_json': json.dumps(safe_metadata, ensure_ascii=False), + }) + + +proxy_handler_instance = CostAllocationCallback() diff --git a/src/ledger.py b/src/ledger.py new file mode 100644 index 0000000..2ef3993 --- /dev/null +++ b/src/ledger.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import sqlite3 +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path +from typing import Iterator, Any + + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS llm_usage ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts TEXT NOT NULL, + month TEXT NOT NULL, + api_id TEXT NOT NULL, + api_name TEXT, + virtual_key_hash TEXT, + model TEXT, + request_id TEXT, + prompt_tokens INTEGER NOT NULL DEFAULT 0, + completion_tokens INTEGER NOT NULL DEFAULT 0, + total_tokens INTEGER NOT NULL DEFAULT 0, + response_cost REAL NOT NULL DEFAULT 0, + latency_ms INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'success', + metadata_json TEXT +); + +CREATE INDEX IF NOT EXISTS idx_llm_usage_month_api ON llm_usage(month, api_id); +CREATE UNIQUE INDEX IF NOT EXISTS uq_llm_usage_request ON llm_usage(request_id) WHERE request_id IS NOT NULL; +""" + + +def now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +class Ledger: + def __init__(self, db_path: str | Path): + self.db_path = Path(db_path) + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self.init() + + @contextmanager + def connect(self) -> Iterator[sqlite3.Connection]: + con = sqlite3.connect(self.db_path) + con.row_factory = sqlite3.Row + try: + yield con + con.commit() + finally: + con.close() + + def init(self) -> None: + with self.connect() as con: + con.executescript(SCHEMA) + + def insert_usage(self, row: dict[str, Any]) -> None: + keys = ','.join(row.keys()) + params = ','.join([f':{k}' for k in row.keys()]) + with self.connect() as con: + con.execute(f'INSERT OR IGNORE INTO llm_usage ({keys}) VALUES ({params})', row) + + def monthly_usage(self, month: str) -> list[dict[str, Any]]: + sql = """ + SELECT + month, + api_id, + MAX(api_name) AS api_name, + COUNT(*) AS requests, + SUM(prompt_tokens) AS input_tokens, + SUM(completion_tokens) AS output_tokens, + SUM(total_tokens) AS total_tokens, + SUM(response_cost) AS litellm_estimated_cost, + AVG(latency_ms) AS avg_latency_ms + FROM llm_usage + WHERE month = ? AND status = 'success' + GROUP BY month, api_id + ORDER BY total_tokens DESC + """ + with self.connect() as con: + return [dict(r) for r in con.execute(sql, (month,)).fetchall()] diff --git a/src/oci_usage_rest.py b/src/oci_usage_rest.py new file mode 100644 index 0000000..910f177 --- /dev/null +++ b/src/oci_usage_rest.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import date, datetime, timezone +from typing import Any + +import oci +import requests +from oci.config import from_file +from oci.signer import Signer + +from settings import Env + + +@dataclass +class OciCostResult: + amount: float + currency: str + raw: dict[str, Any] + + +def _make_signer() -> Signer: + cfg = from_file() + return Signer( + tenancy=cfg['tenancy'], + user=cfg['user'], + fingerprint=cfg['fingerprint'], + private_key_file_location=cfg['key_file'], + pass_phrase=cfg.get('pass_phrase'), + ) + + +def _iso_z(d: date) -> str: + return datetime(d.year, d.month, d.day, tzinfo=timezone.utc).isoformat().replace('+00:00', 'Z') + + +def fetch_monthly_genai_cost_rest(month: str) -> OciCostResult: + """Consulta a OCI Usage API via REST assinado. + + month: YYYY-MM + + Observação: em alguns tenancies o nome exato de service/productDescription pode variar. + Ajuste OCI_USAGE_SERVICE_FILTER e OCI_USAGE_PRODUCT_FILTER no .env conforme a saída real. + """ + env = Env() + year, mon = [int(x) for x in month.split('-')] + start = date(year, mon, 1) + end = date(year + (mon == 12), 1 if mon == 12 else mon + 1, 1) + + endpoint = env.oci_usage_endpoint.rstrip('/') + url = f'{endpoint}/{env.oci_usage_api_version}/usage' + + # RequestSummarizedUsagesDetails. A granularidade mensal reduz ruído de hora/dia. + # GroupBy opcional ajuda a auditar qual produto/sku veio na resposta. + payload: dict[str, Any] = { + 'tenantId': from_file()['tenancy'], + 'timeUsageStarted': _iso_z(start), + 'timeUsageEnded': _iso_z(end), + 'granularity': 'MONTHLY', + 'queryType': 'COST', + 'groupBy': ['service', 'productDescription'], + 'isAggregateByTime': True, + } + + filters: list[dict[str, Any]] = [] + if env.oci_usage_service_filter: + filters.append({'dimension': 'service', 'operator': 'CONTAINS', 'value': env.oci_usage_service_filter}) + if env.oci_usage_product_filter: + filters.append({'dimension': 'productDescription', 'operator': 'CONTAINS', 'value': env.oci_usage_product_filter}) + if filters: + payload['filter'] = {'operator': 'AND', 'dimensions': filters} + + r = requests.post(url, auth=_make_signer(), headers={'content-type': 'application/json'}, data=json.dumps(payload), timeout=60) + r.raise_for_status() + data = r.json() + + # OCI pode retornar itens em data/items dependendo do cliente/endpoint/revisão. + items = data.get('items') or data.get('data') or [] + amount = 0.0 + currency = 'USD' + for item in items: + amount += float(item.get('computedAmount') or item.get('cost') or item.get('amount') or 0) + currency = item.get('currency') or item.get('currencyCode') or currency + + return OciCostResult(amount=amount, currency=currency, raw=data) diff --git a/src/settings.py b/src/settings.py new file mode 100644 index 0000000..dfa04b2 --- /dev/null +++ b/src/settings.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +import yaml +from pydantic import BaseModel, Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Env(BaseSettings): + model_config = SettingsConfigDict(env_file='.env', extra='ignore') + + litellm_base_url: str = 'http://localhost:4000/v1' + litellm_master_key: str = 'sk-master-poc' + litellm_model_alias: str = 'oci-gemini-25-pro' + ledger_db: str = './storage/ledger.sqlite' + config_file: str = './config/apis.yaml' + + oci_usage_endpoint: str = 'https://usageapi.us-ashburn-1.oci.oraclecloud.com' + oci_usage_api_version: str = '20200107' + oci_usage_service_filter: str = 'generative_ai' + oci_usage_product_filter: str | None = None + + +class ApiDef(BaseModel): + id: str + name: str + virtual_key: str + owner: str | None = None + + +class ModelDef(BaseModel): + alias: str + oci_model_name: str + input_weight: float = 1.0 + output_weight: float = 5.0 + + +class AppConfig(BaseModel): + month: str + allocation_metric: Literal['weighted_tokens', 'total_tokens', 'requests'] = 'weighted_tokens' + currency: str = 'USD' + model: ModelDef + apis: list[ApiDef] = Field(default_factory=list) + + +def load_config(path: str | Path) -> AppConfig: + with open(path, 'r', encoding='utf-8') as f: + return AppConfig.model_validate(yaml.safe_load(f)) diff --git a/src/simulate_load.py b/src/simulate_load.py new file mode 100644 index 0000000..6cced82 --- /dev/null +++ b/src/simulate_load.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import argparse +import asyncio +import random +from uuid import uuid4 + +import httpx + +from settings import Env, load_config + + +PROMPTS = [ + 'Explique em uma frase o que é rateio de custo de LLM.', + 'Gere uma resposta curta para cliente perguntando segunda via de fatura.', + 'Classifique a intenção: quero trocar meu plano de internet.', + 'Resuma em tópicos uma política de uso responsável de IA.', + 'Dê 3 métricas para monitorar consumo de tokens por API.', +] + + +async def call_one(client: httpx.AsyncClient, base_url: str, api_id: str) -> None: + prompt = random.choice(PROMPTS) + f' request={uuid4()}' + try: + r = await client.post( + f'{base_url}/apis/{api_id}/ask', + headers={'x-request-id': str(uuid4())}, + json={'prompt': prompt, 'max_tokens': random.choice([64, 128, 256])}, + ) + print(api_id, r.status_code, r.json().get('usage')) + except Exception as e: + print(api_id, 'ERROR', repr(e)) + + +async def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--calls', type=int, default=30) + parser.add_argument('--concurrency', type=int, default=5) + parser.add_argument('--api-base', default='http://localhost:8080') + args = parser.parse_args() + + env = Env() + cfg = load_config(env.config_file) + api_ids = [a.id for a in cfg.apis] + sem = asyncio.Semaphore(args.concurrency) + + async with httpx.AsyncClient(timeout=180) as client: + async def guarded(i: int) -> None: + async with sem: + await call_one(client, args.api_base, random.choice(api_ids)) + await asyncio.gather(*(guarded(i) for i in range(args.calls))) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/storage/.DS_Store b/storage/.DS_Store new file mode 100644 index 0000000..5008ddf Binary files /dev/null and b/storage/.DS_Store differ