From 029adfafc37107db6662c906db969966d4aa345f Mon Sep 17 00:00:00 2001 From: Cristiano Hoshikawa Date: Sat, 27 Jun 2026 09:32:54 -0300 Subject: [PATCH] first commit --- .DS_Store | Bin 0 -> 6148 bytes .env.example | 23 ++ Quick_Test_pt.md | 54 +++ README.md | 384 ++++++++++++++++++ config/apis.yaml | 24 ++ config/genai.pem | 29 ++ config/litellm_config.yaml | 23 ++ docker-compose.yml | 34 ++ requirements.txt | 12 + scripts/run_poc.sh | 14 + src/__pycache__/api_service.cpython-313.pyc | Bin 0 -> 4004 bytes src/__pycache__/ledger.cpython-313.pyc | Bin 0 -> 4666 bytes .../oci_usage_rest.cpython-313.pyc | Bin 0 -> 4565 bytes src/__pycache__/settings.cpython-313.pyc | Bin 0 -> 3037 bytes src/allocate_costs.py | 84 ++++ src/api_service.py | 68 ++++ src/create_litellm_keys.py | 33 ++ src/custom_callbacks.py | 149 +++++++ src/ledger.py | 82 ++++ src/oci_usage_rest.py | 86 ++++ src/settings.py | 51 +++ src/simulate_load.py | 55 +++ storage/.DS_Store | Bin 0 -> 6148 bytes 23 files changed, 1205 insertions(+) create mode 100644 .DS_Store create mode 100644 .env.example create mode 100644 Quick_Test_pt.md create mode 100644 README.md create mode 100644 config/apis.yaml create mode 100644 config/genai.pem create mode 100644 config/litellm_config.yaml create mode 100644 docker-compose.yml create mode 100644 requirements.txt create mode 100644 scripts/run_poc.sh create mode 100644 src/__pycache__/api_service.cpython-313.pyc create mode 100644 src/__pycache__/ledger.cpython-313.pyc create mode 100644 src/__pycache__/oci_usage_rest.cpython-313.pyc create mode 100644 src/__pycache__/settings.cpython-313.pyc create mode 100644 src/allocate_costs.py create mode 100644 src/api_service.py create mode 100644 src/create_litellm_keys.py create mode 100644 src/custom_callbacks.py create mode 100644 src/ledger.py create mode 100644 src/oci_usage_rest.py create mode 100644 src/settings.py create mode 100644 src/simulate_load.py create mode 100644 storage/.DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..ec3d8298c803fc7b7b8e56d4f4296410fe25c726 GIT binary patch literal 6148 zcmeHKu};H447DLcB$kehH!&jh502{ig8o2NDix%bbbtXdkd05{lbDcrwhxpjDkB3t zOTL%0{c_10;#fp<@wl0b%tWMx8_KIKQ?tDJ#7=6Y0jk|&x1BFvGru3ID4miy<6kN3B$aenh-{%|{ZHeG>AqXJZb3Qz$m@GliW&o*n% z0=ZHFDnJFk6|nC^f*aC5mOcnssfk4fB70V?pP6wszD z%L1=dd+X%otk)*^3jSrN^>QrUihbrvv#TV7kz#z;7t< E3C(^f@c;k- literal 0 HcmV?d00001 diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..0c670ad --- /dev/null +++ b/.env.example @@ -0,0 +1,23 @@ +# LiteLLM Proxy +LITELLM_BASE_URL=http://localhost:4000/v1 +LITELLM_MASTER_KEY=sk-master-poc +LITELLM_MODEL_ALIAS=oci-gemini-25-pro + +# Configuração OCI para LiteLLM provider OCI +OCI_REGION=us-chicago-1 +OCI_COMPARTMENT_ID=ocid1.compartment.oc1..replace_me +OCI_AUTH_MODE=api_key +OCI_USER=ocid1.user.oc1..replace_me +OCI_TENANCY=ocid1.tenancy.oc1..replace_me +OCI_FINGERPRINT=replace_me +OCI_KEY_FILE=/home/opc/.oci/oci_api_key.pem + +# OCI Usage API REST +OCI_USAGE_ENDPOINT=https://usageapi.us-ashburn-1.oci.oraclecloud.com +OCI_USAGE_API_VERSION=20200107 +OCI_USAGE_SERVICE_FILTER=generative_ai +OCI_USAGE_PRODUCT_FILTER=Google - Gemini 2.5 Pro + +# Ledger local da PoC +LEDGER_DB=./storage/ledger.sqlite +CONFIG_FILE=./config/apis.yaml diff --git a/Quick_Test_pt.md b/Quick_Test_pt.md new file mode 100644 index 0000000..d4df590 --- /dev/null +++ b/Quick_Test_pt.md @@ -0,0 +1,54 @@ +# PoC - Rateio de custo OCI Generative AI via LiteLLM + +Objetivo: simular várias APIs consumindo Gemini via OCI Generative AI por meio do LiteLLM Proxy, registrar uso por API e ratear o custo total mensal retornado pela OCI Usage API. + +## Fluxo + +1. Cada API consumidora chama o LiteLLM com uma virtual key própria. +2. O request também envia `metadata.api_id` e `metadata.api_name`. +3. Um callback do LiteLLM grava o uso em SQLite: input tokens, output tokens, total tokens, custo estimado LiteLLM e latência. +4. O job `allocate_costs.py` busca o custo mensal total na OCI Usage API via REST assinado. +5. O custo OCI é rateado proporcionalmente pelo consumo registrado no LiteLLM. + +## Rodar + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +cp .env.example .env +# edite .env com OCI_REGION, OCI_COMPARTMENT_ID e credenciais OCI + +docker compose up -d +python src/create_litellm_keys.py +PYTHONPATH=src uvicorn api_service:app --host 0.0.0.0 --port 8080 +``` + +Em outro terminal: + +```bash +python src/simulate_load.py --calls 30 --concurrency 5 +python src/allocate_costs.py --month 2026-06 --oci-cost 1234.56 +``` + +Para usar a OCI de verdade, remova `--oci-cost`: + +```bash +python src/allocate_costs.py --month 2026-06 +``` + +## Ajustes importantes + +- `config/apis.yaml`: cadastro das APIs, virtual keys, dono/squad e pesos de token. +- `config/litellm_config.yaml`: modelo OCI exposto como alias `oci-gemini-25-pro`. +- `.env`: credenciais OCI, endpoint da Usage API, filtros de custo e banco local. + +## Fórmula padrão + +```text +weighted_tokens = input_tokens * input_weight + output_tokens * output_weight +share_api = weighted_tokens_api / sum(weighted_tokens_todas_apis) +allocated_cost_api = oci_total_monthly_cost * share_api +``` + +Para PoC, `input_weight=1` e `output_weight=5`. Ajuste conforme a tabela de preço efetiva do produto/SKU Gemini no contrato/OCI. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0efd2a5 --- /dev/null +++ b/README.md @@ -0,0 +1,384 @@ + +# OCI Generative AI Cost Allocation per API using LiteLLM + + +## Overview + +Organizations commonly expose a single OCI Generative AI endpoint to dozens or hundreds of internal APIs and AI Agents. + +Although OCI provides consolidated billing information for Generative AI consumption, it does **not** provide an out-of-the-box cost breakdown by individual application or API. + +This project solves that problem. + +By introducing LiteLLM Proxy between the applications and OCI Generative AI, every request can be attributed to a specific API, while the official monthly OCI cost is collected through the OCI Usage REST API and proportionally allocated back to each consumer. + +The result is an accurate chargeback/showback model without changing the existing applications. + +--- + +## Business Use Case + +Typical enterprise scenario: + +- Multiple REST APIs +- AI Agents +- LangGraph applications +- MCP Servers +- RAG services + +all consume the same OCI Generative AI endpoint. + +Finance receives only the total monthly OCI invoice. + +Engineering needs answers such as: + +- Which API generated the highest cost? +- Which squad consumed the most tokens? +- Which application should optimize prompts? +- How much does each business unit owe? + +Thi material answers those questions. + +--- + +## Architecture + +Applications -> LiteLLM Proxy -> OCI Generative AI + +LiteLLM records detailed token usage per virtual key/API. + +OCI Usage API provides the official monthly cost. + +The allocation engine proportionally distributes the OCI invoice according to measured consumption. + +--- + +## Why LiteLLM? + +LiteLLM provides several capabilities particularly useful in OCI environments: + +- Unified OpenAI-compatible endpoint +- Multiple LLM providers behind a single API +- Virtual API Keys +- Callbacks +- Usage tracking +- Authentication abstraction +- Gateway functionality +- Rate limiting +- Budget enforcement +- Model routing +- Observability integration + +This makes LiteLLM an ideal component for enterprise cost attribution. + +--- + +## Cost Allocation Strategy + +The project intentionally **does not** trust LiteLLM estimated prices. + +Instead: + +1. LiteLLM measures usage. +2. OCI Usage API provides the official invoice amount. +3. The invoice is proportionally distributed. + +Formula: + +``` +weighted_tokens = +input_tokens × input_weight + +output_tokens × output_weight + +share = +weighted_tokens(API) +/ weighted_tokens(all APIs) + +allocated_cost = +share × official OCI monthly cost +``` + +--- + +## Integration with OCI Usage REST API + +The allocation job authenticates using OCI Request Signing and queries the Usage API. + +Returned information includes: + +- Monthly cost +- SKU +- Service +- Currency +- Usage period + +The value becomes the source of truth. + +--- + +## Components + +## LiteLLM Proxy + +Acts as enterprise LLM Gateway. + +Responsibilities: + +- forwards requests to OCI +- authenticates +- tracks usage +- executes callbacks +- stores request metadata + +## Custom Callback + +Captures every request and stores: + +- API identifier +- request count +- latency +- input tokens +- output tokens +- total tokens + +## SQLite Ledger + +Stores the local consumption ledger used for allocation. + +## Allocation Engine + +Reads: + +- Ledger +- OCI Usage API + +Produces: + +- proportional cost allocation + +## API Service + +Simple REST service used to simulate multiple APIs. + +## Load Simulator + +Generates concurrent traffic for validation. + +--- + +## Repository Structure + +``` +config/ + apis.yaml + litellm_config.yaml + genai.pem + +src/ + api_service.py + custom_callbacks.py + ledger.py + allocate_costs.py + simulate_load.py + create_litellm_keys.py + oci_usage_rest.py + settings.py + +storage/ +scripts/ +docker-compose.yml +requirements.txt +``` + +--- + +## Configuration Files + +## .env + +Contains environment configuration. + +Important parameters include: + +- OCI credentials +- LiteLLM endpoint +- Usage API endpoint +- database path + +## config/apis.yaml + +Defines: + +- monitored APIs +- owners +- virtual keys +- allocation metric +- model alias +- token weights + +## config/litellm_config.yaml + +LiteLLM Proxy configuration. + +Includes: + +- exposed models +- OCI provider +- callbacks +- authentication +- master key +- database + +--- + +## Source Files + +### allocate_costs.py + +Performs monthly allocation. + +### oci_usage_rest.py + +Consumes OCI Usage REST API using OCI request signing. + +### custom_callbacks.py + +Receives LiteLLM callback events. + +### ledger.py + +Persists local usage statistics. + +### api_service.py + +REST service representing client APIs. + +### simulate_load.py + +Generates concurrent traffic. + +### create_litellm_keys.py + +Creates LiteLLM virtual keys. + +### settings.py + +Centralizes application configuration. + +--- + +## Running the Project + +### 1. Create Python environment + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +``` + +### 2. Configure + +``` +cp .env.example .env +``` + +Fill OCI credentials. + +### 3. Start infrastructure + +``` +docker compose up -d +``` + +### 4. Create LiteLLM keys + +``` +python src/create_litellm_keys.py +``` + +### 5. Start API service + +``` +PYTHONPATH=src uvicorn api_service:app --host 0.0.0.0 --port 8080 +``` + +### 6. Generate traffic + +``` +python src/simulate_load.py + +or + +python src/simulate_load.py --calls 30 --concurrency 5 +``` + +### 7. Allocate costs + + +This command will calculate the total cost came from OCI Billing API: + +``` +python src/allocate_costs.py --month 2026-06 +``` + +But, if you have the total cost or want to simmulate, you can run: + +``` +python src/allocate_costs.py --month 2026-06 --oci-cost 1234.56 +``` + +--- + +## Expected Result + +The report shows, for every API: + +- Requests +- Input Tokens +- Output Tokens +- Weighted Tokens +- Consumption Percentage +- Allocated OCI Cost + +--- + +## Reference Documentation + +Oracle Cloud + +- OCI Generative AI +- OCI Usage API +- OCI Request Signing +- OCI SDK for Python + +LiteLLM + +- LiteLLM Documentation +- LiteLLM Proxy +- LiteLLM Callbacks +- LiteLLM Budget Management + +--- + +## Possible Future Improvements + +- PostgreSQL ledger +- Grafana dashboards +- Prometheus metrics +- Langfuse integration +- OpenTelemetry +- Multi-tenancy +- Department cost centers +- Daily allocation +- OCI Object Storage exports + +--- + +## Conclusion + +This material demonstrates a practical enterprise architecture for implementing cost visibility over OCI Generative AI. + +Instead of relying on estimated model pricing, it combines precise request-level telemetry from LiteLLM with the official OCI billing information, enabling transparent chargeback and showback across APIs, business units and AI platforms. + +The approach is lightweight, provider-independent inside OCI Generative AI, and can easily evolve into a production-ready FinOps solution. diff --git a/config/apis.yaml b/config/apis.yaml new file mode 100644 index 0000000..5850c40 --- /dev/null +++ b/config/apis.yaml @@ -0,0 +1,24 @@ +month: "2026-06" +allocation_metric: "weighted_tokens" # weighted_tokens | total_tokens | requests +currency: "USD" + +model: + alias: "oci-gemini-25-pro" + oci_model_name: "google.gemini-2.5-pro" + # Peso lógico para rateio interno. Para Gemini, ajuste conforme o pricing usado no contrato/OCI. + input_weight: 1.0 + output_weight: 5.0 + +apis: + - id: "api-billing" + name: "API Fatura" + virtual_key: "sk-ph3FgX6ph3FgX6ph3FgX6ph3FgX6ph3FgX6ph3FgX6ph3FgX6" + owner: "squad-cobranca" + - id: "api-offers" + name: "API Ofertas" + virtual_key: "sk-ph3FgX6ph3FgX6ph3FgX6ph3FgX6ph3FgX6ph3FgX6ph3FgX6" + owner: "squad-ofertas" + - id: "api-backoffice" + name: "API Backoffice" + virtual_key: "sk-ph3FgX6ph3FgX6ph3FgX6ph3FgX6ph3FgX6ph3FgX6ph3FgX6" + owner: "squad-operacoes" diff --git a/config/genai.pem b/config/genai.pem new file mode 100644 index 0000000..e0711f4 --- /dev/null +++ b/config/genai.pem @@ -0,0 +1,29 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCe9qy2Q5NcgXtM +-----END PRIVATE KEY----- +OCI_API_KEY diff --git a/config/litellm_config.yaml b/config/litellm_config.yaml new file mode 100644 index 0000000..e71aaa1 --- /dev/null +++ b/config/litellm_config.yaml @@ -0,0 +1,23 @@ +model_list: + - model_name: oci-gemini-25-pro + litellm_params: + model: oci/google.gemini-2.5-pro + oci_region: os.environ/OCI_REGION + oci_compartment_id: os.environ/OCI_COMPARTMENT_ID + oci_auth_mode: os.environ/OCI_AUTH_MODE + oci_user: os.environ/OCI_USER + oci_tenancy: os.environ/OCI_TENANCY + oci_fingerprint: os.environ/OCI_FINGERPRINT + oci_key_file: os.environ/OCI_KEY_FILE + timeout: 120 + +litellm_settings: + callbacks: ["custom_callbacks.proxy_handler_instance"] + drop_params: true + set_verbose: false + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + database_url: os.environ/DATABASE_URL + store_model_in_db: true + always_include_stream_usage: true diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b0ca057 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,34 @@ +services: + postgres: + image: postgres:16-alpine + container_name: litellm-postgres + environment: + POSTGRES_DB: litellm + POSTGRES_USER: litellm + POSTGRES_PASSWORD: litellm + ports: + - "5432:5432" + volumes: + - pgdata:/var/lib/postgresql/data + + litellm: + image: litellm/litellm:main-latest + container_name: litellm-proxy + depends_on: + - postgres + ports: + - "4000:4000" + env_file: .env + environment: + DATABASE_URL: postgresql://litellm:litellm@postgres:5432/litellm + PYTHONPATH: /app + volumes: + - ./config/litellm_config.yaml:/app/config.yaml:ro + - ./config/genai.pem:/app/config/genai.pem:ro + - ./src/custom_callbacks.py:/app/custom_callbacks.py:ro + - ./src/ledger.py:/app/ledger.py:ro + - ./storage:/app/storage + command: ["--config", "/app/config.yaml", "--port", "4000", "--host", "0.0.0.0"] + +volumes: + pgdata: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8cab654 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +fastapi==0.115.13 +httpx==0.28.1 +pydantic==2.11.7 +pydantic-settings==2.10.1 +PyYAML==6.0.2 +python-dotenv==1.1.0 +requests==2.32.4 +oci==2.180.0 +pyyaml +uvicorn>=0.33.0,<0.35.0 +pandas>=2.2.0 +openai>=2.20.0,<3.0.0 \ No newline at end of file diff --git a/scripts/run_poc.sh b/scripts/run_poc.sh new file mode 100644 index 0000000..cfa37bf --- /dev/null +++ b/scripts/run_poc.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +cp -n .env.example .env || true +mkdir -p storage + +echo '1) Subindo Postgres + LiteLLM Proxy...' +docker compose up -d + +echo '2) Registrando virtual keys por API consumidora...' +python src/create_litellm_keys.py + +echo '3) Subindo simulador de APIs na porta 8080...' +uvicorn src.api_service:app --host 0.0.0.0 --port 8080 diff --git a/src/__pycache__/api_service.cpython-313.pyc b/src/__pycache__/api_service.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b52bc756e490e31c7c483c82b8a717ef9a1b5034 GIT binary patch literal 4004 zcmbUkU2_xH_3npO-;yPNI~XxaYJG%(<13g3v&V-L~?h4u^#idXWS>$Q=OlPWb)F#nV zxN$bARf-^IgNZ%|D~6g_H@i}_v0}ytJF-&C5c4u4wMj8ty~u2>@118&);;MQNwh!0 z^$ErfD!Oi1ilrI4ISpg(q`s2iZ11RIS|`t)wuNz3$*I&1kDot(?$qV1T0~Z06wVe^ z{p4xr`9~B}oicK2KEc}DDSg=v<_#q$XAON$o4?X?=FF7%I*2b7R4Rf@QPK*fykZ#? z7JDqs%B$Jti*0I)**FT+_sI;Qj6}d%7|Qj-4Jj{iGaiXQ0JD;Z3X%XLuQT#d?>w9E z+u?K6C={)f`c_Fbtw-&^IsNZ|X;JaV;`vsq(#0wv2o!R&TI`5MV(i})_zC~kk*6UvR3zPdQd_#e+srezk_O?QdKh-Fd;O#443A%Tn1!3?J^RFt^f|OU?I)u~N@)qV43Vdomcx9N0d43LbeU zq|nKBK&o>G0fw2b1}%kFy#rDhmUe*T#Z)@TN`W+wZIT7OK#^wE{s_liCf=o7Fg4~(lDJt9c;T15W+aBX>9ZH>6^|GZ>9iljO z3kV3<3EuWNG0656RMS-ERnyJuFti}AKmz$2I6g&pOL!c?^!q410ELdKXIJEcX?sez z5ORQSz76|#(|+Kf`w?N}b;}0u>o$b;~=)ui$KTg znk`TGlcePQ|D!gA&9%atoXtu6d) zosb3QBnwUN2wNUq^M~vHo_qeDntxY$WG&i#dw(tZd_B6q8r@%y9<4@?)}uq!=+IiE z?Rx%Nz8(>)5%FPrZ{=XR8oIFB{@f=*HI!WUaPh#pkF<5&I&t&F!;aoti#HcnyY_q( z`XIF0zU$Vdo0nGiCLjAfz!fDeudb7Di%=eavWvvq-^F)nj|BvAwGu z;%cm8HQrN?_f_M4A0Mg4`|I(6YJ8v`KUR$&dmI!(!a5N=f|J)9X3astV+&NZN>q1> zlLSlO1YKWg!+e>^X;~`+EH8>s@L#%=@VPeo25`_6B1{leLezrDSsHnIRbn?5$8#mv z0*YgbiI0vV8V6(x)=E*+=P71a+dBsZ7&scma6>V@xpkCaz`qDZbE_v`aV4kdP$N?` z7~5y6mIV?_is_AFfk-htQw*c7i66nm%JUiw_UxKky zJ{)!p(f2*Kx<5{5VFp8u`FlX?9K$dVNZ;2a{1pj4ARXV3qg8VB0qK4~c7H?q9}C1A z__G7CT>N0E8ai0^|3wH_ z+74HRBW3Q1z%o5IPCp^gJPGj3G_xKdtmoa%cRH_j)|l7^%P`~221nTVy3+*%gz2t# ux+;O)HD=G3rzR`s->9B?^Pi-b5s*)O%tG(melvix*`Wu3$%wi74ou~}$M5XNueTp_nbXR@oxp&56 zC!{RB5!YwVJ@?GL=Wo8lT7A7A!T7uX9Z%{iwjtwv^P-~_hWxNfCKRC!$Bz^32o>SkLz2} zE+o~pA<2J;wcm+DV{AD1ggk^9E0EPR-OQS*t{F*q4&`*sRHn>QR?ChlILul?K5Htb zT2d^5%rm;CfE+)PHOI+n%v5mJ)Um}ywCU_Uc!|F^6^*2$Vmi_@7!_lu#N=>VjGl=} zX-O;=OLEx&&0?ot6xF;KOQxg!(UdrniVa0lXT?PHtQg6phhs_5G89dwyFkt~#B}sb znkq_W1_ueR1m_%oo}HagWl;N^IN@vAl0sL={wpdr%h{rQS(%o{vxcq8R-9Lgt{7G( z%Zg!EbP|CHtd}NC+0-v9n&Do6T#}qk?2Dd?WCqjX0V10N(IQw+*IrKE1YZ;xzpOOcYsM+00pWs-YKjT*{hv&>E4q z4tJL{vpe5CB*nxLtY1=crqQkE)O|(ORKU*rNKovX&~x1e&UF`+Jmm8JiD}CRie=D- zn?TDh_-E{d=_>jv&^&+q+VRhX+reAGl|aXXK=*Q>dnNGZ68|Rt0f;VwDR!hJd$7H2wk}%_Cj6a>8ei(N*+oHQoZD*R2@_<|CkUCS>8c< z;3uz<7uVXG7>9c0I(_k|V^@SSOvT0-RIGB*X5LDp|yYZL!5wu7#Z4o40$e{6~DM2q!}KrZfViX7k; z4{$)|Y8^~mKr`nfAa0U~u3ZHl85KjaNMpc^Air@`>s-ar6o3F|slW@84eh84bvj$~ z@SXzX(5pyV6+Fp9m~NTpB3&0JDzbUw0-`<&o6vu2q5|K4-as6{qOkJ zl@+o;jdK(Z3uuhl%s87Hq++s)IW(w(|Nri0bec2RZpi&+Gytd3IEl@0A#{)(VTS-k z2EcR%t8wmIcyJ&oh8srZG~hr_#lhXg_D>V#%7K*z;Wcgg_m$? z*qgMNE{l14rahLFqN%hvoDyUG$zk~90?He%e&@x`8Gl$j9U06-C9%_D!eQRxFX^gg z@qoq*J1BfkP=visjzJn?QtvG0GA*Lnl7UH(86>ntI_#PmQ~?!X)5e)rvl}l}=G$c3 zhcH3hG&Idm%}p(Ew}o56r~Z|O-Lw3|#+F~Te%$&+@9zih4E!#>(styx@y{n0-dmVl z$bEX`PJFrT$V%hUS?~Ion@`LomfDWp>$`XO&uw3x`ts&I-s6ixG*UNP_rlg?fiG$#m!Iblri0M(w(k z8R}dI@2RA!O_VEmf1~ts0H5yA%jW;yQj77abS-0}{&gsyN;?QohOR^fM5yFx15mp! za=O2>3i|1$?3FRO%5n{PDm6S*%}bYU<7B(TIw77EBgsC|X%0?^ZX@7|_os$4BVx~4 zyC)dvk8EZ5!>`UG4|-XO>V<%-ndkShV=Onlk}G(T}W$p2yUVz5Ce4MjEE8Xq9bf2+ne=A__IRACHdC-;Dlx z^ltOr@s*B~4>|^xI|lA|#8;XVf9r^U&itZpq4Agf%N_CM<^;{@xeMP~4oPyFe4;+t z9of$PVM}DIcQL>MxwzFs=I#E->%K5Ukwr4h(%%`sVPf$^(P^f=STW03o7rAzsz)7M zvcK1cCD2Enhpj#E7K=}kPMpW2=FfzPrA9$nmk4e6c`AW>VFd(E5NQt9D0il%2*`AF zz5MxbfIv->E&w(CGBjxUa*@zMS)PP8*;#3j<${V0v#4r{rpq!W-2>*yw2MsS`uHfB z-i689BFiLf)SR3(O{`ud-+0*6KmWaj3pRNM8CwzA?idF55xRlgqEH5N!YwcqQH0xJviyL?$p3E? z)r`Wl5Ep^nFB?<3Uu7>TRiEq?y0_vrzK`O!0>iy36TL1d=4`7h324W9r2 literal 0 HcmV?d00001 diff --git a/src/__pycache__/oci_usage_rest.cpython-313.pyc b/src/__pycache__/oci_usage_rest.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..507755b55eb3bbbd4fb00494e4dd45fe08204073 GIT binary patch literal 4565 zcma)9O>7&-6`m!R%jGWrBqkw{IWIE5*J(5`70 zc0uW$cE@Q<+jaD`C(d9d?!lh87klkG&omqNVPCTihdH|~k9jDWX@5L`1GdaN9gK%? z$fnupaJ&t-321@tK?jlO+l@r-xT|5$@kW7gyBHMt-RL5XBVtJO+f;`b76UfbnQ=wi z?i1Z8W%076>85OEb4uq5jbhHc9|nEPvzXIm(_-YjUerv>ZJ5|%l_JKfrj(Y0>8xU= zpzb*V%zfQ*WBIL*(QK5lSV_|4yedf+CrNodUCa^MFG+6}LzwKu;^&rW7-G8CG z@g7I%J0ABzEqI-JxCq3%C`CbGCnroalxnUDLP-c=64g+;nx(s0(v31fifu{s9B=k2 zGABHTP}DnXc}-Q5!S@zjG*oQyi&-tB;sVaX>acI9B|BCuz5o#>6)s@eP@^8aA3DYd zNG3pL?E>{B2s+8pmvXvd$0$As@~9h=$zifVEq6vWNx&(K8G{%j;Ju{UFXiPoRLQUd z5+8=DZupItASdo@t)I9xy2w|4QyAh7RE zT%TB-sdYU2fM@*7CSp8{?QW&zP3X`IzwPPY1F_Z9Nt9@AO%Sva)Y2`cML;dnVp>Fs zb;yK$964O06sZ$*f`rJn4pvfRY znk&@`_N2?vI_sK&O~@kw^fXry7PfFKMZ96BSC5IV6s=k!=6rZiOy|e zV4CC_5#?J6f=NhaQjG7VNCQX9xX;#XIZo-9g{BSls7G`r(k&g8#$5{f^cX9@=@l z13T#3<0V5{XyzAjw;+!J=mI1o3$id5n-Y@b z>K8`OP6`% zfzjZe1~eo0z18zrp}CR-{DDURt``c>-&T$05<40!R$-A06Z}vh(;*1XqKUgKPr8^d z7*-g|Swoc|%xnx^H1HwN!Q_^+5p%Q+A0dt05RVph2|sHyu!~@pBuI!LZLzIsxr;bw zc?)t09u#TI+pyHIC~W&2&ys#yUbAR>Imp#1Y)=gUx^Gc6l?(8+0OXV_Nf`*$tVCAE z#`gFPXi4T|w8I*e(W5ZpgID{?v3l>}n|p5TSr_m04%fm*zw8}e9epRVa_(1M)!yN1 z_~=ik%42u^;gycn(VBmMdHgOHylk$Xzh0__de`0S__w|r!?%pv=RSGsbE?L@T#kJm zYTpR;RYQFnq35ch=hnqqXt+FC4|HCRugI(Jjh^9Z&v2#t=&i#Q?nF8Eh(n$Ge>uJ8 zt9CqlckjWq;~&i2-7Ty|J~&eE7V4dQ>K*$k2cEA)j@9?|J_xY~nDWGC2kLveJW=;@ z8{VhjtFxhN!7IVF*KWrv=jJ!g<*V>@Yj;?EyGo_+#OKSE1!-g9QWd`L(j9jBuijui zI&eEv-Fy17cZIF8yZ+9GuLa%?tSGp)^W4v zM$h`;TJJD$w-V)XcnOqG*FzoU$*;nlm+3kie5Y@f{*Zm2t-ylx9Np-VZ>2xG@X3YS zOSKnH1NX;6e}3jq&s4-zExrJ3%&i~$!^p=Y zw`OZcV!(ZV>23Cb2eoxQVv&E>8gtvM@?#Zx>_6YmQK;>CgY3RvcTR=qs0%inY8J6( zzjvZE9s-fQJ8_OsBnF|D#JWv4YT?2|(Ju11T`WRlc-&$)(Rdv)i*G&hRRaTTMmE@Q z^ywuE44hQ(Zo>eHU<1K6#wkK^Kn;+iJ)5z&d84BF#P$ffiD*SS^Rt+RI&f zW~o#}4+4CM4n62%z?T9+Pi|45J@(Z83vvlK%p^tv1UdK?Kne=v)c1y@>=;hbk@$Ar zZ{B?U-n@Bpm`cS7lrQ2}?)))C$Tz6;3+Pz1!+(N&Ky;#OE2OM(Er<2MN}wF%K~)c~ z&~k`}RGqGb%Ml(?_0UR`N5M9{5-Z1fyqw?(g-2GB$FcV`PTt(Fvl*Hd4z$ zoRJWoQD_{{#76uW9Z+Zz(9{d)phDAtW?n#t^g(@Kg505exE9I}J;P!1noO9k>-nZ{ zd#(UARxy3ka!i2=y=D4a@Qtk6KI5jdmJi6-+ooVQy$W+==(^3E3SddZbmsfETN84q zUwg-Mt9I?0ZTS{X)-T62{t1u=WP@-`Cp@5QJUB}>C~Q8U2X%Tez(ab7hxPDAM32mY zt&ZfO7E@ZB$CQ>(T9U_=mQq@pCzO^^+5k@~ZBS`LJf*Z@rDb_qX(LKI!87_O%rK_3 zaXwHxk)M#kciru0NmxOP7MZ&(!*K{cwj!(@p4&gW)pY|Hl>mr5navrK2p z6aJ-h3kwUS?L`?BK9>UyFzGmTV-pxNn%rrn#N9&O6u@tx;aPI1Z_vezyKJX5=vj85 z#_G0f7nYU_4erU|V>9%GVLG-cTBF61@I7wUSjl0P8skNA5A9^Es~VNf)=;rzDYul& zhAoOaX5EpAE~56)TEqc`J-&Y%?z4tjSc3xp1a@dz%@iEytc)3C6c=i{WyB zm9-z@KwR$kxi!yaFPZ@jm8RwQ%>GRBgEDFuu32Y>A>)Qohh_)sNyE6;G@V{c#(*AN z_>S!|*E0-`@h+2K4Lo*<;6n(3;R~SwhQomJEQ%2nCs2%{7)S9MijyGZh+zl}D9bQ? zpWB;Fp9$zb1#O4$Bl1G2?)s%l>85@ocZ+*>nB|KS?7z@+YQchYgQuY+IQ)8XRc!2V zJcCMLMR*`SA)gP=KHC1rZypa{-(CHB^85kK?ptkozB5%gpeG&%+qBS`UOJ$o4~N@y zsWVdCy|E|SwAdM)-(A@UZ2s%14|B^Chg_c|uG4 z_5KQe6wvg5&OKFh3nArf3W~L0xlnV ze7)giM~|7VkusSgjND+9hdgljfal|(w zGh`#GCoXD`CX#^m6GOa~$q&fb)kdS6pPnh-cWJGAuD>NSUBXz!@V&dt6|M8D-Tb#c z_fpH;ncSQY+0*G+qR!d9CKJAx`$!I&4(0*Otwx=}U0})JZT*^zT20QGYwalBWJtLX z@@%HUs%F!HJI%DQQn)`w&s~Y1cGp|5dwv;36ojNW2EU5>tK4@1 z8&Dt(Vi82&eYea5I&v_2vrWxTc6|4}J-7?TJBi`@?|(i%^~irbes1^L-kB%#T-TjA zhNXe%moQY|V$cphg~|i6e%y8&9!{9!F9rj|)z$<34j=Yj27MjB>o|%kAL4FaDl_D^ zw%i-H{}lmvV8P#VZ`>R?O-_+BEBuNE$V zD@_|;S-Pk9;iRxeHBlTC&>^C4e%{hGyWSZx9T*^q-r zU^&OHJSI4Xos8hB9eb0b4`kTiX~5n&lF#2mB-+@im@XtW9zjf list[dict[str, Any]]: + enriched = [] + for r in rows: + weighted_tokens = (r['input_tokens'] or 0) * input_weight + (r['output_tokens'] or 0) * output_weight + r = dict(r) + r['weighted_tokens'] = weighted_tokens + enriched.append(r) + + denominator = sum(float(r[metric] or 0) for r in enriched) or 1.0 + for r in enriched: + share = float(r[metric] or 0) / denominator + r['allocation_metric'] = metric + r['share_pct'] = round(share * 100, 6) + r['allocated_cost'] = round(total_cost * share, 6) + return enriched + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--month', help='YYYY-MM; default vem de config/apis.yaml') + parser.add_argument('--oci-cost', type=float, help='Permite testar sem chamar OCI Usage API') + parser.add_argument('--out', default='./storage/allocation_report.csv') + args = parser.parse_args() + + env = Env() + cfg = load_config(env.config_file) + month = args.month or cfg.month + rows = Ledger(env.ledger_db).monthly_usage(month) + + if args.oci_cost is None: + cost_result = fetch_monthly_genai_cost_rest(month) + total_cost = cost_result.amount + currency = cost_result.currency or cfg.currency + Path('./storage/oci_usage_raw.json').write_text(json.dumps(cost_result.raw, indent=2, ensure_ascii=False), encoding='utf-8') + else: + total_cost = args.oci_cost + currency = cfg.currency + + allocations = compute_allocations( + rows=rows, + total_cost=total_cost, + metric=cfg.allocation_metric, + input_weight=cfg.model.input_weight, + output_weight=cfg.model.output_weight, + ) + + fieldnames = [ + 'month', 'api_id', 'api_name', 'requests', 'input_tokens', 'output_tokens', 'total_tokens', + 'weighted_tokens', 'litellm_estimated_cost', 'allocation_metric', 'share_pct', 'allocated_cost', + 'avg_latency_ms' + ] + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + with out.open('w', newline='', encoding='utf-8') as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + for r in allocations: + writer.writerow({k: r.get(k) for k in fieldnames}) + + print(json.dumps({ + 'month': month, + 'oci_total_cost': total_cost, + 'currency': currency, + 'allocation_metric': cfg.allocation_metric, + 'report': str(out), + 'allocations': allocations, + }, indent=2, ensure_ascii=False)) + + +if __name__ == '__main__': + main() diff --git a/src/api_service.py b/src/api_service.py new file mode 100644 index 0000000..6c16fe5 --- /dev/null +++ b/src/api_service.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import random +import time +from typing import Any + +from fastapi import FastAPI, Header, HTTPException +from openai import OpenAI +from pydantic import BaseModel + +from settings import Env, load_config + + +env = Env() +config = load_config(env.config_file) +api_by_id = {a.id: a for a in config.apis} + +app = FastAPI(title='LLM Consumer API Simulator') + + +class PromptRequest(BaseModel): + prompt: str | None = None + max_tokens: int = 256 + + +def client_for(api_id: str) -> OpenAI: + api_def = api_by_id.get(api_id) + if not api_def: + raise HTTPException(status_code=404, detail=f'API {api_id} não cadastrada em {env.config_file}') + return OpenAI(api_key=api_def.virtual_key, base_url=env.litellm_base_url) + + +@app.get('/apis') +def list_apis() -> list[dict[str, Any]]: + return [a.model_dump(exclude={'virtual_key'}) for a in config.apis] + + +@app.post('/apis/{api_id}/ask') +def ask(api_id: str, body: PromptRequest, x_request_id: str | None = Header(default=None)) -> dict[str, Any]: + api_def = api_by_id.get(api_id) + if not api_def: + raise HTTPException(status_code=404, detail='api_id inválido') + + prompt = body.prompt or f'Responda em uma frase: qual é uma boa métrica para FinOps de LLM? rand={random.randint(1, 9999)}' + t0 = time.perf_counter() + response = client_for(api_id).chat.completions.create( + model=config.model.alias, + messages=[ + {'role': 'system', 'content': 'Você é um assistente técnico objetivo.'}, + {'role': 'user', 'content': prompt}, + ], + max_tokens=body.max_tokens, + metadata={ + 'api_id': api_def.id, + 'api_name': api_def.name, + 'owner': api_def.owner, + 'external_request_id': x_request_id, + }, + ) + elapsed_ms = int((time.perf_counter() - t0) * 1000) + usage = response.usage.model_dump() if response.usage else {} + return { + 'api_id': api_id, + 'model': config.model.alias, + 'latency_ms': elapsed_ms, + 'usage': usage, + 'answer': response.choices[0].message.content, + } diff --git a/src/create_litellm_keys.py b/src/create_litellm_keys.py new file mode 100644 index 0000000..60cd39c --- /dev/null +++ b/src/create_litellm_keys.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import httpx + +from settings import Env, load_config + + +def main() -> None: + env = Env() + cfg = load_config(env.config_file) + headers = {'Authorization': f'Bearer {env.litellm_master_key}'} + + with httpx.Client(timeout=30) as client: + for api in cfg.apis: + payload = { + 'key': api.virtual_key, + 'aliases': {cfg.model.alias: cfg.model.alias}, + 'models': [cfg.model.alias], + 'metadata': {'api_id': api.id, 'api_name': api.name, 'owner': api.owner}, + 'team_id': api.owner or api.id, + 'user_id': api.id, + } + r = client.post(f'{env.litellm_base_url.replace("/v1", "")}/key/generate', headers=headers, json=payload) + if r.status_code in (200, 201): + print(f'OK key criada/registrada: {api.id}') + elif r.status_code == 400 and 'already' in r.text.lower(): + print(f'OK key já existe: {api.id}') + else: + print(f'ERRO {api.id}: {r.status_code} {r.text}') + + +if __name__ == '__main__': + main() diff --git a/src/custom_callbacks.py b/src/custom_callbacks.py new file mode 100644 index 0000000..59b42af --- /dev/null +++ b/src/custom_callbacks.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import json +import os +from datetime import datetime, timezone +from typing import Any + +from litellm.integrations.custom_logger import CustomLogger + +# Este callback roda dentro do LiteLLM Proxy. +# Ele persiste um ledger local por API consumidora usando metadata enviada na chamada. +try: + from ledger import Ledger, now_iso +except Exception: # pragma: no cover - fallback quando import path muda no container + from src.ledger import Ledger, now_iso + +def _extract_metadata(kwargs: dict[str, Any]) -> dict[str, Any]: + candidates = [] + + candidates.append(kwargs.get("metadata")) + + litellm_params = kwargs.get("litellm_params") or {} + candidates.append(litellm_params.get("metadata")) + + standard_logging_object = kwargs.get("standard_logging_object") or {} + candidates.append(standard_logging_object.get("metadata")) + + proxy_server_request = kwargs.get("proxy_server_request") or {} + candidates.append(proxy_server_request.get("metadata")) + + for candidate in candidates: + if isinstance(candidate, dict) and candidate: + return candidate + + return {} + +def _month_from_ts(dt: datetime) -> str: + return dt.strftime('%Y-%m') + +def _json_safe(value): + """Converte objetos não serializáveis do LiteLLM em tipos seguros para JSON.""" + if value is None: + return None + + if isinstance(value, (str, int, float, bool)): + return value + + if isinstance(value, dict): + return {str(k): _json_safe(v) for k, v in value.items()} + + if isinstance(value, (list, tuple, set)): + return [_json_safe(v) for v in value] + + # Pydantic / dataclass-like + if hasattr(value, "model_dump"): + try: + return _json_safe(value.model_dump()) + except Exception: + pass + + if hasattr(value, "dict"): + try: + return _json_safe(value.dict()) + except Exception: + pass + + # Último fallback: vira string + return str(value) + +def _usage_get(response_obj: Any, key: str, default: int = 0) -> int: + usage = getattr(response_obj, 'usage', None) + if usage is None and isinstance(response_obj, dict): + usage = response_obj.get('usage') + if usage is None: + return default + if isinstance(usage, dict): + return int(usage.get(key, default) or default) + return int(getattr(usage, key, default) or default) + + +def _response_id(response_obj: Any) -> str | None: + if isinstance(response_obj, dict): + return response_obj.get('id') + return getattr(response_obj, 'id', None) + + +class CostAllocationCallback(CustomLogger): + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + metadata = _extract_metadata(kwargs) + + api_id = ( + metadata.get("api_id") + or metadata.get("user_api_key_metadata", {}).get("api_id") + or kwargs.get("user") + or "unknown" + ) + + api_name = ( + metadata.get("api_name") + or metadata.get("user_api_key_metadata", {}).get("api_name") + or api_id + ) + + prompt_tokens = _usage_get(response_obj, 'prompt_tokens') + completion_tokens = _usage_get(response_obj, 'completion_tokens') + total_tokens = _usage_get(response_obj, 'total_tokens', prompt_tokens + completion_tokens) + latency_ms = int((end_time - start_time).total_seconds() * 1000) + + # LiteLLM calcula response_cost quando conhece o modelo/preço. Para rateio final, + # a fonte da verdade será o custo total da OCI Usage API. + response_cost = float(kwargs.get('response_cost') or 0) + + ts = end_time if isinstance(end_time, datetime) else datetime.now(timezone.utc) + ledger = Ledger(os.environ.get('LEDGER_DB', '/app/storage/ledger.sqlite')) + + model = ( + kwargs.get("model") + or kwargs.get("model_name") + or (kwargs.get("standard_logging_object") or {}).get("model") + or (kwargs.get("response_cost_information") or {}).get("model") + or "unknown" + ) + + safe_metadata = { + "api_id": api_id, + "api_name": api_name, + "owner": metadata.get("owner"), + "model": model, + } + + ledger.insert_usage({ + 'ts': ts.isoformat(), + 'month': _month_from_ts(ts), + 'api_id': api_id, + 'api_name': api_name, + 'virtual_key_hash': str(kwargs.get('litellm_api_key') or kwargs.get('api_key') or '')[:12], + 'model': kwargs.get('model'), + 'request_id': _response_id(response_obj), + 'prompt_tokens': prompt_tokens, + 'completion_tokens': completion_tokens, + 'total_tokens': total_tokens, + 'response_cost': response_cost, + 'latency_ms': latency_ms, + 'status': 'success', + 'metadata_json': json.dumps(safe_metadata, ensure_ascii=False), + }) + + +proxy_handler_instance = CostAllocationCallback() diff --git a/src/ledger.py b/src/ledger.py new file mode 100644 index 0000000..2ef3993 --- /dev/null +++ b/src/ledger.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import sqlite3 +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path +from typing import Iterator, Any + + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS llm_usage ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts TEXT NOT NULL, + month TEXT NOT NULL, + api_id TEXT NOT NULL, + api_name TEXT, + virtual_key_hash TEXT, + model TEXT, + request_id TEXT, + prompt_tokens INTEGER NOT NULL DEFAULT 0, + completion_tokens INTEGER NOT NULL DEFAULT 0, + total_tokens INTEGER NOT NULL DEFAULT 0, + response_cost REAL NOT NULL DEFAULT 0, + latency_ms INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'success', + metadata_json TEXT +); + +CREATE INDEX IF NOT EXISTS idx_llm_usage_month_api ON llm_usage(month, api_id); +CREATE UNIQUE INDEX IF NOT EXISTS uq_llm_usage_request ON llm_usage(request_id) WHERE request_id IS NOT NULL; +""" + + +def now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +class Ledger: + def __init__(self, db_path: str | Path): + self.db_path = Path(db_path) + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self.init() + + @contextmanager + def connect(self) -> Iterator[sqlite3.Connection]: + con = sqlite3.connect(self.db_path) + con.row_factory = sqlite3.Row + try: + yield con + con.commit() + finally: + con.close() + + def init(self) -> None: + with self.connect() as con: + con.executescript(SCHEMA) + + def insert_usage(self, row: dict[str, Any]) -> None: + keys = ','.join(row.keys()) + params = ','.join([f':{k}' for k in row.keys()]) + with self.connect() as con: + con.execute(f'INSERT OR IGNORE INTO llm_usage ({keys}) VALUES ({params})', row) + + def monthly_usage(self, month: str) -> list[dict[str, Any]]: + sql = """ + SELECT + month, + api_id, + MAX(api_name) AS api_name, + COUNT(*) AS requests, + SUM(prompt_tokens) AS input_tokens, + SUM(completion_tokens) AS output_tokens, + SUM(total_tokens) AS total_tokens, + SUM(response_cost) AS litellm_estimated_cost, + AVG(latency_ms) AS avg_latency_ms + FROM llm_usage + WHERE month = ? AND status = 'success' + GROUP BY month, api_id + ORDER BY total_tokens DESC + """ + with self.connect() as con: + return [dict(r) for r in con.execute(sql, (month,)).fetchall()] diff --git a/src/oci_usage_rest.py b/src/oci_usage_rest.py new file mode 100644 index 0000000..910f177 --- /dev/null +++ b/src/oci_usage_rest.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import date, datetime, timezone +from typing import Any + +import oci +import requests +from oci.config import from_file +from oci.signer import Signer + +from settings import Env + + +@dataclass +class OciCostResult: + amount: float + currency: str + raw: dict[str, Any] + + +def _make_signer() -> Signer: + cfg = from_file() + return Signer( + tenancy=cfg['tenancy'], + user=cfg['user'], + fingerprint=cfg['fingerprint'], + private_key_file_location=cfg['key_file'], + pass_phrase=cfg.get('pass_phrase'), + ) + + +def _iso_z(d: date) -> str: + return datetime(d.year, d.month, d.day, tzinfo=timezone.utc).isoformat().replace('+00:00', 'Z') + + +def fetch_monthly_genai_cost_rest(month: str) -> OciCostResult: + """Consulta a OCI Usage API via REST assinado. + + month: YYYY-MM + + Observação: em alguns tenancies o nome exato de service/productDescription pode variar. + Ajuste OCI_USAGE_SERVICE_FILTER e OCI_USAGE_PRODUCT_FILTER no .env conforme a saída real. + """ + env = Env() + year, mon = [int(x) for x in month.split('-')] + start = date(year, mon, 1) + end = date(year + (mon == 12), 1 if mon == 12 else mon + 1, 1) + + endpoint = env.oci_usage_endpoint.rstrip('/') + url = f'{endpoint}/{env.oci_usage_api_version}/usage' + + # RequestSummarizedUsagesDetails. A granularidade mensal reduz ruído de hora/dia. + # GroupBy opcional ajuda a auditar qual produto/sku veio na resposta. + payload: dict[str, Any] = { + 'tenantId': from_file()['tenancy'], + 'timeUsageStarted': _iso_z(start), + 'timeUsageEnded': _iso_z(end), + 'granularity': 'MONTHLY', + 'queryType': 'COST', + 'groupBy': ['service', 'productDescription'], + 'isAggregateByTime': True, + } + + filters: list[dict[str, Any]] = [] + if env.oci_usage_service_filter: + filters.append({'dimension': 'service', 'operator': 'CONTAINS', 'value': env.oci_usage_service_filter}) + if env.oci_usage_product_filter: + filters.append({'dimension': 'productDescription', 'operator': 'CONTAINS', 'value': env.oci_usage_product_filter}) + if filters: + payload['filter'] = {'operator': 'AND', 'dimensions': filters} + + r = requests.post(url, auth=_make_signer(), headers={'content-type': 'application/json'}, data=json.dumps(payload), timeout=60) + r.raise_for_status() + data = r.json() + + # OCI pode retornar itens em data/items dependendo do cliente/endpoint/revisão. + items = data.get('items') or data.get('data') or [] + amount = 0.0 + currency = 'USD' + for item in items: + amount += float(item.get('computedAmount') or item.get('cost') or item.get('amount') or 0) + currency = item.get('currency') or item.get('currencyCode') or currency + + return OciCostResult(amount=amount, currency=currency, raw=data) diff --git a/src/settings.py b/src/settings.py new file mode 100644 index 0000000..dfa04b2 --- /dev/null +++ b/src/settings.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +import yaml +from pydantic import BaseModel, Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Env(BaseSettings): + model_config = SettingsConfigDict(env_file='.env', extra='ignore') + + litellm_base_url: str = 'http://localhost:4000/v1' + litellm_master_key: str = 'sk-master-poc' + litellm_model_alias: str = 'oci-gemini-25-pro' + ledger_db: str = './storage/ledger.sqlite' + config_file: str = './config/apis.yaml' + + oci_usage_endpoint: str = 'https://usageapi.us-ashburn-1.oci.oraclecloud.com' + oci_usage_api_version: str = '20200107' + oci_usage_service_filter: str = 'generative_ai' + oci_usage_product_filter: str | None = None + + +class ApiDef(BaseModel): + id: str + name: str + virtual_key: str + owner: str | None = None + + +class ModelDef(BaseModel): + alias: str + oci_model_name: str + input_weight: float = 1.0 + output_weight: float = 5.0 + + +class AppConfig(BaseModel): + month: str + allocation_metric: Literal['weighted_tokens', 'total_tokens', 'requests'] = 'weighted_tokens' + currency: str = 'USD' + model: ModelDef + apis: list[ApiDef] = Field(default_factory=list) + + +def load_config(path: str | Path) -> AppConfig: + with open(path, 'r', encoding='utf-8') as f: + return AppConfig.model_validate(yaml.safe_load(f)) diff --git a/src/simulate_load.py b/src/simulate_load.py new file mode 100644 index 0000000..6cced82 --- /dev/null +++ b/src/simulate_load.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import argparse +import asyncio +import random +from uuid import uuid4 + +import httpx + +from settings import Env, load_config + + +PROMPTS = [ + 'Explique em uma frase o que é rateio de custo de LLM.', + 'Gere uma resposta curta para cliente perguntando segunda via de fatura.', + 'Classifique a intenção: quero trocar meu plano de internet.', + 'Resuma em tópicos uma política de uso responsável de IA.', + 'Dê 3 métricas para monitorar consumo de tokens por API.', +] + + +async def call_one(client: httpx.AsyncClient, base_url: str, api_id: str) -> None: + prompt = random.choice(PROMPTS) + f' request={uuid4()}' + try: + r = await client.post( + f'{base_url}/apis/{api_id}/ask', + headers={'x-request-id': str(uuid4())}, + json={'prompt': prompt, 'max_tokens': random.choice([64, 128, 256])}, + ) + print(api_id, r.status_code, r.json().get('usage')) + except Exception as e: + print(api_id, 'ERROR', repr(e)) + + +async def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--calls', type=int, default=30) + parser.add_argument('--concurrency', type=int, default=5) + parser.add_argument('--api-base', default='http://localhost:8080') + args = parser.parse_args() + + env = Env() + cfg = load_config(env.config_file) + api_ids = [a.id for a in cfg.apis] + sem = asyncio.Semaphore(args.concurrency) + + async with httpx.AsyncClient(timeout=180) as client: + async def guarded(i: int) -> None: + async with sem: + await call_one(client, args.api_base, random.choice(api_ids)) + await asyncio.gather(*(guarded(i) for i in range(args.calls))) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/storage/.DS_Store b/storage/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0