""" AI Agent - Infrastructure & Security Engineer - Backend API v1.1 FastAPI with JWT auth, TOTP MFA, RBAC, OCI GenAI (exact SDK pattern), OCI Account Explorer, MCP Server registry with VectorDB tool integration, Autonomous DB vector storage, CIS reports, chat agent, audit log. """ import os, json, uuid, hashlib, hmac, time, base64, struct, secrets, subprocess import shutil, asyncio, sqlite3, logging, re, concurrent.futures, threading from datetime import datetime, timedelta from pathlib import Path from typing import Optional, List, Dict, Any from contextlib import contextmanager from functools import partial from fastapi import ( FastAPI, HTTPException, Depends, Request, UploadFile, File, Form, Query, Body, BackgroundTasks ) from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, FileResponse, StreamingResponse, HTMLResponse, Response, RedirectResponse from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials from pydantic import BaseModel import jwt as pyjwt # ── Config ──────────────────────────────────────────────────────────────────── APP_SECRET = os.environ.get("APP_SECRET") or secrets.token_hex(32) if not os.environ.get("APP_SECRET"): import warnings warnings.warn("APP_SECRET not set — using random key (tokens will not survive restarts). Set APP_SECRET in .env for production.") JWT_ALG = "HS256" JWT_EXP_H = int(os.environ.get("JWT_EXPIRY_HOURS", "12")) DATA = Path(os.environ.get("DATA_DIR", "/data")) DB_PATH = DATA / "agent.db" OCI_DIR = DATA / "oci_configs" REPORTS = DATA / "reports" MCP_DIR = DATA / "mcp_servers" WALLET_DIR = DATA / "wallets" VERSION = "1.1" _START_TIME = time.time() _MAX_UPLOAD_BYTES = 50 * 1024 * 1024 # 50MB global limit for file uploads for d in [DATA, OCI_DIR, REPORTS, MCP_DIR, WALLET_DIR]: d.mkdir(parents=True, exist_ok=True) _running_reports: dict[str, asyncio.subprocess.Process] = {} # rid → subprocess _running_terraform: dict[str, asyncio.subprocess.Process] = {} # wid → subprocess _EMBED_STATUS_DIR = DATA / ".embed_status" _EMBED_STATUS_DIR.mkdir(parents=True, exist_ok=True) def _set_embed_status(task_id: str, data: dict): """Write embedding status to shared file (visible across all workers).""" (_EMBED_STATUS_DIR / f"{task_id}.json").write_text(json.dumps(data), encoding="utf-8") def _get_embed_status(task_id: str) -> dict | None: """Read embedding status from shared file.""" p = _EMBED_STATUS_DIR / f"{task_id}.json" if p.exists(): try: return json.loads(p.read_text(encoding="utf-8")) except Exception: return None return None def _update_embed_status(task_id: str, updates: dict): """Update embedding status (read-modify-write).""" current = _get_embed_status(task_id) or {} current.update(updates) _set_embed_status(task_id, current) TERRAFORM_DIR = DATA / "terraform" TERRAFORM_DIR.mkdir(parents=True, exist_ok=True) _chat_executor = concurrent.futures.ThreadPoolExecutor(max_workers=10, thread_name_prefix="chat") # ── Chat Memory Compaction Settings ── COMPACT_TOKEN_THRESHOLD = 6000 # estimated tokens before triggering compaction COMPACT_KEEP_RECENT = 20 # recent messages to keep uncompacted (10 interactions = 20 msgs) COMPACT_SUMMARY_MAX_TOKENS = 1500 COMPACT_MIN_MESSAGES = 8 logging.basicConfig(level=logging.INFO) log = logging.getLogger("agent") async def _validate_upload(file: "UploadFile", allowed_exts: set[str] | None = None, max_bytes: int = _MAX_UPLOAD_BYTES): """Validate uploaded file size and extension. Seeks back to 0 after size check.""" data = await file.read() if len(data) > max_bytes: raise HTTPException(400, f"Arquivo excede o limite de {max_bytes // (1024*1024)}MB") if allowed_exts and file.filename: ext = Path(file.filename).suffix.lower() if ext not in allowed_exts: raise HTTPException(400, f"Extensão '{ext}' não permitida. Aceitas: {', '.join(sorted(allowed_exts))}") return data app = FastAPI(title="AI Agent - Infrastructure & Security Engineer", version=VERSION) _CORS_ORIGINS = os.environ.get("CORS_ORIGINS", "").split(",") if os.environ.get("CORS_ORIGINS") else [] app.add_middleware(CORSMiddleware, allow_origins=_CORS_ORIGINS or ["http://localhost:8080", "http://127.0.0.1:8080"], allow_credentials=True, allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"], allow_headers=["Content-Type", "Authorization"]) security = HTTPBearer() # ── OCI GenAI Models Catalog ────────────────────────────────────────────────── # OCIDs are region-specific; "ocids" maps genai_region → OCID. # _call_genai resolves the OCID for the configured region at runtime. GENAI_MODELS = { # ── Meta ── "meta.llama-4-maverick-17b-128e-instruct-fp8": {"provider":"meta","name":"Meta Llama 4 Maverick","api_format":"GENERIC","max_tokens":8192, "ocids":{"us-ashburn-1":"ocid1.generativeaimodel.oc1.iad.amaaaaaask7dceyah6tjdejjashngznsylutuhhvufukzb2g2ls54g2flsfq"}}, "meta.llama-4-scout-17b-16e-instruct": {"provider":"meta","name":"Meta Llama 4 Scout","api_format":"GENERIC","max_tokens":8192, "ocids":{"us-ashburn-1":"ocid1.generativeaimodel.oc1.iad.amaaaaaask7dceyaw23hfc7mtvv5wef3gwvvaguyzqmhb5lx4r5s3y2xzc4a"}}, # ── Google ── "google.gemini-2.5-pro": {"provider":"google","name":"Google Gemini 2.5 Pro","api_format":"GENERIC","max_tokens":65536, "ocids":{"us-ashburn-1":"ocid1.generativeaimodel.oc1.iad.amaaaaaask7dceyargceyuaysrjzo2metq2rinavayxqmpu7tkm6mmfojcvq"}}, "google.gemini-2.5-flash": {"provider":"google","name":"Google Gemini 2.5 Flash","api_format":"GENERIC","max_tokens":8192, "ocids":{"us-ashburn-1":"ocid1.generativeaimodel.oc1.iad.amaaaaaask7dceyaeo4ehrn25guuats5s45hnvswlhxo6riop275l2bkr2vq"}}, # ── OpenAI ── "openai.gpt-5.2": {"provider":"openai","name":"OpenAI GPT-5.2","api_format":"GENERIC","max_tokens":32768, "ocids":{"us-ashburn-1":"ocid1.generativeaimodel.oc1.iad.amaaaaaask7dceya4fw3p5fddnexbfcurnz7spgkqb4mq4a6y5ubyv7777sa"}}, "openai.gpt-5.1": {"provider":"openai","name":"OpenAI GPT-5.1","api_format":"GENERIC","max_tokens":32768, "ocids":{"us-ashburn-1":"ocid1.generativeaimodel.oc1.iad.amaaaaaask7dceya3darth2ozqcfssb2bats5jitpgigllccajasdyqljnkq"}}, "openai.gpt-5-mini": {"provider":"openai","name":"OpenAI GPT-5 Mini","api_format":"GENERIC","max_tokens":16384, "ocids":{"us-ashburn-1":"ocid1.generativeaimodel.oc1.iad.amaaaaaask7dceya3eain4n6v3edm4ryjvze5hnjouujd4vralxntfalwjaq"}}, "openai.gpt-4.1": {"provider":"openai","name":"OpenAI GPT-4.1 (Padrão)","api_format":"GENERIC","max_tokens":32768,"penalties":True, "ocids":{"us-ashburn-1":"ocid1.generativeaimodel.oc1.iad.amaaaaaask7dceyaa6g75r2qqmtzjaooqtlxv4lxkpcqp2jdd6plpq7yq7ea"}}, "openai.gpt-4.1-mini": {"provider":"openai","name":"OpenAI GPT-4.1 Mini","api_format":"GENERIC","max_tokens":16384,"penalties":True, "ocids":{"us-ashburn-1":"ocid1.generativeaimodel.oc1.iad.amaaaaaask7dceya6pk3sxishpiexm2rb5sf4ytb5tsbz4to2g3g23smidaa"}}, "openai.gpt-4o": {"provider":"openai","name":"OpenAI GPT-4o","api_format":"GENERIC","max_tokens":16384,"penalties":True, "ocids":{"us-ashburn-1":"ocid1.generativeaimodel.oc1.iad.amaaaaaask7dceyah7slrtboxdbfdy5cdspsfts62yumoclpdgwydopse7za"}}, "openai.o4-mini": {"provider":"openai","name":"OpenAI o4-mini","api_format":"GENERIC","reasoning":True,"max_tokens":100000, "ocids":{"us-ashburn-1":"ocid1.generativeaimodel.oc1.iad.amaaaaaask7dceya5ivfqp4fxlajoeg2cahlqtuuswagiv6a7dggpigy23bq"}}, "openai.o3": {"provider":"openai","name":"OpenAI o3","api_format":"GENERIC","reasoning":True,"max_tokens":100000, "ocids":{"us-ashburn-1":"ocid1.generativeaimodel.oc1.iad.amaaaaaask7dceyalgnrukpjk6wm5zsf4jzkoneahgswhrk7kukkoagwnzma"}}, # ── xAI ── "xai.grok-4": {"provider":"xai","name":"xAI Grok 4","api_format":"GENERIC","max_tokens":131072, "ocids":{"us-ashburn-1":"ocid1.generativeaimodel.oc1.iad.amaaaaaask7dceyaldmhg25is4nouena4oa2pj4nvwgfeempo4syiaazukia"}}, "xai.grok-3": {"provider":"xai","name":"xAI Grok 3","api_format":"GENERIC","max_tokens":131072, "ocids":{"us-ashburn-1":"ocid1.generativeaimodel.oc1.iad.amaaaaaask7dceyag3w2xk76vlahjujj2gdfeuzhflt25gbo3bxidlsqfjla"}}, "xai.grok-3-mini-fast": {"provider":"xai","name":"xAI Grok 3 Mini Fast","api_format":"GENERIC","max_tokens":131072, "ocids":{"us-ashburn-1":"ocid1.generativeaimodel.oc1.iad.amaaaaaask7dceyauvjoll2repj5pbtkk7pinwj57ex3lkehzpxd6v6rxscq"}}, } EMBEDDING_MODELS = { "cohere.embed-v4.0": {"name":"Cohere Embed v4.0 (Multimodal)","dims":1536, "ocids":{"us-ashburn-1":"ocid1.generativeaimodel.oc1.iad.amaaaaaask7dceyahw4vlsxm7newcqtlgmristnwxlrxox3h7bcnlomjpgwa"}}, "openai.text-embedding-3-large": {"name":"OpenAI Text Embedding 3 Large","dims":3072, "ocids":{"us-ashburn-1":"ocid1.generativeaimodel.oc1.iad.amaaaaaask7dceya3i6o2p5h2mij6unya5bsyc46aqey5hwy3icncawo3vcq"}}, "openai.text-embedding-3-small": {"name":"OpenAI Text Embedding 3 Small","dims":1536, "ocids":{"us-ashburn-1":"ocid1.generativeaimodel.oc1.iad.amaaaaaask7dceyarjpjyniixp4tf7kyhr5bfajxmky4sjmbki2hp55ns2pq"}}, } GENAI_REGIONS = [ "us-chicago-1","us-ashburn-1","us-phoenix-1","uk-london-1", "eu-frankfurt-1","ap-tokyo-1","ap-osaka-1","sa-saopaulo-1", "ca-toronto-1","ap-melbourne-1","ap-mumbai-1","eu-amsterdam-1", "me-jeddah-1","ap-singapore-1","ap-seoul-1","sa-vinhedo-1", ] OCI_REGIONS = { "Americas": [ "us-ashburn-1","us-phoenix-1","us-chicago-1","us-sanjose-1","us-saltlake-2", "ca-toronto-1","ca-montreal-1", "sa-saopaulo-1","sa-vinhedo-1","sa-bogota-1","sa-santiago-1","sa-valparaiso-1", "mx-queretaro-1","mx-monterrey-1", ], "Europe": [ "eu-frankfurt-1","eu-amsterdam-1","eu-zurich-1","eu-madrid-1","eu-marseille-1", "eu-milan-1","eu-paris-1","eu-stockholm-1","eu-jovanovac-1","eu-dcc-rome-1", "uk-london-1","uk-cardiff-1", "il-jerusalem-1", ], "Asia Pacific": [ "ap-tokyo-1","ap-osaka-1","ap-seoul-1","ap-chuncheon-1", "ap-mumbai-1","ap-hyderabad-1", "ap-melbourne-1","ap-sydney-1", "ap-singapore-1","ap-singapore-2", ], "Middle East & Africa": [ "me-jeddah-1","me-abudhabi-1","me-dubai-1","me-riyadh-1", "af-johannesburg-1", ], } # ── Database ────────────────────────────────────────────────────────────────── @contextmanager def db(): conn = sqlite3.connect(str(DB_PATH), timeout=30, check_same_thread=False) conn.row_factory = sqlite3.Row conn.execute("PRAGMA journal_mode=WAL") conn.execute("PRAGMA busy_timeout=30000") conn.execute("PRAGMA foreign_keys=ON") try: yield conn conn.commit() finally: conn.close() def init_db(): with db() as c: c.executescript(""" CREATE TABLE IF NOT EXISTS users ( id TEXT PRIMARY KEY, first_name TEXT NOT NULL, last_name TEXT NOT NULL, username TEXT UNIQUE NOT NULL, email TEXT, password_hash TEXT NOT NULL, role TEXT NOT NULL DEFAULT 'viewer', mfa_secret TEXT, mfa_enabled INTEGER DEFAULT 0, timezone TEXT DEFAULT 'America/Sao_Paulo', is_active INTEGER DEFAULT 1, created_at TEXT DEFAULT (datetime('now')), last_login TEXT ); CREATE TABLE IF NOT EXISTS sessions ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL, created_at TEXT DEFAULT (datetime('now')), expires_at TEXT NOT NULL, is_active INTEGER DEFAULT 1 ); CREATE TABLE IF NOT EXISTS oci_configs ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL, tenancy_name TEXT NOT NULL, tenancy_ocid TEXT NOT NULL, user_ocid TEXT NOT NULL, fingerprint TEXT NOT NULL, region TEXT NOT NULL, key_file_path TEXT NOT NULL, compartment_id TEXT, created_at TEXT DEFAULT (datetime('now')) ); CREATE TABLE IF NOT EXISTS genai_configs ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL, name TEXT NOT NULL DEFAULT 'default', oci_config_id TEXT NOT NULL, model_id TEXT NOT NULL, model_ocid TEXT, compartment_id TEXT NOT NULL, genai_region TEXT NOT NULL, endpoint TEXT NOT NULL, serving_type TEXT DEFAULT 'ON_DEMAND', dedicated_endpoint_id TEXT, temperature REAL DEFAULT 1, max_tokens INTEGER DEFAULT 6000, top_p REAL DEFAULT 0.95, top_k INTEGER DEFAULT 1, frequency_penalty REAL DEFAULT 0, presence_penalty REAL DEFAULT 0, reasoning_effort TEXT, is_default INTEGER DEFAULT 0, system_prompt TEXT DEFAULT '', created_at TEXT DEFAULT (datetime('now')), FOREIGN KEY (oci_config_id) REFERENCES oci_configs(id) ); CREATE TABLE IF NOT EXISTS reports ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL, tenancy_name TEXT NOT NULL, config_id TEXT, level INTEGER DEFAULT 2, obp_checks INTEGER DEFAULT 0, raw_data INTEGER DEFAULT 0, redact_output INTEGER DEFAULT 0, status TEXT DEFAULT 'pending', progress TEXT DEFAULT '', html_path TEXT, json_path TEXT, created_at TEXT DEFAULT (datetime('now')), completed_at TEXT, error_msg TEXT ); CREATE TABLE IF NOT EXISTS report_files ( id TEXT PRIMARY KEY, report_id TEXT NOT NULL, file_name TEXT NOT NULL, file_path TEXT NOT NULL, file_type TEXT NOT NULL, file_category TEXT NOT NULL, file_size INTEGER DEFAULT 0 ); CREATE TABLE IF NOT EXISTS adb_vector_configs ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL, config_name TEXT NOT NULL, dsn TEXT NOT NULL, username TEXT NOT NULL, password_enc TEXT NOT NULL, wallet_dir TEXT, wallet_password_enc TEXT, table_name TEXT DEFAULT '', use_mtls INTEGER DEFAULT 1, is_active INTEGER DEFAULT 1, is_global INTEGER DEFAULT 0, created_at TEXT DEFAULT (datetime('now')) ); CREATE TABLE IF NOT EXISTS adb_vector_tables ( id TEXT PRIMARY KEY, adb_config_id TEXT NOT NULL, table_name TEXT NOT NULL, description TEXT DEFAULT '', is_active INTEGER DEFAULT 1, created_at TEXT DEFAULT (datetime('now')), FOREIGN KEY (adb_config_id) REFERENCES adb_vector_configs(id) ON DELETE CASCADE, UNIQUE(adb_config_id, table_name) ); CREATE TABLE IF NOT EXISTS mcp_servers ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL, name TEXT NOT NULL, description TEXT, server_type TEXT NOT NULL DEFAULT 'stdio', command TEXT, args TEXT, env_vars TEXT, url TEXT, module_path TEXT, tools TEXT, linked_adb_id TEXT, is_active INTEGER DEFAULT 1, is_global INTEGER DEFAULT 0, created_at TEXT DEFAULT (datetime('now')) ); CREATE TABLE IF NOT EXISTS chat_messages ( id TEXT PRIMARY KEY, session_id TEXT NOT NULL, user_id TEXT NOT NULL, role TEXT NOT NULL, content TEXT NOT NULL, model_id TEXT, status TEXT DEFAULT 'done', created_at TEXT DEFAULT (datetime('now')) ); CREATE TABLE IF NOT EXISTS chat_summaries ( id TEXT PRIMARY KEY, session_id TEXT NOT NULL, user_id TEXT NOT NULL, summary TEXT NOT NULL, messages_compacted INTEGER NOT NULL, up_to_message_id TEXT NOT NULL, created_at TEXT DEFAULT (datetime('now')) ); CREATE TABLE IF NOT EXISTS audit_log ( id TEXT PRIMARY KEY, user_id TEXT, username TEXT, action TEXT NOT NULL, resource TEXT, details TEXT, ip TEXT, created_at TEXT DEFAULT (datetime('now')) ); CREATE TABLE IF NOT EXISTS terminal_history ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL, oci_config_id TEXT NOT NULL, command TEXT NOT NULL, exit_code INTEGER, output TEXT, error TEXT, created_at TEXT DEFAULT (datetime('now')) ); CREATE TABLE IF NOT EXISTS config_logs ( id TEXT PRIMARY KEY, config_type TEXT NOT NULL, config_id TEXT NOT NULL, config_name TEXT, severity TEXT NOT NULL, action TEXT NOT NULL, message TEXT NOT NULL, user_id TEXT, username TEXT, created_at TEXT DEFAULT (datetime('now')) ); CREATE TABLE IF NOT EXISTS app_settings ( key TEXT PRIMARY KEY, value TEXT NOT NULL, updated_at TEXT DEFAULT (datetime('now')) ); CREATE TABLE IF NOT EXISTS system_prompts ( id TEXT PRIMARY KEY, name TEXT NOT NULL, agent TEXT NOT NULL DEFAULT 'chat', content TEXT NOT NULL, is_active INTEGER DEFAULT 0, created_at TEXT DEFAULT (datetime('now')) ); CREATE TABLE IF NOT EXISTS chat_logs ( id TEXT PRIMARY KEY, session_id TEXT, message_id TEXT, user_id TEXT, severity TEXT NOT NULL, source TEXT NOT NULL, action TEXT NOT NULL, message TEXT NOT NULL, created_at TEXT DEFAULT (datetime('now')) ); CREATE TABLE IF NOT EXISTS chat_sessions ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL, agent_type TEXT NOT NULL DEFAULT 'chat', title TEXT DEFAULT '', created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now')) ); CREATE TABLE IF NOT EXISTS terraform_workspaces ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL, session_id TEXT NOT NULL, oci_config_id TEXT NOT NULL, compartment_id TEXT, name TEXT DEFAULT 'workspace', tf_code TEXT, status TEXT DEFAULT 'draft', plan_output TEXT, apply_output TEXT, destroy_output TEXT, error TEXT, created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now')) ); CREATE TABLE IF NOT EXISTS explorer_counts_cache ( id INTEGER PRIMARY KEY AUTOINCREMENT, oci_config_id TEXT NOT NULL, compartment_id TEXT NOT NULL, resource_type TEXT NOT NULL, count INTEGER NOT NULL DEFAULT 0, regions TEXT DEFAULT '', updated_at TEXT DEFAULT (datetime('now')), UNIQUE(oci_config_id, compartment_id, resource_type, regions) ); CREATE TABLE IF NOT EXISTS tf_valid_types ( name TEXT PRIMARY KEY, kind TEXT NOT NULL DEFAULT 'resource', category TEXT DEFAULT '', required_args TEXT DEFAULT '', blocks TEXT DEFAULT '' ); CREATE TABLE IF NOT EXISTS tf_resource_docs ( slug TEXT PRIMARY KEY, subcategory TEXT DEFAULT '', example_usage TEXT DEFAULT '', argument_ref TEXT DEFAULT '', fetched_at TEXT DEFAULT (datetime('now')) ); """) c.execute("DELETE FROM config_logs WHERE created_at < datetime('now', '-30 days')") c.execute("DELETE FROM chat_logs WHERE created_at < datetime('now', '-30 days')") c.execute("DELETE FROM audit_log WHERE created_at < datetime('now', '-30 days')") # ── Indexes for performance ── for idx in [ "CREATE INDEX IF NOT EXISTS idx_chat_msg_session ON chat_messages(session_id)", "CREATE INDEX IF NOT EXISTS idx_chat_msg_user ON chat_messages(user_id)", "CREATE INDEX IF NOT EXISTS idx_chat_msg_status ON chat_messages(status)", "CREATE INDEX IF NOT EXISTS idx_chat_sess_user ON chat_sessions(user_id)", "CREATE INDEX IF NOT EXISTS idx_reports_user ON reports(user_id)", "CREATE INDEX IF NOT EXISTS idx_reports_status ON reports(status)", "CREATE INDEX IF NOT EXISTS idx_report_files_report ON report_files(report_id)", "CREATE INDEX IF NOT EXISTS idx_oci_cfg_user ON oci_configs(user_id)", "CREATE INDEX IF NOT EXISTS idx_genai_cfg_user ON genai_configs(user_id)", "CREATE INDEX IF NOT EXISTS idx_mcp_srv_user ON mcp_servers(user_id)", "CREATE INDEX IF NOT EXISTS idx_adb_cfg_user ON adb_vector_configs(user_id)", "CREATE INDEX IF NOT EXISTS idx_adb_tables_cfg ON adb_vector_tables(adb_config_id)", "CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id)", "CREATE INDEX IF NOT EXISTS idx_tf_ws_user ON terraform_workspaces(user_id)", "CREATE INDEX IF NOT EXISTS idx_tf_ws_session ON terraform_workspaces(session_id)", "CREATE INDEX IF NOT EXISTS idx_audit_user ON audit_log(user_id)", ]: c.execute(idx) # ── Migrations ── for col in ["system_prompt TEXT DEFAULT ''", "reasoning_effort TEXT"]: try: c.execute(f"ALTER TABLE genai_configs ADD COLUMN {col}") except sqlite3.OperationalError: pass for col in ["genai_config_id TEXT", "embedding_model_id TEXT DEFAULT 'cohere.embed-v4.0'", "is_global INTEGER DEFAULT 0"]: try: c.execute(f"ALTER TABLE adb_vector_configs ADD COLUMN {col}") except sqlite3.OperationalError: pass for col in ["is_global INTEGER DEFAULT 0"]: try: c.execute(f"ALTER TABLE mcp_servers ADD COLUMN {col}") except sqlite3.OperationalError: pass for col in ["progress TEXT DEFAULT ''", "level INTEGER DEFAULT 2", "obp_checks INTEGER DEFAULT 0", "raw_data INTEGER DEFAULT 0", "redact_output INTEGER DEFAULT 0", "worker_pid INTEGER"]: try: c.execute(f"ALTER TABLE reports ADD COLUMN {col}") except sqlite3.OperationalError: pass for col in ["status TEXT DEFAULT 'done'"]: try: c.execute(f"ALTER TABLE chat_messages ADD COLUMN {col}") except sqlite3.OperationalError: pass for col in ["compartment_id TEXT"]: try: c.execute(f"ALTER TABLE terraform_workspaces ADD COLUMN {col}") except sqlite3.OperationalError: pass for col in ["ssh_public_key TEXT", "auth_type TEXT DEFAULT 'api_key'"]: try: c.execute(f"ALTER TABLE oci_configs ADD COLUMN {col}") except sqlite3.OperationalError: pass for col in ["timezone TEXT DEFAULT 'America/Sao_Paulo'", "auth_provider TEXT DEFAULT 'local'", "oidc_subject TEXT", "must_change_password INTEGER DEFAULT 0"]: try: c.execute(f"ALTER TABLE users ADD COLUMN {col}") except sqlite3.OperationalError: pass # Migrate legacy table_name from adb_vector_configs into adb_vector_tables try: for cfg_row in c.execute("SELECT id, table_name FROM adb_vector_configs WHERE table_name IS NOT NULL AND table_name != ''").fetchall(): if not c.execute("SELECT 1 FROM adb_vector_tables WHERE adb_config_id=? AND table_name=?", (cfg_row["id"], cfg_row["table_name"])).fetchone(): c.execute("INSERT INTO adb_vector_tables (id, adb_config_id, table_name, description) VALUES (?,?,?,?)", (str(uuid.uuid4()), cfg_row["id"], cfg_row["table_name"], "Migrado automaticamente")) except Exception as e: log.warning(f"DB migration: adb_vector_tables backfill failed: {e}") # Backfill chat_sessions from existing chat_messages try: c.execute("""INSERT OR IGNORE INTO chat_sessions (id, user_id, agent_type, title, created_at, updated_at) SELECT cm.session_id, cm.user_id, CASE WHEN tw.id IS NOT NULL THEN 'terraform' ELSE 'chat' END, SUBSTR((SELECT content FROM chat_messages cm2 WHERE cm2.session_id=cm.session_id AND cm2.role='user' ORDER BY cm2.created_at ASC LIMIT 1), 1, 80), MIN(cm.created_at), MAX(cm.created_at) FROM chat_messages cm LEFT JOIN terraform_workspaces tw ON tw.session_id = cm.session_id WHERE cm.session_id NOT IN (SELECT id FROM chat_sessions) GROUP BY cm.session_id""") except Exception as e: log.warning(f"DB migration: chat_sessions backfill failed: {e}") # Migrate legacy base64 fields to Fernet encryption _enc_migrations = [ ("oci_configs", ["tenancy_ocid", "user_ocid", "fingerprint", "compartment_id"]), ("adb_vector_configs", ["password_enc", "wallet_password_enc"]), ] for table, cols in _enc_migrations: for col in cols: try: rows = c.execute(f"SELECT id, {col} FROM {table} WHERE {col} IS NOT NULL AND {col} != ''").fetchall() for row in rows: val = row[col] if val and not val.startswith(_FERNET_PREFIX): try: plaintext = base64.b64decode(val.encode()).decode() c.execute(f"UPDATE {table} SET {col}=? WHERE id=?", (_enc(plaintext), row["id"])) except Exception: pass except Exception as e: log.warning(f"DB migration: encrypt {table}.{col} failed: {e}") # Seed default system prompt if none exists for chat agent if not c.execute("SELECT 1 FROM system_prompts WHERE agent='chat'").fetchone(): c.execute("INSERT INTO system_prompts (id,name,agent,content,is_active) VALUES (?,?,?,?,?)", (str(uuid.uuid4()), "OCI CIS RAG Agent", "chat", RAG_DEFAULT_SYSTEM_PROMPT, 1)) tf_row = c.execute("SELECT id FROM system_prompts WHERE agent='terraform' AND is_active=1").fetchone() if tf_row: # Always sync DB prompt with code default c.execute("UPDATE system_prompts SET content=? WHERE id=?", (TF_DEFAULT_SYSTEM_PROMPT, tf_row["id"])) else: c.execute("INSERT INTO system_prompts (id,name,agent,content,is_active) VALUES (?,?,?,?,?)", (str(uuid.uuid4()), "OCI Terraform Agent", "terraform", TF_DEFAULT_SYSTEM_PROMPT, 1)) # Recover stuck terraform workspaces (interrupted by container restart) stuck = c.execute("SELECT id, status FROM terraform_workspaces WHERE status IN ('planning','applying','destroying')").fetchall() for ws in stuck: prev = 'applied' if ws["status"] == 'destroying' else 'failed' c.execute("UPDATE terraform_workspaces SET status=?, error='Interrompido por restart do container', updated_at=datetime('now') WHERE id=?", (prev, ws["id"])) # Remove stale lock files lock_file = TERRAFORM_DIR / ws["id"] / ".terraform.tfstate.lock.info" if lock_file.exists(): lock_file.unlink() log.info(f"Recovered stuck workspace {ws['id']}: {ws['status']} -> {prev}") adm = c.execute("SELECT id FROM users WHERE username='admin'").fetchone() if not adm: c.execute( "INSERT INTO users (id,first_name,last_name,username,email,password_hash,role,must_change_password) VALUES (?,?,?,?,?,?,?,?)", (str(uuid.uuid4()), "Admin", "Sistema", "admin", "admin@local", _hash_pw("admin123"), "admin", 1) ) log.info("Default admin created — username: admin / password: admin123") # ── Crypto helpers ──────────────────────────────────────────────────────────── def _hash_pw(pw): salt=secrets.token_hex(16); h=hashlib.pbkdf2_hmac("sha256",pw.encode(),salt.encode(),100_000); return f"{salt}:{h.hex()}" def _verify_pw(pw,stored): salt,h=stored.split(":"); return hmac.compare_digest(hashlib.pbkdf2_hmac("sha256",pw.encode(),salt.encode(),100_000).hex(),h) def _totp_secret(): return base64.b32encode(secrets.token_bytes(20)).decode() def _totp_verify(secret,code,window=1): key=base64.b32decode(secret); now=int(time.time())//30 for off in range(-window,window+1): msg=struct.pack(">Q",now+off); h=hmac.new(key,msg,hashlib.sha1).digest(); o=h[-1]&0x0F c=str((struct.unpack(">I",h[o:o+4])[0]&0x7FFFFFFF)%1_000_000).zfill(6) if hmac.compare_digest(c,code): return True return False def _totp_uri(secret,user): return f"otpauth://totp/AI-Agent:{user}?secret={secret}&issuer=AI-Agent" def _make_token(uid,role,sid): return pyjwt.encode({"sub":uid,"role":role,"sid":sid,"exp":datetime.utcnow()+timedelta(hours=JWT_EXP_H),"iat":datetime.utcnow()},APP_SECRET,algorithm=JWT_ALG) # ── Fernet encryption (AES-128-CBC + HMAC-SHA256) ──────────────────────────── from cryptography.fernet import Fernet as _Fernet from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC as _PBKDF2HMAC from cryptography.hazmat.primitives import hashes as _hashes _FERNET_PREFIX = "fernet:" def _derive_fernet_key(secret: str) -> bytes: kdf = _PBKDF2HMAC(algorithm=_hashes.SHA256(), length=32, salt=b"oci-cis-agent-fernet-v1", iterations=100_000) return base64.urlsafe_b64encode(kdf.derive(secret.encode())) _fernet = _Fernet(_derive_fernet_key(APP_SECRET)) def _enc(v): """Encrypt a string using Fernet (AES + HMAC). Returns prefixed token.""" if not v: return v return _FERNET_PREFIX + _fernet.encrypt(v.encode()).decode() def _dec(v): """Decrypt a value. Supports Fernet (prefixed) and legacy base64 (unprefixed).""" if not v: return v if v.startswith(_FERNET_PREFIX): return _fernet.decrypt(v[len(_FERNET_PREFIX):].encode()).decode() # Legacy base64 fallback return base64.b64decode(v.encode()).decode() def _mask(v, show=6): """Mask a sensitive value, showing only the last `show` characters.""" if not v or len(v) <= show: return "•" * 8 return "•" * min(len(v) - show, 20) + v[-show:] def _safe_dec(v): """Decrypt a value, returning as-is if not encrypted (migration compat).""" if not v: return v try: return _dec(v) except Exception: return v # ── OIDC helpers ───────────────────────────────────────────────────────────── _oidc_discovery_cache: dict = {} # {issuer: (data, timestamp)} _oidc_jwks_cache: dict = {} # {jwks_uri: (keys, timestamp)} _OIDC_CACHE_TTL = 3600 # 1 hour _oidc_pending: dict = {} # {state: {"nonce": str, "created": float}} _oidc_pending_lock = threading.Lock() _OIDC_STATE_TTL = 300 # 5 minutes def _get_oidc_config() -> dict: keys = ("auth_mode","oidc_issuer","oidc_client_id","oidc_client_secret", "oidc_redirect_uri","oidc_group_admin","oidc_group_user","oidc_group_viewer") cfg = {} with db() as c: for k in keys: row = c.execute("SELECT value FROM app_settings WHERE key=?", (k,)).fetchone() cfg[k] = row["value"] if row else "" if cfg.get("oidc_client_secret"): cfg["oidc_client_secret"] = _safe_dec(cfg["oidc_client_secret"]) return cfg def _oidc_discover(issuer: str) -> dict: import requests as req now = time.time() cached = _oidc_discovery_cache.get(issuer) if cached and now - cached[1] < _OIDC_CACHE_TTL: return cached[0] url = issuer.rstrip("/") + "/.well-known/openid-configuration" resp = req.get(url, timeout=10) resp.raise_for_status() data = resp.json() _oidc_discovery_cache[issuer] = (data, now) return data def _oidc_get_jwks(jwks_uri: str) -> list: import requests as req now = time.time() cached = _oidc_jwks_cache.get(jwks_uri) if cached and now - cached[1] < _OIDC_CACHE_TTL: return cached[0] resp = req.get(jwks_uri, timeout=10) resp.raise_for_status() keys = resp.json().get("keys", []) _oidc_jwks_cache[jwks_uri] = (keys, now) return keys def _oidc_store_state(state: str, nonce: str): with _oidc_pending_lock: now = time.time() expired = [k for k, v in _oidc_pending.items() if now - v["created"] > _OIDC_STATE_TTL] for k in expired: del _oidc_pending[k] _oidc_pending[state] = {"nonce": nonce, "created": now} def _oidc_pop_state(state: str) -> dict | None: with _oidc_pending_lock: return _oidc_pending.pop(state, None) def _validate_id_token(id_token: str, issuer: str, client_id: str, nonce: str) -> dict: from jwt.algorithms import RSAAlgorithm disco = _oidc_discover(issuer) jwks = _oidc_get_jwks(disco["jwks_uri"]) header = pyjwt.get_unverified_header(id_token) kid = header.get("kid") key_data = None for k in jwks: if k.get("kid") == kid: key_data = k break if not key_data: raise HTTPException(401, "OIDC: chave não encontrada no JWKS") public_key = RSAAlgorithm.from_jwk(json.dumps(key_data)) claims = pyjwt.decode(id_token, public_key, algorithms=["RS256"], audience=client_id, issuer=issuer) if claims.get("nonce") != nonce: raise HTTPException(401, "OIDC: nonce inválido") return claims # ── OCI SDK Client Helper ───────────────────────────────────────────────────── def _get_oci_config(oci_config_id: str) -> dict: import oci config_path = str(OCI_DIR / oci_config_id / "config") config = oci.config.from_file(config_path, "DEFAULT") return config def _get_oci_signer(oci_config_id: str): """Return (config, signer) tuple. For session_token auth, returns SecurityTokenSigner; for api_key returns None (use default).""" import oci config = _get_oci_config(oci_config_id) if config.get("security_token_file"): from oci.auth.signers import SecurityTokenSigner token_path = config["security_token_file"] with open(token_path) as f: token = f.read().strip() signer = SecurityTokenSigner(token, config["key_file"]) return config, signer return config, None def _oci_client(client_class, oci_config_id: str, **kwargs): """Create an OCI client with correct auth (api_key or session_token).""" config, signer = _get_oci_signer(oci_config_id) if signer: return client_class(config=config, signer=signer, **kwargs) return client_class(config, **kwargs) # ── Auth deps ───────────────────────────────────────────────────────────────── async def current_user(cred: HTTPAuthorizationCredentials = Depends(security)): try: p = pyjwt.decode(cred.credentials, APP_SECRET, algorithms=[JWT_ALG]) except pyjwt.ExpiredSignatureError: raise HTTPException(401, "Token expirado") except pyjwt.InvalidTokenError: raise HTTPException(401, "Token inválido") with db() as c: u = c.execute("SELECT * FROM users WHERE id=? AND is_active=1", (p["sub"],)).fetchone() s = c.execute("SELECT * FROM sessions WHERE id=? AND is_active=1", (p["sid"],)).fetchone() if not u or not s: raise HTTPException(401, "Sessão inválida") return dict(u) def _user_from_token(token: str): """Resolve user from a raw JWT token string (for query-param auth on downloads).""" try: p = pyjwt.decode(token, APP_SECRET, algorithms=[JWT_ALG]) except Exception: raise HTTPException(401, "Token inválido") with db() as c: u = c.execute("SELECT * FROM users WHERE id=? AND is_active=1", (p["sub"],)).fetchone() s = c.execute("SELECT * FROM sessions WHERE id=? AND is_active=1", (p["sid"],)).fetchone() if not u or not s: raise HTTPException(401, "Sessão inválida") return dict(u) def require(*roles): async def dep(u=Depends(current_user)): if u["role"] not in roles: raise HTTPException(403, f"Requer role: {', '.join(roles)}") return u return dep def _audit(uid, uname, action, resource=None, details=None, ip=None): with db() as c: c.execute("INSERT INTO audit_log (id,user_id,username,action,resource,details,ip) VALUES (?,?,?,?,?,?,?)", (str(uuid.uuid4()), uid, uname, action, resource, details, ip)) def _config_log(config_type, config_id, config_name, severity, action, message, uid=None, uname=None): with db() as c: c.execute("INSERT INTO config_logs (id,config_type,config_id,config_name,severity,action,message,user_id,username) VALUES (?,?,?,?,?,?,?,?,?)", (str(uuid.uuid4()), config_type, config_id, config_name or "", severity, action, str(message)[:2000], uid, uname)) def _chat_log(sid, mid, uid, severity, source, action, message): with db() as c: c.execute("INSERT INTO chat_logs (id,session_id,message_id,user_id,severity,source,action,message) VALUES (?,?,?,?,?,?,?,?)", (str(uuid.uuid4()), sid, mid, uid, severity, source, action, str(message)[:2000])) # ── Access Control Helpers ─────────────────────────────────────────────────── _CONFIG_TABLES = { "oci": "oci_configs", "genai": "genai_configs", "adb": "adb_vector_configs", "mcp": "mcp_servers", } def _verify_config_access(config_type: str, config_id: str, user: dict, *, require_owner: bool = False) -> dict: """Verify user has access to a config. Returns the config row as dict. - admin: always has access - user/viewer: owns the config OR config is_global=1 (read access) - require_owner=True: only owner or admin can proceed (for write ops on global) Raises 404 if not found, 403 if no access.""" table = _CONFIG_TABLES.get(config_type) if not table: raise HTTPException(400, f"Tipo de config inválido: {config_type}") with db() as c: row = c.execute(f"SELECT * FROM {table} WHERE id=?", (config_id,)).fetchone() if not row: raise HTTPException(404) row = dict(row) if user["role"] == "admin": return row is_owner = row.get("user_id") == user["id"] is_global = row.get("is_global", 0) if not is_owner and not is_global: raise HTTPException(404) if require_owner and not is_owner: raise HTTPException(403, "Apenas o dono pode modificar este recurso") return row def _verify_report_access(report_id: str, user: dict) -> dict: """Verify user owns the report. Reports are 100% private — even admin can't see others'. Raises 404 if not found or not owner.""" with db() as c: row = c.execute("SELECT * FROM reports WHERE id=? AND user_id=?", (report_id, user["id"])).fetchone() if not row: raise HTTPException(404) return dict(row) # ── Models ──────────────────────────────────────────────────────────────────── class LoginReq(BaseModel): username: str; password: str; totp_code: Optional[str] = None class RegisterReq(BaseModel): first_name: str; last_name: str; username: str; email: str; password: str; role: str = "viewer" class TOTPVerify(BaseModel): totp_code: str class ChangePwReq(BaseModel): current_password: str; new_password: str class UserUpdateReq(BaseModel): email: Optional[str] = None; role: Optional[str] = None; is_active: Optional[bool] = None; timezone: Optional[str] = None class ChatMsg(BaseModel): message: str; session_id: Optional[str] = None # Option A: saved preset genai_config_id: Optional[str] = None # Option B: direct model selection (inline) model_id: Optional[str] = None; oci_config_id: Optional[str] = None genai_region: Optional[str] = None; compartment_id: Optional[str] = None temperature: Optional[float] = None; max_tokens: Optional[int] = None top_p: Optional[float] = None; top_k: Optional[int] = None frequency_penalty: Optional[float] = None; presence_penalty: Optional[float] = None reasoning_effort: Optional[str] = None use_tools: Optional[bool] = True class RunReportReq(BaseModel): config_id: str; regions: Optional[List[str]] = None level: int = 2; obp: bool = False; raw: bool = False; redact_output: bool = False class GenAIConfigReq(BaseModel): name: str = "default" oci_config_id: str; model_id: str; model_ocid: Optional[str] = None compartment_id: str; genai_region: str endpoint: Optional[str] = None serving_type: str = "ON_DEMAND"; dedicated_endpoint_id: Optional[str] = None temperature: float = 1; max_tokens: int = 6000; top_p: float = 0.95 top_k: int = 1; frequency_penalty: float = 0; presence_penalty: float = 0 reasoning_effort: Optional[str] = None is_default: bool = False class IngestDocReq(BaseModel): adb_config_id: str; documents: List[Dict[str, Any]]; table_name: Optional[str] = None class MCPServerReq(BaseModel): name: str; description: Optional[str] = None; server_type: str = "stdio" command: Optional[str] = None; args: Optional[List[str]] = None env_vars: Optional[Dict[str,str]] = None; url: Optional[str] = None module_path: Optional[str] = None; tools: Optional[List[dict]] = None linked_adb_id: Optional[str] = None; is_global: bool = False # ── Rate limiting (login brute-force protection) ───────────────────────────── _login_attempts: dict = {} # {ip: [timestamps]} _login_attempts_lock = threading.Lock() _LOGIN_WINDOW = 300 # 5 minutes _LOGIN_MAX = 10 # max attempts per window def _check_rate_limit(ip: str): """Raise 429 if too many login attempts from this IP.""" import time now = time.time() with _login_attempts_lock: attempts = _login_attempts.get(ip, []) attempts = [t for t in attempts if now - t < _LOGIN_WINDOW] if len(attempts) >= _LOGIN_MAX: raise HTTPException(429, "Muitas tentativas de login. Aguarde 5 minutos.") attempts.append(now) _login_attempts[ip] = attempts # ── Auth endpoints ──────────────────────────────────────────────────────────── @app.post("/api/auth/login") async def login(req: LoginReq, request: Request): _check_rate_limit(request.client.host if request.client else "unknown") with db() as c: mode_row = c.execute("SELECT value FROM app_settings WHERE key='auth_mode'").fetchone() if mode_row and mode_row["value"] == "oidc": raise HTTPException(400, "Login local desabilitado. Use Oracle IAM.") with db() as c: u = c.execute("SELECT * FROM users WHERE username=? AND is_active=1", (req.username,)).fetchone() if not u or not _verify_pw(req.password, u["password_hash"]): raise HTTPException(401, "Credenciais inválidas") u = dict(u) if u["mfa_enabled"]: if not req.totp_code: return {"mfa_required": True, "message": "Código MFA necessário"} if not _totp_verify(u["mfa_secret"], req.totp_code): raise HTTPException(401, "Código MFA inválido") sid = str(uuid.uuid4()); exp = (datetime.utcnow()+timedelta(hours=JWT_EXP_H)).isoformat() with db() as c: c.execute("INSERT INTO sessions (id,user_id,expires_at) VALUES (?,?,?)", (sid, u["id"], exp)) c.execute("UPDATE users SET last_login=datetime('now') WHERE id=?", (u["id"],)) _audit(u["id"], u["username"], "login", ip=request.client.host if request.client else None) return {"token":_make_token(u["id"],u["role"],sid), "user":{"id":u["id"],"first_name":u["first_name"],"last_name":u["last_name"],"username":u["username"],"email":u["email"],"role":u["role"],"mfa_enabled":bool(u["mfa_enabled"]),"timezone":u.get("timezone","America/Sao_Paulo")}, "must_change_password": bool(u.get("must_change_password", 0))} @app.post("/api/auth/logout") async def logout_ep(u=Depends(current_user)): with db() as c: c.execute("UPDATE sessions SET is_active=0 WHERE user_id=?", (u["id"],)) return {"ok": True} @app.post("/api/auth/register") async def register(req: RegisterReq, adm=Depends(require("admin"))): if req.role not in ("admin","user","viewer"): raise HTTPException(400, "Role inválida") uid = str(uuid.uuid4()) with db() as c: if c.execute("SELECT 1 FROM users WHERE username=?", (req.username,)).fetchone(): raise HTTPException(400, "Usuário já existe") c.execute("INSERT INTO users (id,first_name,last_name,username,email,password_hash,role) VALUES (?,?,?,?,?,?,?)", (uid, req.first_name, req.last_name, req.username, req.email, _hash_pw(req.password), req.role)) _audit(adm["id"], adm["username"], "create_user", uid, f"user={req.username} role={req.role}") return {"id": uid, "username": req.username, "role": req.role} @app.post("/api/auth/change-password") async def change_pw(req: ChangePwReq, u=Depends(current_user)): if u.get("auth_provider") == "oidc": raise HTTPException(400, "Usuários OIDC não podem alterar senha localmente") if not _verify_pw(req.current_password, u["password_hash"]): raise HTTPException(400, "Senha atual incorreta") pw = req.new_password import re if len(pw) < 8: raise HTTPException(400, "Senha deve ter no mínimo 8 caracteres") if not re.search(r'[A-Z]', pw): raise HTTPException(400, "Senha deve conter ao menos uma letra maiúscula") if not re.search(r'[a-z]', pw): raise HTTPException(400, "Senha deve conter ao menos uma letra minúscula") if not re.search(r'\d', pw): raise HTTPException(400, "Senha deve conter ao menos um número") if not re.search(r'[^A-Za-z0-9]', pw): raise HTTPException(400, "Senha deve conter ao menos um caractere especial") if pw == req.current_password: raise HTTPException(400, "A nova senha deve ser diferente da atual") with db() as c: c.execute("UPDATE users SET password_hash=?, must_change_password=0 WHERE id=?", (_hash_pw(pw), u["id"])) return {"ok": True} # ── OIDC endpoints ─────────────────────────────────────────────────────────── @app.get("/api/auth/oidc/config") async def oidc_public_config(): with db() as c: row = c.execute("SELECT value FROM app_settings WHERE key='auth_mode'").fetchone() return {"auth_mode": row["value"] if row else "local"} @app.get("/api/auth/oidc/login") async def oidc_login(request: Request): _check_rate_limit(request.client.host if request.client else "unknown") cfg = _get_oidc_config() if cfg.get("auth_mode") not in ("oidc", "both"): raise HTTPException(400, "OIDC login não está habilitado") if not cfg.get("oidc_issuer") or not cfg.get("oidc_client_id"): raise HTTPException(500, "OIDC não configurado (issuer/client_id ausente)") disco = _oidc_discover(cfg["oidc_issuer"]) state = secrets.token_urlsafe(32) nonce = secrets.token_urlsafe(32) _oidc_store_state(state, nonce) from urllib.parse import urlencode params = {"response_type": "code", "client_id": cfg["oidc_client_id"], "redirect_uri": cfg["oidc_redirect_uri"], "scope": "openid profile email groups", "state": state, "nonce": nonce} auth_url = disco["authorization_endpoint"] + "?" + urlencode(params) return RedirectResponse(url=auth_url, status_code=302) @app.get("/api/auth/oidc/callback") async def oidc_callback(request: Request, code: str = Query(...), state: str = Query(...)): ip = request.client.host if request.client else "unknown" _check_rate_limit(ip) pending = _oidc_pop_state(state) if not pending: raise HTTPException(400, "OIDC: state inválido ou expirado") nonce = pending["nonce"] cfg = _get_oidc_config() disco = _oidc_discover(cfg["oidc_issuer"]) import requests as req token_resp = req.post(disco["token_endpoint"], data={ "grant_type": "authorization_code", "code": code, "redirect_uri": cfg["oidc_redirect_uri"], "client_id": cfg["oidc_client_id"], "client_secret": cfg["oidc_client_secret"], }, timeout=15) if token_resp.status_code != 200: log.error(f"OIDC token exchange failed: {token_resp.status_code} {token_resp.text[:500]}") raise HTTPException(502, "OIDC: falha ao trocar código por token") tokens = token_resp.json() id_token_raw = tokens.get("id_token") if not id_token_raw: raise HTTPException(502, "OIDC: id_token ausente na resposta") claims = _validate_id_token(id_token_raw, cfg["oidc_issuer"], cfg["oidc_client_id"], nonce) sub = claims.get("sub") email = claims.get("email", "") given_name = claims.get("given_name") or claims.get("name", "").split(" ")[0] or "OIDC" family_name = claims.get("family_name") or " ".join(claims.get("name", "User").split(" ")[1:]) or "User" groups = claims.get("groups", []) if not groups: groups = claims.get("http://www.oracle.com/groups", []) role = "viewer" if cfg.get("oidc_group_admin") and cfg["oidc_group_admin"] in groups: role = "admin" elif cfg.get("oidc_group_user") and cfg["oidc_group_user"] in groups: role = "user" with db() as c: u = c.execute("SELECT * FROM users WHERE oidc_subject=? AND is_active=1", (sub,)).fetchone() if u: u = dict(u) c.execute("UPDATE users SET role=?, last_login=datetime('now'), email=? WHERE id=?", (role, email or u.get("email"), u["id"])) u["role"] = role _audit(u["id"], u["username"], "oidc_login", details=f"sub={sub}", ip=ip) else: uid = str(uuid.uuid4()) username = email.split("@")[0] if email else f"oidc_{sub[:8]}" base_username = username counter = 1 while c.execute("SELECT 1 FROM users WHERE username=?", (username,)).fetchone(): username = f"{base_username}_{counter}" counter += 1 fake_hash = _hash_pw(secrets.token_hex(32)) c.execute( "INSERT INTO users (id,first_name,last_name,username,email,password_hash,role,auth_provider,oidc_subject) " "VALUES (?,?,?,?,?,?,?,?,?)", (uid, given_name, family_name, username, email, fake_hash, role, "oidc", sub)) u = {"id": uid, "first_name": given_name, "last_name": family_name, "username": username, "email": email, "role": role} _audit(uid, username, "oidc_jit_provision", details=f"sub={sub} email={email} role={role}", ip=ip) c.execute("UPDATE users SET last_login=datetime('now') WHERE id=?", (uid,)) sid = str(uuid.uuid4()) exp = (datetime.utcnow() + timedelta(hours=JWT_EXP_H)).isoformat() with db() as c: c.execute("INSERT INTO sessions (id,user_id,expires_at) VALUES (?,?,?)", (sid, u["id"], exp)) token = _make_token(u["id"], u["role"], sid) html = f"""OIDC Login """ return HTMLResponse(content=html) @app.post("/api/auth/oidc/logout") async def oidc_logout(u=Depends(current_user)): with db() as c: c.execute("UPDATE sessions SET is_active=0 WHERE user_id=?", (u["id"],)) _audit(u["id"], u["username"], "oidc_logout") idp_logout_url = None try: cfg = _get_oidc_config() if cfg.get("oidc_issuer"): disco = _oidc_discover(cfg["oidc_issuer"]) idp_logout_url = disco.get("end_session_endpoint") except Exception: pass return {"ok": True, "idp_logout_url": idp_logout_url} @app.post("/api/settings/oidc/test") async def test_oidc(u=Depends(require("admin"))): cfg = _get_oidc_config() if not cfg.get("oidc_issuer"): raise HTTPException(400, "Issuer URL não configurado") import requests as req try: url = cfg["oidc_issuer"].rstrip("/") + "/.well-known/openid-configuration" resp = req.get(url, timeout=10) resp.raise_for_status() disco = resp.json() has_auth = bool(disco.get("authorization_endpoint")) has_token = bool(disco.get("token_endpoint")) has_jwks = bool(disco.get("jwks_uri")) return {"ok": has_auth and has_token and has_jwks, "issuer": disco.get("issuer"), "authorization_endpoint": has_auth, "token_endpoint": has_token, "jwks_uri": has_jwks, "end_session_endpoint": bool(disco.get("end_session_endpoint"))} except Exception as e: raise HTTPException(502, f"Falha ao conectar ao discovery: {str(e)}") # ── MFA ─────────────────────────────────────────────────────────────────────── @app.post("/api/mfa/setup") async def mfa_setup(u=Depends(current_user)): sec = _totp_secret() with db() as c: c.execute("UPDATE users SET mfa_secret=? WHERE id=?", (sec, u["id"])) return {"secret": sec, "uri": _totp_uri(sec, u["username"])} @app.post("/api/mfa/verify") async def mfa_verify(req: TOTPVerify, u=Depends(current_user)): if not u.get("mfa_secret"): raise HTTPException(400, "Chame /api/mfa/setup primeiro") if not _totp_verify(u["mfa_secret"], req.totp_code): raise HTTPException(400, "Código inválido") with db() as c: c.execute("UPDATE users SET mfa_enabled=1 WHERE id=?", (u["id"],)) return {"ok": True, "message": "MFA ativado"} @app.post("/api/mfa/disable/{user_id}") async def mfa_disable(user_id: str, adm=Depends(require("admin"))): with db() as c: c.execute("UPDATE users SET mfa_enabled=0,mfa_secret=NULL WHERE id=?", (user_id,)) _audit(adm["id"], adm["username"], "disable_mfa", user_id) return {"ok": True} # ── Users ───────────────────────────────────────────────────────────────────── @app.get("/api/users") async def list_users(u=Depends(require("admin"))): with db() as c: rows = c.execute("SELECT id,first_name,last_name,username,email,role,mfa_enabled,timezone,is_active,created_at,last_login,auth_provider FROM users").fetchall() return [dict(r) for r in rows] @app.get("/api/users/me") async def me(u=Depends(current_user)): fields = ("id","first_name","last_name","username","email","role","mfa_enabled","timezone","auth_provider","must_change_password") r = {k: u.get(k, "local" if k == "auth_provider" else None) for k in fields} r["must_change_password"] = bool(r.get("must_change_password", 0)) return r @app.put("/api/users/{uid}") async def update_user(uid: str, req: UserUpdateReq, adm=Depends(require("admin"))): sets, vals = [], [] if req.email is not None: sets.append("email=?"); vals.append(req.email) if req.role is not None: if req.role not in ("admin","user","viewer"): raise HTTPException(400, "Role inválida") sets.append("role=?"); vals.append(req.role) if req.is_active is not None: sets.append("is_active=?"); vals.append(int(req.is_active)) if req.timezone is not None: sets.append("timezone=?"); vals.append(req.timezone) if sets: vals.append(uid) with db() as c: c.execute(f"UPDATE users SET {','.join(sets)} WHERE id=?", vals) _audit(adm["id"], adm["username"], "update_user", uid) return {"ok": True} @app.delete("/api/users/{uid}") async def del_user(uid: str, adm=Depends(require("admin"))): if uid == adm["id"]: raise HTTPException(400, "Não pode desativar a si mesmo") with db() as c: c.execute("UPDATE users SET is_active=0 WHERE id=?", (uid,)) _audit(adm["id"], adm["username"], "deactivate_user", uid) return {"ok": True} # ── OCI Config ──────────────────────────────────────────────────────────────── @app.post("/api/oci/config") async def save_oci( tenancy_name: str = Form(...), tenancy_ocid: str = Form(...), user_ocid: str = Form(""), fingerprint: str = Form(""), region: str = Form(...), compartment_id: str = Form(""), key_passphrase: str = Form(""), ssh_public_key: str = Form(""), auth_type: str = Form("api_key"), security_token: str = Form(""), private_key: Optional[UploadFile] = File(None), public_key: Optional[UploadFile] = File(None), u = Depends(require("admin","user")) ): if auth_type not in ("api_key", "session_token"): raise HTTPException(400, "auth_type deve ser 'api_key' ou 'session_token'") _PEM_EXTS = {".pem", ".key"} if private_key and private_key.filename: await _validate_upload(private_key, _PEM_EXTS) private_key.file.seek(0) if public_key and public_key.filename: await _validate_upload(public_key, _PEM_EXTS) public_key.file.seek(0) cid = str(uuid.uuid4()); cdir = OCI_DIR / cid; cdir.mkdir(parents=True) kp = cdir / "oci_api_key.pem" if auth_type == "session_token": # Session token auth: requires tenancy_ocid, region, token, and session key if not security_token.strip(): shutil.rmtree(cdir, ignore_errors=True) raise HTTPException(400, "Session Token é obrigatório para autenticação por token.") if not private_key or not private_key.filename: shutil.rmtree(cdir, ignore_errors=True) raise HTTPException(400, "A chave de sessão (.pem) é obrigatória para autenticação por token.") key_bytes = await private_key.read() kp.write_bytes(key_bytes); kp.chmod(0o600) # Save security token file token_file = cdir / "token" token_file.write_text(security_token.strip()) token_file.chmod(0o600) # Generate config cfg_content = (f"[DEFAULT]\ntenancy={tenancy_ocid}\nregion={region}\n" f"key_file={kp}\nsecurity_token_file={token_file}\n") if user_ocid: cfg_content = f"[DEFAULT]\nuser={user_ocid}\n" + cfg_content.replace("[DEFAULT]\n", "") cfg_file = cdir / "config" cfg_file.write_text(cfg_content); cfg_file.chmod(0o600) # For session_token, fingerprint/user may not be provided — use placeholders user_ocid = user_ocid or "session_token_user" fingerprint = fingerprint or "session_token" else: # API Key auth (original flow) if not user_ocid or not fingerprint: shutil.rmtree(cdir, ignore_errors=True) raise HTTPException(400, "OCID User e Fingerprint são obrigatórios para API Key.") if not private_key or not private_key.filename: shutil.rmtree(cdir, ignore_errors=True) raise HTTPException(400, "Selecione a chave privada (.pem).") key_bytes = await private_key.read() kp.write_bytes(key_bytes); kp.chmod(0o600) key_text = key_bytes.decode("utf-8", errors="ignore") key_is_encrypted = "ENCRYPTED" in key_text or "Proc-Type: 4,ENCRYPTED" in key_text if key_is_encrypted and not key_passphrase: shutil.rmtree(cdir, ignore_errors=True) raise HTTPException(400, "A chave privada está criptografada (ENCRYPTED). Informe a Key Passphrase.") if public_key and public_key.filename: (cdir / "oci_api_key_public.pem").write_bytes(await public_key.read()) cfg_file = cdir / "config" cfg_content = (f"[DEFAULT]\nuser={user_ocid}\nfingerprint={fingerprint}\n" f"tenancy={tenancy_ocid}\nregion={region}\nkey_file={kp}\n") if key_passphrase: cfg_content += f"pass_phrase={key_passphrase}\n" cfg_file.write_text(cfg_content); cfg_file.chmod(0o600) with db() as c: c.execute("INSERT INTO oci_configs (id,user_id,tenancy_name,tenancy_ocid,user_ocid,fingerprint,region,key_file_path,compartment_id,ssh_public_key,auth_type) VALUES (?,?,?,?,?,?,?,?,?,?,?)", (cid, u["id"], tenancy_name, _enc(tenancy_ocid), _enc(user_ocid), _enc(fingerprint), region, str(kp), _enc(compartment_id) if compartment_id else None, ssh_public_key.strip() if ssh_public_key.strip() else None, auth_type)) _audit(u["id"], u["username"], "save_oci_config", cid, f"tenancy={tenancy_name} auth={auth_type}") _config_log("oci", cid, tenancy_name, "success", "save", f"Credencial salva: {tenancy_name} ({region}) [{auth_type}]", u["id"], u["username"]) return {"id": cid, "tenancy_name": tenancy_name, "region": region} @app.get("/api/oci/configs") async def list_oci(u=Depends(current_user)): with db() as c: cols = "id,user_id,tenancy_name,tenancy_ocid,user_ocid,fingerprint,region,compartment_id,ssh_public_key,auth_type,created_at" if u["role"]=="admin": rows=c.execute(f"SELECT {cols} FROM oci_configs").fetchall() else: rows=c.execute(f"SELECT {cols} FROM oci_configs WHERE user_id=?",(u["id"],)).fetchall() result = [] for r in rows: d = dict(r) # Decrypt and mask sensitive fields for display try: d["tenancy_ocid"] = _mask(_dec(d["tenancy_ocid"])) except Exception: d["tenancy_ocid"] = _mask(d["tenancy_ocid"]) try: d["user_ocid"] = _mask(_dec(d["user_ocid"])) except Exception: d["user_ocid"] = _mask(d["user_ocid"]) d["fingerprint"] = "•" * 20 # fingerprint fully masked if d.get("compartment_id"): try: d["compartment_id"] = _mask(_dec(d["compartment_id"])) except Exception: d["compartment_id"] = _mask(d["compartment_id"]) if d.get("ssh_public_key"): # Show just type + first/last few chars for identification parts = d["ssh_public_key"].split() d["ssh_public_key_preview"] = f"{parts[0]} ...{parts[1][-12:]}" if len(parts) >= 2 else "ssh-rsa ..." result.append(d) return result @app.delete("/api/oci/configs/{cid}") async def del_oci(cid: str, u=Depends(require("admin","user"))): with db() as c: cfg=c.execute("SELECT * FROM oci_configs WHERE id=?",(cid,)).fetchone() if not cfg: raise HTTPException(404) if u["role"]!="admin" and cfg["user_id"]!=u["id"]: raise HTTPException(403) c.execute("DELETE FROM oci_configs WHERE id=?",(cid,)) d=OCI_DIR/cid if d.exists(): shutil.rmtree(d) return {"ok": True} @app.put("/api/oci/configs/{cid}") async def update_oci( cid: str, tenancy_name: str = Form(...), tenancy_ocid: str = Form(""), user_ocid: str = Form(""), fingerprint: str = Form(""), region: str = Form(...), compartment_id: str = Form(""), key_passphrase: str = Form(""), ssh_public_key: str = Form(""), auth_type: str = Form(""), security_token: str = Form(""), private_key: Optional[UploadFile] = File(None), public_key: Optional[UploadFile] = File(None), u = Depends(require("admin","user")) ): _PEM_EXTS = {".pem", ".key"} if private_key and private_key.filename: await _validate_upload(private_key, _PEM_EXTS) private_key.file.seek(0) if public_key and public_key.filename: await _validate_upload(public_key, _PEM_EXTS) public_key.file.seek(0) with db() as c: existing = c.execute("SELECT * FROM oci_configs WHERE id=?", (cid,)).fetchone() if not existing: raise HTTPException(404) if u["role"] != "admin" and existing["user_id"] != u["id"]: raise HTTPException(403) # Keep existing encrypted values if not provided (empty = keep current) tenancy_ocid = tenancy_ocid or _safe_dec(existing["tenancy_ocid"]) user_ocid = user_ocid or _safe_dec(existing["user_ocid"]) fingerprint = fingerprint or _safe_dec(existing["fingerprint"]) compartment_id = compartment_id or _safe_dec(existing["compartment_id"]) or "" try: existing_auth = existing["auth_type"] or "api_key" except (IndexError, KeyError): existing_auth = "api_key" auth_type = auth_type or existing_auth try: existing_ssh = existing["ssh_public_key"] or "" except (IndexError, KeyError): existing_ssh = "" ssh_public_key = ssh_public_key.strip() if ssh_public_key.strip() else existing_ssh cdir = OCI_DIR / cid; cdir.mkdir(parents=True, exist_ok=True) kp = cdir / "oci_api_key.pem" if private_key and private_key.filename: key_bytes = await private_key.read() kp.write_bytes(key_bytes); kp.chmod(0o600) key_text = key_bytes.decode("utf-8", errors="ignore") if auth_type == "api_key" and ("ENCRYPTED" in key_text or "Proc-Type: 4,ENCRYPTED" in key_text) and not key_passphrase: raise HTTPException(400, "A chave privada está criptografada (ENCRYPTED). Informe a Key Passphrase.") if public_key and public_key.filename: (cdir / "oci_api_key_public.pem").write_bytes(await public_key.read()) # Update session token file if provided if security_token.strip(): token_file = cdir / "token" token_file.write_text(security_token.strip()); token_file.chmod(0o600) cfg_file = cdir / "config" if auth_type == "session_token": token_path = cdir / "token" cfg_content = f"[DEFAULT]\ntenancy={tenancy_ocid}\nregion={region}\nkey_file={kp}\n" if user_ocid and user_ocid != "session_token_user": cfg_content = f"[DEFAULT]\nuser={user_ocid}\ntenancy={tenancy_ocid}\nregion={region}\nkey_file={kp}\n" cfg_content += f"security_token_file={token_path}\n" else: cfg_content = (f"[DEFAULT]\nuser={user_ocid}\nfingerprint={fingerprint}\n" f"tenancy={tenancy_ocid}\nregion={region}\nkey_file={kp}\n") if key_passphrase: cfg_content += f"pass_phrase={key_passphrase}\n" cfg_file.write_text(cfg_content); cfg_file.chmod(0o600) with db() as c: c.execute("UPDATE oci_configs SET tenancy_name=?,tenancy_ocid=?,user_ocid=?,fingerprint=?,region=?,compartment_id=?,ssh_public_key=?,auth_type=? WHERE id=?", (tenancy_name, _enc(tenancy_ocid), _enc(user_ocid), _enc(fingerprint), region, _enc(compartment_id) if compartment_id else None, ssh_public_key or None, auth_type, cid)) _audit(u["id"], u["username"], "update_oci_config", cid, f"tenancy={tenancy_name} auth={auth_type}") _config_log("oci", cid, tenancy_name, "success", "save", f"Credencial atualizada: {tenancy_name} ({region}) [{auth_type}]", u["id"], u["username"]) return {"id": cid, "tenancy_name": tenancy_name, "region": region} @app.post("/api/oci/test/{cid}") async def test_oci(cid: str, u=Depends(require("admin","user"))): _verify_config_access("oci", cid, u) cp = OCI_DIR / cid / "config" cname = None with db() as c: row = c.execute("SELECT tenancy_name FROM oci_configs WHERE id=?", (cid,)).fetchone() if row: cname = row["tenancy_name"] if not cp.exists(): _config_log("oci", cid, cname, "error", "test", "Config não encontrada", u["id"], u["username"]) return {"status":"error","message":"Config não encontrada"} try: loop = asyncio.get_event_loop() r = await loop.run_in_executor(None, partial( subprocess.run, ["oci","iam","region","list","--config-file",str(cp),"--output","json"], capture_output=True, text=True, timeout=30, stdin=subprocess.DEVNULL)) if r.returncode == 0: _config_log("oci", cid, cname, "success", "test", "Conexão OK", u["id"], u["username"]) return {"status":"success","message":"Conexão OK","data":json.loads(r.stdout)} msg = r.stderr[:500] if "passphrase" in msg.lower() or "getpass" in msg.lower(): msg = "A chave privada requer passphrase. Recadastre a credencial informando a Key Passphrase." _config_log("oci", cid, cname, "error", "test", msg, u["id"], u["username"]) return {"status":"error","message":msg} except FileNotFoundError: _config_log("oci", cid, cname, "error", "test", "OCI CLI não disponível no container.", u["id"], u["username"]) return {"status":"error","message":"OCI CLI não disponível no container."} except subprocess.TimeoutExpired: _config_log("oci", cid, cname, "error", "test", "Timeout na conexão", u["id"], u["username"]) return {"status":"error","message":"Timeout na conexão"} except Exception as e: _config_log("oci", cid, cname, "error", "test", str(e)[:500], u["id"], u["username"]) return {"status":"error","message":str(e)[:500]} # ── OCI CLI Terminal ───────────────────────────────────────────────────────── # OCID resource type → OCI CLI get command mapping _OCID_CMD_MAP = { "instance": "oci compute instance get --instance-id", "image": "oci compute image get --image-id", "vcn": "oci network vcn get --vcn-id", "subnet": "oci network subnet get --subnet-id", "securitylist": "oci network security-list get --security-list-id", "routetable": "oci network route-table get --rt-id", "internetgateway": "oci network internet-gateway get --ig-id", "natgateway": "oci network nat-gateway get --nat-gateway-id", "servicegateway": "oci network service-gateway get --service-gateway-id", "drg": "oci network drg get --drg-id", "drgattachment": "oci network drg-attachment get --drg-attachment-id", "localpeeringgateway": "oci network local-peering-gateway get --local-peering-gateway-id", "networksecuritygroup": "oci network nsg get --nsg-id", "vnic": "oci network vnic get --vnic-id", "privateip": "oci network private-ip get --private-ip-id", "publicip": "oci network public-ip get --public-ip-id", "dhcpoptions": "oci network dhcp-options get --dhcp-id", "cpe": "oci network cpe get --cpe-id", "ipsecconnection": "oci network ip-sec-connection get --ipsc-id", "volume": "oci bv volume get --volume-id", "bootvolume": "oci bv boot-volume get --boot-volume-id", "volumebackup": "oci bv backup get --volume-backup-id", "bootvolumereplica": "oci bv boot-volume-replica get --boot-volume-replica-id", "volumegroup": "oci bv volume-group get --volume-group-id", "bucket": "oci os bucket get --bucket-name", "compartment": "oci iam compartment get --compartment-id", "tenancy": "oci iam tenancy get --tenancy-id", "user": "oci iam user get --user-id", "group": "oci iam group get --group-id", "policy": "oci iam policy get --policy-id", "dynamicgroup": "oci iam dynamic-group get --dynamic-group-id", "identityprovider": "oci iam identity-provider get --identity-provider-id", "autonomousdatabase": "oci db autonomous-database get --autonomous-database-id", "dbsystem": "oci db system get --db-system-id", "database": "oci db database get --database-id", "dbhome": "oci db db-home get --db-home-id", "dbnode": "oci db node get --db-node-id", "mysqldbsystem": "oci mysql db-system get --db-system-id", "loadbalancer": "oci lb load-balancer get --load-balancer-id", "networkloadbalancer": "oci nlb network-load-balancer get --network-load-balancer-id", "certificate": "oci certs-mgmt certificate get --certificate-id", "vault": "oci kms management vault get --vault-id", "key": "oci kms management key get --key-id", "secret": "oci vault secret get --secret-id", "fnapp": "oci fn application get --application-id", "fnfunc": "oci fn function get --function-id", "apigateway": "oci api-gateway gateway get --gateway-id", "containerinstance": "oci container-instances container-instance get --container-instance-id", "cluster": "oci ce cluster get --cluster-id", "nodepool": "oci ce node-pool get --node-pool-id", "filesystem": "oci fs file-system get --file-system-id", "mounttarget": "oci fs mount-target get --mount-target-id", "alarm": "oci monitoring alarm get --alarm-id", "loggroup": "oci logging log-group get --log-group-id", "log": "oci logging log get --log-id", "topic": "oci ons topic get --topic-id", "subscription": "oci ons subscription get --subscription-id", "stream": "oci streaming stream get --stream-id", "streampool": "oci streaming stream-pool get --stream-pool-id", "dnszone": "oci dns zone get --zone-name-or-id", "networkfirewall": "oci network-firewall network-firewall get --network-firewall-id", "networkfirewallpolicy": "oci network-firewall network-firewall-policy get --network-firewall-policy-id", "waaspolicy": "oci waas waas-policy get --waas-policy-id", "instancepool": "oci compute-management instance-pool get --instance-pool-id", "instanceconfiguration": "oci compute-management instance-configuration get --instance-configuration-id", "capacityreservation": "oci compute capacity-reservation get --capacity-reservation-id", "dedicatedvmhost": "oci compute dedicated-vm-host get --dedicated-vm-host-id", } def _resolve_ocid_command(ocid: str) -> str | None: """Parse an OCID and return the corresponding OCI CLI get command.""" parts = ocid.strip().split(".") if len(parts) < 4 or parts[0] != "ocid1": return None resource_type = parts[1].lower() # Extract region from OCID if present (4th part, non-empty for regional resources) region = parts[3] if len(parts) > 3 and parts[3] and parts[3] != parts[2] else None cmd_template = _OCID_CMD_MAP.get(resource_type) if not cmd_template: return None cmd = f"{cmd_template} {ocid}" if region: cmd += f" --region {region}" return cmd _TERMINAL_BLOCKED_PATTERNS = re.compile( r'(rm\s|mv\s|cp\s|chmod\s|chown\s|sudo\s|bash\s|sh\s|curl\s|wget\s|python|pip\s|apt\s|yum\s|>\s|>>\s|\|)', re.IGNORECASE ) @app.post("/api/terminal/execute") async def terminal_execute(req: dict, u=Depends(current_user)): """Execute an OCI CLI command using the selected OCI config. Only 'oci' commands allowed.""" oci_config_id = req.get("oci_config_id", "") command = (req.get("command", "") or "").strip() if not oci_config_id: raise HTTPException(400, "oci_config_id obrigatório") if not command: raise HTTPException(400, "Comando vazio") _verify_config_access("oci", oci_config_id, u) # Auto-lookup: OCID pasted directly → resolve to oci get command resolved_cmd = None if command.startswith("ocid1."): resolved_cmd = _resolve_ocid_command(command) if resolved_cmd: command = resolved_cmd else: raise HTTPException(400, f"Tipo de recurso não reconhecido no OCID: {command.split('.')[1]}") # find → search resources by display name using OCI Search service elif command.startswith("find "): query = command[5:].strip() if not query: raise HTTPException(400, "Uso: find | find %partial% | find 10.0.1.5") # Detect IP address → search by IP if re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', query): search_query = f"query privateip resources where ipAddress = '{query}'" elif '%' in query: # Partial match with LIKE search_query = f"query all resources where displayName =~ '{query}'" else: search_query = f"query all resources where displayName = '{query}'" resolved_cmd = f"oci search resource structured-search --query-text \"{search_query}\" --limit 20" command = resolved_cmd # Security: only allow 'oci' commands if command == "oci": command = "oci --help" if not command.startswith("oci "): raise HTTPException(400, "Apenas comandos 'oci' são permitidos. Ex: oci iam user list") if _TERMINAL_BLOCKED_PATTERNS.search(command): raise HTTPException(400, "Comando contém operações não permitidas") # Build config path cfg_path = OCI_DIR / oci_config_id / "config" if not cfg_path.exists(): raise HTTPException(400, "OCI config file não encontrado. Reconfigure as credenciais.") # Execute with timeout import shlex, subprocess cmd_args = shlex.split(command) + ["--config-file", str(cfg_path)] try: loop = asyncio.get_event_loop() result = await asyncio.wait_for( loop.run_in_executor(_chat_executor, lambda: subprocess.run( cmd_args, capture_output=True, text=True, timeout=60, cwd="/tmp", env={**os.environ, "OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING": "True", "HOME": "/tmp"} )), timeout=65) output = result.stdout or "" error = result.stderr or "" exit_code = result.returncode # Save to history with db() as c: c.execute("INSERT INTO terminal_history (id,user_id,oci_config_id,command,exit_code,output,error) VALUES (?,?,?,?,?,?,?)", (str(uuid.uuid4()), u["id"], oci_config_id, command, exit_code, output[:10000], error[:2000])) return { "exit_code": exit_code, "output": output[:50000], "error": error[:5000] if exit_code != 0 else "", "resolved_cmd": resolved_cmd, } except asyncio.TimeoutError: with db() as c: c.execute("INSERT INTO terminal_history (id,user_id,oci_config_id,command,exit_code,error) VALUES (?,?,?,?,?,?)", (str(uuid.uuid4()), u["id"], oci_config_id, command, -1, "Timeout: 60s")) return {"exit_code": -1, "output": "", "error": "Timeout: comando excedeu 60 segundos"} except Exception as e: return {"exit_code": -1, "output": "", "error": str(e)[:2000]} @app.get("/api/terminal/history") async def terminal_history(oci_config_id: str = Query(""), limit: int = Query(50), u=Depends(current_user)): """Get recent terminal commands for the user.""" with db() as c: if oci_config_id: rows = c.execute( "SELECT * FROM terminal_history WHERE user_id=? AND oci_config_id=? ORDER BY created_at DESC LIMIT ?", (u["id"], oci_config_id, limit)).fetchall() else: rows = c.execute( "SELECT * FROM terminal_history WHERE user_id=? ORDER BY created_at DESC LIMIT ?", (u["id"], limit)).fetchall() return [dict(r) for r in rows] # OCI CLI autocomplete cache _oci_completions_cache: dict = {} def _parse_oci_help(args: list[str]) -> list[str]: """Run oci --help and extract available commands/options.""" import subprocess try: result = subprocess.run( ["oci"] + args + ["--help"], capture_output=True, text=True, timeout=10, env={**os.environ, "OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING": "True", "HOME": "/tmp"}) text = result.stdout + result.stderr commands = [] in_commands = False for line in text.splitlines(): stripped = line.strip() # Detect transition into command/service listing if not in_commands: if re.match(r'^(Commands|Options|Core|Others|Networking|Database|Identity|Storage|Compute|Security)', stripped): in_commands = True continue if not stripped: continue # Section headers (single word or capitalized phrase not indented) — skip if not line.startswith(" "): continue # Capture " command-name Description text" pattern m = re.match(r'^ ([a-z][a-z0-9\-_]+)\s', line) if m and len(m.group(1)) > 1: commands.append(m.group(1)) continue # Capture 2-space indent commands (sub-command help) m = re.match(r'^ ([a-z][a-z0-9\-_]+)\s', line) if m and len(m.group(1)) > 2: commands.append(m.group(1)) continue # Capture --options if stripped.startswith("--"): opt = stripped.split()[0].rstrip(",") if opt.startswith("--"): commands.append(opt) return sorted(set(commands)) except Exception: return [] @app.get("/api/terminal/completions") async def terminal_completions(prefix: str = Query(""), u=Depends(current_user)): """Get OCI CLI autocomplete suggestions for a partial command.""" parts = prefix.strip().split() if not parts or parts[0] != "oci": return {"suggestions": []} # Build cache key from command path (without final partial word) # e.g. "oci iam us" → lookup completions for ["oci", "iam"], filter by "us" if prefix.endswith(" "): cmd_parts = parts[1:] filter_prefix = "" else: cmd_parts = parts[1:-1] filter_prefix = parts[-1] if len(parts) > 1 else "" cache_key = " ".join(cmd_parts) if cache_key not in _oci_completions_cache: loop = asyncio.get_event_loop() completions = await loop.run_in_executor(_chat_executor, lambda: _parse_oci_help(cmd_parts)) _oci_completions_cache[cache_key] = completions suggestions = _oci_completions_cache[cache_key] if filter_prefix: suggestions = [s for s in suggestions if s.startswith(filter_prefix)] # Commands first, then options cmds = [s for s in suggestions if not s.startswith("--")] opts = [s for s in suggestions if s.startswith("--")] return {"suggestions": (cmds + opts)[:50]} # ── OCI Account Explorer ────────────────────────────────────────────────────── @app.get("/api/oci/explore/{cid}/compartments") async def explore_compartments(cid: str, u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) identity = oci.identity.IdentityClient(config) with db() as c: cfg = c.execute("SELECT tenancy_ocid,compartment_id FROM oci_configs WHERE id=?", (cid,)).fetchone() root = _safe_dec(cfg["compartment_id"]) or _safe_dec(cfg["tenancy_ocid"]) comps = identity.list_compartments(root, compartment_id_in_subtree=True).data return [{"id":cp.id,"name":cp.name,"lifecycle_state":cp.lifecycle_state,"description":cp.description or ""} for cp in comps if cp.lifecycle_state == "ACTIVE"] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/regions") async def explore_regions(cid: str, u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) identity = oci.identity.IdentityClient(config) with db() as c: cfg = c.execute("SELECT tenancy_ocid FROM oci_configs WHERE id=?", (cid,)).fetchone() regions = identity.list_region_subscriptions(_safe_dec(cfg["tenancy_ocid"])).data return [{"name":r.region_name,"key":r.region_key,"status":r.status,"is_home":r.is_home_region} for r in regions] except Exception as e: return {"error": str(e)[:500]} _DEAD_STATES = {'TERMINATED','TERMINATING','DELETED','DELETING','PENDING_DELETION','FAULTY'} _EXP_RESOURCE_TYPES = [ 'instances','boot_volumes','instance_pools', 'vcns','subnets','security_lists','nsgs','route_tables','nat_gateways', 'internet_gateways','service_gateways','public_ips','load_balancers', 'buckets','block_volumes','file_systems','mount_targets', 'databases','db_systems','mysql_db_systems', 'container_instances','oke_clusters', 'functions','api_gateways', 'alarms','log_groups','notification_topics','events_rules', 'vaults','dns_zones','network_firewalls','network_firewall_policies', 'iam_users','iam_groups','iam_policies','iam_dynamic_groups', ] def _explore_comp(cid): """Helper: resolve compartment from config.""" with db() as c: cfg = c.execute("SELECT tenancy_ocid,compartment_id FROM oci_configs WHERE id=?", (cid,)).fetchone() return cfg, _safe_dec(cfg["compartment_id"]) or _safe_dec(cfg["tenancy_ocid"]) # ── Explorer Counts Cache ───────────────────────────────────────────────────── @app.get("/api/oci/explore/{cid}/counts") async def explore_counts_cached(cid: str, compartment_id: str = Query(...), regions: str = Query(""), u=Depends(current_user)): """Return cached resource counts from DB (instant).""" _verify_config_access("oci", cid, u) with db() as c: rows = c.execute( "SELECT resource_type, count, updated_at FROM explorer_counts_cache WHERE oci_config_id=? AND compartment_id=? AND regions=?", (cid, compartment_id, regions)).fetchall() return {r["resource_type"]: {"count": r["count"], "updated_at": r["updated_at"]} for r in rows} @app.post("/api/oci/explore/{cid}/counts/refresh") async def explore_counts_refresh(cid: str, req: dict, u=Depends(current_user)): """Refresh all resource counts in background thread and save to DB.""" _verify_config_access("oci", cid, u) compartment_id = req.get("compartment_id") regions_list = req.get("regions", []) if not compartment_id: raise HTTPException(400, "compartment_id required") regions_key = ",".join(sorted(regions_list)) if regions_list else "" import threading def _do_refresh(): import httpx, asyncio _exp_fn_map = {} # will call explore endpoints via internal dispatch results = {} for rt in _EXP_RESOURCE_TYPES: fn_name = f"explore_{rt}" fn = globals().get(fn_name) if not fn: results[rt] = 0 continue try: loop = asyncio.new_event_loop() regs = regions_list if regions_list else [None] total = 0 for reg in regs: # IAM endpoints don't take compartment_id/region if rt.startswith('iam_') and rt != 'iam_policies': data = loop.run_until_complete(fn(cid, u=u)) elif rt == 'iam_policies': data = loop.run_until_complete(fn(cid, compartment_id=compartment_id, u=u)) else: data = loop.run_until_complete(fn(cid, compartment_id=compartment_id, region=reg, u=u)) if isinstance(data, list): total += len(data) elif isinstance(data, dict) and not data.get("error"): total += 0 results[rt] = total loop.close() except Exception as e: log.warning(f"Explorer count refresh error for {rt}: {e}") results[rt] = 0 # Save to DB with db() as c: for rt, cnt in results.items(): c.execute("""INSERT INTO explorer_counts_cache (oci_config_id, compartment_id, resource_type, count, regions, updated_at) VALUES (?,?,?,?,?,datetime('now')) ON CONFLICT(oci_config_id, compartment_id, resource_type, regions) DO UPDATE SET count=excluded.count, updated_at=datetime('now')""", (cid, compartment_id, rt, cnt, regions_key)) log.info(f"Explorer counts refreshed for config={cid} comp={compartment_id[:20]}... total_types={len(results)}") threading.Thread(target=_do_refresh, daemon=True).start() return {"status": "refreshing", "types": len(_EXP_RESOURCE_TYPES)} @app.get("/api/oci/explore/{cid}/compartment-tree") async def explore_compartment_tree(cid: str, u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) identity = oci.identity.IdentityClient(config) cfg, default_comp = _explore_comp(cid) tenancy = _safe_dec(cfg["tenancy_ocid"]) or default_comp user_comp = _safe_dec(cfg["compartment_id"]) # Use tenancy as root for the tree tree = [{"id": tenancy, "name": "root (tenancy)", "parent_id": None}] try: comps = identity.list_compartments(tenancy, compartment_id_in_subtree=True).data seen = {tenancy} for cp in comps: if cp.lifecycle_state == "ACTIVE": tree.append({"id":cp.id, "name":cp.name, "parent_id":cp.compartment_id, "lifecycle_state":cp.lifecycle_state}) seen.add(cp.id) # If user's configured compartment is not in the tree, add it explicitly if user_comp and user_comp != tenancy and user_comp not in seen: try: cp = identity.get_compartment(user_comp).data tree.append({"id":cp.id, "name":cp.name, "parent_id":cp.compartment_id, "lifecycle_state":cp.lifecycle_state}) except Exception as e: log.warning(f"Failed to get compartment {user_comp}: {e}") tree.append({"id":user_comp, "name":"Meu Compartment", "parent_id":tenancy, "lifecycle_state":"ACTIVE"}) except Exception as e: log.warning(f"Failed to list compartments from tenancy: {e}") # If listing from tenancy fails (no permission), list from user's compartment if user_comp and user_comp != tenancy: try: cp = identity.get_compartment(user_comp).data tree = [{"id":cp.id, "name":cp.name, "parent_id": None, "lifecycle_state":cp.lifecycle_state}] sub = identity.list_compartments(user_comp, compartment_id_in_subtree=True).data tree += [{"id":s.id, "name":s.name, "parent_id":s.compartment_id, "lifecycle_state":s.lifecycle_state} for s in sub if s.lifecycle_state == "ACTIVE"] except Exception as e: log.warning(f"Failed to list sub-compartments for {user_comp}: {e}") tree = [{"id":user_comp, "name":"Meu Compartment", "parent_id": None, "lifecycle_state":"ACTIVE"}] return tree except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/vcns") async def explore_vcns(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region vn = oci.core.VirtualNetworkClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp vcns = vn.list_vcns(comp).data return [{"id":v.id,"display_name":v.display_name,"cidr_blocks":v.cidr_blocks,"lifecycle_state":v.lifecycle_state, "dns_label":v.dns_label,"vcn_domain_name":v.vcn_domain_name, "default_route_table_id":v.default_route_table_id,"default_security_list_id":v.default_security_list_id, "time_created":str(v.time_created)} for v in vcns if v.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/instances") async def explore_instances(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region compute = oci.core.ComputeClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp insts = compute.list_instances(comp).data return [{"id":i.id,"display_name":i.display_name,"shape":i.shape,"lifecycle_state":i.lifecycle_state,"region":i.region,"time_created":str(i.time_created)} for i in insts if i.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/databases") async def explore_databases(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region db_client = oci.database.DatabaseClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp adbs = db_client.list_autonomous_databases(comp).data return [{"id":a.id,"display_name":a.display_name,"db_name":a.db_name,"lifecycle_state":a.lifecycle_state, "cpu_core_count":a.cpu_core_count,"data_storage_size_in_tbs":a.data_storage_size_in_tbs, "is_free_tier":a.is_free_tier,"whitelisted_ips":a.whitelisted_ips or []} for a in adbs if a.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/buckets") async def explore_buckets(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region os_client = oci.object_storage.ObjectStorageClient(config) namespace = os_client.get_namespace().data _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp buckets = os_client.list_buckets(namespace, comp, fields=["approximateSize","approximateCount"]).data return [{"name":b.name,"namespace":b.namespace,"time_created":str(b.time_created), "approximate_size":b.approximate_size,"approximate_count":b.approximate_count, "public_access_type":b.public_access_type or "NoPublicAccess", "storage_tier":b.storage_tier,"versioning":b.versioning} for b in buckets] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/subnets") async def explore_subnets(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region vn = oci.core.VirtualNetworkClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp subs = vn.list_subnets(comp).data return [{"id":s.id,"display_name":s.display_name,"cidr_block":s.cidr_block,"vcn_id":s.vcn_id, "lifecycle_state":s.lifecycle_state,"prohibit_public_ip_on_vnic":s.prohibit_public_ip_on_vnic, "availability_domain":s.availability_domain or "Regional","dns_label":s.dns_label, "route_table_id":s.route_table_id,"security_list_ids":s.security_list_ids, "subnet_domain_name":s.subnet_domain_name,"time_created":str(s.time_created)} for s in subs if s.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/security_lists") async def explore_security_lists(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region vn = oci.core.VirtualNetworkClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp sls = vn.list_security_lists(comp).data def _fmt_rule(r, direction, idx): proto_map = {"6":"TCP","17":"UDP","1":"ICMP","all":"ALL"} proto = proto_map.get(r.protocol, r.protocol) src_dst = r.source if direction == "ingress" else r.destination ports = "" if r.tcp_options: dp = r.tcp_options.destination_port_range if dp: ports = f"{dp.min}-{dp.max}" if dp.min != dp.max else str(dp.min) elif r.udp_options: dp = r.udp_options.destination_port_range if dp: ports = f"{dp.min}-{dp.max}" if dp.min != dp.max else str(dp.min) return {"direction":direction,"protocol":proto,"source_dest":src_dst or "","ports":ports, "stateless":r.is_stateless,"description":getattr(r,'description','') or "","rule_index":idx} result = [] for s in sls: if s.lifecycle_state in _DEAD_STATES: continue ingress = [_fmt_rule(r,"ingress",i) for i,r in enumerate(s.ingress_security_rules or [])] egress = [_fmt_rule(r,"egress",i) for i,r in enumerate(s.egress_security_rules or [])] result.append({"id":s.id,"display_name":s.display_name,"vcn_id":s.vcn_id,"lifecycle_state":s.lifecycle_state, "ingress_count":len(ingress),"egress_count":len(egress),"rules":ingress+egress}) return result except Exception as e: return {"error": str(e)[:500]} @app.post("/api/oci/explore/{cid}/security_lists/{sl_id}/rules") async def add_security_list_rule(cid: str, sl_id: str, req: dict, u=Depends(require("admin", "user"))): """Add a rule to a Security List (appends to existing rules).""" _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) region = req.get("region") if region: config["region"] = region vn = oci.core.VirtualNetworkClient(config) sl = vn.get_security_list(sl_id).data direction = req.get("direction", "ingress").lower() protocol = str(req.get("protocol", "6")) source_dest = req.get("source_dest", "0.0.0.0/0") port_min = req.get("port_min") port_max = req.get("port_max") description = req.get("description", "").strip() or None is_stateless = req.get("is_stateless", False) tcp_opts = None udp_opts = None if protocol == "6" and port_min is not None: tcp_opts = oci.core.models.TcpOptions( destination_port_range=oci.core.models.PortRange(min=int(port_min), max=int(port_max or port_min))) elif protocol == "17" and port_min is not None: udp_opts = oci.core.models.UdpOptions( destination_port_range=oci.core.models.PortRange(min=int(port_min), max=int(port_max or port_min))) ingress_rules = list(sl.ingress_security_rules or []) egress_rules = list(sl.egress_security_rules or []) if direction == "ingress": ingress_rules.append(oci.core.models.IngressSecurityRule( protocol=protocol, source=source_dest, source_type="CIDR_BLOCK", is_stateless=is_stateless, description=description, tcp_options=tcp_opts, udp_options=udp_opts)) else: egress_rules.append(oci.core.models.EgressSecurityRule( protocol=protocol, destination=source_dest, destination_type="CIDR_BLOCK", is_stateless=is_stateless, description=description, tcp_options=tcp_opts, udp_options=udp_opts)) update = oci.core.models.UpdateSecurityListDetails( ingress_security_rules=ingress_rules, egress_security_rules=egress_rules) vn.update_security_list(sl_id, update) _audit(u["id"], u["username"], "seclist_add_rule", sl_id, f"{direction} {protocol} {source_dest} port={port_min or 'ALL'}") return {"ok": True, "message": "Regra adicionada com sucesso"} except Exception as e: raise HTTPException(400, str(e)[:500]) @app.delete("/api/oci/explore/{cid}/security_lists/{sl_id}/rules") async def remove_security_list_rule(cid: str, sl_id: str, req: dict, u=Depends(require("admin", "user"))): """Remove a rule from a Security List by index.""" _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) region = req.get("region") if region: config["region"] = region vn = oci.core.VirtualNetworkClient(config) sl = vn.get_security_list(sl_id).data direction = req.get("direction", "ingress").lower() rule_index = req.get("rule_index") if rule_index is None: raise HTTPException(400, "rule_index obrigatório") ingress_rules = list(sl.ingress_security_rules or []) egress_rules = list(sl.egress_security_rules or []) if direction == "ingress": if 0 <= rule_index < len(ingress_rules): removed = ingress_rules.pop(rule_index) desc = f"ingress idx={rule_index} src={removed.source}" else: raise HTTPException(400, "Índice inválido") else: if 0 <= rule_index < len(egress_rules): removed = egress_rules.pop(rule_index) desc = f"egress idx={rule_index} dst={removed.destination}" else: raise HTTPException(400, "Índice inválido") update = oci.core.models.UpdateSecurityListDetails( ingress_security_rules=ingress_rules, egress_security_rules=egress_rules) vn.update_security_list(sl_id, update) _audit(u["id"], u["username"], "seclist_remove_rule", sl_id, desc) return {"ok": True, "message": "Regra removida com sucesso"} except Exception as e: raise HTTPException(400, str(e)[:500]) @app.get("/api/oci/explore/{cid}/block_volumes") async def explore_block_volumes(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region bs = oci.core.BlockstorageClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp vols = bs.list_volumes(compartment_id=comp).data return [{"id":v.id,"display_name":v.display_name,"size_in_gbs":v.size_in_gbs, "lifecycle_state":v.lifecycle_state,"vpus_per_gb":v.vpus_per_gb} for v in vols if v.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/load_balancers") async def explore_load_balancers(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region lb = oci.load_balancer.LoadBalancerClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp lbs = lb.list_load_balancers(comp).data result = [] for l in lbs: if l.lifecycle_state in _DEAD_STATES: continue listeners = [] for name, lis in (l.listeners or {}).items(): listeners.append({"name": name, "port": lis.port, "protocol": lis.protocol, "backend_set": lis.default_backend_set_name}) backend_sets = [] for name, bs in (l.backend_sets or {}).items(): backends = [{"ip": b.ip_address, "port": b.port, "weight": b.weight} for b in (bs.backends or [])] backend_sets.append({"name": name, "backends": backends, "policy": bs.policy}) result.append({"id":l.id,"display_name":l.display_name,"lifecycle_state":l.lifecycle_state, "shape_name":l.shape_name,"ip_addresses":[ip.ip_address for ip in (l.ip_addresses or [])], "is_private":l.is_private,"listeners":listeners,"backend_sets":backend_sets}) return result except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/functions") async def explore_functions(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region fn = oci.functions.FunctionsManagementClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp apps = fn.list_applications(comp).data result = [] for a in apps: if a.lifecycle_state in _DEAD_STATES: continue fns = fn.list_functions(a.id).data for f in fns: if f.lifecycle_state in _DEAD_STATES: continue result.append({"id":f.id,"display_name":f.display_name,"application":a.display_name, "lifecycle_state":f.lifecycle_state,"memory_in_mbs":f.memory_in_mbs, "timeout_in_seconds":f.timeout_in_seconds,"image":f.image}) return result except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/container_instances") async def explore_container_instances(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region ci = oci.container_instances.ContainerInstanceClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp cis = ci.list_container_instances(comp).data return [{"id":c.id,"display_name":c.display_name,"lifecycle_state":c.lifecycle_state, "container_count":c.container_count,"shape":c.shape, "time_created":str(c.time_created)} for c in cis if c.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/nsgs") async def explore_nsgs(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region vn = oci.core.VirtualNetworkClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp nsgs = vn.list_network_security_groups(compartment_id=comp).data result = [] proto_map = {"1": "ICMP", "6": "TCP", "17": "UDP", "all": "ALL"} for n in nsgs: if n.lifecycle_state in _DEAD_STATES: continue rules_data = oci.pagination.list_call_get_all_results( vn.list_network_security_group_security_rules, n.id).data rules = [] for r in rules_data: ports = "" if r.tcp_options and r.tcp_options.destination_port_range: pr = r.tcp_options.destination_port_range ports = str(pr.min) if pr.min == pr.max else f"{pr.min}-{pr.max}" elif r.udp_options and r.udp_options.destination_port_range: pr = r.udp_options.destination_port_range ports = str(pr.min) if pr.min == pr.max else f"{pr.min}-{pr.max}" src_dst = "" src_dst_type = "" if r.direction == "INGRESS": src_dst = r.source or "" src_dst_type = r.source_type or "" else: src_dst = r.destination or "" src_dst_type = r.destination_type or "" rules.append({ "id": r.id, "direction": r.direction, "protocol": proto_map.get(r.protocol, r.protocol), "protocol_num": r.protocol, "source_dest": src_dst, "source_dest_type": src_dst_type, "ports": ports, "is_stateless": r.is_stateless or False, "description": r.description or "" }) result.append({"id": n.id, "display_name": n.display_name, "vcn_id": n.vcn_id, "lifecycle_state": n.lifecycle_state, "time_created": str(n.time_created), "rule_count": len(rules), "rules": rules}) return result except Exception as e: return {"error": str(e)[:500]} @app.post("/api/oci/explore/{cid}/nsgs/{nsg_id}/rules") async def add_nsg_rule(cid: str, nsg_id: str, req: dict, u=Depends(require("admin", "user"))): """Add a security rule to an NSG.""" _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) region = req.get("region") if region: config["region"] = region vn = oci.core.VirtualNetworkClient(config) direction = req.get("direction", "INGRESS").upper() protocol = str(req.get("protocol", "6")) source_dest = req.get("source_dest", "0.0.0.0/0") source_dest_type = req.get("source_dest_type", "CIDR_BLOCK") port_min = req.get("port_min") port_max = req.get("port_max") description = req.get("description", "") is_stateless = req.get("is_stateless", False) rule = {"direction": direction, "protocol": protocol, "is_stateless": is_stateless, "description": description} if direction == "INGRESS": rule["source"] = source_dest rule["source_type"] = source_dest_type else: rule["destination"] = source_dest rule["destination_type"] = source_dest_type if protocol == "6" and port_min is not None: rule["tcp_options"] = oci.core.models.TcpOptions( destination_port_range=oci.core.models.PortRange(min=int(port_min), max=int(port_max or port_min))) elif protocol == "17" and port_min is not None: rule["udp_options"] = oci.core.models.UdpOptions( destination_port_range=oci.core.models.PortRange(min=int(port_min), max=int(port_max or port_min))) details = oci.core.models.AddNetworkSecurityGroupSecurityRulesDetails( security_rules=[oci.core.models.AddSecurityRuleDetails(**rule)]) resp = vn.add_network_security_group_security_rules(nsg_id, details) _audit(u["id"], u["username"], "nsg_add_rule", nsg_id, f"{direction} {protocol} {source_dest} {port_min or 'ALL'}") added = [{"id": r.id} for r in (resp.data.security_rules or [])] return {"ok": True, "message": "Regra adicionada com sucesso", "rules_added": added} except Exception as e: raise HTTPException(400, str(e)[:500]) @app.delete("/api/oci/explore/{cid}/nsgs/{nsg_id}/rules/{rule_id}") async def remove_nsg_rule(cid: str, nsg_id: str, rule_id: str, region: str = Query(None), u=Depends(require("admin", "user"))): """Remove a security rule from an NSG.""" _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region vn = oci.core.VirtualNetworkClient(config) details = oci.core.models.RemoveNetworkSecurityGroupSecurityRulesDetails(security_rule_ids=[rule_id]) vn.remove_network_security_group_security_rules(nsg_id, details) _audit(u["id"], u["username"], "nsg_remove_rule", nsg_id, f"rule_id={rule_id}") return {"ok": True, "message": "Regra removida com sucesso"} except Exception as e: raise HTTPException(400, str(e)[:500]) @app.get("/api/oci/explore/{cid}/route_tables") async def explore_route_tables(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region vn = oci.core.VirtualNetworkClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp rts = vn.list_route_tables(comp).data result = [] for rt in rts: if rt.lifecycle_state in _DEAD_STATES: continue routes = [] for r in (rt.route_rules or []): target_type = r.network_entity_id.split(".")[1] if r.network_entity_id and "." in r.network_entity_id else "unknown" routes.append({"destination":r.destination or "","destination_type":r.destination_type or "", "target_id":r.network_entity_id or "","target_type":target_type, "description":getattr(r,'description','') or ""}) result.append({"id":rt.id,"display_name":rt.display_name,"vcn_id":rt.vcn_id,"lifecycle_state":rt.lifecycle_state, "route_count":len(routes),"routes":routes}) return result except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/nat_gateways") async def explore_nat_gateways(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region vn = oci.core.VirtualNetworkClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp ngs = vn.list_nat_gateways(comp).data return [{"id":n.id,"display_name":n.display_name,"vcn_id":n.vcn_id,"lifecycle_state":n.lifecycle_state, "nat_ip":n.nat_ip,"block_traffic":n.block_traffic} for n in ngs if n.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/internet_gateways") async def explore_internet_gateways(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region vn = oci.core.VirtualNetworkClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp igs = vn.list_internet_gateways(comp).data return [{"id":g.id,"display_name":g.display_name,"vcn_id":g.vcn_id,"lifecycle_state":g.lifecycle_state, "is_enabled":g.is_enabled} for g in igs if g.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/service_gateways") async def explore_service_gateways(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region vn = oci.core.VirtualNetworkClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp sgs = vn.list_service_gateways(comp).data return [{"id":g.id,"display_name":g.display_name,"vcn_id":g.vcn_id,"lifecycle_state":g.lifecycle_state, "services":[s.service_name for s in (g.services or [])]} for g in sgs if g.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/public_ips") async def explore_public_ips(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region vn = oci.core.VirtualNetworkClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp ips = vn.list_public_ips(scope="REGION", compartment_id=comp).data return [{"id":ip.id,"display_name":ip.display_name or "—","ip_address":ip.ip_address, "lifecycle_state":ip.lifecycle_state,"lifetime":ip.lifetime, "assigned_entity_type":ip.assigned_entity_type} for ip in ips if ip.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/boot_volumes") async def explore_boot_volumes(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region identity = oci.identity.IdentityClient(config) bs = oci.core.BlockstorageClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp ads = identity.list_availability_domains(comp).data result = [] for ad in ads: bvs = bs.list_boot_volumes(availability_domain=ad.name, compartment_id=comp).data for v in bvs: if v.lifecycle_state in _DEAD_STATES: continue result.append({"id":v.id,"display_name":v.display_name,"size_in_gbs":v.size_in_gbs, "lifecycle_state":v.lifecycle_state,"availability_domain":ad.name, "vpus_per_gb":v.vpus_per_gb}) return result except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/instance_pools") async def explore_instance_pools(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region cm = oci.core.ComputeManagementClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp pools = cm.list_instance_pools(comp).data return [{"id":p.id,"display_name":p.display_name,"lifecycle_state":p.lifecycle_state, "size":p.size,"time_created":str(p.time_created)} for p in pools if p.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/oke_clusters") async def explore_oke_clusters(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region ce = oci.container_engine.ContainerEngineClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp clusters = ce.list_clusters(comp).data return [{"id":c.id,"name":c.name,"lifecycle_state":c.lifecycle_state, "kubernetes_version":c.kubernetes_version,"vcn_id":c.vcn_id, "endpoint_config":c.endpoint_config.is_public_ip_enabled if c.endpoint_config else None} for c in clusters if c.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/file_systems") async def explore_file_systems(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region identity = oci.identity.IdentityClient(config) fss = oci.file_storage.FileStorageClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp ads = identity.list_availability_domains(comp).data result = [] for ad in ads: fs_list = fss.list_file_systems(comp, ad.name).data for f in fs_list: if f.lifecycle_state in _DEAD_STATES: continue result.append({"id":f.id,"display_name":f.display_name,"lifecycle_state":f.lifecycle_state, "availability_domain":ad.name,"metered_bytes":f.metered_bytes, "time_created":str(f.time_created)}) return result except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/mount_targets") async def explore_mount_targets(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region identity = oci.identity.IdentityClient(config) fss = oci.file_storage.FileStorageClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp ads = identity.list_availability_domains(comp).data result = [] for ad in ads: mts = fss.list_mount_targets(comp, ad.name).data for m in mts: if m.lifecycle_state in _DEAD_STATES: continue result.append({"id":m.id,"display_name":m.display_name,"lifecycle_state":m.lifecycle_state, "availability_domain":ad.name,"subnet_id":m.subnet_id, "private_ip_ids":m.private_ip_ids}) return result except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/db_systems") async def explore_db_systems(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region db_client = oci.database.DatabaseClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp dbs = db_client.list_db_systems(comp).data return [{"id":d.id,"display_name":d.display_name,"lifecycle_state":d.lifecycle_state, "shape":d.shape,"database_edition":d.database_edition, "cpu_core_count":d.cpu_core_count,"node_count":d.node_count} for d in dbs if d.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/mysql_db_systems") async def explore_mysql_db_systems(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region mysql = oci.mysql.DbSystemClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp dbs = mysql.list_db_systems(comp).data return [{"id":d.id,"display_name":d.display_name,"lifecycle_state":d.lifecycle_state, "shape_name":d.shape_name,"mysql_version":d.mysql_version, "is_highly_available":d.is_highly_available} for d in dbs if d.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/alarms") async def explore_alarms(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region mon = oci.monitoring.MonitoringClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp alarms = mon.list_alarms(comp).data return [{"id":a.id,"display_name":a.display_name,"lifecycle_state":a.lifecycle_state, "severity":a.severity,"namespace":a.namespace,"is_enabled":a.is_enabled} for a in alarms if a.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/log_groups") async def explore_log_groups(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region logging_client = oci.logging.LoggingManagementClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp lgs = logging_client.list_log_groups(comp).data return [{"id":g.id,"display_name":g.display_name,"time_created":str(g.time_created), "description":g.description or ""} for g in lgs] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/iam_users") async def explore_iam_users(cid: str, u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) identity = oci.identity.IdentityClient(config) cfg, _ = _explore_comp(cid) tenancy = _safe_dec(cfg["tenancy_ocid"]) users = identity.list_users(tenancy).data return [{"id":u2.id,"name":u2.name,"email":u2.email or "","lifecycle_state":u2.lifecycle_state, "is_mfa_activated":u2.is_mfa_activated,"time_created":str(u2.time_created), "last_successful_login_time":str(u2.last_successful_login_time) if u2.last_successful_login_time else None} for u2 in users if u2.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/iam_groups") async def explore_iam_groups(cid: str, u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) identity = oci.identity.IdentityClient(config) cfg, _ = _explore_comp(cid) tenancy = _safe_dec(cfg["tenancy_ocid"]) groups = identity.list_groups(tenancy).data return [{"id":g.id,"name":g.name,"description":g.description or "","lifecycle_state":g.lifecycle_state, "time_created":str(g.time_created)} for g in groups if g.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/iam_policies") async def explore_iam_policies(cid: str, compartment_id: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) identity = oci.identity.IdentityClient(config) cfg, default_comp = _explore_comp(cid) comp = compartment_id or _safe_dec(cfg["tenancy_ocid"]) or default_comp policies = identity.list_policies(comp).data return [{"id":p.id,"name":p.name,"description":p.description or "","lifecycle_state":p.lifecycle_state, "statements":p.statements,"time_created":str(p.time_created)} for p in policies if p.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/iam_dynamic_groups") async def explore_iam_dynamic_groups(cid: str, u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) identity = oci.identity.IdentityClient(config) cfg, _ = _explore_comp(cid) tenancy = _safe_dec(cfg["tenancy_ocid"]) dgs = identity.list_dynamic_groups(tenancy).data return [{"id":g.id,"name":g.name,"description":g.description or "","lifecycle_state":g.lifecycle_state, "matching_rule":g.matching_rule,"time_created":str(g.time_created)} for g in dgs if g.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/vaults") async def explore_vaults(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region kms = oci.key_management.KmsVaultClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp vaults = kms.list_vaults(comp).data return [{"id":v.id,"display_name":v.display_name,"lifecycle_state":v.lifecycle_state, "vault_type":v.vault_type,"crypto_endpoint":v.crypto_endpoint, "time_created":str(v.time_created)} for v in vaults if v.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/dns_zones") async def explore_dns_zones(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region dns = oci.dns.DnsClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp zones = dns.list_zones(comp).data return [{"id":z.id,"name":z.name,"lifecycle_state":z.lifecycle_state, "zone_type":z.zone_type,"serial":z.serial, "time_created":str(z.time_created)} for z in zones if z.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/api_gateways") async def explore_api_gateways(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region apigw = oci.apigateway.GatewayClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp gws = apigw.list_gateways(comp).data return [{"id":g.id,"display_name":g.display_name,"lifecycle_state":g.lifecycle_state, "endpoint_type":g.endpoint_type,"hostname":g.hostname, "subnet_id":g.subnet_id,"time_created":str(g.time_created)} for g in gws if g.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/notification_topics") async def explore_notification_topics(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region ons = oci.ons.NotificationControlPlaneClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp topics = ons.list_topics(comp).data return [{"id":t.topic_id,"name":t.name,"lifecycle_state":t.lifecycle_state, "description":t.description or "","api_endpoint":t.api_endpoint, "time_created":str(t.time_created)} for t in topics if t.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/events_rules") async def explore_events_rules(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region events = oci.events.EventsClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp rules = events.list_rules(comp).data return [{"id":r.id,"display_name":r.display_name,"lifecycle_state":r.lifecycle_state, "condition":r.condition,"is_enabled":r.is_enabled, "description":r.description or "","time_created":str(r.time_created)} for r in rules if r.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/network_firewalls") async def explore_network_firewalls(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region fw = oci.network_firewall.NetworkFirewallClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp resp = fw.list_network_firewalls(compartment_id=comp).data firewalls = resp.items if hasattr(resp, 'items') else (resp if isinstance(resp, list) else []) return [{"id":f.id,"display_name":f.display_name,"lifecycle_state":f.lifecycle_state, "subnet_id":f.subnet_id,"network_firewall_policy_id":f.network_firewall_policy_id, "time_created":str(f.time_created)} for f in firewalls if f.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} @app.get("/api/oci/explore/{cid}/network_firewall_policies") async def explore_network_firewall_policies(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)): _verify_config_access("oci", cid, u) try: import oci config = _get_oci_config(cid) if region: config["region"] = region fw = oci.network_firewall.NetworkFirewallClient(config) _, default_comp = _explore_comp(cid) comp = compartment_id or default_comp resp = fw.list_network_firewall_policies(compartment_id=comp).data policies = resp.items if hasattr(resp, 'items') else (resp if isinstance(resp, list) else []) return [{"id":p.id,"display_name":p.display_name,"lifecycle_state":p.lifecycle_state, "time_created":str(p.time_created)} for p in policies if p.lifecycle_state not in _DEAD_STATES] except Exception as e: return {"error": str(e)[:500]} # ── OCI GenAI Config & Chat ─────────────────────────────────────────────────── @app.get("/api/genai/models") async def list_genai_models(u=Depends(current_user)): return JSONResponse( content={"models": GENAI_MODELS, "regions": GENAI_REGIONS, "embedding_models": EMBEDDING_MODELS, "oci_regions": OCI_REGIONS}, headers={"Cache-Control": "max-age=3600"}, ) @app.post("/api/genai/config") async def save_genai(req: GenAIConfigReq, u=Depends(require("admin","user"))): gid = str(uuid.uuid4()) ep = req.endpoint or f"https://inference.generativeai.{req.genai_region}.oci.oraclecloud.com" with db() as c: if req.is_default: c.execute("UPDATE genai_configs SET is_default=0 WHERE user_id=?", (u["id"],)) c.execute( """INSERT INTO genai_configs (id,user_id,name,oci_config_id,model_id,model_ocid,compartment_id, genai_region,endpoint,serving_type,dedicated_endpoint_id,temperature,max_tokens,top_p,top_k, frequency_penalty,presence_penalty,reasoning_effort,is_default) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", (gid, u["id"], req.name, req.oci_config_id, req.model_id, req.model_ocid, req.compartment_id, req.genai_region, ep, req.serving_type, req.dedicated_endpoint_id, req.temperature, req.max_tokens, req.top_p, req.top_k, req.frequency_penalty, req.presence_penalty, req.reasoning_effort, int(req.is_default))) _audit(u["id"], u["username"], "save_genai_config", gid, req.model_id) _config_log("genai", gid, req.name, "success", "save", f"Modelo salvo: {req.model_id} ({req.genai_region})", u["id"], u["username"]) return {"id": gid, "model_id": req.model_id, "endpoint": ep} @app.get("/api/genai/configs") async def list_genai(u=Depends(current_user)): with db() as c: if u["role"]=="admin": rows=c.execute("SELECT * FROM genai_configs").fetchall() else: rows=c.execute("SELECT * FROM genai_configs WHERE user_id=?",(u["id"],)).fetchall() return [dict(r) for r in rows] @app.delete("/api/genai/configs/{gid}") async def del_genai(gid: str, u=Depends(require("admin","user"))): _verify_config_access("genai", gid, u, require_owner=True) with db() as c: c.execute("DELETE FROM genai_configs WHERE id=?", (gid,)) return {"ok": True} @app.put("/api/genai/configs/{gid}") async def update_genai(gid: str, req: GenAIConfigReq, u=Depends(require("admin","user"))): with db() as c: existing = c.execute("SELECT * FROM genai_configs WHERE id=?", (gid,)).fetchone() if not existing: raise HTTPException(404) if u["role"] != "admin" and existing["user_id"] != u["id"]: raise HTTPException(403) ep = req.endpoint or f"https://inference.generativeai.{req.genai_region}.oci.oraclecloud.com" with db() as c: if req.is_default: c.execute("UPDATE genai_configs SET is_default=0 WHERE user_id=?", (u["id"],)) c.execute( """UPDATE genai_configs SET name=?,oci_config_id=?,model_id=?,model_ocid=?,compartment_id=?, genai_region=?,endpoint=?,serving_type=?,dedicated_endpoint_id=?,temperature=?,max_tokens=?, top_p=?,top_k=?,frequency_penalty=?,presence_penalty=?,reasoning_effort=?,is_default=? WHERE id=?""", (req.name, req.oci_config_id, req.model_id, req.model_ocid, req.compartment_id, req.genai_region, ep, req.serving_type, req.dedicated_endpoint_id, req.temperature, req.max_tokens, req.top_p, req.top_k, req.frequency_penalty, req.presence_penalty, req.reasoning_effort, int(req.is_default), gid)) _audit(u["id"], u["username"], "update_genai_config", gid, req.model_id) _config_log("genai", gid, req.name, "success", "save", f"Modelo atualizado: {req.model_id} ({req.genai_region})", u["id"], u["username"]) return {"id": gid, "model_id": req.model_id, "endpoint": ep} @app.post("/api/genai/test/{gid}") async def test_genai(gid: str, u=Depends(require("admin","user"))): _verify_config_access("genai", gid, u) with db() as c: gc = c.execute("SELECT * FROM genai_configs WHERE id=?",(gid,)).fetchone() if not gc: raise HTTPException(404) gname = gc["name"] try: resp, _, _ = _call_genai(dict(gc), "Say 'connection successful' in one short sentence.") _config_log("genai", gid, gname, "success", "test", f"GenAI OK: {resp[:200]}", u["id"], u["username"]) return {"status":"success","message":"GenAI OK","response":resp[:300]} except Exception as e: msg = str(e)[:500] _config_log("genai", gid, gname, "error", "test", msg, u["id"], u["username"]) return {"status":"error","message":msg} # ── Chat Memory Compaction ───────────────────────────────────────────────────── def _estimate_tokens(text: str) -> int: return len(text) // 4 def _estimate_history_tokens(history: list) -> int: return sum(_estimate_tokens(h["content"]) for h in history) def _should_compact(history: list) -> bool: if len(history) < COMPACT_MIN_MESSAGES: return False return _estimate_history_tokens(history) > COMPACT_TOKEN_THRESHOLD def _generate_summary(genai_cfg: dict, messages_to_summarize: list) -> str: """Use the same GenAI model to summarize older conversation messages.""" conversation_text = "" for m in messages_to_summarize: role_label = "Usuário" if m["role"] == "user" else "Assistente" # Truncate very long messages in the summary input content = m["content"][:3000] if len(m["content"]) > 3000 else m["content"] conversation_text += f"{role_label}: {content}\n\n" summary_prompt = ( "Resuma a conversa abaixo de forma compacta (máximo 2-3 parágrafos curtos) que capture:\n" "- Tópicos discutidos e perguntas do usuário\n" "- Decisões tomadas, conclusões e resultados importantes\n" "- Resultados de ferramentas/tools executadas (checks PASS/FAIL, scores, recursos encontrados)\n" "- Contexto OCI/CIS relevante (compartments, regiões, config_ids usados)\n" "- Dados numéricos importantes (scores, contagens, OCIDs mencionados)\n\n" "Seja conciso mas preserve TODA informação acionável e dados concretos. " "Não inclua saudações ou formalidades.\n\n" "CONVERSA:\n" + conversation_text ) summary_cfg = dict(genai_cfg) summary_cfg["system_prompt"] = "Você é um sumarizador. Responda apenas com o resumo." summary_cfg["max_tokens"] = COMPACT_SUMMARY_MAX_TOKENS try: text, _, _ = _call_genai(summary_cfg, summary_prompt, history=None, tools=None) return text.strip() except Exception as e: log.warning(f"Summary generation failed: {e}") return "" def _compact_history(session_id: str, user_id: str, genai_cfg: dict, history: list) -> list: """Compact conversation history by summarizing older messages.""" if len(history) <= COMPACT_KEEP_RECENT: return history messages_to_summarize = history[:-COMPACT_KEEP_RECENT] recent_messages = history[-COMPACT_KEEP_RECENT:] # Check for existing summary with db() as c: row = c.execute( "SELECT summary, messages_compacted FROM chat_summaries " "WHERE session_id=? ORDER BY created_at DESC LIMIT 1", (session_id,) ).fetchone() if row: existing_summary = row["summary"] prev_compacted = row["messages_compacted"] # Reuse if no new messages to summarize if len(messages_to_summarize) <= prev_compacted: return [{"role": "assistant", "content": f"[Resumo da conversa anterior: {existing_summary}]"}] + recent_messages else: existing_summary = None # Include existing summary as prefix for incremental compaction if existing_summary: messages_to_summarize = [{"role": "assistant", "content": f"[Resumo anterior: {existing_summary}]"}] + messages_to_summarize summary_text = _generate_summary(genai_cfg, messages_to_summarize) if not summary_text: # Fallback: truncate keeping recent messages that fit in budget truncated = [] budget = 6000 for m in reversed(history): t = _estimate_tokens(m["content"]) if budget - t < 0: break truncated.insert(0, m) budget -= t return truncated # Save summary to DB actual_count = len(messages_to_summarize) - (1 if existing_summary else 0) with db() as c: last_msg = c.execute( "SELECT id FROM chat_messages WHERE session_id=? AND role IN ('user','assistant') " "ORDER BY created_at ASC LIMIT 1 OFFSET ?", (session_id, max(0, actual_count - 1)) ).fetchone() last_id = last_msg["id"] if last_msg else "unknown" c.execute( "INSERT INTO chat_summaries (id,session_id,user_id,summary,messages_compacted,up_to_message_id) VALUES (?,?,?,?,?,?)", (str(uuid.uuid4()), session_id, user_id, summary_text, actual_count, last_id) ) log.info(f"Generated summary for session {session_id}: {len(summary_text)} chars, {actual_count} msgs compacted") return [{"role": "assistant", "content": f"[Resumo da conversa anterior: {summary_text}]"}] + recent_messages # ── GenAI Call ───────────────────────────────────────────────────────────────── def _call_genai(gc: dict, message: str, history: list = None, tools: list = None, tool_results_cohere: list = None, extra_messages: list = None, attachments: list = None) -> tuple: """ Call OCI Generative AI with optional tool use support. Returns (text, tool_calls, tool_calls_raw) tuple. tool_calls is a list of dicts or None. tool_calls_raw is the raw OCI SDK objects (for Generic format continuations). extra_messages: list of raw OCI SDK message objects to append after the user message (for tool use loop accumulation). """ import oci # Load OCI config from stored credentials (same as ~/.oci/config) config_path = str(OCI_DIR / gc["oci_config_id"] / "config") config = oci.config.from_file(config_path, "DEFAULT") # Service endpoint - built from region endpoint = gc["endpoint"] # Create inference client with retry strategy and timeout generative_ai_inference_client = oci.generative_ai_inference.GenerativeAiInferenceClient( config=config, service_endpoint=endpoint, retry_strategy=oci.retry.NoneRetryStrategy(), timeout=(10, 600) ) # System prompt system_prompt = gc.get("system_prompt", "") # Build ChatDetails chat_detail = oci.generative_ai_inference.models.ChatDetails() # Determine API format from model catalog model_info = GENAI_MODELS.get(gc["model_id"], {}) api_format = model_info.get("api_format", "GENERIC") if api_format == "COHERE": # ── Cohere models (CohereChatRequest) ── chat_request = oci.generative_ai_inference.models.CohereChatRequest() chat_request.api_format = oci.generative_ai_inference.models.BaseChatRequest.API_FORMAT_COHERE if system_prompt: chat_request.preamble_override = system_prompt chat_request.message = message cohere_max = model_info.get("max_tokens", 4096) chat_request.max_tokens = min(int(gc.get("max_tokens", 4096)), cohere_max) chat_request.temperature = float(gc.get("temperature", 1)) chat_request.frequency_penalty = float(gc.get("frequency_penalty", 0)) chat_request.presence_penalty = float(gc.get("presence_penalty", 0)) chat_request.top_p = float(gc.get("top_p", 0.95)) chat_request.top_k = int(gc.get("top_k", 1)) if history: chat_history = [] for h in history: entry = oci.generative_ai_inference.models.CohereMessage() entry.role = "USER" if h["role"] == "user" else "CHATBOT" entry.message = h["content"] chat_history.append(entry) chat_request.chat_history = chat_history # Tool use support for Cohere if tools: cohere_tools = [] for t in tools: props = t.get("input_schema", {}).get("properties", {}) required = t.get("input_schema", {}).get("required", []) param_defs = {} for k, v in props.items(): pd = oci.generative_ai_inference.models.CohereParameterDefinition() pd.type = v.get("type", "str") pd.description = v.get("description", "") pd.is_required = k in required param_defs[k] = pd ct = oci.generative_ai_inference.models.CohereTool() ct.name = t["name"] ct.description = t.get("description", "") ct.parameter_definitions = param_defs if param_defs else None cohere_tools.append(ct) chat_request.tools = cohere_tools if tool_results_cohere: chat_request.tool_results = tool_results_cohere else: # ── Generic format (Meta Llama, Google, xAI, OpenAI) ── chat_request = oci.generative_ai_inference.models.GenericChatRequest() chat_request.api_format = oci.generative_ai_inference.models.BaseChatRequest.API_FORMAT_GENERIC provider = model_info.get("provider", "") is_openai = provider == "openai" is_reasoning = model_info.get("reasoning", False) # Clamp max_tokens to model limit model_max = model_info.get("max_tokens", 16384) requested_tokens = int(gc.get("max_tokens", 6000)) clamped_tokens = min(requested_tokens, model_max) # ── Parameter matrix per provider ── # OpenAI reasoning (o3, o4-mini): max_completion_tokens + reasoning_effort only if is_reasoning: chat_request.max_completion_tokens = clamped_tokens re = gc.get("reasoning_effort") if re and hasattr(chat_request, "reasoning_effort"): chat_request.reasoning_effort = re.upper() if isinstance(re, str) else re # Explicitly unset unsupported params chat_request.temperature = None chat_request.top_p = None chat_request.frequency_penalty = None chat_request.presence_penalty = None # OpenAI standard (GPT-4.1, GPT-5.x, GPT-4o): max_completion_tokens, temperature, top_p # freq/pres penalty only for models with "penalties":True (GPT-4.1, GPT-4.1-mini, GPT-4o) elif is_openai: chat_request.max_completion_tokens = clamped_tokens chat_request.temperature = float(gc.get("temperature", 1)) chat_request.top_p = float(gc.get("top_p", 0.95)) if model_info.get("penalties"): chat_request.frequency_penalty = float(gc.get("frequency_penalty", 0)) chat_request.presence_penalty = float(gc.get("presence_penalty", 0)) else: chat_request.frequency_penalty = None chat_request.presence_penalty = None # xAI Grok: max_tokens, temperature, top_p (no top_k, no freq/pres penalty) elif provider == "xai": chat_request.max_tokens = clamped_tokens chat_request.temperature = float(gc.get("temperature", 1)) chat_request.top_p = float(gc.get("top_p", 0.95)) # Explicitly unset unsupported params to prevent SDK serialization chat_request.frequency_penalty = None chat_request.presence_penalty = None # Meta Llama: max_tokens, temperature, top_p, top_k (no freq/pres penalty) # Google Gemini: max_tokens, temperature, top_p, top_k (no freq/pres penalty) else: chat_request.max_tokens = clamped_tokens chat_request.temperature = float(gc.get("temperature", 1)) chat_request.top_p = float(gc.get("top_p", 0.95)) chat_request.top_k = int(gc.get("top_k", 1)) # Explicitly unset unsupported params to prevent SDK serialization chat_request.frequency_penalty = None chat_request.presence_penalty = None # Log applied parameters params_log = f"provider={provider}, reasoning={is_reasoning}" params_log += f", max_tokens={getattr(chat_request, 'max_completion_tokens', None) or getattr(chat_request, 'max_tokens', None)}" params_log += f", temp={getattr(chat_request, 'temperature', None)}, top_p={getattr(chat_request, 'top_p', None)}" params_log += f", top_k={getattr(chat_request, 'top_k', None)}" params_log += f", freq_pen={getattr(chat_request, 'frequency_penalty', None)}, pres_pen={getattr(chat_request, 'presence_penalty', None)}" if is_reasoning: params_log += f", reasoning_effort={getattr(chat_request, 'reasoning_effort', None)}" log.info(f"GenAI params: {params_log}") messages = [] if system_prompt: sys_content = oci.generative_ai_inference.models.TextContent() sys_content.text = system_prompt sys_msg = oci.generative_ai_inference.models.Message() sys_msg.role = "SYSTEM" sys_msg.content = [sys_content] messages.append(sys_msg) if history: for h in history: content = oci.generative_ai_inference.models.TextContent() content.text = h["content"] msg = oci.generative_ai_inference.models.Message() msg.role = "USER" if h["role"] == "user" else "ASSISTANT" msg.content = [content] messages.append(msg) # Current user message (with optional multimodal attachments) content_parts = [] if attachments: for att in attachments: if att["type"] == "image": img_url = oci.generative_ai_inference.models.ImageUrl() img_url.url = att["data_uri"] img_url.detail = "AUTO" img_content = oci.generative_ai_inference.models.ImageContent() img_content.image_url = img_url content_parts.append(img_content) elif att["type"] == "document": doc_url = oci.generative_ai_inference.models.DocumentUrl() doc_url.url = att["data_uri"] doc_content = oci.generative_ai_inference.models.DocumentContent() doc_content.document_url = doc_url content_parts.append(doc_content) text_content = oci.generative_ai_inference.models.TextContent() text_content.text = message content_parts.append(text_content) user_message = oci.generative_ai_inference.models.Message() user_message.role = "USER" user_message.content = content_parts messages.append(user_message) # Append accumulated tool use loop messages (assistant+tool_calls → tool_results → ...) if extra_messages: messages.extend(extra_messages) chat_request.messages = messages # Tool definitions for Generic format if tools: generic_tools = [] for t in tools: fd = oci.generative_ai_inference.models.FunctionDefinition() fd.name = t["name"] fd.description = t.get("description", "") fd.parameters = t.get("input_schema") generic_tools.append(fd) chat_request.tools = generic_tools # Serving mode - resolve OCID: explicit model_ocid > catalog ocid for region > short model_id model_ref = gc.get("model_ocid") or "" if not model_ref: region = gc.get("genai_region", "") ocids = model_info.get("ocids", {}) model_ref = ocids.get(region) or gc["model_id"] log.info(f"GenAI call: model_id={gc.get('model_id')}, model_ref={model_ref[:60]}...") if gc.get("serving_type") == "DEDICATED" and gc.get("dedicated_endpoint_id"): chat_detail.serving_mode = oci.generative_ai_inference.models.DedicatedServingMode( endpoint_id=gc["dedicated_endpoint_id"] ) else: chat_detail.serving_mode = oci.generative_ai_inference.models.OnDemandServingMode( model_id=model_ref ) chat_detail.chat_request = chat_request chat_detail.compartment_id = gc["compartment_id"] # Execute chat_response = generative_ai_inference_client.chat(chat_detail) # Extract text and tool_calls from response resp = chat_response.data.chat_response if api_format == "COHERE": text = resp.text if hasattr(resp, 'text') else "" tool_calls = None if hasattr(resp, 'tool_calls') and resp.tool_calls: tool_calls = [] for tc in resp.tool_calls: tool_calls.append({ "id": tc.name, # Cohere uses name as identifier "name": tc.name, "arguments": tc.parameters if isinstance(tc.parameters, dict) else {} }) return (text or "", tool_calls, None) else: text = "" tool_calls = None tool_calls_raw = None # raw content list from assistant message for continuation if hasattr(resp, 'choices') and resp.choices: choice = resp.choices[0] if hasattr(choice, 'message') and choice.message: # Check for tool calls if hasattr(choice.message, 'tool_calls') and choice.message.tool_calls: tool_calls = [] for tc in choice.message.tool_calls: args = {} if hasattr(tc, 'arguments') and tc.arguments: try: args = json.loads(tc.arguments) if isinstance(tc.arguments, str) else tc.arguments except Exception as e: log.warning(f"Failed to parse tool call arguments: {e}"); args = {} tool_calls.append({ "id": tc.id if hasattr(tc, 'id') else tc.name, "name": tc.name if hasattr(tc, 'name') else "", "arguments": args }) # Preserve raw tool_calls for building assistant message in next iteration tool_calls_raw = choice.message.tool_calls # Extract text from content if hasattr(choice.message, 'content') and choice.message.content: contents = choice.message.content if isinstance(contents, str): text = contents elif isinstance(contents, list): text_parts = [] for c in contents: if isinstance(c, str): text_parts.append(c) elif hasattr(c, 'text') and c.text: text_parts.append(c.text) elif isinstance(c, dict) and c.get('text'): text_parts.append(c['text']) text = "\n".join(text_parts) else: text = str(contents) # Fallback: check reasoning_content (some models like GPT-5.2 use this) if not text and hasattr(resp, 'choices') and resp.choices: choice = resp.choices[0] if hasattr(choice, 'message') and choice.message: msg_obj = choice.message if hasattr(msg_obj, 'reasoning_content') and msg_obj.reasoning_content: rc = msg_obj.reasoning_content if isinstance(rc, str): text = rc elif isinstance(rc, list): text_parts = [] for c in rc: if isinstance(c, str): text_parts.append(c) elif hasattr(c, 'text') and c.text: text_parts.append(c.text) elif isinstance(c, dict) and c.get('text'): text_parts.append(c['text']) text = "\n".join(text_parts) else: text = str(rc) if text: log.info(f"GenAI: extracted text from reasoning_content ({len(text)} chars)") # Last resort: try choice.text if not text: if hasattr(choice, 'text') and choice.text: text = choice.text if not text and not tool_calls: raw_content = getattr(getattr(resp.choices[0], 'message', None), 'content', None) if hasattr(resp, 'choices') and resp.choices else None # Also dump message attrs for debugging msg_attrs = [] if hasattr(resp, 'choices') and resp.choices and hasattr(resp.choices[0], 'message') and resp.choices[0].message: msg_attrs = [a for a in dir(resp.choices[0].message) if not a.startswith('_')] reasoning = getattr(resp.choices[0].message, 'reasoning_content', None) if hasattr(resp, 'choices') and resp.choices and hasattr(resp.choices[0], 'message') else None refusal = getattr(resp.choices[0].message, 'refusal', None) if hasattr(resp, 'choices') and resp.choices and hasattr(resp.choices[0], 'message') else None finish_reason = getattr(resp.choices[0], 'finish_reason', None) if hasattr(resp, 'choices') and resp.choices else None choice_attrs = [a for a in dir(resp.choices[0]) if not a.startswith('_')] if hasattr(resp, 'choices') and resp.choices else [] log.warning(f"GenAI returned empty response. model={gc.get('model_id')}, finish_reason={finish_reason}, content_repr: {repr(raw_content)[:500]}, reasoning: {repr(reasoning)[:500] if reasoning else None}, refusal: {repr(refusal)[:200] if refusal else None}, choice_attrs: {choice_attrs}") return (text or "", tool_calls, tool_calls_raw) # ── RAG Helpers ─────────────────────────────────────────────────────────────── RAG_CONTEXT_TEMPLATE = """--- CONTEXTO RECUPERADO (Vector Search) --- {context} --- FIM DO CONTEXTO --- Pergunta do usuário: {question}""" RAG_DEFAULT_SYSTEM_PROMPT = """Você é um assistente RAG especializado em Oracle Cloud Infrastructure (OCI). ### Escopo e restrições - Responda **SOMENTE** perguntas relacionadas a Oracle Cloud (OCI), CIS Benchmark, CIS Report e itens de inventário OCI (IAM/AssetManagement/Networking/StorageBlock/FileStorageService/Objectstore/Compute/LoggingandMonitoring). - Se a pergunta não for desse escopo, recuse educadamente e peça uma pergunta dentro do tema. ### Bases vetorizadas disponíveis (contexto recuperado automaticamente) **Tabelas de findings (dados do scan da tenancy):** - **summaryreportcsvvector**: resumo geral do CIS report (compliance scores por seção) - **identityandaccess**: findings de IAM (users, policies, MFA, API keys) - **networking**: findings de rede (security lists, NSGs, VCNs) - **computeinstances**: findings de compute (instances, metadata, boot) - **loggingandmonitoring**: findings de logging (alarms, events, notifications) - **objectstorage**: findings de Object Storage (buckets, visibility, encryption) - **storageblockvolume**: findings de Block Volume (encryption, CMK) - **filestorageservice**: findings de File Storage (encryption, CMK) - **assetmanagement**: findings de Asset Management (compartments, tagging) **Tabelas de referência (conhecimento genérico):** - **cisrecom**: recomendações oficiais do CIS Benchmark — contém passos detalhados de remediação, auditoria e rationale para cada controle CIS - **engineerknowledgebase**: base de conhecimento complementar (blogs, documentações, PDFs) ### Quando usar RAG (dados armazenados) vs MCP Tools (tempo real) **Use RAG (contexto recuperado abaixo) quando o usuário:** - Perguntar sobre **relatórios já gerados**, resultados de scans anteriores, histórico - Quiser saber **como corrigir** um problema (remediação do cisrecom) - Pedir **resumo de compliance**, scores, comparações entre datas - Perguntar sobre **documentação, boas práticas, referências** (engineerknowledgebase) - Palavras-chave: "no último scan", "no report", "segundo o CIS", "como corrigir", "remediação", "recomendação" **Use MCP Tools (scan em tempo real) quando o usuário:** - Pedir para **verificar agora**, **validar agora**, **checar o estado atual** - Quiser dados **atualizados/em tempo real** da tenancy - Palavras-chave: "verifica agora", "valida agora", "estado atual", "scan", "checa", "como está hoje" - Se tiver dúvida se os dados do RAG estão desatualizados **Se não tiver certeza:** pergunte ao usuário se quer consultar os dados armazenados (mais rápido) ou verificar em tempo real (scan ao vivo). ### Hierarquia de fontes RAG (IMPORTANTE) 1. Para **identificar problemas e status**: use as tabelas de findings (identityandaccess, networking, etc.) 2. Para **como corrigir (remediação)**: use SEMPRE a tabela **cisrecom** como fonte principal — ela contém os passos oficiais do CIS 3. Para **informações complementares**: use **engineerknowledgebase** como plus, se tiver algo relevante a acrescentar 4. Para **resumo de compliance**: use **summaryreportcsvvector** - Se houver conflito entre fontes, priorize: findings (dados reais) > cisrecom (recomendações oficiais) > engineerknowledgebase (complementar) ### Tratamento temporal (datas de extração) - Cada documento possui um campo **date** no cabeçalho indicando quando os dados foram coletados. - **Por padrão, priorize SEMPRE os dados mais recentes** (data mais nova). - Se o usuário pedir comparação ou evolução, apresente os dados de ambas as datas lado a lado, indicando claramente qual é o mais recente e qual é o anterior. - Informe a data de extração nas respostas para que o usuário saiba de quando são os dados. ### Regras de fidelidade - Responda **somente** com informações suportadas por evidências do contexto recuperado. - Se não houver evidência suficiente: **"Não encontrei nas minhas bases"** e peça dados para refinar (Recommendation #, seção, recurso, OCID, etc.). - **Não invente** páginas, comandos, políticas, valores ou números. - Use somente **1–3 linhas** de evidência por item para manter respostas compactas. - **Sempre informe a data de extração** dos dados apresentados. ### Formato de resposta - **Tenancy:** - **Data de extração:** - **Recomendação — ** - **Seção/Capítulo:** … - **Nível/Tipo:** Manual/Automated (se disponível) - **O que isso exige (CIS):** breve explicação - **Recursos afetados:** listar recursos non-compliant com nome/OCID (se disponíveis no contexto) - **Como corrigir (remediação):** passos COMPLETOS e DETALHADOS - Passos via Console OCI (se estiverem nas evidências) - Passos via OCI CLI com comandos completos em code blocks (```bash) - Passos via API (se disponíveis) - Observações/alertas importantes - **Como auditar:** como verificar se a correção foi aplicada - **Fontes:** tabela e data de extração de cada informação usada ### Regras de formatação - Comandos CLI e policies devem ser apresentados em **code blocks** (```bash ou ```hcl) - Listar TODOS os passos de remediação disponíveis no contexto — nunca resumir ou omitir passos - Se a remediação tem passos via Console E via CLI, apresentar AMBOS - Usar tabelas markdown para listar recursos afetados quando houver múltiplos - Ser completo na remediação mas conciso nas explicações""" CONSULT_SYSTEM_PROMPT = """Você é um consultor de dados vetorizados. Sua função é responder perguntas com base nos documentos recuperados do banco vetorial. ### Regras - Responda **somente** com informações presentes nos documentos fornecidos. - Se não encontrar informação relevante, diga claramente. - Seja direto e objetivo. Apresente os dados de forma organizada. - Use markdown para formatação (negrito, listas, tabelas quando apropriado). - Cite de qual tabela/fonte veio cada informação. - Não invente dados que não estejam nos documentos.""" TF_DEFAULT_SYSTEM_PROMPT = """Você gera somente Terraform HCL para Oracle Cloud Infrastructure (OCI) usando o provider oracle/oci. Se pedirem AWS, Azure, GCP ou outro provider, recuse educadamente. ### Regras - Gere código production-ready, com variáveis, defaults sensatos, tags e outputs úteis. Evite ficar solicitando ao usuário para ficar interagindo a todo momento, entregue o que ele precisa e faça o mínimo de interações para entender o que o usuário precisa. Explique de forma clara o que foi gerado e o que foi arrumado para que fosse possível gerar o código do terraform. - Use sempre a sintaxe mais recente do provider OCI. - Não inclua `terraform { required_providers }` nem `provider "oci"` principal. - Em multi-região, use provider aliases para regiões adicionais e `provider = oci.alias` nos recursos. - Para novas gerações, gere sempre múltiplos arquivos: no mínimo `variables.tf`, 1 arquivo de recursos e `outputs.tf`. - Cada bloco deve começar com `// filename: nome.tf`. - Nunca declare o mesmo recurso (tipo + nome) em mais de um arquivo. - Use `var.compartment_id` nos recursos e declare sempre `variable "region"` em `variables.tf`. - Se houver "RECURSOS OCI EXISTENTES NO COMPARTMENT", reutilize recursos com data sources sempre que possível. - Se houver "ARQUIVOS TERRAFORM ATUAIS NO WORKSPACE", gere **SOMENTE os arquivos que precisam de correção**. Arquivos sem erro NÃO devem ser regenerados. Mantenha os mesmos nomes de arquivo. Se o erro está em `networking.tf`, gere APENAS `// filename: networking.tf` com o conteúdo corrigido. NUNCA regenere `variables.tf`, `outputs.tf` ou outros arquivos que não têm erro, a menos que o erro exija alteração neles (ex: variável faltando). - Sempre responda com blocos `hcl`, depois **Resource Plan** (`+` criar, `~` reutilizar), uma breve explicação e uma seção `✅ Validação`. - Valide: referências cruzadas, CIDRs, route tables, security lists, dependências, segurança e variáveis. - Use apenas resource types válidos do OCI. Exemplos corretos: - `oci_network_firewall_network_firewall` - `oci_network_firewall_network_firewall_policy` - `oci_core_drg_attachment` - `oci_core_drg_attachment_management` - `oci_core_drg_route_table` - `oci_core_drg_route_distribution` - `oci_core_drg_route_distribution_statement` - `oci_core_drg_route_table_route_rule` - `oci_core_remote_peering_connection` - Nunca invente recursos inexistentes. - Use sempre HCL multi-line, nunca blocos inline com múltiplos argumentos. ### Correção de erros (IMPORTANTE) - Quando o usuário envia um erro de plan/apply com arquivos existentes, gere SOMENTE o(s) arquivo(s) que corrigem o erro. - Exemplo: se o erro é em `service_gateways.tf`, retorne APENAS `// filename: service_gateways.tf` com o conteúdo corrigido. - Se a correção exige criar/alterar uma variável, inclua também `// filename: variables.tf` apenas com as variáveis novas/alteradas + todas existentes. - NUNCA regenere todos os arquivos do zero. Isso descarta o trabalho anterior e cria loops infinitos de correção. - Preserve os nomes de arquivo exatos. Não renomeie `drg_attachments_and_routes.tf` para `drg.tf`. ### Regras críticas - `oci_core_drg_route_distribution_statement` deve usar `match_criteria`. - Remote peering: APENAS UM LADO do par define `peer_id` e `peer_region_name`. O outro lado é criado SEM `peer_id`. Nunca gere dois RPCs apontando um para o outro (causa Cycle error). Exemplo: RPC_mad1 (sem peer_id) e RPC_mad3 (com peer_id = RPC_mad1.id, peer_region_name = var.region). - Associação de route table com subnet deve ser via `route_table_id` na subnet ou `oci_core_route_table_attachment`. - `oci_network_firewall_network_firewall` exige `network_firewall_policy_id`. - DRG attachments: use `oci_core_drg_attachment` para VCN attachments (tipo padrão). Use `oci_core_drg_attachment_management` SOMENTE para REMOTE_PEERING_CONNECTION (único tipo suportado para management). Nunca duplique resource type+name entre arquivos (ex: dois `oci_core_drg_attachment.vcn_app_x`). Sempre associe `drg_route_table_id` nos VCN attachments para garantir roteamento correto. ### Proibições absolutas — módulos e variáveis - NUNCA use `module` blocks. Este workspace é flat (um único diretório). Não existem subdiretórios `modules/`. Toda a infraestrutura deve ser definida diretamente com `resource` e `data` blocks. - NUNCA declare a mesma `variable` em mais de um arquivo. Todas as variáveis devem estar em `variables.tf` e SOMENTE lá. Os outros arquivos (.tf) apenas referenciam `var.nome`. - Antes de gerar, verifique se uma variável já foi declarada. Se já existe em `variables.tf`, NÃO redeclare. """ # ── Terraform OCI Resource Reference (generated at build time, updatable at runtime) ── _TF_RESOURCE_REF_PATH = "/data/oci_tf_resource_reference.txt" _tf_resource_ref_cache: dict = {"content": None, "mtime": 0} def _populate_tf_valid_types(): """Parse the OCI TF resource reference file and populate tf_valid_types table in SQLite.""" p = Path(_TF_RESOURCE_REF_PATH) if not p.exists(): return 0 content = p.read_text() import re as _re entries = [] current_category = "" for line in content.split('\n'): if line.startswith('## '): current_category = line[3:].strip() continue # Resource lines: - **oci_xxx** required: ... blocks: ... m = _re.match(r'^- \*\*(\w+)\*\*\s*(.*)', line) if m: name = m.group(1) rest = m.group(2).strip() required = "" blocks = "" # Split by "blocks:" to separate required from blocks blk_parts = rest.split('blocks:') if blk_parts[0].strip().startswith('required:'): required = blk_parts[0].replace('required:', '').strip() if len(blk_parts) > 1: blocks = blk_parts[1].strip() entries.append((name, 'resource', current_category, required, blocks)) continue # Data source lines (in "All Resource Types" or "All Data Sources" section): - oci_xxx m2 = _re.match(r'^- (\w+)\s*$', line) if m2: name = m2.group(1) if current_category.startswith('All Data'): entries.append((name, 'data', '', '', '')) elif current_category.startswith('All Resource'): entries.append((name, 'resource', '', '', '')) if not entries: return 0 with db() as c: c.execute("DELETE FROM tf_valid_types") # Insert detailed entries first (with required_args), then bulk lists with IGNORE to not overwrite detailed = [e for e in entries if e[3] or e[4]] # has required_args or blocks bulk = [e for e in entries if not e[3] and not e[4]] if detailed: c.executemany( "INSERT OR REPLACE INTO tf_valid_types (name, kind, category, required_args, blocks) VALUES (?,?,?,?,?)", detailed) if bulk: c.executemany( "INSERT OR IGNORE INTO tf_valid_types (name, kind, category, required_args, blocks) VALUES (?,?,?,?,?)", bulk) log.info(f"Populated tf_valid_types: {len(entries)} entries") return len(entries) def _validate_tf_resource_types(tf_code: str) -> list: """Validate resource/data types in generated TF code against the tf_valid_types table. Returns list of dicts with invalid types and suggestions.""" import re as _re from difflib import get_close_matches # Extract only resource type declarations (data sources are read-only and may not be in the schema) used_types = set() for m in _re.finditer(r'resource\s+"(\w+)"', tf_code): used_types.add(m.group(1)) if not used_types: return [] # Load valid types from DB with db() as c: rows = c.execute("SELECT name, kind, required_args FROM tf_valid_types").fetchall() valid_names = {r["name"] for r in rows} valid_required = {r["name"]: r["required_args"] for r in rows} if not valid_names: return [] # No reference loaded # Check each used type errors = [] for t in sorted(used_types): if t not in valid_names: # Find closest matches suggestions = get_close_matches(t, valid_names, n=3, cutoff=0.6) errors.append({ "type": t, "suggestions": suggestions, "message": f"Resource type '{t}' NÃO EXISTE no provider oracle/oci." }) return errors def _load_tf_resource_reference() -> str: """Load the OCI Terraform resource reference file, cached by mtime.""" try: p = Path(_TF_RESOURCE_REF_PATH) if not p.exists(): return "" mtime = p.stat().st_mtime if _tf_resource_ref_cache["content"] and _tf_resource_ref_cache["mtime"] == mtime: return _tf_resource_ref_cache["content"] content = p.read_text() _tf_resource_ref_cache["content"] = content _tf_resource_ref_cache["mtime"] = mtime log.info(f"Loaded TF resource reference: {len(content)} chars, {content.count(chr(10))} lines") return content except Exception as e: log.warning(f"Failed to load TF resource reference: {e}") return "" def _regenerate_tf_reference() -> dict: """Regenerate the OCI Terraform resource reference file.""" try: result = subprocess.run( ["python", "/app/gen_tf_reference.py"], capture_output=True, text=True, timeout=300 ) _tf_resource_ref_cache["content"] = None # invalidate cache if result.returncode == 0: ref = _load_tf_resource_reference() return {"ok": True, "output": result.stdout.strip(), "lines": ref.count('\n')} return {"ok": False, "error": result.stderr.strip()} except Exception as e: return {"ok": False, "error": str(e)} # ── Terraform Resource Docs (on-demand from GitHub, cached in SQLite) ── _TF_DOC_BASE_URL = "https://raw.githubusercontent.com/oracle/terraform-provider-oci/master/website/docs/r/" def _fetch_tf_resource_doc(slug: str) -> dict | None: """Fetch a single resource doc from GitHub, parse Example Usage + Argument Reference, cache in SQLite.""" import re as _re # Check cache first with db() as c: row = c.execute("SELECT example_usage, argument_ref FROM tf_resource_docs WHERE slug=?", (slug,)).fetchone() if row and (row["example_usage"] or row["argument_ref"]): return {"slug": slug, "example": row["example_usage"], "args": row["argument_ref"]} # Fetch from GitHub try: import urllib.request url = f"{_TF_DOC_BASE_URL}{slug}.html.markdown" req = urllib.request.Request(url, headers={"User-Agent": "oci-cis-agent/2.1"}) with urllib.request.urlopen(req, timeout=15) as resp: if resp.status != 200: return None md = resp.read().decode("utf-8") # Extract Example Usage section example = "" m = _re.search(r'## Example Usage\s*\n(.*?)(?=\n## )', md, _re.DOTALL) if m: example = m.group(1).strip() # Extract Argument Reference section args = "" m2 = _re.search(r'## Argument Reference\s*\n(.*?)(?=\n## )', md, _re.DOTALL) if m2: args = m2.group(1).strip() # Cache in SQLite subcategory = "" m3 = _re.search(r'subcategory:\s*"([^"]+)"', md) if m3: subcategory = m3.group(1) with db() as c: c.execute( "INSERT OR REPLACE INTO tf_resource_docs (slug, subcategory, example_usage, argument_ref) VALUES (?,?,?,?)", (slug, subcategory, example, args)) log.info(f"Fetched TF doc: {slug} (example={len(example)} chars, args={len(args)} chars)") return {"slug": slug, "example": example, "args": args} except Exception as e: log.warning(f"Failed to fetch TF doc for {slug}: {e}") return None def _get_tf_docs_for_resources(resource_types: list[str]) -> str: """Given a list of oci_xxx resource types, fetch their docs and build a context string.""" docs_parts = [] for rtype in resource_types[:10]: # limit to 10 to keep context manageable # Convert oci_core_vcn -> core_vcn slug = rtype.replace("oci_", "", 1) if rtype.startswith("oci_") else rtype doc = _fetch_tf_resource_doc(slug) if doc and (doc["example"] or doc["args"]): part = f"#### {rtype}\n" if doc["example"]: part += f"**Example Usage:**\n{doc['example']}\n\n" if doc["args"]: part += f"**Arguments:**\n{doc['args']}\n" docs_parts.append(part) if not docs_parts: return "" return "### Documentação Oficial dos Recursos (registry.terraform.io)\n" + \ "Use exatamente os argumentos e estrutura mostrados nos exemplos abaixo.\n\n" + \ "\n---\n".join(docs_parts) def _detect_tf_resource_types(text: str) -> list[str]: """Detect OCI resource types mentioned or implied in a user message + conversation.""" import re as _re # Direct mentions of oci_ resource types explicit = set(_re.findall(r'\boci_\w+', text)) # Keyword-to-resource mapping for common infra requests keyword_map = { 'vcn': ['oci_core_vcn', 'oci_core_subnet', 'oci_core_internet_gateway', 'oci_core_nat_gateway', 'oci_core_route_table', 'oci_core_security_list'], 'subnet': ['oci_core_subnet'], 'instance': ['oci_core_instance', 'oci_core_instance_configuration'], 'compute': ['oci_core_instance'], 'load.?balancer': ['oci_load_balancer_load_balancer', 'oci_load_balancer_backend_set', 'oci_load_balancer_listener'], 'nlb': ['oci_network_load_balancer_network_load_balancer', 'oci_network_load_balancer_backend_set', 'oci_network_load_balancer_listener'], 'firewall': ['oci_network_firewall_network_firewall', 'oci_network_firewall_network_firewall_policy'], 'drg': ['oci_core_drg', 'oci_core_drg_attachment', 'oci_core_drg_route_table'], 'peering|rpc': ['oci_core_remote_peering_connection'], 'bucket|object.?storage': ['oci_objectstorage_bucket', 'oci_objectstorage_namespace_metadata'], 'autonomous|adb': ['oci_database_autonomous_database'], 'db.?system': ['oci_database_db_system'], 'mysql': ['oci_mysql_mysql_db_system'], 'oke|kubernetes|cluster': ['oci_containerengine_cluster', 'oci_containerengine_node_pool'], 'dns': ['oci_dns_zone', 'oci_dns_record'], 'vault|kms': ['oci_kms_vault', 'oci_kms_key'], 'waf': ['oci_waf_web_app_firewall', 'oci_waf_web_app_firewall_policy'], 'api.?gateway': ['oci_apigateway_gateway', 'oci_apigateway_deployment'], 'function|serverless': ['oci_functions_application', 'oci_functions_function'], 'nsg': ['oci_core_network_security_group', 'oci_core_network_security_group_security_rule'], 'bastion': ['oci_bastion_bastion', 'oci_bastion_session'], 'file.?storage': ['oci_file_storage_file_system', 'oci_file_storage_mount_target', 'oci_file_storage_export'], 'block.?volume': ['oci_core_volume', 'oci_core_volume_attachment'], 'policy|iam': ['oci_identity_policy', 'oci_identity_compartment', 'oci_identity_group', 'oci_identity_dynamic_group'], } text_lower = text.lower() for pattern, resources in keyword_map.items(): if _re.search(pattern, text_lower): explicit.update(resources) return sorted(explicit) def _get_adb_connection(cfg: dict): """Create an oracledb connection from an adb_vector_configs row.""" import oracledb params = {"user": cfg["username"], "password": _dec(cfg["password_enc"]), "dsn": cfg["dsn"]} if cfg["use_mtls"] and cfg.get("wallet_dir"): params["config_dir"] = cfg["wallet_dir"] params["wallet_location"] = cfg["wallet_dir"] params["wallet_password"] = _dec(cfg["wallet_password_enc"]) if cfg.get("wallet_password_enc") else "" params["tcp_connect_timeout"] = 15 # 15s connection timeout return oracledb.connect(**params) def _resolve_embed_config(oci_config_id: str = None, genai_cfg: dict = None, user_id: str = None) -> dict: """Resolve embedding config from genai_cfg or oci_config. Scoped to user_id when provided. Returns dict with oci_config_id, endpoint, genai_region, compartment_id.""" if genai_cfg: return genai_cfg if oci_config_id: with db() as c: # Try linked genai config first (scoped to user if provided) if user_id: gc = c.execute("SELECT * FROM genai_configs WHERE oci_config_id=? AND user_id=? ORDER BY is_default DESC, created_at DESC", (oci_config_id, user_id)).fetchone() else: gc = c.execute("SELECT * FROM genai_configs WHERE oci_config_id=? ORDER BY is_default DESC, created_at DESC", (oci_config_id,)).fetchone() if gc: return dict(gc) # Fallback: build from oci_config directly oc = c.execute("SELECT * FROM oci_configs WHERE id=?", (oci_config_id,)).fetchone() if oc: return { "oci_config_id": oc["id"], "genai_region": oc["region"], "endpoint": f"https://inference.generativeai.{oc['region']}.oci.oraclecloud.com", "compartment_id": _safe_dec(dict(oc).get("compartment_id") or oc["tenancy_ocid"]), } # Last resort: user's own genai/oci config (never cross-user) with db() as c: if user_id: gc = c.execute("SELECT * FROM genai_configs WHERE user_id=? ORDER BY is_default DESC, created_at DESC", (user_id,)).fetchone() else: gc = c.execute("SELECT * FROM genai_configs ORDER BY is_default DESC, created_at DESC").fetchone() if gc: return dict(gc) if user_id: oc = c.execute("SELECT * FROM oci_configs WHERE user_id=? ORDER BY created_at DESC", (user_id,)).fetchone() else: oc = c.execute("SELECT * FROM oci_configs ORDER BY created_at DESC").fetchone() if oc: return { "oci_config_id": oc["id"], "genai_region": oc["region"], "endpoint": f"https://inference.generativeai.{oc['region']}.oci.oraclecloud.com", "compartment_id": _safe_dec(dict(oc).get("compartment_id") or oc["tenancy_ocid"]), } raise HTTPException(400, "Nenhuma credencial OCI/GenAI configurada para gerar embeddings.") def _embed_text(text: str, genai_cfg: dict, embedding_model_id: str, max_chars: int = 8000) -> list: """Generate embedding using OCI GenAI embed endpoint.""" import oci # Truncate text to fit model token limit (~8192 tokens) if len(text) > max_chars: text = text[:max_chars] config_path = str(OCI_DIR / genai_cfg["oci_config_id"] / "config") config = oci.config.from_file(config_path, "DEFAULT") endpoint = genai_cfg.get("endpoint") or f"https://inference.generativeai.{genai_cfg.get('genai_region','us-ashburn-1')}.oci.oraclecloud.com" client = oci.generative_ai_inference.GenerativeAiInferenceClient( config=config, service_endpoint=endpoint, retry_strategy=oci.retry.NoneRetryStrategy(), timeout=(10, 120) ) embed_detail = oci.generative_ai_inference.models.EmbedTextDetails() embed_detail.inputs = [text] emb_info = EMBEDDING_MODELS.get(embedding_model_id, {}) region = genai_cfg.get("genai_region", "") emb_ref = emb_info.get("ocids", {}).get(region) or embedding_model_id embed_detail.serving_mode = oci.generative_ai_inference.models.OnDemandServingMode(model_id=emb_ref) embed_detail.compartment_id = genai_cfg.get("compartment_id", "") embed_detail.truncate = "END" embed_detail.input_type = "SEARCH_QUERY" response = client.embed_text(embed_detail) return response.data.embeddings[0] _DIM_TO_MODEL = {1536: "openai.text-embedding-3-small", 3072: "openai.text-embedding-3-large"} def _get_table_embedding_dim(cfg: dict, table_name: str) -> int: """Detect the embedding dimension of a table by sampling one row, or from column definition if empty.""" conn = _get_adb_connection(cfg) try: cur = conn.cursor() # Try from data first cur.execute(f'SELECT VECTOR_DIMENSION_COUNT(EMBEDDING) FROM "{table_name}" FETCH FIRST 1 ROWS ONLY') row = cur.fetchone() if row and row[0]: cur.close() return int(row[0]) # Table is empty — check column definition from DDL try: cur.execute(f"""SELECT DBMS_METADATA.GET_DDL('TABLE', :1) FROM DUAL""", [table_name]) ddl_row = cur.fetchone() cur.close() if ddl_row and ddl_row[0]: import re m = re.search(r'VECTOR\((\d+)', str(ddl_row[0])) if m: return int(m.group(1)) except Exception: pass return 0 except Exception as e: log.warning(f"Failed to get embedding dimension for table '{table_name}': {e}") return 0 finally: conn.close() _GLOBAL_TABLES = {"cisrecom", "engineerknowledgebase"} # Tables without tenancy filter (generic knowledge) def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name: str = None, tenancy: str = None, text_filter: str = None, user_id: str = None) -> list: """Search ADB vector store using cosine similarity. Returns top-K documents. If tenancy is provided and table is not global, filters by tenancy. If user_id is provided and table is not global, filters by user_id (legacy docs without user_id are included). If text_filter is provided, also filters TEXT content with LIKE.""" import array table_name = table_name or cfg.get("table_name", "") conn = _get_adb_connection(cfg) try: conn.call_timeout = 30000 # 30s query timeout (milliseconds) cur = conn.cursor() vec = array.array('f', query_embedding) limit = int(top_k) is_non_global = table_name.lower() not in _GLOBAL_TABLES use_tenancy_filter = tenancy and is_non_global use_user_filter = user_id and is_non_global # Build WHERE clauses conditions = [] params = [vec] param_idx = 2 if use_tenancy_filter: conditions.append(f"JSON_VALUE(METADATA, '$.tenancy') = :{param_idx}") params.append(tenancy) param_idx += 1 if use_user_filter: conditions.append(f"(JSON_VALUE(METADATA, '$.user_id') = :{param_idx} OR JSON_VALUE(METADATA, '$.user_id') IS NULL)") params.append(user_id) param_idx += 1 if text_filter: conditions.append(f"TEXT LIKE :{param_idx}") params.append(f"%{text_filter}%") param_idx += 1 where = f"WHERE {' AND '.join(conditions)}" if conditions else "" cur.execute(f""" SELECT ID, TEXT, METADATA, VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance FROM "{table_name}" {where} ORDER BY distance ASC FETCH FIRST {limit} ROWS ONLY """, params) results = [] for row in cur: content = row[1] if hasattr(content, 'read'): content = content.read() results.append({ "id": row[0], "content": content or "", "metadata": row[2], "distance": float(row[3]) }) cur.close() return results finally: conn.close() # ── Table relevance mapping for smart skip ── _TABLE_KEYWORDS = { "identityandaccess": {"iam", "user", "users", "policy", "policies", "mfa", "api key", "group", "password", "credential", "authentication", "identity", "access"}, "networking": {"network", "vcn", "subnet", "security list", "nsg", "firewall", "route", "gateway", "nat", "internet", "port", "ingress", "egress", "cidr", "ip"}, "computeinstances": {"compute", "instance", "vm", "boot", "metadata", "secure boot", "encryption"}, "loggingandmonitoring": {"log", "logging", "monitor", "alarm", "event", "notification", "audit", "cloud guard", "flow"}, "objectstorage": {"bucket", "object storage", "storage", "visibility", "public"}, "storageblockvolume": {"block volume", "volume", "disk", "cmk", "encryption"}, "filestorageservice": {"file storage", "file system", "mount", "nfs"}, "assetmanagement": {"compartment", "tag", "asset", "resource"}, "summaryreportcsvvector": {"summary", "score", "compliance", "report", "overview", "total"}, "cisrecom": {"remediation", "fix", "correct", "resolve", "recommendation", "how to", "como corrigir", "remediação"}, "engineerknowledgebase": {"documentation", "guide", "tutorial", "blog", "best practice", "knowledge"}, } def _relevant_tables(query: str, tables: list[str]) -> list[str]: """Filter tables to only those relevant to the query. Always includes cisrecom and engineerknowledgebase.""" q = query.lower() # If query mentions a CIS number, include all tables (the text filter handles precision) import re as _re if _re.search(r'cis\s*\d+\.\d+', q): return tables relevant = [] for tbl in tables: tbl_lower = tbl.lower() # Always include global knowledge tables if tbl_lower in _GLOBAL_TABLES or tbl_lower == "summaryreportcsvvector": relevant.append(tbl) continue keywords = _TABLE_KEYWORDS.get(tbl_lower, set()) if any(kw in q for kw in keywords): relevant.append(tbl) # If no specific table matched, search all (generic question) return relevant if len(relevant) > 2 else tables def _vector_search_multi(cfg: dict, query_embedding: list, tables: list[str], top_k_per_table: int = 3, tenancy: str = None, text_filter: str = None, user_id: str = None) -> list: """Search multiple tables using a SINGLE ADB connection. Returns all documents with source.""" import array conn = _get_adb_connection(cfg) all_results = [] try: conn.call_timeout = 30000 vec = array.array('f', query_embedding) for table_name in tables: try: cur = conn.cursor() limit = int(top_k_per_table) is_non_global = table_name.lower() not in _GLOBAL_TABLES use_tenancy = tenancy and is_non_global use_user_filter = user_id and is_non_global conditions = [] params = [vec] param_idx = 2 if use_tenancy: conditions.append(f"JSON_VALUE(METADATA, '$.tenancy') = :{param_idx}") params.append(tenancy) param_idx += 1 if use_user_filter: conditions.append(f"(JSON_VALUE(METADATA, '$.user_id') = :{param_idx} OR JSON_VALUE(METADATA, '$.user_id') IS NULL)") params.append(user_id) param_idx += 1 tbl_filter = text_filter if (text_filter and is_non_global) else None if tbl_filter: conditions.append(f"TEXT LIKE :{param_idx}") params.append(f"%{tbl_filter}%") param_idx += 1 where = f"WHERE {' AND '.join(conditions)}" if conditions else "" cur.execute(f""" SELECT ID, TEXT, METADATA, VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance FROM "{table_name}" {where} ORDER BY distance ASC FETCH FIRST {limit} ROWS ONLY """, params) for row in cur: content = row[1] if hasattr(content, 'read'): content = content.read() all_results.append({ "id": row[0], "content": content or "", "metadata": row[2], "distance": float(row[3]), "source": table_name, }) cur.close() except Exception as e: log.warning(f"Multi-search failed for {table_name}: {str(e)[:120]}") finally: conn.close() return all_results def _enrich_doc_content(doc: dict) -> str: """Extract the best text content from a document, checking metadata.text as fallback.""" content = doc.get("content", "") meta = doc.get("metadata", "") if isinstance(meta, str) and meta: try: meta = json.loads(meta) except Exception as e: log.warning(f"Failed to parse document metadata JSON: {e}") meta = {} if isinstance(meta, dict): # If TEXT column is short/empty, use metadata.text instead meta_text = meta.get("text", "") if meta_text and len(meta_text) > len(content): header_parts = [] if meta.get("recommendationNumber"): header_parts.append(f"Recommendation: {meta['recommendationNumber']}") if meta.get("chapter"): header_parts.append(f"Chapter: {meta['chapter']}") if meta.get("header"): header_parts.append(meta["header"]) header = " | ".join(header_parts) content = (header + "\n" + meta_text) if header else meta_text # Also enrich with section/tenancy info if available elif meta.get("section") or meta.get("tenancy"): extra = [] if meta.get("tenancy"): extra.append(f"Tenancy: {meta['tenancy']}") if meta.get("section"): extra.append(f"Section: {meta['section']}") if extra: content = " | ".join(extra) + "\n" + content return content def _build_rag_context(documents: list, max_total_chars: int = 12000) -> str: """Format retrieved documents into a context string for the LLM prompt. Includes extract_date from metadata for temporal awareness. Limits total context to max_total_chars to prevent token overflow.""" if not documents: return "" parts = [] total = 0 max_per_doc = max_total_chars // min(len(documents), 8) for i, doc in enumerate(documents, 1): source = doc.get("source", "unknown") dist = doc.get("distance", 0) # Extract date from metadata for temporal context extract_date = "" meta_raw = doc.get("metadata", "") if isinstance(meta_raw, str) and meta_raw: try: meta_obj = json.loads(meta_raw) extract_date = meta_obj.get("extract_date", "") or meta_obj.get("report_date", "") except Exception: pass content = _enrich_doc_content(doc) if len(content) > max_per_doc: content = content[:max_per_doc] + "..." date_tag = f" | date: {extract_date}" if extract_date else "" part = f"[Doc {i} | {source}{date_tag} | relevance: {1 - dist:.2f}]\n{content}" if total + len(part) > max_total_chars: break parts.append(part) total += len(part) return "\n\n---\n\n".join(parts) def _get_active_adb_configs(user_id: str) -> list[dict]: """Get all active ADB vector configs accessible to this user (own + global). No cross-user fallback.""" with db() as c: rows = c.execute( "SELECT * FROM adb_vector_configs WHERE is_active=1 AND (user_id=? OR is_global=1) ORDER BY is_global DESC, created_at DESC", (user_id,) ).fetchall() return [dict(r) for r in rows] def _get_tables_for_config(adb_config_id: str, active_only: bool = True) -> list[dict]: """Get all registered vector tables for an ADB config.""" with db() as c: sql = "SELECT * FROM adb_vector_tables WHERE adb_config_id=?" if active_only: sql += " AND is_active=1" sql += " ORDER BY created_at ASC" rows = c.execute(sql, (adb_config_id,)).fetchall() return [dict(r) for r in rows] # ── MCP Servers ─────────────────────────────────────────────────────────────── @app.post("/api/mcp/servers") async def register_mcp(req: MCPServerReq, u=Depends(require("admin","user"))): mid = str(uuid.uuid4()) is_global_val = 1 if (req.is_global and u["role"] == "admin") else 0 with db() as c: c.execute( "INSERT INTO mcp_servers (id,user_id,name,description,server_type,command,args,env_vars,url,module_path,tools,linked_adb_id,is_global) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", (mid, u["id"], req.name, req.description, req.server_type, req.command, json.dumps(req.args) if req.args else None, json.dumps(req.env_vars) if req.env_vars else None, req.url, req.module_path, json.dumps(req.tools) if req.tools else None, req.linked_adb_id, is_global_val)) _audit(u["id"], u["username"], "register_mcp", mid, req.name) _config_log("mcp", mid, req.name, "success", "save", f"MCP registrado: {req.name} ({req.server_type})", u["id"], u["username"]) return {"id": mid, "name": req.name, "server_type": req.server_type} @app.get("/api/mcp/servers") async def list_mcp(u=Depends(current_user)): with db() as c: if u["role"]=="admin": rows=c.execute("SELECT * FROM mcp_servers ORDER BY created_at DESC").fetchall() else: rows=c.execute("SELECT * FROM mcp_servers WHERE user_id=? OR is_global=1 ORDER BY created_at DESC",(u["id"],)).fetchall() res = [] for r in rows: d = dict(r) for k in ("args","env_vars","tools"): if d.get(k): try: d[k] = json.loads(d[k]) except Exception as e: log.warning(f"Failed to parse MCP server field '{k}': {e}") res.append(d) return res @app.delete("/api/mcp/servers/{mid}") async def del_mcp(mid: str, u=Depends(require("admin","user"))): _verify_config_access("mcp", mid, u, require_owner=True) with db() as c: c.execute("DELETE FROM mcp_servers WHERE id=?", (mid,)) return {"ok": True} @app.put("/api/mcp/servers/{mid}") async def update_mcp(mid: str, req: MCPServerReq, u=Depends(require("admin","user"))): with db() as c: existing = c.execute("SELECT * FROM mcp_servers WHERE id=?", (mid,)).fetchone() if not existing: raise HTTPException(404) if u["role"] != "admin" and existing["user_id"] != u["id"]: raise HTTPException(403) is_global_val = 1 if (req.is_global and u["role"] == "admin") else (existing["is_global"] if u["role"] != "admin" else (1 if req.is_global else 0)) with db() as c: c.execute( "UPDATE mcp_servers SET name=?,description=?,server_type=?,command=?,args=?,env_vars=?,url=?,tools=?,linked_adb_id=?,is_global=? WHERE id=?", (req.name, req.description, req.server_type, req.command, json.dumps(req.args) if req.args else None, json.dumps(req.env_vars) if req.env_vars else None, req.url, json.dumps(req.tools) if req.tools else None, req.linked_adb_id, is_global_val, mid)) _audit(u["id"], u["username"], "update_mcp", mid, req.name) _config_log("mcp", mid, req.name, "success", "save", f"MCP atualizado: {req.name} ({req.server_type})", u["id"], u["username"]) return {"id": mid, "name": req.name, "server_type": req.server_type} @app.put("/api/mcp/servers/{mid}/toggle") async def toggle_mcp(mid: str, u=Depends(require("admin","user"))): _verify_config_access("mcp", mid, u, require_owner=True) with db() as c: cur = c.execute("SELECT is_active FROM mcp_servers WHERE id=?",(mid,)).fetchone() if not cur: raise HTTPException(404) c.execute("UPDATE mcp_servers SET is_active=? WHERE id=?",(0 if cur["is_active"] else 1, mid)) return {"ok": True, "is_active": not cur["is_active"]} @app.post("/api/mcp/servers/{mid}/upload") async def upload_mcp_file(mid: str, file: UploadFile = File(...), u=Depends(require("admin","user"))): await _validate_upload(file, {".py", ".js", ".ts", ".json", ".yaml", ".yml", ".toml", ".sh"}) file.file.seek(0) sdir = MCP_DIR / mid; sdir.mkdir(parents=True, exist_ok=True) fp = sdir / file.filename fp.write_bytes(await file.read()) with db() as c: c.execute("UPDATE mcp_servers SET module_path=? WHERE id=?", (str(fp), mid)) return {"ok": True, "path": str(fp)} @app.put("/api/mcp/servers/{mid}/link-adb") async def link_mcp_adb(mid: str, adb_id: str = Query(...), u=Depends(require("admin","user"))): with db() as c: c.execute("UPDATE mcp_servers SET linked_adb_id=? WHERE id=?", (adb_id, mid)) return {"ok": True, "mcp_id": mid, "linked_adb_id": adb_id} @app.post("/api/mcp/servers/{mid}/discover-tools") async def discover_mcp_tools(mid: str, u=Depends(require("admin","user"))): """Connect to MCP server and discover available tools.""" with db() as c: row = c.execute("SELECT * FROM mcp_servers WHERE id=?", (mid,)).fetchone() if not row: raise HTTPException(404) mcp_srv = dict(row) try: discovered = await _discover_mcp_tools(mcp_srv) except Exception as e: raise HTTPException(500, f"Erro ao descobrir tools: {str(e)[:500]}") # Merge with existing tools (keep manually added ones) existing = [] if mcp_srv.get("tools"): try: existing = json.loads(mcp_srv["tools"]) if isinstance(mcp_srv["tools"], str) else mcp_srv["tools"] except Exception as e: log.warning(f"Failed to parse existing MCP tools JSON: {e}") existing_names = {t["name"] for t in existing if isinstance(t, dict)} for dt in discovered: if dt["name"] not in existing_names: existing.append(dt) else: # Update existing tool with discovered schema for i, et in enumerate(existing): if isinstance(et, dict) and et["name"] == dt["name"]: existing[i] = dt break with db() as c: c.execute("UPDATE mcp_servers SET tools=? WHERE id=?", (json.dumps(existing), mid)) _audit(u["id"], u["username"], "discover_mcp_tools", mid, f"{len(discovered)} tools found") return {"ok": True, "discovered": len(discovered), "total": len(existing), "tools": existing} @app.put("/api/mcp/servers/{mid}/tools") async def update_mcp_tools(mid: str, req: dict, u=Depends(require("admin","user"))): """Manually update tools for an MCP server.""" with db() as c: row = c.execute("SELECT id FROM mcp_servers WHERE id=?", (mid,)).fetchone() if not row: raise HTTPException(404) tools = req.get("tools", []) if not isinstance(tools, list): raise HTTPException(400, "tools must be a list") for t in tools: if not isinstance(t, dict) or "name" not in t: raise HTTPException(400, "Each tool must have a 'name' field") with db() as c: c.execute("UPDATE mcp_servers SET tools=? WHERE id=?", (json.dumps(tools), mid)) return {"ok": True, "tools": tools} async def _discover_mcp_tools(mcp_srv: dict) -> list[dict]: """Connect to MCP server and list available tools.""" from mcp import ClientSession if mcp_srv["server_type"] in ("stdio", "module"): from mcp.client.stdio import stdio_client, StdioServerParameters cmd = mcp_srv.get("command") or "python3" args_raw = mcp_srv.get("args") args = json.loads(args_raw) if isinstance(args_raw, str) else (args_raw or []) env_raw = mcp_srv.get("env_vars") env = json.loads(env_raw) if isinstance(env_raw, str) else (env_raw or None) params = StdioServerParameters(command=cmd, args=args, env=env) async with stdio_client(params) as streams: async with ClientSession(*streams) as session: await session.initialize() result = await session.list_tools() return [{"name": t.name, "description": t.description or "", "input_schema": t.inputSchema if hasattr(t, 'inputSchema') else {}} for t in result.tools] elif mcp_srv["server_type"] == "sse": from mcp.client.sse import sse_client async with sse_client(mcp_srv["url"]) as streams: async with ClientSession(*streams) as session: await session.initialize() result = await session.list_tools() return [{"name": t.name, "description": t.description or "", "input_schema": t.inputSchema if hasattr(t, 'inputSchema') else {}} for t in result.tools] return [] async def _execute_mcp_tool(mcp_srv: dict, tool_name: str, arguments: dict) -> str: """Connect to MCP server and execute a specific tool.""" from mcp import ClientSession if mcp_srv["server_type"] in ("stdio", "module"): from mcp.client.stdio import stdio_client, StdioServerParameters cmd = mcp_srv.get("command") or "python3" args_raw = mcp_srv.get("args") args = json.loads(args_raw) if isinstance(args_raw, str) else (args_raw or []) env_raw = mcp_srv.get("env_vars") env = json.loads(env_raw) if isinstance(env_raw, str) else (env_raw or None) params = StdioServerParameters(command=cmd, args=args, env=env) async with stdio_client(params) as streams: async with ClientSession(*streams) as session: await session.initialize() result = await session.call_tool(tool_name, arguments) parts = [] for c in result.content: if hasattr(c, 'text'): parts.append(c.text) elif hasattr(c, 'data'): parts.append(str(c.data)) return "\n".join(parts) if parts else "Tool executed successfully (no output)" elif mcp_srv["server_type"] == "sse": from mcp.client.sse import sse_client async with sse_client(mcp_srv["url"]) as streams: async with ClientSession(*streams) as session: await session.initialize() result = await session.call_tool(tool_name, arguments) parts = [] for c in result.content: if hasattr(c, 'text'): parts.append(c.text) elif hasattr(c, 'data'): parts.append(str(c.data)) return "\n".join(parts) if parts else "Tool executed successfully (no output)" raise ValueError(f"Unsupported MCP server type: {mcp_srv['server_type']}") def _get_active_mcp_tools(user_id: str) -> list[dict]: """Get all tools from active MCP servers for a user, with server reference.""" with db() as c: rows = c.execute( "SELECT * FROM mcp_servers WHERE is_active=1 AND (user_id=? OR is_global=1)", (user_id,) ).fetchall() result = [] for r in rows: srv = dict(r) tools_raw = srv.get("tools") if not tools_raw: continue try: tools = json.loads(tools_raw) if isinstance(tools_raw, str) else tools_raw except Exception as e: log.warning(f"Failed to parse tools JSON for MCP server '{srv.get('name', '?')}': {e}") continue for t in tools: if isinstance(t, dict) and t.get("name"): result.append({"server": srv, "tool": t}) return result # ── ADB Vector DB Config ───────────────────────────────────────────────────── @app.post("/api/adb/parse-wallet") async def parse_wallet(wallet: UploadFile = File(...), u=Depends(require("admin","user"))): """Extract DSN names from tnsnames.ora inside wallet ZIP (temporary parse, no save).""" await _validate_upload(wallet, {".zip"}) wallet.file.seek(0) import zipfile, tempfile try: data = await wallet.read() with tempfile.TemporaryDirectory() as tmp: zp = Path(tmp) / "wallet.zip" zp.write_bytes(data) with zipfile.ZipFile(str(zp), 'r') as z: z.extractall(tmp) tns = Path(tmp) / "tnsnames.ora" if not tns.exists(): raise HTTPException(400, "Wallet ZIP não contém tnsnames.ora") tns_text = tns.read_text(errors="ignore") dsn_names = re.findall(r'^(\w[\w\-.]*)\s*=', tns_text, re.MULTILINE) if not dsn_names: raise HTTPException(400, "Nenhum DSN encontrado no tnsnames.ora") return {"dsn_names": dsn_names, "files": [n for n in os.listdir(tmp) if n != "wallet.zip"]} except zipfile.BadZipFile: raise HTTPException(400, "Arquivo não é um ZIP válido") @app.post("/api/adb/config") async def save_adb( config_name: str = Form(...), dsn: str = Form(...), username: str = Form(...), password: str = Form(...), wallet_password: str = Form(""), table_name: str = Form(""), use_mtls: str = Form("true"), genai_config_id: str = Form(""), embedding_model_id: str = Form("cohere.embed-v4.0"), is_global: str = Form("false"), wallet: Optional[UploadFile] = File(None), u=Depends(require("admin","user")) ): if wallet and wallet.filename: await _validate_upload(wallet, {".zip"}) wallet.file.seek(0) vid = str(uuid.uuid4()) use_mtls_bool = use_mtls.lower() in ("true", "1", "yes") is_global_val = 1 if (is_global.lower() in ("true", "1", "yes") and u["role"] == "admin") else 0 with db() as c: c.execute( "INSERT INTO adb_vector_configs (id,user_id,config_name,dsn,username,password_enc,wallet_password_enc,table_name,use_mtls,genai_config_id,embedding_model_id,is_global) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", (vid, u["id"], config_name, dsn, username, _enc(password), _enc(wallet_password) if wallet_password else None, table_name, int(use_mtls_bool), genai_config_id or None, embedding_model_id, is_global_val)) # Auto-save wallet if provided if wallet and wallet.filename: import zipfile wdir = WALLET_DIR / vid; wdir.mkdir(parents=True, exist_ok=True) zp = wdir / "wallet.zip" zp.write_bytes(await wallet.read()) with zipfile.ZipFile(str(zp), 'r') as z: z.extractall(str(wdir)) with db() as c: c.execute("UPDATE adb_vector_configs SET wallet_dir=? WHERE id=?", (str(wdir), vid)) _config_log("adb", vid, config_name, "success", "upload", f"Wallet enviada com a conexão", u["id"], u["username"]) _audit(u["id"], u["username"], "save_adb_config", vid, config_name) _config_log("adb", vid, config_name, "success", "save", f"Conexão salva: {config_name} ({dsn})", u["id"], u["username"]) return {"id": vid, "config_name": config_name} @app.post("/api/adb/{vid}/upload-wallet") async def upload_wallet(vid: str, wallet: UploadFile = File(...), u=Depends(require("admin","user"))): _verify_config_access("adb", vid, u, require_owner=True) await _validate_upload(wallet, {".zip"}) wallet.file.seek(0) cname = None with db() as c: row = c.execute("SELECT config_name FROM adb_vector_configs WHERE id=?", (vid,)).fetchone() if row: cname = row["config_name"] try: wdir = WALLET_DIR / vid; wdir.mkdir(parents=True, exist_ok=True) zp = wdir / "wallet.zip" zp.write_bytes(await wallet.read()) import zipfile with zipfile.ZipFile(str(zp), 'r') as z: z.extractall(str(wdir)) with db() as c: c.execute("UPDATE adb_vector_configs SET wallet_dir=? WHERE id=?", (str(wdir), vid)) files = os.listdir(str(wdir)) _config_log("adb", vid, cname, "success", "upload", f"Wallet enviada: {', '.join(files)}", u["id"], u["username"]) return {"ok": True, "wallet_dir": str(wdir), "files": files} except Exception as e: _config_log("adb", vid, cname, "error", "upload", str(e)[:500], u["id"], u["username"]) raise HTTPException(500, str(e)[:500]) @app.get("/api/adb/configs") async def list_adb(u=Depends(current_user)): with db() as c: cols = "id,user_id,config_name,dsn,username,table_name,use_mtls,is_active,is_global,wallet_dir,genai_config_id,embedding_model_id,created_at" if u["role"]=="admin": rows=c.execute(f"SELECT {cols} FROM adb_vector_configs").fetchall() else: rows=c.execute(f"SELECT {cols} FROM adb_vector_configs WHERE user_id=? OR is_global=1",(u["id"],)).fetchall() result = [] for r in rows: d = dict(r) d["tables"] = _get_tables_for_config(d["id"], active_only=False) result.append(d) return result @app.post("/api/adb/test/{vid}") async def test_adb(vid: str, u=Depends(require("admin","user"))): _verify_config_access("adb", vid, u) with db() as c: cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?",(vid,)).fetchone() if not cfg: raise HTTPException(404) cname = cfg["config_name"] try: conn = _get_adb_connection(dict(cfg)) cur = conn.cursor(); cur.execute("SELECT 1 FROM DUAL"); cur.close(); conn.close() _config_log("adb", vid, cname, "success", "test", "Conexão Autonomous DB OK!", u["id"], u["username"]) return {"status":"success","message":"Conexão Autonomous DB OK!"} except ImportError: _config_log("adb", vid, cname, "error", "test", "python-oracledb não disponível no container.", u["id"], u["username"]) return {"status":"error","message":"python-oracledb não disponível no container."} except Exception as e: msg = str(e)[:500] _config_log("adb", vid, cname, "error", "test", msg, u["id"], u["username"]) return {"status":"error","message":msg} @app.post("/api/adb/{vid}/tables/check") async def check_adb_tables(vid: str, u=Depends(require("admin","user"))): """Check which registered tables exist in ADB and list all user tables.""" _verify_config_access("adb", vid, u) with db() as c: cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone() if not cfg: raise HTTPException(404) registered = _get_tables_for_config(vid, active_only=False) reg_names_upper = {t["table_name"].upper() for t in registered} try: conn = _get_adb_connection(dict(cfg)) cur = conn.cursor() cur.execute("SELECT USER FROM DUAL") current_user_name = cur.fetchone()[0] cur.execute("SELECT table_name FROM user_tables ORDER BY table_name") adb_tables = [r[0] for r in cur.fetchall()] adb_lookup = {t.upper(): t for t in adb_tables} results = [] for t in registered: tname_reg = t["table_name"] tname_upper = tname_reg.upper() real_name = adb_lookup.get(tname_upper) if real_name: try: cur.execute(f'SELECT COUNT(*) FROM "{real_name}"') cnt = cur.fetchone()[0] case_note = f" (ADB: {real_name})" if real_name != tname_reg else "" results.append({"table": tname_reg, "adb_name": real_name, "status": "ok", "rows": cnt, "message": f"{cnt} registros{case_note}"}) except Exception as e: results.append({"table": tname_reg, "status": "error", "message": str(e)[:200]}) else: results.append({"table": tname_reg, "status": "not_found", "message": f"Não existe no schema {current_user_name}"}) unregistered = [t for t in adb_tables if t.upper() not in reg_names_upper] cur.close() conn.close() return { "status": "success", "schema_user": current_user_name, "adb_tables_total": len(adb_tables), "registered": results, "unregistered_in_adb": unregistered[:50] } except Exception as e: return {"status": "error", "message": f"Erro de conexão: {str(e)[:500]}"} @app.delete("/api/adb/configs/{vid}") async def del_adb(vid: str, u=Depends(require("admin","user"))): _verify_config_access("adb", vid, u, require_owner=True) with db() as c: c.execute("DELETE FROM adb_vector_configs WHERE id=?", (vid,)) d = WALLET_DIR / vid if d.exists(): shutil.rmtree(d) return {"ok": True} @app.put("/api/adb/configs/{vid}") async def update_adb( vid: str, config_name: str = Form(...), dsn: str = Form(...), username: str = Form(...), password: str = Form(""), wallet_password: str = Form(""), table_name: str = Form(""), use_mtls: str = Form("true"), genai_config_id: str = Form(""), embedding_model_id: str = Form("cohere.embed-v4.0"), is_global: str = Form(""), wallet: Optional[UploadFile] = File(None), u=Depends(require("admin","user")) ): if wallet and wallet.filename: await _validate_upload(wallet, {".zip"}) wallet.file.seek(0) with db() as c: existing = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone() if not existing: raise HTTPException(404) if u["role"] != "admin" and existing["user_id"] != u["id"]: raise HTTPException(403) if u["role"] != "admin" and existing.get("is_global"): raise HTTPException(403, "Não é possível editar configuração global") use_mtls_bool = use_mtls.lower() in ("true", "1", "yes") sets = "config_name=?,dsn=?,username=?,table_name=?,use_mtls=?,genai_config_id=?,embedding_model_id=?" vals = [config_name, dsn, username, table_name, int(use_mtls_bool), genai_config_id or None, embedding_model_id] if u["role"] == "admin" and is_global: is_global_val = 1 if is_global.lower() in ("true", "1", "yes") else 0 sets += ",is_global=?" vals.append(is_global_val) if password: sets += ",password_enc=?" vals.append(_enc(password)) if wallet_password: sets += ",wallet_password_enc=?" vals.append(_enc(wallet_password)) vals.append(vid) with db() as c: c.execute(f"UPDATE adb_vector_configs SET {sets} WHERE id=?", vals) if wallet and wallet.filename: import zipfile wdir = WALLET_DIR / vid; wdir.mkdir(parents=True, exist_ok=True) zp = wdir / "wallet.zip" zp.write_bytes(await wallet.read()) with zipfile.ZipFile(str(zp), 'r') as z: z.extractall(str(wdir)) with db() as c: c.execute("UPDATE adb_vector_configs SET wallet_dir=? WHERE id=?", (str(wdir), vid)) _config_log("adb", vid, config_name, "success", "upload", "Wallet atualizado", u["id"], u["username"]) _audit(u["id"], u["username"], "update_adb_config", vid, config_name) _config_log("adb", vid, config_name, "success", "save", f"Conexão atualizada: {config_name} ({dsn})", u["id"], u["username"]) return {"id": vid, "config_name": config_name} _TABLE_NAME_RE = re.compile(r'^[A-Za-z][A-Za-z0-9_]{0,127}$') @app.get("/api/adb/{vid}/tables") async def list_adb_tables(vid: str, u=Depends(current_user)): _verify_config_access("adb", vid, u) with db() as c: cfg = c.execute("SELECT id FROM adb_vector_configs WHERE id=?", (vid,)).fetchone() if not cfg: raise HTTPException(404) return _get_tables_for_config(vid, active_only=False) @app.post("/api/adb/{vid}/tables") async def add_adb_table(vid: str, req: dict, u=Depends(require("admin","user"))): _verify_config_access("adb", vid, u, require_owner=True) table_name = req.get("table_name", "").strip() description = req.get("description", "").strip() if not table_name: raise HTTPException(400, "table_name é obrigatório") if not _TABLE_NAME_RE.match(table_name): raise HTTPException(400, "Nome da tabela inválido. Use letras maiúsculas, números e underscores.") with db() as c: cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone() if not cfg: raise HTTPException(404, "ADB config not found") tid = str(uuid.uuid4()) try: with db() as c: c.execute("INSERT INTO adb_vector_tables (id, adb_config_id, table_name, description) VALUES (?,?,?,?)", (tid, vid, table_name, description)) except sqlite3.IntegrityError: raise HTTPException(409, f"Tabela '{table_name}' já registrada nesta conexão") _config_log("adb", vid, cfg["config_name"], "success", "add_table", f"Tabela '{table_name}' registrada", u["id"], u["username"]) return {"id": tid, "table_name": table_name} @app.delete("/api/adb/{vid}/tables/{tid}") async def remove_adb_table(vid: str, tid: str, u=Depends(require("admin","user"))): _verify_config_access("adb", vid, u, require_owner=True) with db() as c: row = c.execute("SELECT * FROM adb_vector_tables WHERE id=? AND adb_config_id=?", (tid, vid)).fetchone() if not row: raise HTTPException(404, "Table entry not found") with db() as c: c.execute("DELETE FROM adb_vector_tables WHERE id=?", (tid,)) return {"ok": True} @app.put("/api/adb/{vid}/tables/{tid}") async def update_adb_table(vid: str, tid: str, req: dict, u=Depends(require("admin","user"))): _verify_config_access("adb", vid, u, require_owner=True) with db() as c: row = c.execute("SELECT * FROM adb_vector_tables WHERE id=? AND adb_config_id=?", (tid, vid)).fetchone() if not row: raise HTTPException(404, "Table entry not found") sets, vals = [], [] if "table_name" in req: tn = req["table_name"].strip() if not tn: raise HTTPException(400, "table_name é obrigatório") if not _TABLE_NAME_RE.match(tn): raise HTTPException(400, "Nome da tabela inválido") sets.append("table_name=?"); vals.append(tn) if "description" in req: sets.append("description=?"); vals.append(req["description"]) if "is_active" in req: sets.append("is_active=?"); vals.append(int(req["is_active"])) if not sets: return {"ok": True} vals.append(tid) try: with db() as c: c.execute(f"UPDATE adb_vector_tables SET {','.join(sets)} WHERE id=?", vals) except sqlite3.IntegrityError: raise HTTPException(409, "Tabela com este nome já existe nesta conexão") return {"ok": True} @app.post("/api/adb/{vid}/ingest") async def ingest_documents(vid: str, req: IngestDocReq, bg: BackgroundTasks, u=Depends(require("admin","user"))): _verify_config_access("adb", vid, u) with db() as c: cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone() if not cfg: raise HTTPException(404, "ADB config not found") cfg = dict(cfg) if not cfg.get("genai_config_id"): raise HTTPException(400, "GenAI config not linked to this ADB connection") with db() as c: gc = c.execute("SELECT * FROM genai_configs WHERE id=?", (cfg["genai_config_id"],)).fetchone() if not gc: raise HTTPException(400, "Linked GenAI config not found") bg.add_task(_ingest_documents_task, cfg, dict(gc), req.documents, u["id"], u["username"], table_name=req.table_name) return {"ok": True, "message": f"Ingestão iniciada para {len(req.documents)} documentos", "config_id": vid} def _build_metadata_json(tenancy: str = "", compartments: str = "", section: str = "", report_date: str = "", user_id: str = "", extra: dict = None) -> str: """Build a structured JSON metadata string for vector embeddings.""" meta = {} if tenancy: meta["tenancy"] = tenancy if compartments: meta["compartments"] = compartments if section: meta["section"] = section if report_date: meta["report_date"] = report_date if user_id: meta["user_id"] = user_id if extra: meta.update(extra) return json.dumps(meta, ensure_ascii=False) if meta else "" def _auto_register_table(adb_config_id: str, table_name: str, description: str = ""): """Auto-register a table in adb_vector_tables if not already present.""" if not table_name: return with db() as c: exists = c.execute("SELECT 1 FROM adb_vector_tables WHERE adb_config_id=? AND table_name=? COLLATE NOCASE", (adb_config_id, table_name)).fetchone() if not exists: c.execute("INSERT INTO adb_vector_tables (id, adb_config_id, table_name, description) VALUES (?,?,?,?)", (str(uuid.uuid4()), adb_config_id, table_name, description)) log.info(f"Auto-registered table '{table_name}' for ADB config {adb_config_id}") def _ingest_documents_task(cfg: dict, genai_cfg: dict, documents: list, user_id: str, username: str, table_name: str = None, tenancy: str = None, compartments: str = None, report_date: str = None, task_id: str = None): """Background task: embed and insert documents into ADB via OCI GenAI. Tenancy and compartments are stored in METADATA as structured JSON for filtering.""" import array emb_model = cfg.get("embedding_model_id", "cohere.embed-v4.0") table_name = table_name or cfg.get("table_name", "") # Auto-detect embedding dimension from existing table data and use matching model try: actual_dim = _get_table_embedding_dim(cfg, table_name) if actual_dim and actual_dim in _DIM_TO_MODEL: detected_model = _DIM_TO_MODEL[actual_dim] if detected_model != emb_model: log.info(f"Ingest: table '{table_name}' has {actual_dim} dims, switching model from {emb_model} to {detected_model}") emb_model = detected_model except Exception as e: log.warning(f"Ingest: failed to detect dimension for '{table_name}': {e}") total = len(documents) # Track status if task_id: _set_embed_status(task_id, {"status": "running", "table": table_name, "tenancy": tenancy or "", "inserted": 0, "total": total, "user_id": user_id, "message": "Iniciando embedding..."}) # Auto-register table so it appears in multi-table RAG search _auto_register_table(cfg["id"], table_name) conn = _get_adb_connection(cfg) try: cur = conn.cursor() inserted = 0 for i, doc in enumerate(documents): try: content = doc.get("content", "") if not content: continue embedding = _embed_text(content, genai_cfg, emb_model) vec = array.array('f', [float(x) for x in embedding]) # Build structured metadata with tenancy isolation doc_tenancy = tenancy or doc.get("tenancy", "") doc_compartments = compartments or doc.get("compartments", "") metadata = _build_metadata_json( tenancy=doc_tenancy, compartments=doc_compartments, section=doc.get("section", ""), report_date=report_date or "", user_id=user_id, extra={"legacy_metadata": doc.get("metadata", "")} if doc.get("metadata") else None ) cur.execute(f""" INSERT INTO "{table_name}" (ID, TEXT, EMBEDDING, METADATA) VALUES (HEXTORAW(:1), :2, :3, :4) """, [uuid.uuid4().hex.upper(), content, vec, metadata]) inserted += 1 if task_id: _update_embed_status(task_id, {"inserted": inserted, "message": f"Embedding {inserted}/{total}..."}) except Exception as e: log.error(f"Failed to ingest document: {e}") conn.commit() cur.close() msg = f"{inserted}/{total} documentos ingeridos em {table_name}" + (f" (tenancy: {tenancy})" if tenancy else "") log.info(f"Ingested {inserted}/{total} documents into {table_name}" + (f" (tenancy={tenancy})" if tenancy else "")) _audit(user_id, username, "ingest_documents", cfg["id"], f"{inserted} documents") _config_log("adb", cfg["id"], cfg.get("config_name"), "success", "ingest", msg, user_id, username) if task_id: _update_embed_status(task_id, {"status": "done", "inserted": inserted, "message": msg}) except Exception as e: log.error(f"Ingestion task failed: {e}") _config_log("adb", cfg["id"], cfg.get("config_name"), "error", "ingest", str(e)[:500], user_id, username) if task_id: _update_embed_status(task_id, {"status": "error", "message": str(e)[:300]}) finally: conn.close() # ── Embeddings ──────────────────────────────────────────────────────────────── def _chunk_report_by_section(report_data: dict) -> list: """Chunk a CIS report into documents grouped by section.""" if isinstance(report_data, str): report_data = json.loads(report_data) if isinstance(report_data, list): report_data = {"findings": {str(i): item for i, item in enumerate(report_data)}, "tenancy": "unknown"} findings = report_data.get("findings", {}) tenancy = report_data.get("tenancy", "unknown") generated_at = report_data.get("generated_at", "") regions = report_data.get("regions", []) compartments = report_data.get("compartments", []) # Build context header for each chunk ctx_parts = [f"Tenancy: {tenancy}"] if regions: ctx_parts.append(f"Regions: {', '.join(regions)}") if compartments: ctx_parts.append(f"Compartments: {', '.join(compartments[:50])}") ctx_header = "\n".join(ctx_parts) sections = {} for cid, check in findings.items(): sec = check.get("section", "Other") sections.setdefault(sec, []) sections[sec].append(check) documents = [] for section_name, checks in sections.items(): passed = sum(1 for c in checks if c.get("status") == "PASS") failed = sum(1 for c in checks if c.get("status") == "FAIL") review = sum(1 for c in checks if c.get("status") == "REVIEW") lines = [ctx_header, "", f"Section: {section_name}", f"Total checks: {len(checks)}, Passed: {passed}, Failed: {failed}, Review: {review}", ""] for c in checks: status = c.get("status", "REVIEW") lines.append(f"- [{c.get('id', '')}] {c.get('title', '')} — Status: {status}") if c.get("findings"): for f in c["findings"]: lines.append(f" Finding: {f}") documents.append({ "content": "\n".join(lines), "source": f"CIS Report - {tenancy} - {generated_at}", "section": section_name, "tenancy": tenancy, "compartments": ", ".join(compartments[:50]), "metadata": f"tenancy: {tenancy}, section: {section_name}, total: {len(checks)}, passed: {passed}, failed: {failed}, review: {review}" }) return documents def _chunk_cis_pdf(text: str, filename: str, target_chars: int = 7000, overlap_chars: int = 500) -> list: """Chunk a CIS Foundations Benchmark PDF by recommendation number. Each recommendation (1.1, 1.2, etc.) becomes one or more chunks with overlap. Port of the JavaScript embedding pipeline.""" import re as _re def normalize(t): t = t.replace('\r', '\n') t = _re.sub(r'[ \t]+\n', '\n', t) t = _re.sub(r'\n{3,}', '\n\n', t) return t.strip() def strip_page_headers(t): # Remove "Page XX" both standalone and at start of lines t = _re.sub(r'^\s*Page\s+\d+\s*$', '', t, flags=_re.MULTILINE | _re.IGNORECASE) t = _re.sub(r'^Page\s+\d+\s+', '', t, flags=_re.MULTILINE | _re.IGNORECASE) return t def remove_toc(t): # Remove everything from "Table of Contents" up to the actual recommendations section # The real content starts with "Recommendations\n1 Identity" or "Profile Applicability" toc_start = _re.search(r'\bTable of Contents\b', t, _re.IGNORECASE) if not toc_start: return t # Find where actual recommendation content begins content_start = _re.search(r'\bRecommendations\s*\n\s*1\s+Identity', t[toc_start.start():], _re.IGNORECASE) if not content_start: content_start = _re.search(r'\bProfile Applicability\b', t[toc_start.start():], _re.IGNORECASE) if not content_start: content_start = _re.search(r'\bOverview\b', t[toc_start.start():], _re.IGNORECASE) if not content_start: return t end_pos = toc_start.start() + content_start.start() if end_pos <= toc_start.start(): return t return normalize(t[:toc_start.start()] + '\n\n' + t[end_pos:]) def is_chapter_header(line): l = line.strip() return bool(_re.match(r'^\d+\s+[A-Za-z].+', l)) and not _re.match(r'^\d+\.\d+', l) def is_rec_header_start(line): l = line.strip() # Must be "1.1 Word..." but NOT a TOC line (with dots/page numbers) if not _re.match(r'^\d+\.\d+(\.\d+)?\s+[A-Z]', l): return False # Skip TOC lines: contain "...." or end with a page number if '....' in l or _re.search(r'\.\s*\d+\s*$', l): return False return True def header_looks_complete(h): # Complete if has (Manual)/(Automated) or ends with a closing paren if _re.search(r'\(\s*(Manual|Automated)\s*\)', h, _re.IGNORECASE): return True # Also stop if next line starts a known section like "Profile Applicability" return False def chunk_text(t): if not t: return [] paragraphs = [p.strip() for p in t.split('\n\n') if p.strip()] chunks = [] buf = "" def push(): nonlocal buf b = buf.strip() if b: chunks.append(b) buf = "" for p in paragraphs: if len(p) > target_chars: push() i = 0 while i < len(p): chunks.append(p[i:i + target_chars].strip()) i += max(1, target_chars - overlap_chars) continue candidate = f"{buf}\n\n{p}" if buf else p if len(candidate) <= target_chars: buf = candidate else: push() if chunks and overlap_chars > 0: prev = chunks[-1] overlap = prev[max(0, len(prev) - overlap_chars):] buf = f"{overlap}\n\n{p}".strip() else: buf = p push() return chunks def remove_appendix(t): """Remove appendix sections (Assessment Status, Change History, etc.) that pollute embeddings.""" for marker in [r'\bAppendix\b', r'\bAssessment Status\b', r'\bChange History\b', r'\bCIS Controls v\d', r'\bDate\s+Version\s+Changes']: m = _re.search(marker, t, _re.IGNORECASE) if m and m.start() > len(t) * 0.7: # only cut if in last 30% of doc t = t[:m.start()].rstrip() break return t # Pipeline text = normalize(text) text = strip_page_headers(text) text = remove_toc(text) text = remove_appendix(text) lines = text.split('\n') # Segment by recommendation segments = [] current = None current_chapter = "" i = 0 while i < len(lines): line = lines[i] if is_chapter_header(line): current_chapter = line.strip() if is_rec_header_start(line): if current: segments.append(current) header = line.strip() j = i + 1 while j < len(lines) and not header_looks_complete(header): next_line = lines[j].strip() if is_rec_header_start(next_line): break # Stop consuming if we hit a known section start if next_line.startswith('Profile Applicability') or next_line.startswith('Description:'): break if next_line: header = _re.sub(r'\s+', ' ', f"{header} {next_line}").strip() j += 1 i = j - 1 current = {"header": header, "chapter": current_chapter, "body_lines": []} i += 1 continue if current: current["body_lines"].append(line) i += 1 if current: segments.append(current) # Generate chunks documents = [] for seg in segments: body = normalize('\n'.join(seg["body_lines"])) if not body: continue rec_match = _re.match(r'^(\d+(\.\d+)+)', seg["header"]) rec_number = rec_match.group(1) if rec_match else "unknown" canonical = normalize('\n'.join(filter(None, [ f"Recommendation: {seg['header']}", f"Chapter: {seg['chapter']}" if seg['chapter'] else "", "", body, ]))) chunks = chunk_text(canonical) for idx, chunk in enumerate(chunks): documents.append({ "content": chunk, "source": filename, "metadata": json.dumps({ "filename": filename, "recommendationNumber": rec_number, "chapter": seg["chapter"], "source": "CIS-OCI-PDF", "chunkIndex": idx + 1, "chunkCount": len(chunks), }), }) log.info(f"CIS PDF chunking: {len(segments)} recommendations → {len(documents)} chunks from {filename}") return documents def _chunk_text_file(text: str, filename: str, chunk_size: int = 1000, overlap: int = 200) -> list: """Split text into chunks by paragraphs with overlap to avoid losing context at boundaries.""" paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()] documents = [] current_chunk = "" prev_tail = "" # last N chars of previous chunk for overlap chunk_num = 1 for para in paragraphs: if len(current_chunk) + len(para) + 2 > chunk_size and current_chunk: documents.append({"content": current_chunk, "source": filename, "metadata": f"chunk: {chunk_num}"}) chunk_num += 1 # Keep overlap from end of current chunk prev_tail = current_chunk[-overlap:] if len(current_chunk) > overlap else current_chunk current_chunk = prev_tail + "\n\n" + para else: current_chunk = current_chunk + "\n\n" + para if current_chunk else para if current_chunk: documents.append({"content": current_chunk, "source": filename, "metadata": f"chunk: {chunk_num}"}) return documents def _get_adb_and_genai(vid: str, oci_config_id: str = None, user_id: str = None): """Load ADB config and resolve embed config (scoped to user_id). Priority: ADB.genai_config_id → genai by oci_config_id → oci_config directly → user's default.""" with db() as c: cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone() if not cfg: raise HTTPException(404, "ADB config not found") cfg = dict(cfg) genai_cfg = None if cfg.get("genai_config_id"): with db() as c: row = c.execute("SELECT * FROM genai_configs WHERE id=?", (cfg["genai_config_id"],)).fetchone() if row: genai_cfg = dict(row) gc = _resolve_embed_config(oci_config_id=oci_config_id, genai_cfg=genai_cfg, user_id=user_id or cfg.get("user_id")) return cfg, gc @app.get("/api/embeddings/preview/{rid}") async def preview_report_chunks(rid: str, u=Depends(current_user)): """Preview the chunks that will be generated from a CIS report before embedding.""" _verify_report_access(rid, u) with db() as c: r = c.execute("SELECT json_path,tenancy_name FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone() if not r: raise HTTPException(404, "Report not found or not completed") json_path = r["json_path"] if not json_path or not Path(json_path).exists(): raise HTTPException(400, "Report JSON file not found") try: report_data = json.loads(Path(json_path).read_text()) except Exception: raise HTTPException(400, "Invalid report data") documents = _chunk_report_by_section(report_data) rd = report_data if isinstance(report_data, dict) else {} return {"tenancy": rd.get("tenancy", "unknown"), "regions": rd.get("regions", []), "compartments": rd.get("compartments", []), "total_chunks": len(documents), "chunks": documents} def _chunk_summary_csv(csv_path: str, tenancy: str, extract_date: str) -> list: """Chunk the cis_summary_report.csv into documents with tenancy/date metadata. Each row becomes a document with structured content for vector search.""" import csv as csvmod p = Path(csv_path) if not p.exists(): return [] documents = [] with open(p, "r", encoding="utf-8") as f: rows = list(csvmod.DictReader(f)) if not rows: return [] # Group rows by section for richer context sections: dict = {} for row in rows: sec = row.get("Section", "Unknown") sections.setdefault(sec, []).append(row) for sec_name, sec_rows in sections.items(): lines = [] for r in sec_rows: rec = r.get("Recommendation #", "") title = r.get("Title", "") compliant = r.get("Compliant", "") pct = r.get("Compliance Percentage Per Recommendation", "") findings = r.get("Findings", "0") total = r.get("Total", "0") lines.append(f"Recommendation {rec}: {title} | Status: {compliant} | Compliance: {pct}% | Findings: {findings}/{total}") content = ( f"Tenancy: {tenancy}\n" f"Extract Date: {extract_date}\n" f"Section: {sec_name}\n" f"Total Recommendations: {len(sec_rows)}\n\n" + "\n".join(lines) ) documents.append({ "content": content, "section": sec_name, "tenancy": tenancy, "metadata": json.dumps({"tenancy": tenancy, "extract_date": extract_date, "section": sec_name}) }) return documents # ── CIS Report CSV → ADB Table Mapping ── _CIS_TABLE_MAP = { "Identity_and_Access_Management": "identityandaccess", "Networking": "networking", "Compute": "computeinstances", "Logging_and_Monitoring": "loggingandmonitoring", "Storage_Object_Storage": "objectstorage", "Storage_Block_Volumes": "storageblockvolume", "Storage_File_Storage_Service": "filestorageservice", "Asset_Management": "assetmanagement", } def _purge_table_by_tenancy(cfg: dict, table_name: str, tenancy: str, extract_date: str = "") -> int: """Delete existing embeddings from a table for a specific tenancy (and optionally extract_date). Returns number of rows deleted.""" try: conn = _get_adb_connection(cfg) cur = conn.cursor() if extract_date: cur.execute(f"""DELETE FROM "{table_name}" WHERE JSON_VALUE(METADATA, '$.tenancy') = :1 AND JSON_VALUE(METADATA, '$.extract_date') = :2""", [tenancy, extract_date]) else: cur.execute(f"""DELETE FROM "{table_name}" WHERE JSON_VALUE(METADATA, '$.tenancy') = :1""", [tenancy]) deleted = cur.rowcount conn.commit() cur.close() conn.close() if deleted: log.info(f"Purged {deleted} rows from {table_name} (tenancy={tenancy}, date={extract_date or 'all'})") return deleted except Exception as e: log.warning(f"Purge failed for {table_name}: {e}") return 0 def _resolve_table_for_csv(filename: str) -> str | None: """Map a CIS report CSV filename to its ADB vector table.""" if filename == "cis_summary_report.csv": return "summaryreportcsvvector" for pattern, table in _CIS_TABLE_MAP.items(): if pattern in filename: return table return None def _chunk_findings_csv(csv_path: str, tenancy: str, extract_date: str, max_chars: int = 8000) -> list: """Chunk a CIS findings CSV into documents. Each row becomes one or more documents. If a row exceeds max_chars (~6000 tokens), it's split into smaller chunks with a context header (tenancy, resource name, ID) repeated in each part.""" import csv as csvmod, re as _re p = Path(csv_path) if not p.exists(): return [] documents = [] with open(p, "r", encoding="utf-8") as f: rows = list(csvmod.DictReader(f)) if not rows: return [] skip_cols = {"extract_date", "deep_link", "domain_deeplink", "defined_tags", "freeform_tags", "system_tags", "external_identifier"} # Extract CIS recommendation number from filename (e.g., cis_Identity_and_Access_Management_1-1.csv → 1.1) rec_match = _re.search(r'_(\d+)-(\d+(?:\.\d+)?)\.csv$', p.name) cis_rec = f"{rec_match.group(1)}.{rec_match.group(2)}" if rec_match else "" # Extract section name from filename sec_match = _re.search(r'^cis_(.+?)_\d+-', p.name) cis_section = sec_match.group(1).replace("_", " ") if sec_match else "" meta = json.dumps({"tenancy": tenancy, "extract_date": extract_date, "source": p.name, "cis_recommendation": cis_rec}) for row in rows: # Build context header (always repeated in each chunk) header_parts = [f"Tenancy: {tenancy}", f"Extract Date: {extract_date}"] if cis_rec: header_parts.append(f"CIS Recommendation: {cis_rec}") if cis_section: header_parts.append(f"Section: {cis_section}") header_parts.append(f"Status: Non-Compliant") body_parts = [] # Identify key fields for the header name = row.get("name") or row.get("display_name") or row.get("username") or "" rid = row.get("id", "") if name: header_parts.append(f"Resource: {name}") if rid: header_parts.append(f"ID: {rid}") for col, val in row.items(): if col.lower() in skip_cols or not val or not val.strip(): continue if col.lower() in ("name", "display_name", "username", "id"): continue # already in header # Clean HYPERLINK formulas if val.startswith("=HYPERLINK"): m = _re.search(r',\s*"([^"]+)"', val) val = m.group(1) if m else val body_parts.append(f"{col}: {val}") header = "\n".join(header_parts) body = "\n".join(body_parts) full_content = header + "\n" + body if len(full_content) <= max_chars: if len(full_content) > 50: documents.append({"content": full_content, "tenancy": tenancy, "metadata": meta}) else: # Split body into chunks, each prefixed with context header chunk_size = max_chars - len(header) - 50 # reserve space for header + part label chunks = [] current = "" for line in body_parts: if len(current) + len(line) + 2 > chunk_size and current: chunks.append(current) current = line else: current = current + "\n" + line if current else line if current: chunks.append(current) for i, chunk in enumerate(chunks): part_label = f"(part {i + 1}/{len(chunks)})" if len(chunks) > 1 else "" content = f"{header}\n{part_label}\n{chunk}".strip() if len(content) > 50: documents.append({"content": content, "tenancy": tenancy, "metadata": meta}) return documents @app.post("/api/embeddings/report/{rid}") async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(require("admin","user"))): """Auto-embed all CIS report CSVs into their mapped ADB vector tables.""" _verify_report_access(rid, u) vid = req.get("adb_config_id") if not vid: raise HTTPException(400, "adb_config_id is required") _verify_config_access("adb", vid, u) with db() as c: r = c.execute("SELECT tenancy_name, config_id FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone() if not r: raise HTTPException(404, "Report not found or not completed") rdir = REPORTS / rid tenancy = r["tenancy_name"] or "unknown" # Read extract_date from summary CSV import csv as csvmod summary_csv = rdir / "cis_summary_report.csv" extract_date = "" if summary_csv.exists(): with open(summary_csv, "r", encoding="utf-8") as f: first = next(csvmod.DictReader(f), {}) extract_date = first.get("extract_date", "") cfg, gc = _get_adb_and_genai(vid, oci_config_id=r["config_id"] if r["config_id"] else None, user_id=u["id"]) # Load registered tables for validation with db() as c: registered = {t["table_name"].lower() for t in c.execute("SELECT table_name FROM adb_vector_tables WHERE adb_config_id=? AND is_active=1", (vid,)).fetchall()} # Scan all CSVs and map to tables task_id = str(uuid.uuid4()) table_docs: dict[str, list] = {} missing_tables: list[str] = [] skipped_tables: list[str] = [] for csv_file in sorted(rdir.glob("cis_*.csv")): table = _resolve_table_for_csv(csv_file.name) if not table: continue if table.lower() not in registered: if table not in skipped_tables: skipped_tables.append(table) continue if csv_file.name == "cis_summary_report.csv": docs = _chunk_summary_csv(str(csv_file), tenancy, extract_date) else: docs = _chunk_findings_csv(str(csv_file), tenancy, extract_date) if docs: table_docs.setdefault(table, []).extend(docs) if not table_docs: if skipped_tables: raise HTTPException(400, f"Tabela(s) não registrada(s) no ADB: {', '.join(skipped_tables)}. Registre em Configurações > ADB Vector.") raise HTTPException(400, "No CSV files found to embed") # Calculate totals for status tracking total_docs = sum(len(d) for d in table_docs.values()) tables_used = list(table_docs.keys()) _set_embed_status(task_id, { "status": "running", "table": ", ".join(tables_used), "tenancy": tenancy, "inserted": 0, "total": total_docs, "user_id": u["id"], "message": f"Embedding {total_docs} documentos em {len(tables_used)} tabelas..." }) # Build queue info: [(table, doc_count), ...] table_queue = [(tbl, len(docs)) for tbl, docs in table_docs.items()] def _bg_embed_all(): """Background: embed documents into their respective tables sequentially.""" import array default_model = cfg.get("embedding_model_id", "cohere.embed-v4.0") inserted_total = 0 skipped_total = 0 processed_total = 0 errors = [] for tbl_idx, (tbl, docs) in enumerate(table_docs.items()): remaining = table_queue[tbl_idx + 1:] if tbl_idx + 1 < len(table_queue) else [] _auto_register_table(cfg["id"], tbl) purged = _purge_table_by_tenancy(cfg, tbl, tenancy, extract_date) if purged: _update_embed_status(task_id, {"message": f"Purged {purged} old docs from {tbl}...", "current_table": tbl, "current_inserted": 0, "current_total": len(docs), "queue": [{"table": t, "docs": n} for t, n in remaining]}) emb_model = default_model try: actual_dim = _get_table_embedding_dim(cfg, tbl) if actual_dim and actual_dim in _DIM_TO_MODEL: emb_model = _DIM_TO_MODEL[actual_dim] except Exception: pass current_inserted = 0 current_skipped = 0 try: conn = _get_adb_connection(cfg) cur = conn.cursor() for doc in docs: try: content = doc.get("content", "") if not content: skipped_total += 1 current_skipped += 1 processed_total += 1 continue embedding = _embed_text(content, gc, emb_model) vec = array.array('f', [float(x) for x in embedding]) metadata = _build_metadata_json( tenancy=doc.get("tenancy", tenancy), section=doc.get("section", ""), report_date=extract_date, user_id=u["id"], ) cur.execute(f'INSERT INTO "{tbl}" (ID, TEXT, EMBEDDING, METADATA) VALUES (HEXTORAW(:1), :2, :3, :4)', [uuid.uuid4().hex.upper(), content, vec, metadata]) inserted_total += 1 current_inserted += 1 except Exception as e: err_str = str(e) if "message" in err_str: import re as _re m = _re.search(r'"message"\s*:\s*"(.*?)"', err_str) log.warning(f"Embed skip in {tbl}: {m.group(1)[:200] if m else err_str[:200]}") else: log.warning(f"Embed skip in {tbl}: {err_str[:200]}") skipped_total += 1 current_skipped += 1 processed_total += 1 _update_embed_status(task_id, { "inserted": inserted_total, "skipped": skipped_total, "processed": processed_total, "current_table": tbl, "current_inserted": current_inserted, "current_skipped": current_skipped, "current_total": len(docs), "queue": [{"table": t, "docs": n} for t, n in remaining], "message": f"{tbl}: {current_inserted}/{len(docs)} — global: {inserted_total}/{total_docs}" }) conn.commit() cur.close() conn.close() log.info(f"Embedded {current_inserted}/{len(docs)} docs into {tbl} (tenancy={tenancy})") except Exception as e: log.error(f"Failed to connect/embed to {tbl}: {e}") errors.append(f"{tbl}: {str(e)[:100]}") msg = f"{inserted_total}/{total_docs} documentos em {len(tables_used)} tabelas ({', '.join(tables_used)})" if tenancy: msg += f" — tenancy: {tenancy}" if errors: msg += f" | Erros: {'; '.join(errors)}" _update_embed_status(task_id, { "status": "done" if not errors else "done", "inserted": inserted_total, "message": msg }) _audit(u["id"], u["username"], "embed_report_auto", rid, msg) _config_log("adb", cfg["id"], cfg.get("config_name"), "success" if not errors else "error", "ingest", msg, u["id"], u["username"]) _chat_executor.submit(_bg_embed_all) msg = f"Embedding iniciado — {total_docs} documentos em {len(tables_used)} tabelas ({', '.join(tables_used)})" if skipped_tables: msg += f". Ignoradas (não registradas): {', '.join(skipped_tables)}" return { "ok": True, "task_id": task_id, "message": msg, "tables": tables_used, "skipped": skipped_tables, "total_documents": total_docs, "tenancy": tenancy } @app.get("/api/embeddings/status/{task_id}") async def embedding_status(task_id: str, u=Depends(current_user)): """Check embedding task progress.""" st = _get_embed_status(task_id) if not st: return {"status": "unknown", "message": "Task not found"} if st.get("user_id") and st["user_id"] != u["id"] and u["role"] != "admin": return {"status": "unknown", "message": "Task not found"} return {k: v for k, v in st.items() if k != "user_id"} @app.post("/api/embeddings/report/{rid}/section") async def embed_report_section(rid: str, req: dict, bg: BackgroundTasks, u=Depends(require("admin","user"))): """Embed all CSV files from a specific report section into the mapped ADB table.""" _verify_report_access(rid, u) import csv as csvmod vid = req.get("adb_config_id") file_names: list = req.get("file_names", []) if not vid: # Auto-detect first active ADB with db() as c: adb = c.execute("SELECT id FROM adb_vector_configs WHERE is_active=1 LIMIT 1").fetchone() if not adb: raise HTTPException(400, "No active ADB config found") vid = adb["id"] if not file_names: raise HTTPException(400, "file_names is required") with db() as c: r = c.execute("SELECT tenancy_name, config_id FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone() if not r: raise HTTPException(404) rdir = REPORTS / rid tenancy = r["tenancy_name"] or "unknown" # Read extract_date summary = rdir / "cis_summary_report.csv" extract_date = "" if summary.exists(): with open(summary, "r", encoding="utf-8") as f: first = next(csvmod.DictReader(f), {}) extract_date = first.get("extract_date", "") cfg, gc = _get_adb_and_genai(vid, oci_config_id=r["config_id"] if r["config_id"] else None, user_id=u["id"]) # Load registered tables for validation with db() as c: registered = {t["table_name"].lower() for t in c.execute("SELECT table_name FROM adb_vector_tables WHERE adb_config_id=? AND is_active=1", (vid,)).fetchall()} # Group files by target table and validate import array table_docs: dict[str, list] = {} missing_tables: list[str] = [] for fname in file_names: csv_path = rdir / fname if not csv_path.exists(): continue table = _resolve_table_for_csv(fname) if not table: continue if table.lower() not in registered: if table not in missing_tables: missing_tables.append(table) continue if fname == "cis_summary_report.csv": docs = _chunk_summary_csv(str(csv_path), tenancy, extract_date) else: docs = _chunk_findings_csv(str(csv_path), tenancy, extract_date) if docs: table_docs.setdefault(table, []).extend(docs) if missing_tables and not table_docs: raise HTTPException(400, f"Tabela(s) não registrada(s) no ADB: {', '.join(missing_tables)}. Registre em Configurações > ADB Vector.") if not table_docs: raise HTTPException(400, "No embeddable content found in the selected files") total_docs = sum(len(d) for d in table_docs.values()) tables_used = list(table_docs.keys()) task_id = str(uuid.uuid4()) _set_embed_status(task_id, { "status": "running", "table": ", ".join(tables_used), "tenancy": tenancy, "inserted": 0, "total": total_docs, "user_id": u["id"], "message": f"Embedding {total_docs} docs em {', '.join(tables_used)}..." }) def _bg(): default_model = cfg.get("embedding_model_id", "cohere.embed-v4.0") inserted = 0 skipped = 0 processed = 0 for tbl, docs in table_docs.items(): purged = _purge_table_by_tenancy(cfg, tbl, tenancy, extract_date) if purged: _update_embed_status(task_id, {"message": f"Purged {purged} old docs from {tbl}..."}) _auto_register_table(cfg["id"], tbl) emb_model = default_model try: actual_dim = _get_table_embedding_dim(cfg, tbl) if actual_dim and actual_dim in _DIM_TO_MODEL: emb_model = _DIM_TO_MODEL[actual_dim] except Exception: pass try: conn = _get_adb_connection(cfg) cur = conn.cursor() for doc in docs: try: content = doc.get("content", "") if not content: skipped += 1 processed += 1 continue embedding = _embed_text(content, gc, emb_model) vec = array.array('f', [float(x) for x in embedding]) metadata = _build_metadata_json(tenancy=doc.get("tenancy", tenancy), section=doc.get("section", ""), report_date=extract_date, user_id=u["id"]) cur.execute(f'INSERT INTO "{tbl}" (ID, TEXT, EMBEDDING, METADATA) VALUES (HEXTORAW(:1), :2, :3, :4)', [uuid.uuid4().hex.upper(), content, vec, metadata]) inserted += 1 except Exception as e: skipped += 1 err_str = str(e) if "message" in err_str: import re as _re m = _re.search(r'"message"\s*:\s*"(.*?)"', err_str) log.warning(f"Embed skip in {tbl}: {m.group(1)[:200] if m else err_str[:200]}") else: log.warning(f"Embed skip in {tbl}: {err_str[:200]}") processed += 1 _update_embed_status(task_id, { "inserted": inserted, "skipped": skipped, "processed": processed, "total": total_docs, "message": f"{tbl}: {inserted} OK, {skipped} falhas ({processed}/{total_docs})" }) conn.commit(); cur.close(); conn.close() except Exception as e: log.error(f"Embed connection error {tbl}: {e}") msg = f"{inserted}/{total_docs} embeddings OK" if skipped: msg += f", {skipped} falhas" msg += f" em {', '.join(tables_used)} (tenancy: {tenancy})" _update_embed_status(task_id, {"status": "done", "inserted": inserted, "skipped": skipped, "processed": processed, "message": msg}) _audit(u["id"], u["username"], "embed_section", rid, msg) _chat_executor.submit(_bg) return {"ok": True, "task_id": task_id, "tables": tables_used, "total": total_docs, "message": f"Embedding de {total_docs} docs iniciado ({', '.join(tables_used)})"} @app.post("/api/embeddings/report/{rid}/file/{fid}") async def embed_report_file(rid: str, fid: str, req: dict, bg: BackgroundTasks, u=Depends(require("admin","user"))): """Embed a specific report file (CSV, JSON, TXT, etc.) into ADB vector store.""" _verify_report_access(rid, u) vid = req.get("adb_config_id") if not vid: raise HTTPException(400, "adb_config_id is required") _verify_config_access("adb", vid, u) with db() as c: r = c.execute("SELECT tenancy_name, config_id FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone() if not r: raise HTTPException(404, "Report not found or not completed") f = c.execute("SELECT * FROM report_files WHERE id=? AND report_id=?", (fid, rid)).fetchone() if not f: raise HTTPException(404, "File not found") p = Path(f["file_path"]) if not p.exists(): raise HTTPException(404, "File not found on disk") fname = f["file_name"].lower() allowed = ('.txt', '.csv', '.json', '.md', '.pdf') if not any(fname.endswith(ext) for ext in allowed): raise HTTPException(400, f"Formatos aceitos para embedding: {', '.join(allowed)}") raw = p.read_bytes() if fname.endswith('.pdf'): content = _extract_pdf_text(raw) else: content = raw.decode("utf-8", errors="replace") if not content.strip(): raise HTTPException(400, "File is empty") documents = _chunk_text_file(content, f["file_name"]) if not documents: raise HTTPException(400, "No content chunks found") cfg, gc = _get_adb_and_genai(vid, oci_config_id=r["config_id"] if r["config_id"] else None, user_id=u["id"]) target_table = req.get("table_name") or None tenancy = r["tenancy_name"] or "unknown" task_id = str(uuid.uuid4()) bg.add_task(_ingest_documents_task, cfg, gc, documents, u["id"], u["username"], table_name=target_table, tenancy=tenancy, task_id=task_id) _audit(u["id"], u["username"], "embed_report_file", f"{rid}/{fid}", f"{f['file_name']}, {len(documents)} chunks, tenancy={tenancy}") return {"ok": True, "task_id": task_id, "message": f"Embedding de {f['file_name']} iniciado ({len(documents)} chunks)", "chunks": len(documents)} def _extract_pdf_text(file_bytes: bytes) -> str: """Extract text from a PDF file using PyPDF2 or pdfplumber.""" import io try: import PyPDF2 reader = PyPDF2.PdfReader(io.BytesIO(file_bytes)) pages = [] for page in reader.pages: text = page.extract_text() if text: pages.append(text.strip()) return "\n\n".join(pages) except ImportError: pass try: import pdfplumber pages = [] with pdfplumber.open(io.BytesIO(file_bytes)) as pdf: for page in pdf.pages: text = page.extract_text() if text: pages.append(text.strip()) return "\n\n".join(pages) except ImportError: raise HTTPException(400, "PDF support requires PyPDF2 or pdfplumber. Install: pip install PyPDF2") @app.post("/api/embeddings/upload") async def embed_upload(adb_config_id: str = Form(...), table_name: str = Form(""), file: UploadFile = File(...), bg: BackgroundTasks = None, u=Depends(require("admin","user"))): _verify_config_access("adb", adb_config_id, u) fname = file.filename.lower() allowed = ('.txt', '.pdf', '.csv', '.json', '.md') if not any(fname.endswith(ext) for ext in allowed): raise HTTPException(400, f"Formatos aceitos: {', '.join(allowed)}") await _validate_upload(file, allowed) file.file.seek(0) raw = await file.read() if fname.endswith('.pdf'): content = _extract_pdf_text(raw) else: content = raw.decode("utf-8", errors="replace") if not content.strip(): raise HTTPException(400, "File is empty") target_table = table_name.strip() or None # Use CIS-specific chunking for cisrecom table (segments by recommendation number with overlap) if target_table and 'cisrecom' in target_table.lower(): documents = _chunk_cis_pdf(content, file.filename) if not documents: documents = _chunk_text_file(content, file.filename) # fallback else: documents = _chunk_text_file(content, file.filename) if not documents: raise HTTPException(400, "No content chunks found") cfg, gc = _get_adb_and_genai(adb_config_id, user_id=u["id"]) bg.add_task(_ingest_documents_task, cfg, gc, documents, u["id"], u["username"], table_name=target_table) _audit(u["id"], u["username"], "embed_upload", file.filename, f"{len(documents)} chunks") return {"ok": True, "message": f"Embedding de {len(documents)} chunks iniciado", "chunks": len(documents), "filename": file.filename} def _extract_text_from_html(html: str) -> str: """Extract readable text from HTML, stripping tags and scripts.""" import re as _re text = _re.sub(r']*>[\s\S]*?', ' ', html, flags=_re.IGNORECASE) text = _re.sub(r']*>[\s\S]*?', ' ', text, flags=_re.IGNORECASE) text = _re.sub(r'<[^>]+>', ' ', text) text = _re.sub(r'&[a-zA-Z]+;', ' ', text) text = _re.sub(r'&#\d+;', ' ', text) text = _re.sub(r'\s+', ' ', text).strip() return text @app.post("/api/embeddings/upload-url") async def embed_upload_url( adb_config_id: str = Form(...), table_name: str = Form(""), url: str = Form(...), bg: BackgroundTasks = None, u=Depends(require("admin", "user")) ): _verify_config_access("adb", adb_config_id, u) import requests as req url = url.strip() if not url.startswith(("http://", "https://")): raise HTTPException(400, "URL inválida — deve começar com http:// ou https://") try: resp = req.get(url, timeout=30, headers={"User-Agent": "Mozilla/5.0 AI-Agent/1.0"}) resp.raise_for_status() except Exception as e: raise HTTPException(400, f"Erro ao acessar URL: {str(e)[:300]}") ct = resp.headers.get("content-type", "") if "pdf" in ct: content = _extract_pdf_text(resp.content) elif "html" in ct or "text" in ct: content = _extract_text_from_html(resp.text) else: content = resp.text if not content or not content.strip(): raise HTTPException(400, "Nenhum conteúdo extraído da URL") documents = _chunk_text_file(content, url) if not documents: raise HTTPException(400, "Nenhum chunk gerado do conteúdo") cfg, gc = _get_adb_and_genai(adb_config_id, user_id=u["id"]) target_table = table_name.strip() or None bg.add_task(_ingest_documents_task, cfg, gc, documents, u["id"], u["username"], table_name=target_table) _audit(u["id"], u["username"], "embed_url", url, f"{len(documents)} chunks") return {"ok": True, "message": f"Embedding de {len(documents)} chunks iniciado", "chunks": len(documents), "url": url} class ConsultQuery(BaseModel): query: str table_name: str = "" top_k: int = 10 oci_config_id: str = "" @app.post("/api/embeddings/consult") async def consult_embeddings(req: ConsultQuery, u=Depends(current_user)): """Query embeddings via vector search + GenAI to get a formatted answer.""" if not req.query.strip(): raise HTTPException(400, "Query não pode ser vazia") adb_configs = _get_active_adb_configs(u["id"]) if not adb_configs: raise HTTPException(400, "Nenhuma conexão ADB ativa configurada") # Resolve tenancy for filtered search rag_tenancy = None if req.oci_config_id: with db() as c: oci_row = c.execute("SELECT tenancy_name FROM oci_configs WHERE id=?", (req.oci_config_id,)).fetchone() if oci_row: rag_tenancy = oci_row["tenancy_name"] log.info(f"Consult: filtering by tenancy '{rag_tenancy}'") # Detect CIS recommendation number in query for exact text filtering import re as _re cis_match = _re.search(r'(?:cis|recommendation)\s*(\d+\.\d+)', req.query, _re.IGNORECASE) cis_text_filter = f"CIS Recommendation: {cis_match.group(1)}" if cis_match else None if cis_text_filter: log.info(f"Consult: detected CIS filter '{cis_text_filter}'") # Collect results from all active ADB configs + tables all_docs = [] rag_errors = [] for adb_cfg in adb_configs: try: emb_genai = _resolve_embed_config(oci_config_id=adb_cfg.get("oci_config_id"), user_id=u["id"]) except Exception as e: log.warning(f"Consult: resolve config failed for {adb_cfg['id']}: {e}") continue tables = _get_tables_for_config(adb_cfg["id"], active_only=True) if req.table_name: tables = [t for t in tables if t["table_name"] == req.table_name] all_table_names = [t["table_name"] for t in tables if t["table_name"]] # Smart skip relevant = _relevant_tables(req.query, all_table_names) if not req.table_name else all_table_names skipped = set(all_table_names) - set(relevant) if skipped: log.info(f"Consult: skipped {', '.join(skipped)}") # Auto-detect model emb_model = adb_cfg.get("embedding_model_id", "cohere.embed-v4.0") for tbl_name in relevant: try: dim = _get_table_embedding_dim(adb_cfg, tbl_name) if dim and dim in _DIM_TO_MODEL: emb_model = _DIM_TO_MODEL[dim] break except Exception: pass try: query_embedding = _embed_text(req.query, emb_genai, emb_model) tbl_top_k = 10 if cis_text_filter else 3 docs = _vector_search_multi(adb_cfg, query_embedding, relevant, top_k_per_table=tbl_top_k, tenancy=rag_tenancy, text_filter=cis_text_filter, user_id=u["id"]) all_docs.extend(docs) if docs: sources = {} for d in docs: sources[d["source"]] = sources.get(d["source"], 0) + 1 log.info(f"Consult: {len(docs)} docs — {', '.join(f'{k}:{v}' for k,v in sources.items())}") except Exception as e: err = str(e)[:150] log.warning(f"Consult: search failed: {err}") if "DPY-6001" in str(e) or "DPY-6005" in str(e) or "timeout" in str(e).lower(): rag_errors.append(f"ADB offline ou timeout ({adb_cfg.get('config_name','?')})") if not all_docs: if rag_errors: return {"answer": "⚠️ " + "; ".join(set(rag_errors)) + ". A base de conhecimento não está disponível no momento.", "documents": [], "total": 0} return {"answer": "Nenhum resultado encontrado nas bases vetoriais.", "documents": [], "total": 0} # Sort by distance and take top results all_docs.sort(key=lambda d: d.get("distance", 999)) top_limit = 15 if cis_text_filter else 8 top_docs = all_docs[:top_limit] # Build context with dates and sources rag_context = _build_rag_context(top_docs, max_total_chars=16000 if cis_text_filter else 12000) if rag_errors: rag_context += "\n\n⚠️ Algumas bases não puderam ser consultadas: " + "; ".join(set(rag_errors)) augmented = RAG_CONTEXT_TEMPLATE.format(context=rag_context, question=req.query) # Get GenAI config for answering — try saved config first, then auto-resolve from OCI gc = None with db() as c: gc_row = c.execute("SELECT * FROM genai_configs WHERE user_id=? AND is_default=1 ORDER BY created_at DESC", (u["id"],)).fetchone() if not gc_row: gc_row = c.execute("SELECT * FROM genai_configs WHERE user_id=? ORDER BY created_at DESC", (u["id"],)).fetchone() if gc_row: gc = dict(gc_row) else: # Auto-resolve: build GenAI config from OCI credentials + default model try: resolved = _resolve_embed_config(user_id=u["id"]) gc = { "oci_config_id": resolved["oci_config_id"], "endpoint": resolved.get("endpoint", f"https://inference.generativeai.{resolved.get('genai_region','us-ashburn-1')}.oci.oraclecloud.com"), "compartment_id": resolved.get("compartment_id", ""), "genai_region": resolved.get("genai_region", "us-ashburn-1"), "model_id": "openai.gpt-5.2", "model_ocid": "", "serving_type": "ON_DEMAND", "temperature": 0.3, "max_tokens": 8000, "top_p": 0.9, "top_k": 1, "frequency_penalty": 0, "presence_penalty": 0, } log.info(f"Consult: auto-resolved GenAI config from OCI, model=openai.gpt-4.1") except Exception as e: log.warning(f"Consult: no GenAI config available: {e}") doc_list = [{"content": d.get("content", "")[:500], "source": d.get("source", ""), "distance": round(d.get("distance", 0), 4), "metadata": d.get("metadata", "")} for d in top_docs] parts = [] for i, d in enumerate(top_docs, 1): content = d.get("content", "") if len(content) > 800: content = content[:800] + "..." parts.append(f"**Documento {i}** — `{d.get('source', '?')}` (distância: {d.get('distance', 0):.4f})\n\n{content}") return {"answer": "\n\n---\n\n".join(parts), "documents": doc_list, "total": len(all_docs)} try: gc["system_prompt"] = CONSULT_SYSTEM_PROMPT answer, _, _ = _call_genai(gc, augmented) except Exception as e: log.error(f"Consult GenAI error: {e}") answer = f"Erro ao consultar GenAI: {str(e)[:300]}" doc_list = [{"content": d.get("content", "")[:500], "source": d.get("source", ""), "distance": round(d.get("distance", 0), 4), "metadata": d.get("metadata", "")} for d in top_docs] return {"answer": answer, "documents": doc_list, "total": len(all_docs)} @app.get("/api/embeddings/{vid}/list") async def list_embeddings(vid: str, table_name: str = Query(""), limit: int = Query(50), offset: int = Query(0), u=Depends(current_user)): _verify_config_access("adb", vid, u) with db() as c: cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone() if not cfg: raise HTTPException(404) try: conn = _get_adb_connection(dict(cfg)) cur = conn.cursor() table_name = table_name.strip() or cfg["table_name"] if not table_name: raise HTTPException(400, "Nenhuma tabela selecionada") cur.execute(f'SELECT COUNT(*) FROM "{table_name}"') total = cur.fetchone()[0] cur.execute(f""" SELECT ID, METADATA FROM "{table_name}" OFFSET :1 ROWS FETCH NEXT :2 ROWS ONLY """, [offset, limit]) rows = [] for row in cur: rid = row[0].hex() if isinstance(row[0], bytes) else str(row[0]) meta = row[1] if hasattr(meta, 'read'): meta = meta.read() rows.append({"id": rid, "metadata": meta}) cur.close(); conn.close() return {"total": total, "offset": offset, "limit": limit, "documents": rows} except Exception as e: raise HTTPException(500, f"Erro ao listar embeddings: {str(e)[:500]}") @app.delete("/api/embeddings/{vid}/{doc_id}") async def delete_embedding(vid: str, doc_id: str, table_name: str = Query(""), u=Depends(require("admin","user"))): _verify_config_access("adb", vid, u, require_owner=True) with db() as c: cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone() if not cfg: raise HTTPException(404) try: conn = _get_adb_connection(dict(cfg)) cur = conn.cursor() table_name = table_name.strip() or cfg["table_name"] if not table_name: raise HTTPException(400, "Nenhuma tabela selecionada") cur.execute(f'DELETE FROM "{table_name}" WHERE ID = :1', [doc_id]) conn.commit() cur.close(); conn.close() return {"ok": True} except Exception as e: raise HTTPException(500, f"Erro ao deletar: {str(e)[:500]}") @app.post("/api/embeddings/{vid}/purge") async def purge_embeddings(vid: str, req: dict, u=Depends(require("admin","user"))): """Delete old embeddings from a table, optionally filtered by tenancy.""" _verify_config_access("adb", vid, u, require_owner=True) table_name = req.get("table_name", "").strip() tenancy = req.get("tenancy", "").strip() if not table_name: raise HTTPException(400, "table_name é obrigatório") with db() as c: cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone() if not cfg: raise HTTPException(404) try: conn = _get_adb_connection(dict(cfg)) cur = conn.cursor() if tenancy: cur.execute(f'SELECT COUNT(*) FROM "{table_name}" WHERE METADATA LIKE :1', [f'%"tenancy":"{tenancy}"%']) count = cur.fetchone()[0] cur.execute(f'DELETE FROM "{table_name}" WHERE METADATA LIKE :1', [f'%"tenancy":"{tenancy}"%']) else: cur.execute(f'SELECT COUNT(*) FROM "{table_name}"') count = cur.fetchone()[0] cur.execute(f'DELETE FROM "{table_name}"') conn.commit() cur.close(); conn.close() _audit(u["id"], u["username"], "purge_embeddings", vid, f"table={table_name}, tenancy={tenancy or 'ALL'}, deleted={count}") return {"ok": True, "deleted": count, "table": table_name, "tenancy": tenancy or "ALL"} except Exception as e: raise HTTPException(500, f"Erro ao limpar: {str(e)[:500]}") # ── Reports ─────────────────────────────────────────────────────────────────── def _classify_report_file(fname: str) -> str: """Classify a report file into a category based on its filename.""" fl = fname.lower() if "summary_report" in fl: return "summary" if "error_report" in fl or "error" in fl and fl.endswith(".csv"): return "error" if fl.startswith("obp_") and "findings" in fl: return "obp_finding" if fl.startswith("obp_") and "best_practices" in fl: return "obp_best_practice" if fl.startswith("obp_"): return "obp_finding" if fl.startswith("raw_data_"): return "raw_data" if fl.startswith("cis_"): return "cis_finding" if "consolidated_report" in fl: return "consolidated" if fl.endswith(".png"): return "diagram" return "other" @app.post("/api/reports/run") async def run_report(req: RunReportReq, bg: BackgroundTasks, u=Depends(require("admin","user"))): if req.level not in (1, 2): raise HTTPException(400, "Level must be 1 or 2") _verify_config_access("oci", req.config_id, u) with db() as c: cfg = c.execute("SELECT * FROM oci_configs WHERE id=?",(req.config_id,)).fetchone() if not cfg: raise HTTPException(404, "Config não encontrada") rid = str(uuid.uuid4()) c.execute("INSERT INTO reports (id,user_id,tenancy_name,config_id,level,obp_checks,raw_data,redact_output,status) VALUES (?,?,?,?,?,?,?,?,?)", (rid, u["id"], cfg["tenancy_name"], req.config_id, req.level, int(req.obp), int(req.raw), int(req.redact_output), "running")) bg.add_task(_exec_report, rid, dict(cfg), req.regions, req.level, req.obp, req.raw, req.redact_output) _audit(u["id"], u["username"], "run_report", rid) return {"report_id": rid, "status": "running"} async def _exec_report(rid, cfg, regions, level=2, obp=False, raw=False, redact_output=False): rdir = REPORTS / rid; rdir.mkdir(parents=True, exist_ok=True) config_path = str(OCI_DIR / cfg["id"] / "config") try: cmd = ["python3", "-u", "/app/cis_reports.py", "-c", config_path, "--report-directory", str(rdir), "--level", str(level), "--print-to-screen", "True", "--report-summary-json"] if regions: cmd += ["--regions", ",".join(regions)] if obp: cmd.append("--obp") if raw: cmd.append("--raw") if redact_output: cmd.append("--redact-output") proc = await asyncio.create_subprocess_exec(*cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) _running_reports[rid] = proc with db() as c: c.execute("UPDATE reports SET worker_pid=? WHERE id=?", (proc.pid, rid)) progress_lines = [] try: while True: line = await proc.stdout.readline() if not line: break text = line.decode(errors="replace").strip() if text: progress_lines.append(text) with db() as c: c.execute("UPDATE reports SET progress=? WHERE id=?", ("\n".join(progress_lines[-50:]), rid)) await proc.wait() finally: _running_reports.pop(rid, None) stderr_data = await proc.stderr.read() # Check if cancelled with db() as c: cur_status = c.execute("SELECT status FROM reports WHERE id=?", (rid,)).fetchone() if cur_status and cur_status["status"] == "cancelled": return if proc.returncode == 0: # Scan output directory for all generated files html_path = None; json_path = None with db() as c: for fpath in rdir.iterdir(): if fpath.is_file(): fname = fpath.name ftype = fpath.suffix.lstrip(".") category = _classify_report_file(fname) fsize = fpath.stat().st_size c.execute("INSERT INTO report_files (id,report_id,file_name,file_path,file_type,file_category,file_size) VALUES (?,?,?,?,?,?,?)", (str(uuid.uuid4()), rid, fname, str(fpath), ftype, category, fsize)) if "summary_report" in fname and fname.endswith(".html"): html_path = str(fpath) elif "summary_report" in fname and fname.endswith(".json"): json_path = str(fpath) c.execute("UPDATE reports SET status='completed',progress=?,html_path=?,json_path=?,completed_at=datetime('now') WHERE id=?", ("\n".join(progress_lines), html_path, json_path, rid)) _config_log("oci", cfg["id"], cfg["tenancy_name"], "success", "report", f"Report completed: {rid}") else: err = (stderr_data.decode(errors="replace") if stderr_data else "Unknown")[:2000] with db() as c: c.execute("UPDATE reports SET status='failed',progress=?,error_msg=?,completed_at=datetime('now') WHERE id=?", ("\n".join(progress_lines), err, rid)) _config_log("oci", cfg["id"], cfg["tenancy_name"], "error", "report", err) except Exception as e: _running_reports.pop(rid, None) with db() as c: c.execute("UPDATE reports SET status='failed',error_msg=?,completed_at=datetime('now') WHERE id=?", (str(e)[:2000], rid)) _config_log("oci", cfg["id"], cfg["tenancy_name"], "error", "report", str(e)[:2000]) @app.get("/api/reports") async def list_reports(skip: int = Query(0, ge=0), limit: int = Query(50, ge=1, le=200), u=Depends(current_user)): with db() as c: q = "SELECT id,user_id,tenancy_name,config_id,status,progress,level,obp_checks,raw_data,redact_output,created_at,completed_at,error_msg FROM reports" rows = c.execute(q+" WHERE user_id=? ORDER BY created_at DESC LIMIT ? OFFSET ?",(u["id"], limit, skip)).fetchall() return [dict(r) for r in rows] @app.get("/api/reports/{rid}/progress") async def get_report_progress(rid: str, u=Depends(current_user)): _verify_report_access(rid, u) with db() as c: r = c.execute("SELECT status,progress,created_at,completed_at,error_msg FROM reports WHERE id=?", (rid,)).fetchone() if not r: raise HTTPException(404) return dict(r) @app.post("/api/reports/{rid}/cancel") async def cancel_report(rid: str, u=Depends(require("admin", "user"))): _verify_report_access(rid, u) with db() as c: r = c.execute("SELECT status, worker_pid FROM reports WHERE id=?", (rid,)).fetchone() if not r: raise HTTPException(404) if r["status"] != "running": raise HTTPException(400, "Report is not running") proc = _running_reports.get(rid) if proc: try: proc.terminate() try: await asyncio.wait_for(proc.wait(), timeout=5) except asyncio.TimeoutError: proc.kill() except ProcessLookupError: pass _running_reports.pop(rid, None) elif r["worker_pid"]: import signal try: os.kill(r["worker_pid"], signal.SIGTERM) except (ProcessLookupError, OSError): pass with db() as c: c.execute("UPDATE reports SET status='cancelled',error_msg='Cancelled by user',completed_at=datetime('now') WHERE id=?", (rid,)) _audit(u["id"], u["username"], "cancel_report", rid) return {"ok": True, "status": "cancelled"} @app.get("/api/reports/{rid}") async def get_report(rid, u=Depends(current_user)): with db() as c: r=c.execute("SELECT * FROM reports WHERE id=?",(rid,)).fetchone() if not r: raise HTTPException(404) if u["role"]!="admin" and r["user_id"]!=u["id"]: raise HTTPException(403) return dict(r) @app.delete("/api/reports/{rid}") async def delete_report(rid: str, u=Depends(current_user)): with db() as c: r = c.execute("SELECT id, user_id, status, json_path, html_path FROM reports WHERE id=?", (rid,)).fetchone() if not r: raise HTTPException(404) if u["role"] != "admin" and r["user_id"] != u["id"]: raise HTTPException(403) if r["status"] == "running": raise HTTPException(400, "Cannot delete a running report") # Remove associated files for path_col in ("json_path", "html_path"): p = r[path_col] if p and Path(p).exists(): try: Path(p).unlink() except OSError: pass c.execute("DELETE FROM reports WHERE id=?", (rid,)) _audit(u["id"], u["username"], "delete_report", rid) return {"ok": True} @app.get("/api/reports/{rid}/summary") async def report_summary(rid: str, u=Depends(current_user)): with db() as c: r = c.execute("SELECT user_id,json_path,tenancy_name,level,created_at FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone() if not r: raise HTTPException(404) if u["role"] != "admin" and r["user_id"] != u["id"]: raise HTTPException(403) jp = r["json_path"] if not jp or not Path(jp).exists(): raise HTTPException(400, "Report JSON not found") data = json.loads(Path(jp).read_text()) total = passed = failed = 0 sections = {} total_findings = 0 total_compliant = 0 # Format: list of checks with Compliant=Yes/No, Section, Findings, Compliant Items, Total checks = data if isinstance(data, list) else data.get("findings", []) if isinstance(checks, dict): checks = list(checks.values()) for check in checks: total += 1 compliant = str(check.get("Compliant", check.get("status", ""))).strip().lower() is_pass = compliant in ("yes", "pass") if is_pass: passed += 1 else: failed += 1 fi = str(check.get("Findings", "0")).strip() ci = str(check.get("Compliant Items", "0")).strip() total_findings += int(fi) if fi.isdigit() else 0 total_compliant += int(ci) if ci.isdigit() else 0 sec = check.get("Section", check.get("section", "Other")) s = sections.setdefault(sec, {"passed": 0, "failed": 0, "total": 0}) s["total"] += 1 if is_pass: s["passed"] += 1 else: s["failed"] += 1 score = round((passed / total * 100), 1) if total else 0 return { "tenancy": data.get("tenancy", r["tenancy_name"]) if isinstance(data, dict) else r["tenancy_name"], "level": r["level"] or 2, "regions": data.get("regions", []) if isinstance(data, dict) else [], "generated_at": r["created_at"], "total": total, "passed": passed, "failed": failed, "total_findings": total_findings, "total_compliant": total_compliant, "score": score, "sections": sections } @app.get("/api/reports/{rid}/files") async def list_report_files(rid: str, u=Depends(current_user)): with db() as c: r = c.execute("SELECT user_id FROM reports WHERE id=?", (rid,)).fetchone() if not r: raise HTTPException(404) if u["role"] != "admin" and r["user_id"] != u["id"]: raise HTTPException(403) files = c.execute( "SELECT id,file_name,file_type,file_category,file_size FROM report_files WHERE report_id=? ORDER BY file_category,file_name", (rid,) ).fetchall() return [dict(f) for f in files] @app.get("/api/reports/{rid}/files/{fid}/download") async def download_report_file(rid: str, fid: str, token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))): u = _user_from_token(token) if token else (await current_user(cred) if cred else None) if not u: raise HTTPException(401, "Not authenticated") with db() as c: r = c.execute("SELECT user_id FROM reports WHERE id=?", (rid,)).fetchone() if not r: raise HTTPException(404) if u["role"] != "admin" and r["user_id"] != u["id"]: raise HTTPException(403) f = c.execute("SELECT * FROM report_files WHERE id=? AND report_id=?", (fid, rid)).fetchone() if not f: raise HTTPException(404) p = Path(f["file_path"]) if not p.exists(): raise HTTPException(404, "File not found on disk") return FileResponse(p, filename=f["file_name"]) @app.api_route("/api/reports/{rid}/html", methods=["GET", "HEAD"]) async def report_html(rid, token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))): u = _user_from_token(token) if token else (await current_user(cred) if cred else None) if not u: raise HTTPException(401, "Not authenticated") _verify_report_access(rid, u) with db() as c: r=c.execute("SELECT html_path FROM reports WHERE id=?",(rid,)).fetchone() if not r or not r["html_path"] or not Path(r["html_path"]).exists(): raise HTTPException(404) return FileResponse(r["html_path"], media_type="text/html") # ═══════════════════════════════════════════════════════════════════════ # LAD A-Team CIS Compliance Report # ═══════════════════════════════════════════════════════════════════════ def _extract_section(text: str, section_name: str) -> str: """Extract a named section from a CIS Benchmark chunk (Description, Remediation, etc.).""" import re as _re pattern = _re.compile(r'\n' + _re.escape(section_name) + r'[:\s]*\n', _re.IGNORECASE) match = pattern.search(text) if not match: return "" start = match.end() end_match = _re.search( r'\n(?:Description|Rationale|Audit|Remediation|Additional Information|CIS Controls|Controls\s*\nVersion|References|Default Value|Impact|Recommendation|Profile Applicability)[:\s]*\n', text[start:], _re.IGNORECASE ) if end_match: return text[start:start + end_match.start()].strip() return text[start:].strip() def _extract_remediation_section(text: str) -> str: """Extract only the Remediation section.""" return _extract_section(text, "Remediation") def _extract_audit_section(text: str) -> str: """Extract the Audit section (verification steps).""" return _extract_section(text, "Audit") def _extract_rationale_section(text: str) -> str: """Extract the Rationale section (why this matters).""" return _extract_section(text, "Rationale") def _extract_description_section(text: str) -> str: """Extract the Description section (what this CIS finding is about).""" return _extract_section(text, "Description") def _format_remediation_html(text: str) -> str: """Format remediation text with code snippets and proper indentation.""" import re as _re if not text: return "" def esc(v): return (v or "").replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """) # Pre-process: join broken lines from PDF extraction # PDF text wraps at column width, creating artificial line breaks mid-sentence processed_lines = [] _BLOCK_STARTERS = _re.compile( r'^(?:\d+\.\s|From |From:|Note:|Remediation:|Audit:|Description:|Rationale:|' r'oci |Allow |Define |for |do$|done$|\$ |query |where |#|' r'Cloud Guard:|Using |Remediation |Default Value|Impact|References|' r'Profile Applicability|Additional Information|CIS Controls)', _re.IGNORECASE ) for line in text.split('\n'): stripped = line.strip() if not stripped: processed_lines.append('') continue prev = processed_lines[-1] if processed_lines else '' # Join if previous line doesn't end with sentence terminator AND # current line is NOT a new block/section/command should_join = ( prev and not prev.endswith(('.', ':', ';', ')', '}', '\\', '|', '"')) and not _BLOCK_STARTERS.match(stripped) ) if should_join: processed_lines[-1] = prev + ' ' + stripped else: processed_lines.append(stripped) text = '\n'.join(processed_lines) # Code line detection patterns _CODE_STARTERS = re.compile( r'^(?:oci |Allow |Define |Endorse |Admit |terraform |curl |openssl |ssh |python|' r'for |do$|done$|if |fi$|echo |export |eval |jq |grep |awk |sed |cat |' r'\$ |#!|output=|[a-z_]+=\$|\{\{|query |where |bucket |' r'select |filter=)', re.IGNORECASE ) _CODE_CONTINUATIONS = re.compile( r'^(?:--[a-z]|[|>]|\)$|\}|&&|\|\||resources\b|\.data|"query |' r'IngressSecurityRules|EgressSecurityRules|tcpOptions|udpOptions|' r'destinationPortRange|sourcePortRange|"region-|"\.region|' r'publicAccessType|ObjectRead|ObjectReadWithoutList|' r'\.source |\.direction |\.protocol |' r'[a-z]+\.[a-z]+\.|2>/dev/null|\()', re.IGNORECASE ) lines = text.split('\n') html_parts = [] in_code = False code_buf = [] def flush_code(): nonlocal in_code, code_buf if code_buf: html_parts.append('
' + esc('\n'.join(code_buf)) + '
') code_buf = [] in_code = False for line in lines: stripped = line.strip() if not stripped: if in_code: code_buf.append('') continue is_code_start = bool(_CODE_STARTERS.match(stripped)) is_code_cont = in_code and ( bool(_CODE_CONTINUATIONS.match(stripped)) or stripped.endswith('\\') or stripped.endswith('|') or stripped.endswith('{') or (not stripped[0].isupper() and not _re.match(r'^\d+\.', stripped) and not stripped.startswith('From ') and not stripped.startswith('To ') and not stripped.startswith('Note') and not stripped.startswith('Ensure') and not stripped.startswith('Cloud ') and not stripped.startswith('The ') and not stripped.startswith('Navigate') and not stripped.startswith('Click') and not stripped.startswith('Select') and not stripped.startswith('In the') and not stripped.startswith('Go to') and not stripped.startswith('Open') and not stripped.startswith('This ') and not stripped.startswith('For ') and not stripped.startswith('If ') and not stripped.startswith('Are ') and not stripped.startswith('Check ') and not stripped.startswith('Verify')) ) if is_code_start or is_code_cont: if not in_code: in_code = True code_buf = [] code_buf.append(stripped) else: flush_code() # "From Console:", "From CLI:", "From Command Line:", "Audit / Verification:" → bold header header_match = _re.match(r'^(From\s+(?:Console|CLI|Command Line|the Console|the CLI|OCI Console|OCI CLI)[:\s]*|Audit\s*/?\s*Verification[:\s]*)(.*)', stripped, _re.IGNORECASE) if header_match: rest = esc(header_match.group(2)) if header_match.group(2) else '' html_parts.append(f'

{esc(header_match.group(1).strip())} {rest}

') else: step_match = _re.match(r'^(\d+)\.\s+(.+)', stripped) if step_match: html_parts.append(f'
{step_match.group(1)}. {esc(step_match.group(2))}
') else: html_parts.append(f'

{esc(stripped)}

') flush_code() return '\n'.join(html_parts) def _search_cis_remediation(title: str, rec_num: str, user_id: str) -> dict: """Search CIS Recommendations in vector DB. Returns {'description': str, 'remediation': str}.""" try: with db() as c: adb_cfgs = c.execute( "SELECT * FROM adb_vector_configs WHERE is_active=1 AND (user_id=? OR is_global=1)", (user_id,) ).fetchall() if not adb_cfgs: return {"description": "", "remediation": ""} query = f"Recommendation: {rec_num} {title} Remediation Audit From Console From Command Line" for adb_cfg in adb_cfgs: adb_cfg = dict(adb_cfg) try: # Resolve embedding config: use the ADB's own embedding model emb_model = adb_cfg.get("embedding_model_id", "cohere.embed-v4.0") tables = _get_tables_for_config(adb_cfg["id"], active_only=True) if not tables: continue # Resolve GenAI config for embedding generation genai_linked = None if adb_cfg.get("genai_config_id"): with db() as c: row = c.execute("SELECT * FROM genai_configs WHERE id=?", (adb_cfg["genai_config_id"],)).fetchone() if row: genai_linked = dict(row) emb_genai = _resolve_embed_config(genai_cfg=genai_linked, user_id=user_id) # Search across cisrecom + section-specific tables all_docs = [] search_tables = [t["table_name"] for t in tables if "cisrecom" in t["table_name"].lower()] if not search_tables: search_tables = [t["table_name"] for t in tables] for tbl_name in search_tables: try: # Detect dimension per table tbl_model = emb_model try: actual_dim = _get_table_embedding_dim(adb_cfg, tbl_name) if actual_dim and actual_dim in _DIM_TO_MODEL: tbl_model = _DIM_TO_MODEL[actual_dim] except Exception as e: log.warning(f"Failed to detect embedding dimension for table '{tbl_name}': {e}") query_embedding = _embed_text(query, emb_genai, tbl_model) docs = _vector_search(adb_cfg, query_embedding, top_k=10, table_name=tbl_name, user_id=user_id) if docs: all_docs.extend(docs) except Exception as e: log.warning(f"Vector search failed for table '{tbl_name}': {e}") if all_docs: import re as _re all_docs.sort(key=lambda d: d.get("distance", 999)) # FILTER: keep chunks that mention THIS recommendation number in text OR metadata exact_pattern = _re.compile( r'(?:Recommendation[:\s]*' + _re.escape(rec_num) + r'|^' + _re.escape(rec_num) + r')\s', _re.MULTILINE | _re.IGNORECASE ) matched = [] for d in all_docs: content = d.get("content", "") if not content or len(content) < 200: continue # Match by text content if exact_pattern.search(content): matched.append(content) continue # Match by metadata recommendationNumber (for overlap chunks) meta = d.get("metadata", "") if isinstance(meta, str) and f'"recommendationNumber": "{rec_num}"' in meta: matched.append(content) continue if isinstance(meta, str) and f'"recommendationNumber":"{rec_num}"' in meta: matched.append(content) # Also fetch ALL chunks for this rec from ADB directly (catches overlap chunks) try: conn2 = _get_adb_connection(adb_cfg) cur2 = conn2.cursor() for tbl_name in search_tables: cur2.execute(f'''SELECT "TEXT" FROM "{tbl_name}" WHERE JSON_VALUE("METADATA", '$.recommendationNumber') = :1 ORDER BY JSON_VALUE("METADATA", '$.chunkIndex')''', [rec_num]) for row in cur2: txt = row[0].read() if hasattr(row[0], 'read') else row[0] if txt and txt not in matched: matched.append(txt) cur2.close() conn2.close() except Exception as e: log.warning(f"Direct chunk fetch for {rec_num}: {e}") if matched: # Concatenate all chunks for the best possible extraction full_text = '\n\n'.join(matched) desc = _extract_description_section(full_text) rationale = _extract_rationale_section(full_text) rem = _extract_remediation_section(full_text) audit = _extract_audit_section(full_text) # Fallback: try individual chunks if full concatenation missed something for extra in matched[1:4]: if not rem: rem = _extract_remediation_section(extra) elif len(rem) < 200: extra_rem = _extract_remediation_section(extra) if extra_rem and extra_rem not in rem: rem = rem.rstrip() + "\n\n" + extra_rem if not audit: audit = _extract_audit_section(extra) if not desc: desc = _extract_description_section(extra) elif desc and not desc.rstrip().endswith(('.', ':', '!', '?', ';')): # Description looks truncated — append continuation from next chunk extra_desc = _extract_description_section(extra) if extra_desc: desc = desc.rstrip() + "\n" + extra_desc if not rationale: rationale = _extract_rationale_section(extra) # Also check non-matched docs for continuation (same CIS but different chunk) all_contents = [d["content"] for d in all_docs if d.get("content") and len(d["content"]) > 200] for extra_content in all_contents[:6]: if extra_content in matched: continue if not rem: rem = _extract_remediation_section(extra_content) if desc and not desc.rstrip().endswith(('.', ':', '!', '?', ';')): extra_desc = _extract_description_section(extra_content) if extra_desc and extra_desc not in desc: desc = desc.rstrip() + "\n" + extra_desc # Combine remediation + audit for complete guidance full_rem = rem or "" if audit and len(audit) > 50: full_rem = (full_rem + "\n\nAudit / Verification:\n" + audit).strip() if full_rem else audit return {"description": desc, "rationale": rationale, "remediation": full_rem} except Exception as e: log.warning(f"CIS remediation search failed for {adb_cfg.get('config_name','?')}: {e}") return {"description": "", "remediation": ""} except Exception as e: log.warning(f"CIS remediation search error: {e}") return {"description": "", "remediation": ""} def _build_findings_table(csv_path: str, max_rows: int = 10) -> str: """Read a findings CSV and return a compact HTML table with up to max_rows.""" import csv as csvmod, re as _re p = Path(csv_path) if isinstance(csv_path, str) else csv_path if not p.exists(): return "" try: with open(p, "r", encoding="utf-8") as f: all_rows = list(csvmod.DictReader(f)) if not all_rows: return "" def esc(v): return (v or "").replace("&","&").replace("<","<").replace(">",">") def fmt(v, col=""): v = (v or "").strip() if not v: return "—" if v.startswith("=HYPERLINK"): m = _re.search(r',\s*"([^"]+)"', v) return m.group(1) if m else "—" if v.startswith("ocid1.") and len(v) > 30: return v[:25] + "..." + v[-8:] # Format time fields as date + age in days if col.lower() in ("time_created", "time_expires", "last_successful_login_date"): try: from datetime import datetime dt = datetime.fromisoformat(v.replace("Z", "+00:00").split("+")[0].split(".")[0]) age = (datetime.utcnow() - dt).days return f"{v[:10]} ({age}d)" except Exception: return v[:16] # Truncate long values but keep enough to identify if len(v) > 60: return v[:57] + "..." return v # Show all useful columns (skip only internal/redundant ones) all_cols = list(all_rows[0].keys()) skip = {"extract_date", "deep_link", "domain_deeplink", "defined_tags", "domain_id", "external_identifier", "freeform_tags", "system_tags", "email_verified", "is_federated", "api_keys", "auth_tokens", "customer_secret_keys", "database_passwords", "can_use_api_keys", "can_use_auth_tokens", "can_use_console_password", "can_use_customer_secret_keys", "can_use_db_credentials", "can_use_o_auth2_client_credentials", "can_use_smtp_credentials"} display = [c for c in all_cols if c.lower() not in skip] # Limit to max 7 columns to keep table readable if len(display) > 7: # Prioritize: identity cols first, then status, then details priority_order = ["user_name", "name", "display_name", "username", "id", "user_id", "key_id", "lifecycle_state", "status", "is_mfa_activated", "domain_name", "email", "description", "fingerprint", "time_created", "last_successful_login_date", "groups", "compartment_id", "region", "cidr_block", "protocol", "port_range", "visibility", "encryption", "versioning", "age", "time_expires", "statements", "type", "severity"] ordered = [] for p in priority_order: for d in display: if d.lower() == p and d not in ordered: ordered.append(d) # Add any remaining not in priority for d in display: if d not in ordered: ordered.append(d) display = ordered[:7] # Pretty column names _COL_LABELS = { "user_name": "User", "user_id": "User ID", "key_id": "Key ID", "fingerprint": "Fingerprint", "time_created": "Created (Age)", "display_name": "Name", "lifecycle_state": "State", "is_mfa_activated": "MFA", "domain_name": "Domain", "cidr_block": "CIDR", "port_range": "Ports", } def col_label(c): return _COL_LABELS.get(c, c.replace("_", " ").title()) total = len(all_rows) show = all_rows[:max_rows] header = "".join(f'{esc(col_label(c))}' for c in display) rows_html = "" for row in show: cells = "".join(f'{esc(fmt(row.get(c, ""), c))}' for c in display) rows_html += f"{cells}" note = f'
{total} non-compliant resource{"s" if total != 1 else ""}.' if total > max_rows: note += f' Showing first {max_rows} — see CSV for complete list.' note += '
' return f'
{header}{rows_html}
{note}
' except Exception as e: log.warning(f"Failed to build findings table from {p}: {e}") return "" def _check_oci_services(config_id: str) -> list[dict]: """Check status of OCI security services (VSS, Data Safe, Cloud Guard, Bastion, Security Zones, Vault).""" import oci results = [] try: oci_config = _get_oci_config(config_id) tenancy_ocid = oci_config.get("tenancy", "") # Use compartment_id from DB if available, otherwise root tenancy with db() as c: cfg = c.execute("SELECT compartment_id FROM oci_configs WHERE id=?", (config_id,)).fetchone() # DB stores base64 — decode if needed compartment_raw = (cfg["compartment_id"] or "") if cfg else "" if compartment_raw and not compartment_raw.startswith("ocid"): try: compartment_raw = base64.b64decode(compartment_raw).decode("utf-8") except Exception: pass root_compartment = compartment_raw if compartment_raw.startswith("ocid") else tenancy_ocid def _safe_list(resp_data): """Extract list from OCI response data (handles both .items and direct list).""" if hasattr(resp_data, 'items'): return resp_data.items if isinstance(resp_data, list): return resp_data return [] # 1. Vulnerability Scanning Service (VSS) try: vss_client = _oci_client(oci.vulnerability_scanning.VulnerabilityScanningClient, config_id) targets = _safe_list(vss_client.list_host_scan_targets(compartment_id=root_compartment).data) if targets: results.append({"service": "Vulnerability Scanning Service", "status": "OK", "description": f"All instances in the compartment root and its subcompartments are scanned ({len(targets)} target(s))"}) else: results.append({"service": "Vulnerability Scanning Service", "status": "WARNING", "description": "No VSS scan targets found in Tenant"}) except Exception as e: log.warning(f"OCI Services check - VSS: {e}") results.append({"service": "Vulnerability Scanning Service", "status": "WARNING", "description": "Unable to verify VSS status or service not enabled"}) # 2. Data Safe try: ds_client = _oci_client(oci.data_safe.DataSafeClient, config_id) targets = _safe_list(ds_client.list_target_databases(compartment_id=root_compartment, lifecycle_state="ACTIVE").data) if targets: results.append({"service": "Data Safe", "status": "OK", "description": f"{len(targets)} target database(s) registered in Data Safe"}) else: results.append({"service": "Data Safe", "status": "WARNING", "description": "No target databases registered in Data Safe"}) except Exception as e: log.warning(f"OCI Services check - Data Safe: {e}") results.append({"service": "Data Safe", "status": "WARNING", "description": "Unable to verify Data Safe status or service not enabled"}) # 3. Cloud Guard (needs tenancy root) + recommendations try: cg_client = _oci_client(oci.cloud_guard.CloudGuardClient, config_id) cg_config = cg_client.get_configuration(tenancy_ocid).data cg_recommendations = [] if cg_config.status == "ENABLED": try: recs = _safe_list(cg_client.list_recommendations( compartment_id=tenancy_ocid, lifecycle_state="ACTIVE", sort_by="riskLevel", sort_order="DESC", limit=50).data) cg_recommendations = [{"name": r.name, "count": r.problem_count, "risk": r.risk_level} for r in recs if r.problem_count and r.problem_count > 0] except Exception as re: log.warning(f"Cloud Guard recommendations fetch failed: {re}") results.append({"service": "Cloud Guard", "status": "OK", "description": "Cloud Guard service is enabled in Tenant", "recommendations": cg_recommendations}) else: results.append({"service": "Cloud Guard", "status": "WARNING", "description": "Cloud Guard service is not enabled in Tenant"}) except Exception as e: log.warning(f"OCI Services check - Cloud Guard: {e}") results.append({"service": "Cloud Guard", "status": "WARNING", "description": "Unable to verify Cloud Guard status"}) # 4. Bastion try: bastion_client = _oci_client(oci.bastion.BastionClient, config_id) bastions = _safe_list(bastion_client.list_bastions(compartment_id=root_compartment).data) if bastions: results.append({"service": "Bastion", "status": "OK", "description": f"Some OCI Bastion instances were found in Tenant ({len(bastions)})"}) else: results.append({"service": "Bastion", "status": "WARNING", "description": "No OCI Bastion instances found in Tenant"}) except Exception as e: log.warning(f"OCI Services check - Bastion: {e}") results.append({"service": "Bastion", "status": "WARNING", "description": "Unable to verify Bastion status"}) # 5. Security Zones (needs tenancy root) try: cg_client_sz = _oci_client(oci.cloud_guard.CloudGuardClient, config_id) sz = _safe_list(cg_client_sz.list_security_zones(compartment_id=tenancy_ocid).data) if sz: results.append({"service": "Security Zones", "status": "OK", "description": f"Security Zones were created in Tenant ({len(sz)})"}) else: results.append({"service": "Security Zones", "status": "WARNING", "description": "No Security Zones found in Tenant"}) except Exception as e: log.warning(f"OCI Services check - Security Zones: {e}") results.append({"service": "Security Zones", "status": "WARNING", "description": "Unable to verify Security Zones status"}) # 6. Vault try: vault_client = _oci_client(oci.key_management.KmsVaultClient, config_id) vaults = _safe_list(vault_client.list_vaults(compartment_id=root_compartment).data) active_vaults = [v for v in vaults if v.lifecycle_state == "ACTIVE"] if active_vaults: results.append({"service": "Vault", "status": "OK", "description": f"Vault instances were found in Tenant ({len(active_vaults)} active)"}) else: results.append({"service": "Vault", "status": "WARNING", "description": "No active Vault instances found in Tenant"}) except Exception as e: log.warning(f"OCI Services check - Vault: {e}") results.append({"service": "Vault", "status": "WARNING", "description": "Unable to verify Vault status"}) except Exception as e: log.error(f"OCI Services check failed: {e}") return results # ── Static descriptions for OCI Services section ── _OCI_SERVICE_DESCRIPTIONS = { "Vulnerability Scanning Service": { "title": "Vulnerability Scanning Service", "description": ( "Oracle Cloud Infrastructure Vulnerability Scanning Service (OCI VSS) is simple, prescriptive, and tightly integrated " "with the OCI platform. VSS is available to all OCI customers that have paid accounts at no additional cost. The " "scanning platform includes default plugins and engines for instance and container scanning. The service scans " "installed packages and artifacts looking for the existence of known vulnerabilities. The service scans for open ports " "on an instance and it reports on publicly and privately available open ports. These findings will highlight what ports " "an attacker might use or what application could be shut down by the customer to reduce their attack surface. The " "scanning agent also reviews the configuration of each instance against specific OS CIS benchmarks enabling " "customers to see immediately what Operating System (OS) security hardening opportunities exist on their instances." ), "recommendation": "Create a recipe for scanning instances/containers and targets to use those recipes. Collect all results in Cloud Guard.", }, "Data Safe": { "title": "Data Safe", "description": ( "An audit trail represents the collection of audit records from the target database such as " "UNIFIED_AUDIT_TRAIL, which provides documentary evidence of the sequence of activities that happen. " "A database audit trail is the source of audit records showing what has happened in the target database. When audit " "data collection is enabled for the specified database audit trail in an audit trail resource, the audit records are " "copied from the database's audit trail into Oracle Data Safe in near-real time. You can manage the audit records " "volume in the target database using the auto purge feature." ), "recommendation": ( "Start Oracle Data Safe audit trail for all Databases. " "Take advantage of Activity Auditing Dashboard and Audit Insights." ), }, "Cloud Guard": { "title": "Cloud Guard", "description": ( "Misconfigured resources and insecure activity in the cloud represent one of the most difficult problems for security " "professionals. Misconfigured cloud tenancies around cloud resources can present themselves in many ways; from " "publicly accessible object storage buckets, unencrypted data storage, sensitive ports open to the internet. " "Oracle Cloud Infrastructure Cloud Guard is a cloud security service that detects misconfigured resources and " "insecure activities. Cloud Guard acts as a log and events aggregator that directly integrates with all major Oracle " "Cloud Infrastructure services (Compute, Networking, Storage, etc.) providing actionable results. Cloud Guard offers " "the flexibility to take action on security issues manually or automatically with conditional operators." ), "recommendation": "Based on its monitoring (aligned with CIS baselines), Cloud Guard provides in its console recommendations, some already highlighted in this document.", }, "Bastion": { "title": "OCI Bastion", "description": ( "Provide restricted and time-limited secure access to resources that don't have public endpoints and require strict " "resource access controls. Examples include compute instances, bare metal and virtual machines, MySQL, ATP, OKE, " "and any other resource that allows Secure Shell Protocol (SSH) access. With Oracle Cloud Infrastructure (OCI) " "Bastion service, customers can enable access to private hosts without deploying and maintaining a jump host. In " "addition, customers gain improved security posture with identity-based permissions and a centralized, audited, and " "time-bound SSH session. OCI Bastion removes the need for a public IP for bastion access, eliminating the hassle and " "potential attack surface from remote access." ), "recommendation": "We recommend using Bastion to remediate findings from CIS Controls 2.1, 2.2, 2.3 and 2.4 in public subnets, if the permission is related to remote access use cases without restricted sources, VPN or Fast Connect.", }, "Security Zones": { "title": "Security Zones", "description": ( "Security Zones let you be confident that your resources in Oracle Cloud Infrastructure, including Compute, " "Networking, Object Storage, Block Volume and Database resources, comply with your security policies. " "A security zone is associated with one or more compartments and a security zone recipe. When you create and " "update resources in a security zone, Oracle Cloud Infrastructure validates these operations against the list of " "policies that are defined in the security zone recipe. If any security zone policy is violated, then the operation is " "denied. By default, a compartment and any subcompartments are in the same security zone, but you can also create " "a different security zone for a subcompartment." ), "recommendation": "", }, "Vault": { "title": "Vault", "description": ( "Vaults are logical entities where Vault service creates and durably stores vault keys and secrets. The type of vault " "you have determines features and functionality such as degrees of storage isolation, access to management and " "encryption, scalability, and the ability to back up. The type of vault you have also affects pricing. You cannot change " "a vault's type after you create the vault. " "The Vault service offers different vault types to accommodate your organization's needs and budget. All vault types " "ensure the security and integrity of the encryption keys and secrets that vaults store. A virtual private vault is an " "isolated partition on a hardware security module (HSM). Vaults otherwise share partitions on the HSM with other vaults." ), "recommendation": "", }, } def _build_compliance_html(filtered_rows: list, tenancy_name: str, extract_date: str, sections_data: dict, rid: str = "", file_id_map: dict = None, file_path_map: dict = None, oci_services: list = None) -> str: """Build the full LAD A-Team CIS Compliance Report HTML.""" import re as _re from datetime import datetime as _dt now = _dt.utcnow() months_pt = ["Janeiro","Fevereiro","Março","Abril","Maio","Junho","Julho","Agosto","Setembro","Outubro","Novembro","Dezembro"] month_year = f"{months_pt[now.month - 1]}, {now.year}" def esc(v): return (v or "").replace("&","&").replace("<","<").replace(">",">").replace('"',""") def slug(s): return _re.sub(r"[^\w-]","", _re.sub(r"\s+","-",(s or "").strip().lower())) def _join_broken_lines(text): """Join single-newline broken lines into flowing paragraphs. Preserves: double-newline paragraph breaks, numbered lists, bullet lists, headers.""" if not text: return text lines = text.split('\n') result = [] for line in lines: line = line.strip() if not line: result.append('\n') continue # Never join if this line starts a list item or header starts_new = ( _re.match(r'^\d+[\.\)]\s', line) or # "1. ", "2) " _re.match(r'^[a-z][\.\)]\s', line) or # "a. ", "b) " line.startswith('- ') or line.startswith('* ') or # bullets line.startswith('From ') or # "From Console:", "From CLI:" line.startswith('Audit') or # "Audit / Verification:" line.startswith('Note:') or line.startswith('See ') ) if starts_new: result.append(line) elif result and result[-1] != '\n' and not result[-1].endswith(('.', ':', '!', '?', ';')): result[-1] = result[-1] + ' ' + line else: result.append(line) return '\n'.join(result).strip() total_all = len(filtered_rows) total_passed = sum(1 for r in filtered_rows if r["__compliant"]) total_failed = total_all - total_passed pct_global = round(total_passed / total_all * 100) if total_all else 0 # ── Domains table rows ── domains_rows = "" for sec_name, sd in sections_data.items(): domains_rows += f'{esc(sec_name)}{sd["total"]}{sd["failed"]}{sd["passed"]}' # ── CIS Benchmark Summary table ── summary_rows = "" current_section = "" for r in filtered_rows: sec = r["Section"] rec = r.get("Recommendation #", "") title = r.get("Title", "") level = (r.get("Level", "") or "").strip() is_compliant = r["__compliant"] if sec != current_section: current_section = sec summary_rows += f'{esc(sec)}' status_cls = "pass" if is_compliant else "fail" status_txt = "OK" if is_compliant else "FAILED" level_txt = f'L{level}' if level else '' summary_rows += f'{esc(rec)}{esc(title)}{level_txt}{status_txt}' # ── Detailed sections ── sections_html = "" for sec_name, sd in sections_data.items(): sec_id = f"section-{slug(sec_name)}" sections_html += f'
' sections_html += '
ORACLE
' sections_html += f'

{esc(sec_name)}

' for r in sd["items"]: rec = r.get("Recommendation #", "") title = r.get("Title", "") level = (r.get("Level", "") or "").strip() cis_v8 = (r.get("CIS v8", "") or "").strip().strip("[]'\"") cccs = (r.get("CCCS Guard Rail", "") or "").strip().strip("[]'\"") rec_id = f"rec-{slug(sec_name)}-{slug(rec)}" is_compliant = r["__compliant"] findings = r.get("Findings", "").strip() findings_num = r["__findings"] total_items = r.get("Total", "").strip() compliant_items = r.get("Compliant Items", "").strip() pct = r["__pct"] csv_remediation = (r.get("Remediation", "") or "").strip() rag_data = r.get("__rag_remediation", {}) if isinstance(rag_data, str): rag_data = {"description": "", "rationale": "", "remediation": rag_data} rag_desc = (rag_data.get("description", "") or "").strip() rag_rationale = (rag_data.get("rationale", "") or "").strip() rag_rem = (rag_data.get("remediation", "") or "").strip() # Status line if is_compliant: result_html = f'
Result: Tenancy {esc(tenancy_name)}: Compliant
' else: detail = "" if findings_num is not None and total_items: detail = f" ({findings_num} of {esc(total_items)} items)" result_html = f'
Result: Tenancy {esc(tenancy_name)}: Non-Compliant{detail}
' def _text_to_html_paras(text): """Convert text to HTML paragraphs, rendering numbered lists as
    and bullets as
      .""" cleaned = _join_broken_lines(text) parts = [] lines = cleaned.split('\n') ol_items = [] ul_items = [] def flush_ol(): if ol_items: parts.append('
        ' + ''.join(f'
      1. {esc(i)}
        2. ' for i in ol_items) + '
      ') ol_items.clear() def flush_ul(): if ul_items: parts.append('
        ' + ''.join(f'
      • {esc(i)}
        • ' for i in ul_items) + '
      ') ul_items.clear() for line in lines: line = line.strip() if not line: flush_ol() flush_ul() continue num_match = _re.match(r'^(\d+)[\.\)]\s+(.+)', line) if num_match: flush_ul() ol_items.append(num_match.group(2)) elif line.startswith('- ') or line.startswith('* '): flush_ol() ul_items.append(line[2:]) else: flush_ol() flush_ul() parts.append(f'

      {esc(line)}

      ') flush_ol() flush_ul() return ''.join(parts) # Description description_html = "" if not is_compliant and rag_desc: desc_body = _text_to_html_paras(rag_desc) description_html = f'

      Description:

      {desc_body}
      ' # Rationale rationale_html = "" if not is_compliant and rag_rationale: rat_body = _text_to_html_paras(rag_rationale) rationale_html = f'

      Rationale:

      {rat_body}
      ' # Remediation — formatted with code snippets remediation_html = "" if not is_compliant: rem_text = rag_rem or csv_remediation if rem_text: formatted = _format_remediation_html(rem_text) remediation_html = f'

      Remediation:

      {formatted}
      ' # Compliance bar bar_html = "" if pct is not None: cls = "bad" if pct <= 20 else "warn" if pct <= 60 else "ok" bar_html = f'
      {pct:.0f}% Compliant
      ' # Findings file link + resource table findings_file_html = "" findings_table_html = "" csv_filename = (r.get("Filename", "") or "").strip().strip('"').strip("'") if csv_filename and not is_compliant: if file_id_map: fid = file_id_map.get(csv_filename, "") if fid and rid: dl_url = f"/api/reports/{rid}/files/{fid}/download" findings_file_html = f'
      [CSV] {esc(csv_filename)}Affected resources details
      ' if file_path_map: csv_path = file_path_map.get(csv_filename, "") if csv_path: findings_table_html = _build_findings_table(csv_path, max_rows=10) level_badge = "" if level: lvl_cls = "level-1" if level == "1" else "level-2" level_badge = f'Level {esc(level)}' refs_html = "" ref_parts = [] if cis_v8: ref_parts.append(f"CIS v8: {esc(cis_v8)}") if cccs: ref_parts.append(f"CCCS: {esc(cccs)}") if ref_parts: refs_html = f'
      {" | ".join(ref_parts)}
      ' sections_html += f'''

      {esc(rec)} {esc(title)} {level_badge}

      {refs_html} {bar_html} {result_html} {findings_table_html} {findings_file_html} {description_html} {rationale_html} {remediation_html}
      ''' sections_html += '
' # ── OCI Services section ── oci_services_html = "" if oci_services: svc_rows = "" for svc in oci_services: st = svc["status"] st_cls = "pass" if st == "OK" else "warn" svc_rows += f'{esc(st)}{esc(svc["service"])}{esc(svc["description"])}' # Build service detail pages (each pair of services on a new page with ORACLE header) svc_detail_pages = "" svc_list = [s for s in oci_services if _OCI_SERVICE_DESCRIPTIONS.get(s["service"])] for idx, svc in enumerate(svc_list): svc_name = svc["service"] info = _OCI_SERVICE_DESCRIPTIONS[svc_name] # Start new page every 2 services (or first one) if idx % 2 == 0: if idx > 0: svc_detail_pages += '' # close previous page div svc_detail_pages += f'
' svc_detail_pages += '
ORACLE
' svc_detail_pages += f'
' svc_detail_pages += f'

{esc(info.get("title", svc_name))}

' desc = info.get("description", "") for para in desc.split("\n"): para = para.strip() if para: svc_detail_pages += f'

{esc(para)}

' rec = info.get("recommendation", "") if rec: svc_detail_pages += f'

Recommendation:

{esc(rec)}

' cg_recs = svc.get("recommendations", []) if cg_recs: svc_detail_pages += '' svc_detail_pages += '' for cr in cg_recs: svc_detail_pages += f'' svc_detail_pages += '
RecommendationsTotal
{esc(cr["name"])}{cr["count"]}
' svc_detail_pages += '
' if svc_list: svc_detail_pages += '
' # close last page div oci_services_html = f'''
ORACLE

OCI Services

Recommendations:

The recommended services are already enabled. This way we recommend further development of their usage:

{svc_rows}
STATUSSERVICEDescription
{svc_detail_pages}''' # ── TOC ── toc_lines = '
CIS Security Assessment Summary
' toc_lines += '
CIS Benchmark Recommendation Summary
' for sec_name, sd in sections_data.items(): sec_id = f"section-{slug(sec_name)}" toc_lines += f'
{esc(sec_name)}{sd["total"]}
' for r in sd["items"]: rec = r.get("Recommendation #", "-") title = r.get("Title", "-") rec_id = f"rec-{slug(sec_name)}-{slug(rec)}" toc_lines += f'
{esc(rec)} {esc(title)}
' if oci_services: toc_lines += '
OCI Services
' DISCLAIMER_TEXT = "This document provides prescriptive guidance for establishing a secure baseline configuration for the Oracle Cloud Infrastructure environment based on the CIS Oracle Cloud Infrastructure Foundations Benchmark. The scope of this benchmark is to establish a base level of security for anyone utilizing the included Oracle Cloud Infrastructure services. The benchmark is, however, not an exhaustive list of all possible security configurations and architectures. You should take the benchmark as a starting point and do the required site-specific tailoring wherever needed and when it is prudent to do so.\n\nThis document is provided \"as is\" for informational purposes only, without warranties of any kind, whether express or implied. The findings and recommendations reflect the conditions observed at the time of the assessment and do not guarantee the prevention of future security incidents or compliance outcomes. This document does not form part of any Oracle license, service contract, or other agreement." PROFILE_DEFINITIONS = """Level 1 — Items in this profile intend to be practical and prudent, provide a clear security benefit, and not inhibit the utility of the technology beyond acceptable means. Level 2 — Items in this profile extend the Level 1 profile. Items in this profile exhibit one or more of the following characteristics: are intended for environments or use cases where security is paramount, act as defense in depth measure, and may negatively inhibit the utility or performance of the technology.""" SECURITY_OVERVIEW = """Oracle's mission is to build cloud infrastructure and platform services for your business to have effective and manageable security to run your mission-critical workloads and store your data with confidence. Oracle Cloud Infrastructure's security approach is based on seven core pillars: Customer Isolation — Allow customers to deploy their application and data assets in an environment that commits full isolation from other tenants and Oracle's staff. Data Encryption — Protect customer data at-rest and in-transit in a way that allows customers to meet their security and compliance requirements for cryptographic algorithms and key management. Security Controls — Offer customers effective and easy-to-use security management solutions that allow them to constrain access to their services and segregate operational responsibilities. Visibility — Offer customers comprehensive log data and security analytics that they can use to audit and monitor actions on their resources. Secure Hybrid Cloud — Enable customers to use their existing security assets, such as user accounts and policies, as well as third-party security solutions when accessing their cloud resources. High Availability — Offer fault-independent data centers that enable high availability scale-out architectures and are resilient against network attacks. Verifiably Secure Infrastructure — Follow rigorous processes and use effective security controls in all phases of cloud service development and operation.""" return f''' CIS Foundations Benchmark Assessment — {esc(tenancy_name)}
ORACLE
CIS Foundations Benchmark Assessment
{esc(tenancy_name)} – Oracle Cloud Infrastructure
Tenancy: {esc(tenancy_name)}
{esc(month_year)}, Version [1.0]
{f'
Extract date: {esc(extract_date)}
' if extract_date else ''}
Copyright © {now.year}, Oracle and/or its affiliates
Confidential – Oracle Restricted
ORACLE

Disclaimer

{esc(DISCLAIMER_TEXT)}

Profile Definitions

{PROFILE_DEFINITIONS}

ORACLE
Table of contents
{toc_lines}
ORACLE

Security Overview

{SECURITY_OVERVIEW}

ORACLE

CIS Security Assessment

CIS Oracle Cloud Infrastructure Foundations Benchmark v3.0.0

Number of controls per domain and non-compliant controls on tenancy {esc(tenancy_name)}:

{domains_rows}
DOMAINSTOTAL CONTROLSFAILEDPASSED
Total: {total_all} • Passed: {total_passed} • Failed: {total_failed} • Compliance: {pct_global}%
ORACLE

CIS Benchmark Recommendation Summary

Status per recommendation
{summary_rows}
ItemDescriptionLevelStatus
{sections_html} {oci_services_html}
ORACLE

Connect with us

Call +1.800.ORACLE1 or visit oracle.com. Outside North America, find your local office at: oracle.com/contact.

blogs.oracle.com facebook.com/oracle twitter.com/oracle

Copyright © {now.year}, Oracle and/or its affiliates. This document is provided for information purposes only, and the contents hereof are subject to change without notice. This document is not warranted to be error-free, nor subject to any other warranties or conditions, whether expressed orally or implied in law, including implied warranties and conditions of merchantability or fitness for a particular purpose. We specifically disclaim any liability with respect to this document, and no contractual obligations are formed either directly or indirectly by this document. This document may not be reproduced or transmitted in any form or by any means, electronic or mechanical, for any purpose, without our prior written permission.

Oracle, Java, MySQL, and NetSuite are registered trademarks of Oracle and/or its affiliates. Other names may be trademarks of their respective owners.

''' def _generate_compliance_report(rid: str, user_id: str, force_rag: bool = True) -> str: """Generate and cache the LAD A-Team CIS Compliance Report HTML.""" import csv as csvmod import re as _re with db() as c: report = c.execute("SELECT * FROM reports WHERE id=?", (rid,)).fetchone() if not report: raise HTTPException(404, "Report not found") rdir = REPORTS / rid csv_path = rdir / "cis_summary_report.csv" if not csv_path.exists(): raise HTTPException(404, "cis_summary_report.csv not found") rows = [] with open(csv_path, "r", encoding="utf-8") as f: for row in csvmod.DictReader(f): rows.append(row) if not rows: raise HTTPException(404, "No data in summary CSV") tenancy_name = report["tenancy_name"] or "Unknown" extract_date = rows[0].get("extract_date", "") if rows else "" # Build filename → file_id and filename → file_path maps file_id_map = {} file_path_map = {} with db() as c: rf = c.execute("SELECT id, file_name, file_path FROM report_files WHERE report_id=?", (rid,)).fetchall() for f in rf: file_id_map[f["file_name"]] = f["id"] file_path_map[f["file_name"]] = f["file_path"] def norm_section(raw): s = _re.sub(r"\s+", " ", (raw or "").strip()) if not s: return None if _re.match(r"^Logging and Monitoring\s*\d*$", s, _re.I): return "Logging and Monitoring" if _re.match(r"^Storage$", s, _re.I): return None if _re.match(r"^Storage\s*-\s*Block Volumes$", s, _re.I): return "Storage - Block Volumes" if _re.match(r"^Storage\s*-\s*File Storage Service$", s, _re.I): return "Storage - File Storage Service" if _re.match(r"^Storage\s*-\s*Object Storage$", s, _re.I): return "Storage - Object Storage" return s seen = set() filtered = [] for r in rows: sec = norm_section(r.get("Section", "")) if not sec: continue key = f"{sec}||{r.get('Recommendation #', '')}||{r.get('Title', '')}".lower() if key in seen: continue seen.add(key) r["Section"] = sec status_raw = (r.get("Compliant", "") or "").strip().lower() r["__compliant"] = status_raw in ("yes", "true", "compliant", "ok") pct_str = (r.get("Compliance Percentage Per Recommendation", "") or "").replace(",", ".") m = _re.search(r"(\d+(\.\d+)?)", pct_str) r["__pct"] = max(0, min(100, float(m.group(1)))) if m else None f_str = (r.get("Findings", "") or "").replace(",", "") fm = _re.search(r"\d+", f_str) r["__findings"] = int(fm.group(0)) if fm else None filtered.append(r) # RAG remediations (with per-item timeout) for r in filtered: r["__rag_remediation"] = {"description": "", "remediation": ""} if force_rag: from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeout import json as _pjson non_compliant = [r for r in filtered if not r["__compliant"]] total_rag = len(non_compliant) log.info(f"Compliance report: searching RAG for {total_rag} non-compliant items") progress_marker = rdir / ".compliance_generating" def _update_progress(step, current, rec=""): try: progress_marker.write_text(_pjson.dumps({ "step": step, "current": current, "total": total_rag, "rec": rec }), encoding="utf-8") except Exception: pass _update_progress("RAG", 0) with ThreadPoolExecutor(max_workers=2) as pool: for idx, r in enumerate(non_compliant): rec_num = r.get('Recommendation #', '?') title = r.get("Title", "") _update_progress("RAG", idx + 1, rec_num) for attempt in range(1, 3): try: fut = pool.submit(_search_cis_remediation, title, rec_num, user_id) r["__rag_remediation"] = fut.result(timeout=60) log.info(f" RAG OK: {rec_num}") break except FuturesTimeout: if attempt == 1: log.warning(f" RAG timeout {rec_num} (attempt 1/2, retrying...)") else: log.warning(f" RAG timeout {rec_num} (attempt 2/2, giving up)") except Exception as e: log.warning(f" RAG error {rec_num}: {e}") break _update_progress("OCI Services", total_rag) SECTION_ORDER = [ "Identity and Access Management", "Networking", "Compute", "Logging and Monitoring", "Storage - Object Storage", "Storage - Block Volumes", "Storage - File Storage Service", "Asset Management", ] from collections import OrderedDict sections = OrderedDict() for sec in SECTION_ORDER: items = [r for r in filtered if r["Section"] == sec] if items: items.sort(key=lambda x: [int(p) if p.isdigit() else 0 for p in (x.get("Recommendation #", "0") or "0").split(".")]) passed = sum(1 for x in items if x["__compliant"]) sections[sec] = {"items": items, "total": len(items), "passed": passed, "failed": len(items) - passed} # Check OCI security services status (with user overrides) oci_services = [] config_id = report["config_id"] if report else None if config_id: try: import json as _json log.info(f"Compliance report: checking OCI security services status...") detected = _check_oci_services(config_id) # Load user overrides with db() as c: ov_row = c.execute("SELECT value FROM app_settings WHERE key=?", (f"oci_services_{config_id}",)).fetchone() overrides = _json.loads(ov_row["value"]) if ov_row else {} # Merge: build final list with overrides taking priority auto_map = {s["service"]: s for s in detected} for name in ["Vulnerability Scanning Service", "Data Safe", "Cloud Guard", "Bastion", "Security Zones", "Vault"]: ov = overrides.get(name, {}) au = auto_map.get(name, {"service": name, "status": "WARNING", "description": "Unable to verify"}) svc = { "service": name, "status": ov.get("status", au["status"]), "description": ov.get("description", au["description"]), } if "recommendations" in au: svc["recommendations"] = au["recommendations"] oci_services.append(svc) log.info(f"Compliance report: OCI services check done ({len(oci_services)} services, {len(overrides)} overrides)") except Exception as e: log.warning(f"OCI services check failed: {e}") html = _build_compliance_html(filtered, tenancy_name, extract_date, sections, rid, file_id_map, file_path_map, oci_services) # Cache to disk cache_path = rdir / "compliance_report.html" cache_path.write_text(html, encoding="utf-8") # Save processed data as JSON (used by DOCX generator to produce identical output) import json as _json report_data = { "tenancy_name": tenancy_name, "extract_date": extract_date, "filtered": [{k: v for k, v in r.items() if not k.startswith("__")} | { "__compliant": r["__compliant"], "__pct": r["__pct"], "__findings": r["__findings"], "__rag_remediation": r.get("__rag_remediation", {}), } for r in filtered], "sections": {name: {"total": sd["total"], "passed": sd["passed"], "failed": sd["failed"], "items": [{k: v for k, v in r.items() if not k.startswith("__")} | { "__compliant": r["__compliant"], "__pct": r["__pct"], "__findings": r["__findings"], "__rag_remediation": r.get("__rag_remediation", {}), } for r in sd["items"]]} for name, sd in sections.items()}, "oci_services": oci_services, "file_id_map": file_id_map or {}, "file_path_map": file_path_map or {}, } data_path = rdir / "compliance_data.json" data_path.write_text(_json.dumps(report_data, ensure_ascii=False, default=str), encoding="utf-8") log.info(f"Compliance data cached: {data_path}") log.info(f"Compliance report cached: {cache_path} ({len(html)} bytes)") return html @app.get("/api/oci-health") async def oci_health(u=Depends(current_user)): """Proxy Oracle Cloud Infrastructure public status API.""" import httpx try: async with httpx.AsyncClient(timeout=15) as client: r = await client.get("https://ocistatus.oraclecloud.com/api/v2/components.json") r.raise_for_status() return r.json() except Exception as e: log.warning(f"OCI Health API failed: {e}") raise HTTPException(502, f"Failed to fetch OCI status: {e}") _OCI_SERVICE_NAMES = ["Vulnerability Scanning Service", "Data Safe", "Cloud Guard", "Bastion", "Security Zones", "Vault"] @app.get("/api/oci-services/{config_id}") async def get_oci_services_status(config_id: str, detect: int = Query(1), u=Depends(current_user)): """Get OCI services status (auto-detected + user overrides). Use detect=0 to skip auto-detect.""" _verify_config_access("oci", config_id, u) import json as _json # Load user overrides with db() as c: row = c.execute("SELECT value FROM app_settings WHERE key=?", (f"oci_services_{config_id}",)).fetchone() overrides = _json.loads(row["value"]) if row else {} # Auto-detect (skip if detect=0) auto = {} if detect: try: detected = _check_oci_services(config_id) for svc in detected: auto[svc["service"]] = {"status": svc["status"], "description": svc["description"]} except Exception as e: log.warning(f"OCI services auto-detect failed: {e}") # Merge: override wins result = [] for name in _OCI_SERVICE_NAMES: ov = overrides.get(name, {}) au = auto.get(name, {"status": "WARNING", "description": "Unable to verify"}) result.append({ "service": name, "auto_status": au["status"], "auto_description": au["description"], "status": ov.get("status", au["status"]), "description": ov.get("description", au["description"]), "overridden": bool(ov), }) return result @app.put("/api/oci-services/{config_id}") async def set_oci_services_status(config_id: str, request: Request, u=Depends(current_user)): """Save user overrides for OCI services status.""" _verify_config_access("oci", config_id, u) import json as _json body = await request.json() overrides = {} for svc in body: name = svc.get("service", "") if name in _OCI_SERVICE_NAMES and svc.get("overridden"): overrides[name] = {"status": svc.get("status", "OK"), "description": svc.get("description", "")} with db() as c: c.execute("INSERT INTO app_settings (key,value,updated_at) VALUES (?,?,datetime('now')) ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at", (f"oci_services_{config_id}", _json.dumps(overrides))) _audit(u["id"], u["username"], "set_oci_services", config_id) return {"ok": True} @app.api_route("/api/reports/{rid}/compliance-report", methods=["GET", "HEAD"]) async def compliance_report(rid: str, regen: int = Query(0), token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))): """Serve cached LAD A-Team CIS Compliance Report, or return 404 if not generated yet.""" u = _user_from_token(token) if token else (await current_user(cred) if cred else None) if not u: raise HTTPException(401, "Not authenticated") _verify_report_access(rid, u) rdir = REPORTS / rid cache_path = rdir / "compliance_report.html" if regen == 1 and cache_path.exists(): cache_path.unlink() if cache_path.exists(): return FileResponse(cache_path, media_type="text/html", headers={"Cache-Control": "no-cache, no-store, must-revalidate", "Pragma": "no-cache", "Expires": "0"}) # Return a friendly HTML page instead of JSON 404 (shown in iframe) marker = rdir / ".compliance_generating" if marker.exists(): msg = "Generating compliance report... Please wait." else: msg = "Compliance report not generated yet." return HTMLResponse( f'' f'

{msg}

', status_code=200, headers={"Cache-Control": "no-cache"}) @app.post("/api/reports/{rid}/compliance-report/generate") async def generate_compliance_report(rid: str, bg: BackgroundTasks, u=Depends(current_user)): """Start generating the compliance report in background.""" _verify_report_access(rid, u) with db() as c: report = c.execute("SELECT * FROM reports WHERE id=?", (rid,)).fetchone() if not report: raise HTTPException(404, "Report not found") rdir = REPORTS / rid csv_path = rdir / "cis_summary_report.csv" if not csv_path.exists(): raise HTTPException(404, "cis_summary_report.csv not found") # Reject if already generating marker = rdir / ".compliance_generating" if marker.exists(): age = time.time() - marker.stat().st_mtime if age < 600: return {"status": "already_generating", "has_rag": False} # Mark as generating BEFORE deleting cache marker.write_text(str(uuid.uuid4()), encoding="utf-8") # Delete existing cache cache_path = rdir / "compliance_report.html" if cache_path.exists(): cache_path.unlink() has_adb = False with db() as c: adb_cfgs = c.execute("SELECT * FROM adb_vector_configs WHERE user_id=? AND is_active=1", (u["id"],)).fetchall() if adb_cfgs: has_adb = True # Pre-check ADB connection before starting (fail fast instead of timeout loop) if has_adb: adb_cfg = dict(adb_cfgs[0]) try: conn = _get_adb_connection(adb_cfg) conn.ping() conn.close() log.info(f"ADB connection OK for compliance report {rid}") except Exception as e: marker.unlink(missing_ok=True) log.warning(f"ADB connection failed for compliance report {rid}: {e}") raise HTTPException(503, f"Conexao com ADB falhou. Verifique a whitelist e a configuracao do ADB. Erro: {e}") user_id = u["id"] log.info(f"Compliance report generation started for {rid} (has_adb={has_adb})") def _bg_generate(): try: log.info(f"Compliance report thread running for {rid}") _generate_compliance_report(rid, user_id, force_rag=has_adb) log.info(f"Compliance report generation completed for {rid}") except Exception as e: log.error(f"Compliance report generation failed for {rid}: {e}", exc_info=True) finally: # Remove generating marker m = REPORTS / rid / ".compliance_generating" if m.exists(): m.unlink(missing_ok=True) _chat_executor.submit(_bg_generate) return {"status": "generating", "has_rag": has_adb} @app.get("/api/reports/{rid}/compliance-report/status") async def compliance_report_status(rid: str, u=Depends(current_user)): """Check compliance report status: ready, generating, or not started.""" _verify_report_access(rid, u) rdir = REPORTS / rid cache_path = rdir / "compliance_report.html" marker = rdir / ".compliance_generating" if cache_path.exists(): # Clean stale marker if report is ready if marker.exists(): marker.unlink(missing_ok=True) return {"ready": True, "generating": False} if marker.exists(): age = time.time() - marker.stat().st_mtime if age > 600: marker.unlink(missing_ok=True) log.warning(f"Expired stale compliance marker for {rid} (age={age:.0f}s)") return {"ready": False, "generating": False} # Read progress from marker file try: import json as _json progress = _json.loads(marker.read_text(encoding="utf-8")) except Exception: progress = {} return {"ready": False, "generating": True, "current": progress.get("current", 0), "total": progress.get("total", 0), "step": progress.get("step", ""), "rec": progress.get("rec", "")} return {"ready": False, "generating": False} def _generate_camo_strip() -> bytes: """Generate a camouflage-style decorative strip PNG (like Oracle PDF reports).""" from PIL import Image, ImageDraw, ImageFilter import random, io as _io W, H = 36, 1500 # Exact colors from the HTML page-strip gradient — olive/sage/gold tones C1 = (107, 127, 58) # #6b7f3a olive green C2 = (138, 154, 78) # #8a9a4e sage green C3 = (74, 103, 65) # #4a6741 forest C4 = (45, 59, 42) # #2d3b2a dark forest C5 = (196, 169, 77) # #c4a94d gold C6 = (139, 109, 59) # #8b6d3b brown/tan C7 = (61, 92, 58) # #3d5c3a deep green ALL = [C1, C2, C3, C4, C5, C6, C7] img = Image.new('RGB', (W, H), C4) draw = ImageDraw.Draw(img) random.seed(42) import math def _poly(cx, cy, r, sides=5): pts = [] for i in range(sides): a = 2 * math.pi * i / sides + random.uniform(-0.5, 0.5) rx = r * random.uniform(0.5, 1.4) ry = r * random.uniform(0.8, 2.2) pts.append((cx + rx * math.cos(a), cy + ry * math.sin(a))) return pts # Vertical gradient base (like the CSS linear-gradient) for y in range(H): t = y / H # Cycle through gradient stops if t < 0.16: c = C1 if t < 0.08 else C2 elif t < 0.32: c = C3 if t < 0.24 else C4 elif t < 0.48: c = C5 if t < 0.40 else C6 elif t < 0.64: c = C7 if t < 0.56 else C1 elif t < 0.80: c = C5 if t < 0.72 else C3 else: c = C4 if t < 0.88 else C2 draw.line([(0, y), (W, y)], fill=c) # Organic patches on top for camo texture for _ in range(35): c = random.choice(ALL) cx = random.randint(-8, W + 8) cy = random.randint(-20, H + 20) draw.polygon(_poly(cx, cy, random.randint(12, 30), random.randint(4, 6)), fill=c) for _ in range(25): c = random.choice([C5, C6, C2]) cx = random.randint(0, W) cy = random.randint(0, H) draw.polygon(_poly(cx, cy, random.randint(6, 16), random.randint(3, 5)), fill=c) img = img.filter(ImageFilter.GaussianBlur(radius=1.2)) buf = _io.BytesIO() img.save(buf, format='PNG') return buf.getvalue() def _add_floating_image_to_header(header, image_bytes: bytes, width_emu: int, height_emu: int): """Add a floating behind-text image to a DOCX header (positioned at left edge, full page height).""" from docx.oxml.ns import nsdecls from docx.oxml import parse_xml, OxmlElement import io as _io rId, _ = header.part.get_or_add_image(_io.BytesIO(image_bytes)) sw, sh = str(width_emu), str(height_emu) anchor_xml = f''' 0 0 ''' anchor_el = parse_xml(anchor_xml) drawing = OxmlElement('w:drawing') drawing.append(anchor_el) # Add to the first run in header paragraph p = header.paragraphs[0] runs = p.runs if runs: runs[0]._r.append(drawing) else: r = p.add_run() r._r.append(drawing) def _generate_compliance_docx(rid: str, tenancy: str, file_map: dict) -> bytes: """Generate a professionally styled DOCX version of the compliance report matching the HTML/PDF.""" import csv as csvmod, io as _io, re as _re from datetime import datetime as _dt from docx import Document from docx.shared import Inches, Pt, Cm, RGBColor, Emu from docx.enum.text import WD_ALIGN_PARAGRAPH from docx.enum.table import WD_TABLE_ALIGNMENT from docx.oxml.ns import qn, nsdecls from docx.oxml import parse_xml, OxmlElement import json as _json rdir = REPORTS / rid # Read the same processed data used by the HTML report (ensures identical content) data_path = rdir / "compliance_data.json" if not data_path.exists(): log.warning(f"DOCX: compliance_data.json not found for {rid} — generate HTML report first") return None with open(data_path, "r", encoding="utf-8") as f: report_data = _json.load(f) tenancy_name = report_data.get("tenancy_name", tenancy) extract_date = report_data.get("extract_date", "") filtered = report_data.get("filtered", []) sections_data = report_data.get("sections", {}) oci_services = report_data.get("oci_services", []) file_path_map = report_data.get("file_path_map", file_map) if not filtered: return None now = _dt.utcnow() months_en = ["January","February","March","April","May","June","July","August","September","October","November","December"] month_year = f"{months_en[now.month - 1]}, {now.year}" # ── Exact colors from HTML/PDF spec ── ORACLE_RED = RGBColor(0xC7, 0x46, 0x34) HEADER_GREEN = '4A7C59' FINDINGS_HEADER_GREEN = '4A6741' ROW_ALT = 'EEF3F0' GREEN = RGBColor(0x0F, 0x7B, 0x0F) RED = RGBColor(0xC0, 0x00, 0x00) MUTED = RGBColor(0x64, 0x74, 0x8B) WHITE = RGBColor(0xFF, 0xFF, 0xFF) BODY_TEXT = RGBColor(0x1A, 0x1A, 0x1A) TITLE_DARK = RGBColor(0x11, 0x18, 0x27) SUBTITLE_GRAY = RGBColor(0x37, 0x41, 0x51) ORANGE = RGBColor(0xD9, 0x77, 0x06) # ── Helpers ── def _cell_shading(cell, color_hex): shading = parse_xml(f'') cell._tc.get_or_add_tcPr().append(shading) def _set_cell_width(cell, width_inches): tcPr = cell._tc.get_or_add_tcPr() tcW = OxmlElement('w:tcW') tcW.set(qn('w:w'), str(int(width_inches * 1440))) tcW.set(qn('w:type'), 'dxa') tcPr.append(tcW) def _cell_margins(cell, top=40, bottom=40, left=80, right=80): """Set cell margins in twips for tighter padding.""" tcPr = cell._tc.get_or_add_tcPr() tcMar = OxmlElement('w:tcMar') for side, val in [('top', top), ('bottom', bottom), ('start', left), ('end', right)]: el = OxmlElement(f'w:{side}') el.set(qn('w:w'), str(val)) el.set(qn('w:type'), 'dxa') tcMar.append(el) tcPr.append(tcMar) def _cell_text(cell, text, bold=False, color=None, size=Pt(9), align=WD_ALIGN_PARAGRAPH.LEFT, uppercase=False): cell.text = "" p = cell.paragraphs[0] p.alignment = align p.paragraph_format.space_before = Pt(2) p.paragraph_format.space_after = Pt(2) p.paragraph_format.line_spacing = 1.0 display = str(text).upper() if uppercase else str(text) r = p.add_run(display) r.font.size = size r.font.name = 'Calibri' r.bold = bold if color: r.font.color.rgb = color else: r.font.color.rgb = BODY_TEXT def _table_borders(table, color='D0D5DD', size='4'): tbl = table._tbl tblPr = tbl.tblPr if tbl.tblPr is not None else OxmlElement('w:tblPr') borders = OxmlElement('w:tblBorders') for edge in ('top', 'left', 'bottom', 'right', 'insideH', 'insideV'): el = OxmlElement(f'w:{edge}') el.set(qn('w:val'), 'single') el.set(qn('w:sz'), size) el.set(qn('w:space'), '0') el.set(qn('w:color'), color) borders.append(el) tblPr.append(borders) def _set_repeat_header(table): """Make first row repeat as header on page breaks (like PDF).""" tr = table.rows[0]._tr trPr = tr.get_or_add_trPr() tblHeader = OxmlElement('w:tblHeader') trPr.append(tblHeader) def _no_cell_spacing(table): """Remove cell spacing for tight table look.""" tbl = table._tbl tblPr = tbl.tblPr if tbl.tblPr is not None else OxmlElement('w:tblPr') spacing = OxmlElement('w:tblCellSpacing') spacing.set(qn('w:w'), '0') spacing.set(qn('w:type'), 'dxa') tblPr.append(spacing) def _styled_table(doc, headers, data_rows, col_widths=None, header_align=None, header_color=None): """Create a table matching the PDF style: green header, alternating rows.""" hdr_clr = header_color or HEADER_GREEN tbl = doc.add_table(rows=1 + len(data_rows), cols=len(headers)) tbl.autofit = False tbl.allow_autofit = False tbl.alignment = WD_TABLE_ALIGNMENT.CENTER # Set column widths if provided if col_widths: for i, w in enumerate(col_widths): for row in tbl.rows: _set_cell_width(row.cells[i], w) # Header row — green background, white text, uppercase for i, h in enumerate(headers): cell = tbl.rows[0].cells[i] _cell_shading(cell, hdr_clr) _cell_margins(cell) h_align = WD_ALIGN_PARAGRAPH.CENTER if header_align and i < len(header_align): h_align = header_align[i] elif i == 0: h_align = WD_ALIGN_PARAGRAPH.LEFT _cell_text(cell, h, bold=True, color=WHITE, size=Pt(9), align=h_align, uppercase=True) # Data rows — alternating green tint / white for ridx, row_data in enumerate(data_rows): bg = ROW_ALT if ridx % 2 == 0 else 'FFFFFF' for cidx, val in enumerate(row_data): cell = tbl.rows[ridx + 1].cells[cidx] _cell_shading(cell, bg) _cell_margins(cell) if isinstance(val, tuple): txt, bld, clr = val[0], val[1], val[2] a = WD_ALIGN_PARAGRAPH.CENTER if cidx > 0 and len(str(txt)) < 10 else WD_ALIGN_PARAGRAPH.LEFT _cell_text(cell, txt, bold=bld, color=clr, size=Pt(9), align=a) else: a = WD_ALIGN_PARAGRAPH.CENTER if cidx > 0 and len(str(val)) < 10 else WD_ALIGN_PARAGRAPH.LEFT _cell_text(cell, val, size=Pt(9), align=a) _table_borders(tbl) _no_cell_spacing(tbl) _set_repeat_header(tbl) return tbl def _add_heading_red(doc, text, level=1): h = doc.add_heading(text, level=level) for r in h.runs: r.font.color.rgb = ORACLE_RED r.font.name = 'Calibri' return h def _clean_text(text): """Join artificially broken lines into flowing paragraphs. Preserves: paragraph breaks, numbered lists, bullets, headers.""" if not text: return text lines = text.split('\n') result = [] for line in lines: line = line.strip() if not line: result.append('\n') continue starts_new = ( _re.match(r'^\d+[\.\)]\s', line) or _re.match(r'^[a-z][\.\)]\s', line) or line.startswith('- ') or line.startswith('* ') or line.startswith('From ') or line.startswith('Note:') or line.startswith('See ') ) if starts_new: result.append(line) elif result and result[-1] != '\n' and not result[-1].endswith(('.', ':', '!', '?', ';')): result[-1] = result[-1] + ' ' + line else: result.append(line) return '\n'.join(result).strip() def _add_body_para(doc, text, size=Pt(10), color=None, bold=False, italic=False, space_after=Pt(8), space_before=Pt(0), align=WD_ALIGN_PARAGRAPH.JUSTIFY): """Add paragraph(s) with consistent body styling. Detects numbered/bullet lists and indents them.""" text = _clean_text(text) if not text: return None last_p = None for chunk in text.split('\n'): chunk = chunk.strip() if not chunk: continue # Detect list items: "1. text", "a. text", "- text", "* text" list_match = _re.match(r'^(\d+[\.\)]|[a-z][\.\)]|-|\*)\s+(.+)', chunk) p = doc.add_paragraph() p.alignment = WD_ALIGN_PARAGRAPH.LEFT if list_match else align p.paragraph_format.space_before = Pt(1) if list_match else space_before p.paragraph_format.space_after = Pt(3) if list_match else space_after p.paragraph_format.line_spacing = 1.25 if list_match: # Indent list items p.paragraph_format.left_indent = Inches(0.3) p.paragraph_format.first_line_indent = Inches(-0.3) # Number/bullet in bold r = p.add_run(list_match.group(1) + ' ') r.font.name = 'Calibri' r.font.size = size r.font.color.rgb = RGBColor(0x2F, 0x55, 0x97) r.bold = True # Content r = p.add_run(list_match.group(2)) r.font.name = 'Calibri' r.font.size = size r.font.color.rgb = color or BODY_TEXT else: r = p.add_run(chunk) r.font.name = 'Calibri' r.font.size = size r.font.color.rgb = color or BODY_TEXT r.bold = bold r.italic = italic last_p = p return last_p def _is_code_line(line): """Detect if a line is a CLI command or code snippet.""" line = line.strip() if not line: return False # Strong indicators: starts with known CLI commands cli_starts = ['oci ', 'aws ', 'az ', 'gcloud ', 'terraform ', 'kubectl ', 'curl ', 'grep ', 'cat ', 'echo ', 'export ', 'sudo ', '#!/'] if any(line.startswith(p) for p in cli_starts): return True # Continuation lines (start with --flag or -flag=value) if _re.match(r'^--\w+', line) or _re.match(r'^-\w+=', line): return True # Shell constructs if line in ('do', 'done', 'fi', 'then', 'else', 'esac'): return True if line.startswith('for ') and (' in ' in line or line.endswith('do')): return True return False def _is_section_header(line): """Detect "From Console:", "From Command Line:", "Audit / Verification:", etc.""" low = line.strip().lower() return (low.startswith('from console') or low.startswith('from oci console') or low.startswith('from command') or low.startswith('from cli') or low.startswith('from the command') or low.startswith('audit') and 'verification' in low) def _add_code_block(doc, code_text): """Add a code block with monospace font and gray background.""" p = doc.add_paragraph() p.paragraph_format.space_before = Pt(6) p.paragraph_format.space_after = Pt(8) p.paragraph_format.line_spacing = 1.1 p.alignment = WD_ALIGN_PARAGRAPH.LEFT pPr = p._p.get_or_add_pPr() pShd = OxmlElement('w:shd') pShd.set(qn('w:val'), 'clear') pShd.set(qn('w:color'), 'auto') pShd.set(qn('w:fill'), 'F3F4F6') pPr.append(pShd) pBdr = OxmlElement('w:pBdr') for edge_name, color in [('left', '9CA3AF'), ('top', 'E5E7EB'), ('bottom', 'E5E7EB'), ('right', 'E5E7EB')]: edge = OxmlElement(f'w:{edge_name}') edge.set(qn('w:val'), 'single') edge.set(qn('w:sz'), '6' if edge_name == 'left' else '4') edge.set(qn('w:space'), '6' if edge_name == 'left' else '2') edge.set(qn('w:color'), color) pBdr.append(edge) pPr.append(pBdr) r = p.add_run(code_text) r.font.name = 'Consolas' r.font.size = Pt(8) r.font.color.rgb = RGBColor(0x1F, 0x29, 0x37) def _add_remediation_docx(doc, remediation_text): """Add remediation text with code blocks and section headers, matching HTML output.""" # Split into paragraphs first (double newline), then process each paragraphs = _re.split(r'\n{2,}', remediation_text) for para_text in paragraphs: para_text = para_text.strip() if not para_text: continue lines = para_text.split('\n') # Check if entire paragraph is code non_empty = [l.strip() for l in lines if l.strip()] if non_empty and all(_is_code_line(l) for l in non_empty): _add_code_block(doc, '\n'.join(non_empty)) continue # Mixed content: process line by line code_buffer = [] text_buffer = [] def flush_code(): if code_buffer: _add_code_block(doc, '\n'.join(code_buffer)) code_buffer.clear() def flush_text(): if text_buffer: joined = ' '.join(text_buffer) _add_body_para(doc, joined, size=Pt(9), space_after=Pt(6)) text_buffer.clear() for line in lines: stripped = line.strip() if not stripped: continue if _is_section_header(stripped): flush_text() flush_code() p = doc.add_paragraph() p.paragraph_format.space_before = Pt(8) p.paragraph_format.space_after = Pt(4) r = p.add_run(stripped) r.bold = True r.font.size = Pt(10) r.font.name = 'Calibri' r.font.color.rgb = BODY_TEXT elif _is_code_line(stripped): flush_text() code_buffer.append(stripped) else: flush_code() # If line ends with sentence punctuation, flush as separate paragraph text_buffer.append(stripped) if stripped.endswith(('.', ':', '!', '?', ';')): flush_text() flush_code() flush_text() def _result_box(doc, tenancy_name, is_compliant, findings="0", total="0"): """Add a result box with colored background and left border (matching PDF).""" p = doc.add_paragraph() pPr = p._p.get_or_add_pPr() # Background shading pShd = OxmlElement('w:shd') pShd.set(qn('w:val'), 'clear') pShd.set(qn('w:color'), 'auto') pShd.set(qn('w:fill'), 'F0FDF4' if is_compliant else 'FEF2F2') pPr.append(pShd) # Left border pBdr = OxmlElement('w:pBdr') left_b = OxmlElement('w:left') left_b.set(qn('w:val'), 'single') left_b.set(qn('w:sz'), '18') left_b.set(qn('w:space'), '6') left_b.set(qn('w:color'), '0F7B0F' if is_compliant else 'C00000') pBdr.append(left_b) pPr.append(pBdr) # Padding via indentation pInd = OxmlElement('w:ind') pInd.set(qn('w:left'), '120') pPr.append(pInd) p.paragraph_format.space_before = Pt(8) p.paragraph_format.space_after = Pt(8) run = p.add_run("Result: ") run.bold = True run.font.size = Pt(10) run.font.color.rgb = BODY_TEXT run.font.name = 'Calibri' run = p.add_run(f"Tenancy {tenancy_name}: ") run.bold = True run.font.size = Pt(10) run.font.color.rgb = BODY_TEXT run.font.name = 'Calibri' if is_compliant: run = p.add_run("Compliant") run.font.color.rgb = GREEN run.bold = True run.font.size = Pt(10) run.font.name = 'Calibri' else: detail = f" ({findings} of {total} items)" if findings and total else "" run = p.add_run(f"Non-Compliant{detail}") run.font.color.rgb = RED run.bold = True run.font.size = Pt(10) run.font.name = 'Calibri' return p doc = Document() # ── Global styles ── style = doc.styles['Normal'] style.font.name = 'Calibri' style.font.size = Pt(10) style.font.color.rgb = BODY_TEXT style.paragraph_format.space_after = Pt(4) style.paragraph_format.line_spacing = 1.15 for i in range(1, 4): hs = doc.styles[f'Heading {i}'] hs.font.color.rgb = ORACLE_RED hs.font.name = 'Calibri' doc.styles['Heading 1'].font.size = Pt(22) doc.styles['Heading 1'].font.bold = True doc.styles['Heading 2'].font.size = Pt(16) doc.styles['Heading 3'].font.size = Pt(12) # ── Page size A4 + Margins ── sect = doc.sections[0] sect.page_width = Cm(21.0) sect.page_height = Cm(29.7) sect.top_margin = Cm(2.5) sect.bottom_margin = Cm(2.0) sect.left_margin = Cm(2.5) sect.right_margin = Cm(2.5) # ── Header / Footer ── hdr = sect.header hdr.is_linked_to_previous = False hp = hdr.paragraphs[0] hp.alignment = WD_ALIGN_PARAGRAPH.LEFT r = hp.add_run("ORACLE") r.font.color.rgb = ORACLE_RED r.font.size = Pt(14) r.bold = True r.font.name = 'Calibri' # Add decorative camouflage strip to header (floating, behind text, left edge) try: camo_bytes = _generate_camo_strip() strip_w = int(1.0 * 360000) # ~1cm in EMU strip_h = int(29.7 * 360000) # A4 height (297mm) in EMU _add_floating_image_to_header(hdr, camo_bytes, strip_w, strip_h) except Exception as e: log.warning(f"DOCX: failed to add camo strip: {e}") ftr = sect.footer ftr.is_linked_to_previous = False fp = ftr.paragraphs[0] fp.alignment = WD_ALIGN_PARAGRAPH.CENTER r = fp.add_run("Confidential \u2013 Oracle Restricted") r.font.size = Pt(8) r.font.color.rgb = MUTED r.font.name = 'Calibri' # ═══════════════════════════════════════════════ # ── COVER PAGE ── # ═══════════════════════════════════════════════ # Spacer to push title to center of page for _ in range(10): sp = doc.add_paragraph() sp.paragraph_format.space_before = Pt(0) sp.paragraph_format.space_after = Pt(0) sp.paragraph_format.line_spacing = 1.0 # Title — centered, serif font p = doc.add_paragraph() p.alignment = WD_ALIGN_PARAGRAPH.CENTER p.paragraph_format.space_before = Pt(0) p.paragraph_format.space_after = Pt(0) r = p.add_run("CIS Foundations") r.font.size = Pt(36) r.font.color.rgb = TITLE_DARK r.font.name = 'Georgia' r.bold = False p = doc.add_paragraph() p.alignment = WD_ALIGN_PARAGRAPH.CENTER p.paragraph_format.space_before = Pt(0) p.paragraph_format.space_after = Pt(12) r = p.add_run("Benchmark Assessment") r.font.size = Pt(36) r.font.color.rgb = TITLE_DARK r.font.name = 'Georgia' r.bold = False # Subtitle — centered p = doc.add_paragraph() p.alignment = WD_ALIGN_PARAGRAPH.CENTER p.paragraph_format.space_before = Pt(8) p.paragraph_format.space_after = Pt(8) r = p.add_run(f"{tenancy} \u2013 Oracle Cloud Infrastructure") r.font.size = Pt(13) r.font.color.rgb = SUBTITLE_GRAY r.font.name = 'Calibri' # Extra spacer before metadata for _ in range(6): sp = doc.add_paragraph() sp.paragraph_format.space_before = Pt(0) sp.paragraph_format.space_after = Pt(0) sp.paragraph_format.line_spacing = 1.0 # Metadata — identical to PDF cover def _meta_line(doc, text, bold_prefix=None, sz=Pt(10)): p = doc.add_paragraph() p.alignment = WD_ALIGN_PARAGRAPH.CENTER p.paragraph_format.space_before = Pt(1) p.paragraph_format.space_after = Pt(1) p.paragraph_format.line_spacing = 1.3 if bold_prefix: r = p.add_run(f"{bold_prefix} ") r.bold = True r.font.size = sz r.font.color.rgb = BODY_TEXT r.font.name = 'Calibri' r = p.add_run(text) else: r = p.add_run(text) r.font.size = sz r.font.color.rgb = SUBTITLE_GRAY r.font.name = 'Calibri' _meta_line(doc, tenancy, bold_prefix="Tenancy:") _meta_line(doc, f"{month_year}, Version [1.0]", sz=Pt(9)) if extract_date: _meta_line(doc, extract_date, bold_prefix="Extract date:", sz=Pt(9)) _meta_line(doc, f"Copyright \u00A9 {now.year}, Oracle and/or its affiliates", sz=Pt(9)) _meta_line(doc, "Confidential \u2013 Oracle Restricted", sz=Pt(9)) doc.add_page_break() # ═══════════════════════════════════════════════ # ── DISCLAIMER ── # ═══════════════════════════════════════════════ _add_heading_red(doc, "Disclaimer", level=1) _add_body_para(doc, 'This document provides prescriptive guidance for establishing a secure baseline configuration ' 'for the Oracle Cloud Infrastructure environment based on the CIS Oracle Cloud Infrastructure ' 'Foundations Benchmark. The scope of this benchmark is to establish a base level of security for ' 'anyone utilizing the included Oracle Cloud Infrastructure services. The benchmark is, however, not ' 'an exhaustive list of all possible security configurations and architectures.', space_after=Pt(8)) _add_body_para(doc, 'This document is provided "as is" for informational purposes only, without warranties of any kind, ' 'whether express or implied. The findings and recommendations reflect the conditions observed at the ' 'time of the assessment and do not guarantee the prevention of future security incidents or compliance outcomes.', space_after=Pt(8)) # ═══════════════════════════════════════════════ # ── PROFILE DEFINITIONS ── # ═══════════════════════════════════════════════ _add_heading_red(doc, "Profile Definitions", level=1) p = doc.add_paragraph() p.paragraph_format.line_spacing = 1.15 r = p.add_run("Level 1") r.bold = True r.font.name = 'Calibri' r.font.color.rgb = BODY_TEXT r = p.add_run(" \u2014 Items in this profile intend to be practical and prudent, provide a clear security benefit, " "and not inhibit the utility of the technology beyond acceptable means.") r.font.name = 'Calibri' r.font.color.rgb = BODY_TEXT p = doc.add_paragraph() p.paragraph_format.line_spacing = 1.15 r = p.add_run("Level 2") r.bold = True r.font.name = 'Calibri' r.font.color.rgb = BODY_TEXT r = p.add_run(" \u2014 Items in this profile extend the Level 1 profile. Items in this profile exhibit one or more " "of the following characteristics: are intended for environments or use cases where security is paramount, " "act as defense in depth measure, and may negatively inhibit the utility or performance of the technology.") r.font.name = 'Calibri' r.font.color.rgb = BODY_TEXT # ═══════════════════════════════════════════════ # ── TABLE OF CONTENTS ── # ═══════════════════════════════════════════════ doc.add_page_break() _add_heading_red(doc, "Table of Contents", level=1) p = doc.add_paragraph() r = p.add_run() fld1 = OxmlElement('w:fldChar'); fld1.set(qn('w:fldCharType'), 'begin'); r._r.append(fld1) inst = OxmlElement('w:instrText'); inst.set(qn('xml:space'), 'preserve'); inst.text = ' TOC \\o "1-3" \\h \\z \\u '; r._r.append(inst) fld2 = OxmlElement('w:fldChar'); fld2.set(qn('w:fldCharType'), 'separate'); r._r.append(fld2) ph = OxmlElement('w:r'); pht = OxmlElement('w:t'); pht.text = "Right-click and select 'Update Field' to generate TOC"; ph.append(pht); r._r.append(ph) fld3 = OxmlElement('w:fldChar'); fld3.set(qn('w:fldCharType'), 'end'); r._r.append(fld3) # Auto-update fields on open settings = doc.settings.element uf = OxmlElement('w:updateFields'); uf.set(qn('w:val'), 'true'); settings.append(uf) doc.add_page_break() # ═══════════════════════════════════════════════ # ── SECURITY OVERVIEW ── # ═══════════════════════════════════════════════ _add_heading_red(doc, "Security Overview", level=1) _add_body_para(doc, 'Oracle Cloud Infrastructure (OCI) is designed with security as a foundational pillar. ' 'The CIS Oracle Cloud Infrastructure Foundations Benchmark provides prescriptive guidance for ' 'establishing a secure baseline configuration. This assessment evaluates the tenancy against the ' 'benchmark controls across key security domains including Identity and Access Management, Networking, ' 'Logging and Monitoring, Object Storage, and other critical infrastructure services.', space_after=Pt(8)) _add_body_para(doc, 'The following sections present a summary of findings by domain, a detailed benchmark recommendation ' 'summary, and individual findings with remediation guidance for non-compliant controls.', space_after=Pt(8)) doc.add_page_break() # ═══════════════════════════════════════════════ # ── DOMAINS SUMMARY TABLE (data from compliance_data.json — identical to HTML) ── # ═══════════════════════════════════════════════ _add_heading_red(doc, "CIS Security Assessment", level=1) _add_body_para(doc, f"Number of controls per domain and non-compliant controls on tenancy {tenancy}:", space_after=Pt(8), align=WD_ALIGN_PARAGRAPH.LEFT) domain_data = [ (sec, str(sd["total"]), (str(sd["failed"]), True, RED), (str(sd["passed"]), True, GREEN)) for sec, sd in sections_data.items() ] _styled_table(doc, ["DOMAINS", "TOTAL CONTROLS", "FAILED", "PASSED"], domain_data, col_widths=[3.5, 1.2, 0.9, 0.9]) # Totals summary line total_all = sum(sd["total"] for sd in sections_data.values()) total_passed = sum(sd["passed"] for sd in sections_data.values()) total_failed = total_all - total_passed pct = round(total_passed / total_all * 100) if total_all else 0 p = doc.add_paragraph() p.paragraph_format.space_before = Pt(10) p.paragraph_format.space_after = Pt(4) r = p.add_run(f"Total: {total_all} controls | ") r.font.size = Pt(10) r.font.color.rgb = BODY_TEXT r.font.name = 'Calibri' r = p.add_run(f"Passed: {total_passed} ({pct}%)") r.font.color.rgb = GREEN r.font.size = Pt(10) r.bold = True r.font.name = 'Calibri' r = p.add_run(" | ") r.font.size = Pt(10) r.font.color.rgb = BODY_TEXT r.font.name = 'Calibri' r = p.add_run(f"Failed: {total_failed}") r.font.color.rgb = RED r.font.size = Pt(10) r.bold = True r.font.name = 'Calibri' doc.add_page_break() # ═══════════════════════════════════════════════ # ── BENCHMARK SUMMARY TABLE ── # ═══════════════════════════════════════════════ _add_heading_red(doc, "CIS Benchmark Recommendation Summary", level=1) bench_data = [] for r in filtered: rec = r.get("Recommendation #", "") title_txt = r.get("Title", "") level = r.get("Level", "") compliant = r.get("__compliant", False) st_text = "OK" if compliant else "FAILED" st_color = GREEN if compliant else RED bench_data.append((rec, title_txt, f"L{level}" if level else "", (st_text, True, st_color))) _styled_table(doc, ["Item", "Description", "Level", "Status"], bench_data, col_widths=[0.6, 4.0, 0.5, 0.8]) doc.add_page_break() # ═══════════════════════════════════════════════ # ── DETAILED FINDINGS ── # ═══════════════════════════════════════════════ for sec_name, sd in sections_data.items(): _add_heading_red(doc, sec_name, level=1) for r in sd["items"]: compliant = r.get("__compliant", False) rec = r.get("Recommendation #", "") title_txt = r.get("Title", "") level = r.get("Level", "") findings = r.get("Findings", "0") total = r.get("Total", "0") csv_remediation = (r.get("Remediation", "") or "").strip() rag_data = r.get("__rag_remediation", {}) if isinstance(rag_data, str): rag_data = {"description": "", "rationale": "", "remediation": rag_data} rag_desc = (rag_data.get("description", "") or "").strip() if isinstance(rag_data, dict) else "" rag_rationale = (rag_data.get("rationale", "") or "").strip() if isinstance(rag_data, dict) else "" rag_rem = (rag_data.get("remediation", "") or "").strip() if isinstance(rag_data, dict) else "" # Recommendation heading in Oracle Red _add_heading_red(doc, f"{rec} {title_txt}", level=2) # Level badge if level: p = doc.add_paragraph() p.paragraph_format.space_before = Pt(0) p.paragraph_format.space_after = Pt(4) r_run = p.add_run(f" Level {level} ") r_run.font.size = Pt(8) r_run.bold = True r_run.font.color.rgb = MUTED r_run.font.name = 'Calibri' rPr = r_run._r.get_or_add_rPr() shd = OxmlElement('w:shd') shd.set(qn('w:val'), 'clear') shd.set(qn('w:color'), 'auto') shd.set(qn('w:fill'), 'E8E8E8') rPr.append(shd) # Result box with colored background and left border _result_box(doc, tenancy_name, compliant, findings, total) # Description (from RAG — same as HTML) if not compliant and rag_desc: _add_heading_red(doc, "Description:", level=3) _add_body_para(doc, rag_desc, size=Pt(9), space_after=Pt(4)) # Rationale (from RAG — same as HTML) if not compliant and rag_rationale: _add_heading_red(doc, "Rationale:", level=3) _add_body_para(doc, rag_rationale, size=Pt(9), space_after=Pt(4)) # Remediation (from RAG or CSV — same as HTML, with code blocks) remediation = rag_rem or csv_remediation if not compliant and remediation: _add_heading_red(doc, "Remediation:", level=3) _add_remediation_docx(doc, remediation) # Affected Resources table if not compliant: csv_filename = (r.get("Filename", "") or "").strip().strip('"').strip("'") if csv_filename and csv_filename in file_path_map: csv_file_path = Path(file_path_map[csv_filename]) if csv_file_path.exists(): try: with open(csv_file_path, "r", encoding="utf-8") as cf: csv_rows = list(csvmod.DictReader(cf)) if csv_rows: cols = [c for c in csv_rows[0].keys() if c.lower() not in ("extract_date", "deep_link", "domain_deeplink", "defined_tags", "domain_id", "external_identifier", "freeform_tags", "system_tags")][:5] res_data = [] for crow in csv_rows[:10]: row_vals = [] for c in cols: val = crow.get(c, "") if val.startswith("=HYPERLINK"): m = _re.search(r',\s*"([^"]+)"', val) val = m.group(1) if m else "" if val.startswith("ocid1.") and len(val) > 30: val = val[:20] + "..." + val[-8:] if len(val) > 50: val = val[:47] + "..." row_vals.append(val) res_data.append(tuple(row_vals)) if res_data: _add_heading_red(doc, "Affected Resources:", level=3) n = len(cols) cw = [6.5 / n] * n _styled_table(doc, [c.replace("_", " ").title() for c in cols], res_data, col_widths=cw, header_color=FINDINGS_HEADER_GREEN) if len(csv_rows) > 10: p = doc.add_paragraph(f"Showing 10 of {len(csv_rows)} resources.") p.runs[0].font.size = Pt(8) p.runs[0].font.color.rgb = MUTED except Exception as e: log.warning(f"DOCX: resource table error for {csv_filename}: {e}") # ═══════════════════════════════════════════════ # ── OCI SERVICES (from same data as HTML report) ── # ═══════════════════════════════════════════════ try: if oci_services: svc_data = [] for svc in oci_services: st = svc.get("status", "WARNING") desc = svc.get("description", "") st_color = GREEN if st == "OK" else ORANGE svc_data.append(((st, True, st_color), svc["service"], desc)) doc.add_page_break() _add_heading_red(doc, "OCI SERVICES", level=1) _add_heading_red(doc, "Recommendations:", level=2) _add_body_para(doc, "The recommended services are already enabled. This way we recommend further development of their usage:", space_after=Pt(8), align=WD_ALIGN_PARAGRAPH.LEFT) _styled_table(doc, ["STATUS", "SERVICE", "Description"], svc_data, col_widths=[0.8, 2.0, 3.7]) # Service detail descriptions for svc in oci_services: info = _OCI_SERVICE_DESCRIPTIONS.get(svc["service"], {}) if not info: continue _add_heading_red(doc, info.get("title", svc["service"]), level=2) desc = info.get("description", "") for para in desc.split("\n"): para = para.strip() if para: _add_body_para(doc, para, space_after=Pt(4)) rec_text = info.get("recommendation", "") if rec_text: p = doc.add_paragraph() p.paragraph_format.line_spacing = 1.15 r = p.add_run("Recommendation: ") r.bold = True r.font.name = 'Calibri' r.font.color.rgb = BODY_TEXT r = p.add_run(rec_text) r.font.name = 'Calibri' r.font.color.rgb = BODY_TEXT # Cloud Guard recommendations table cg_recs = svc.get("recommendations", []) if cg_recs: cg_data = [(cr["name"], (str(cr["count"]), True, None)) for cr in cg_recs] _styled_table(doc, ["Recommendations", "Total"], cg_data, col_widths=[5.5, 0.8]) except Exception as e: log.warning(f"DOCX: OCI services section error: {e}") # ═══════════════════════════════════════════════ # ── BACK PAGE ── # ═══════════════════════════════════════════════ doc.add_page_break() _add_heading_red(doc, "Connect with us", level=1) _add_body_para(doc, "Call +1.800.ORACLE1 or visit oracle.com.\nOutside North America, find your local office at: oracle.com/contact.", space_after=Pt(12), align=WD_ALIGN_PARAGRAPH.LEFT) p = doc.add_paragraph() p.paragraph_format.space_before = Pt(8) p.paragraph_format.space_after = Pt(4) for icon, link in [("Blog", "blogs.oracle.com"), ("Facebook", "facebook.com/oracle"), ("Twitter", "twitter.com/oracle")]: r = p.add_run(f" {icon}: {link} ") r.font.size = Pt(9) r.font.color.rgb = MUTED r.font.name = 'Calibri' p = doc.add_paragraph() p.paragraph_format.space_before = Pt(24) p.paragraph_format.space_after = Pt(0) r = p.add_run( f"Copyright \u00A9 {now.year}, Oracle and/or its affiliates. This document is provided for information purposes only, " "and the contents hereof are subject to change without notice. This document is not warranted to be error-free, " "nor subject to any other warranties or conditions, whether expressed orally or implied in law. " "Oracle, Java, MySQL, and NetSuite are registered trademarks of Oracle and/or its affiliates." ) r.font.size = Pt(7) r.font.color.rgb = MUTED r.font.name = 'Calibri' buf = _io.BytesIO() doc.save(buf) return buf.getvalue() @app.get("/api/reports/{rid}/compliance-report/docx") async def compliance_report_docx(rid: str, token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))): """Download DOCX compliance report directly.""" import re as _re u = _user_from_token(token) if token else (await current_user(cred) if cred else None) if not u: raise HTTPException(401, "Not authenticated") _verify_report_access(rid, u) with db() as c: report = c.execute("SELECT tenancy_name FROM reports WHERE id=?", (rid,)).fetchone() if not report: raise HTTPException(404) files = c.execute("SELECT file_name, file_path FROM report_files WHERE report_id=?", (rid,)).fetchall() tenancy = report["tenancy_name"] or "report" safe_name = _re.sub(r'[^\w\-]', '_', tenancy) try: file_map = {f["file_name"]: f["file_path"] for f in files} docx_bytes = _generate_compliance_docx(rid, tenancy, file_map) if not docx_bytes: raise HTTPException(500, "DOCX generation failed") log.info(f"DOCX direct download: {len(docx_bytes)} bytes") except HTTPException: raise except Exception as e: log.error(f"DOCX generation failed: {e}", exc_info=True) raise HTTPException(500, f"DOCX generation failed: {e}") return Response( content=docx_bytes, media_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document", headers={"Content-Disposition": f'attachment; filename="CIS_Compliance_Report_{safe_name}.docx"'} ) @app.get("/api/reports/{rid}/compliance-report/download") async def compliance_report_download(rid: str, token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))): """Download ZIP with compliance report PDF (Chromium headless) + CSV files.""" import zipfile, io, re as _re, subprocess, tempfile u = _user_from_token(token) if token else (await current_user(cred) if cred else None) if not u: raise HTTPException(401, "Not authenticated") _verify_report_access(rid, u) rdir = REPORTS / rid html_path = rdir / "compliance_report.html" if not html_path.exists(): raise HTTPException(404, "Compliance report not generated yet") with db() as c: report = c.execute("SELECT tenancy_name FROM reports WHERE id=?", (rid,)).fetchone() if not report: raise HTTPException(404) files = c.execute("SELECT file_name, file_path FROM report_files WHERE report_id=?", (rid,)).fetchall() tenancy = report["tenancy_name"] or "report" safe_name = _re.sub(r'[^\w\-]', '_', tenancy) file_map = {f["file_name"]: f["file_path"] for f in files} # Prepare HTML for PDF: rewrite CSV links to relative paths html = html_path.read_text(encoding="utf-8") html = _re.sub(r'', '', html, flags=_re.DOTALL) html = _re.sub(r'', '', html, flags=_re.DOTALL) for fname in file_map: html = _re.sub( r'href="[^"]*?/download[^"]*?"([^>]*>)\s*\[CSV\]\s*' + _re.escape(fname), f'href="csv/{fname}"\\1[CSV] {fname}', html ) # Generate PDF via Chromium headless (same engine as browser print) pdf_bytes = None try: with tempfile.TemporaryDirectory() as tmpdir: tmp_html = Path(tmpdir) / "report.html" tmp_pdf = Path(tmpdir) / "report.pdf" tmp_html.write_text(html, encoding="utf-8") loop = asyncio.get_event_loop() result = await asyncio.wait_for( loop.run_in_executor(None, partial( subprocess.run, [ "chromium", "--headless", "--no-sandbox", "--disable-gpu", "--disable-software-rasterizer", "--run-all-compositor-stages-before-draw", f"--print-to-pdf={tmp_pdf}", "--no-pdf-header-footer", f"file://{tmp_html}" ], capture_output=True, timeout=120)), timeout=120) if tmp_pdf.exists(): pdf_bytes = tmp_pdf.read_bytes() log.info(f"Chromium PDF generated: {len(pdf_bytes)} bytes") else: log.error(f"Chromium PDF failed: {result.stderr.decode()[:500]}") except Exception as e: log.error(f"PDF generation failed: {e}", exc_info=True) if not pdf_bytes: raise HTTPException(500, "PDF generation failed") # Generate DOCX docx_bytes = None try: docx_bytes = _generate_compliance_docx(rid, tenancy, file_map) if docx_bytes: log.info(f"DOCX generated: {len(docx_bytes)} bytes") except Exception as e: log.warning(f"DOCX generation failed (non-fatal): {e}") # Build ZIP: PDF + DOCX + CSVs buf = io.BytesIO() with zipfile.ZipFile(buf, 'w', zipfile.ZIP_DEFLATED) as zf: zf.writestr(f"CIS_Compliance_Report_{safe_name}.pdf", pdf_bytes) if docx_bytes: zf.writestr(f"CIS_Compliance_Report_{safe_name}.docx", docx_bytes) for fname, fpath in file_map.items(): p = Path(fpath) if p.exists() and fname.lower().endswith('.csv'): zf.write(str(p), f"csv/{fname}") buf.seek(0) return Response( content=buf.getvalue(), media_type="application/zip", headers={"Content-Disposition": f'attachment; filename="CIS_Compliance_Report_{safe_name}.zip"'} ) @app.get("/api/reports/{rid}/download") async def report_dl(rid, fmt: str = Query("json"), token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))): u = _user_from_token(token) if token else (await current_user(cred) if cred else None) if not u: raise HTTPException(401, "Not authenticated") _verify_report_access(rid, u) with db() as c: r=c.execute("SELECT * FROM reports WHERE id=?",(rid,)).fetchone() if not r: raise HTTPException(404) p = r["json_path"] if fmt=="json" else r["html_path"] if not p or not Path(p).exists(): raise HTTPException(404) return FileResponse(p, filename=f"cis_{r['tenancy_name']}_{rid[:8]}.{fmt}") # ── Chat Agent ──────────────────────────────────────────────────────────────── # (endpoints defined below _chat_core, after _agent_respond) def _chat_start(msg: ChatMsg, u, attachments: list = None, agent_type: str = "chat"): """Start a chat: save user msg, resolve config, return (sid, mid, genai_cfg) or immediate response. If genai_cfg is None, returns immediate fallback response in mid field as dict.""" is_new = not msg.session_id sid = msg.session_id or str(uuid.uuid4()) with db() as c: c.execute("INSERT INTO chat_messages (id,session_id,user_id,role,content,model_id,status) VALUES (?,?,?,?,?,?,?)", (str(uuid.uuid4()), sid, u["id"], "user", msg.message, None, "done")) if is_new: title = (msg.message or "Nova conversa")[:80].strip() c.execute("INSERT OR IGNORE INTO chat_sessions (id,user_id,agent_type,title) VALUES (?,?,?,?)", (sid, u["id"], agent_type, title)) else: c.execute("UPDATE chat_sessions SET updated_at=datetime('now') WHERE id=?", (sid,)) genai_cfg = None if msg.genai_config_id: _verify_config_access("genai", msg.genai_config_id, u) with db() as c: row = c.execute("SELECT * FROM genai_configs WHERE id=?", (msg.genai_config_id,)).fetchone() if row: genai_cfg = dict(row) if msg.temperature is not None: genai_cfg["temperature"] = msg.temperature if msg.max_tokens is not None: genai_cfg["max_tokens"] = msg.max_tokens if msg.top_p is not None: genai_cfg["top_p"] = msg.top_p if msg.top_k is not None: genai_cfg["top_k"] = msg.top_k if msg.frequency_penalty is not None: genai_cfg["frequency_penalty"] = msg.frequency_penalty if msg.presence_penalty is not None: genai_cfg["presence_penalty"] = msg.presence_penalty if msg.reasoning_effort is not None: genai_cfg["reasoning_effort"] = msg.reasoning_effort elif msg.model_id and msg.oci_config_id: _verify_config_access("oci", msg.oci_config_id, u) with db() as c: oci_row = c.execute("SELECT * FROM oci_configs WHERE id=?", (msg.oci_config_id,)).fetchone() if not oci_row: raise HTTPException(400, "OCI config not found") region = msg.genai_region or oci_row["region"] compartment = _safe_dec(oci_row["compartment_id"]) if oci_row["compartment_id"] else "" if not compartment: raise HTTPException(400, "compartment_id required") genai_cfg = { "oci_config_id": msg.oci_config_id, "model_id": msg.model_id, "model_ocid": None, "compartment_id": compartment, "genai_region": region, "endpoint": f"https://inference.generativeai.{region}.oci.oraclecloud.com", "serving_type": "ON_DEMAND", "dedicated_endpoint_id": None, "temperature": msg.temperature if msg.temperature is not None else 1.0, "max_tokens": msg.max_tokens if msg.max_tokens is not None else 6000, "top_p": msg.top_p if msg.top_p is not None else 0.95, "top_k": msg.top_k if msg.top_k is not None else 1, "frequency_penalty": msg.frequency_penalty if msg.frequency_penalty is not None else 0.0, "presence_penalty": msg.presence_penalty if msg.presence_penalty is not None else 0.0, "reasoning_effort": msg.reasoning_effort, } if not genai_cfg: # No GenAI config — return immediate fallback resp = _agent_respond(msg.message, u) with db() as c: c.execute("INSERT INTO chat_messages (id,session_id,user_id,role,content,model_id,status) VALUES (?,?,?,?,?,?,?)", (str(uuid.uuid4()), sid, u["id"], "assistant", resp, None, "done")) return sid, {"session_id": sid, "response": resp, "model_id": None, "status": "done"}, None # Create placeholder assistant message for background processing mid = str(uuid.uuid4()) with db() as c: c.execute("INSERT INTO chat_messages (id,session_id,user_id,role,content,model_id,status) VALUES (?,?,?,?,?,?,?)", (mid, sid, u["id"], "assistant", "", genai_cfg.get("model_id"), "processing")) return sid, mid, genai_cfg async def _chat_background(mid: str, sid: str, msg: ChatMsg, user: dict, genai_cfg: dict, attachments: list = None, agent_type: str = "chat"): """Background worker — processes GenAI chat, updates DB when done.""" log.info(f"Chat background started: mid={mid}, sid={sid}") try: history = [] with db() as c: prev = c.execute("SELECT role,content FROM chat_messages WHERE session_id=? AND role IN ('user','assistant') AND status='done' ORDER BY created_at ASC", (sid,)).fetchall() history = [{"role":r["role"],"content":r["content"]} for r in prev] # ── RAG: augment with vector context from ALL active ADB configs ── # Resolve active tenancy for filtered vector search rag_tenancy = None active_oci_for_rag = genai_cfg.get("oci_config_id") or (msg.oci_config_id if hasattr(msg, 'oci_config_id') and msg.oci_config_id else None) if active_oci_for_rag: with db() as c: oci_for_rag = c.execute("SELECT tenancy_name FROM oci_configs WHERE id=?", (active_oci_for_rag,)).fetchone() if oci_for_rag: rag_tenancy = oci_for_rag["tenancy_name"] log.info(f"RAG: filtering by tenancy '{rag_tenancy}'") rag_context = "" # For short follow-up questions, enrich with previous context for better RAG search import re as _re_chat rag_query = msg.message if len(msg.message.split()) <= 8 and history: # Short question — get previous user messages (excluding current which is already in history) prev_user_msgs = [h["content"] for h in history if h["role"] == "user" and h["content"] != msg.message] if prev_user_msgs: rag_query = prev_user_msgs[-1] + " " + msg.message log.info(f"RAG: enriched short query → '{rag_query[:100]}'") elif len(history) >= 2: # Fallback: use last assistant response for context last_assistant = [h["content"][:200] for h in history if h["role"] == "assistant"] if last_assistant: rag_query = last_assistant[-1] + " " + msg.message log.info(f"RAG: enriched from assistant context") # Detect CIS recommendation number for exact text filtering cis_chat_match = _re_chat.search(r'(?:cis|recommendation)\s*(\d+\.\d+)', rag_query, _re_chat.IGNORECASE) cis_chat_filter = f"CIS Recommendation: {cis_chat_match.group(1)}" if cis_chat_match else None if cis_chat_filter: log.info(f"RAG: detected CIS filter '{cis_chat_filter}'") adb_cfgs = _get_active_adb_configs(user["id"]) if agent_type != "terraform" else [] rag_errors = [] if adb_cfgs: all_documents = [] for adb_cfg in adb_cfgs: try: genai_linked = None if adb_cfg.get("genai_config_id"): with db() as c: row = c.execute("SELECT * FROM genai_configs WHERE id=?", (adb_cfg["genai_config_id"],)).fetchone() if row: genai_linked = dict(row) emb_genai = _resolve_embed_config(oci_config_id=active_oci_for_rag, genai_cfg=genai_linked, user_id=user["id"]) if not emb_genai: continue default_model = adb_cfg.get("embedding_model_id", "cohere.embed-v4.0") tables = _get_tables_for_config(adb_cfg["id"], active_only=True) if not tables: tables = [{"table_name": adb_cfg.get("table_name", "")}] all_table_names = [t["table_name"] for t in tables if t["table_name"]] # Smart skip: only search relevant tables relevant = _relevant_tables(rag_query, all_table_names) skipped = set(all_table_names) - set(relevant) if skipped: log.info(f"RAG: skipped {', '.join(skipped)} (not relevant)") # Auto-detect embedding model (use first table's dim as representative) emb_model = default_model for tbl_name in relevant: try: dim = _get_table_embedding_dim(adb_cfg, tbl_name) if dim and dim in _DIM_TO_MODEL: emb_model = _DIM_TO_MODEL[dim] break except Exception: pass query_embedding = _embed_text(rag_query, emb_genai, emb_model) # Single connection, multi-table search tbl_top_k = 10 if cis_chat_filter else 3 docs = _vector_search_multi(adb_cfg, query_embedding, relevant, top_k_per_table=tbl_top_k, tenancy=rag_tenancy, text_filter=cis_chat_filter, user_id=user["id"]) if docs: all_documents.extend(docs) sources = {} for d in docs: sources[d["source"]] = sources.get(d["source"], 0) + 1 log.info(f"RAG: {len(docs)} docs — {', '.join(f'{k}:{v}' for k,v in sources.items())}") except Exception as e: err_short = str(e)[:150] log.warning(f"RAG retrieval failed for {adb_cfg.get('config_name','?')}: {err_short}") if "DPY-6001" in str(e) or "DPY-6005" in str(e) or "timeout" in str(e).lower() or "connect" in str(e).lower(): rag_errors.append(f"Não foi possível conectar ao ADB ({adb_cfg.get('config_name','?')})") if all_documents: all_documents.sort(key=lambda d: d["distance"]) top_limit = 15 if cis_chat_filter else 8 rag_context = _build_rag_context(all_documents[:top_limit], max_total_chars=16000 if cis_chat_filter else 12000) # Append connection errors to context so LLM can inform the user if rag_errors and not rag_context: rag_context = "⚠️ AVISO: " + "; ".join(set(rag_errors)) + ". A base de conhecimento não está disponível no momento. Responda com base no seu conhecimento geral, informando que os dados do ADB não puderam ser consultados." elif rag_errors: rag_context += "\n\n⚠️ AVISO: Algumas bases não puderam ser consultadas: " + "; ".join(set(rag_errors)) cfg_dict = dict(genai_cfg) with db() as c: sp_row = c.execute("SELECT content FROM system_prompts WHERE agent=? AND is_active=1 LIMIT 1", (agent_type,)).fetchone() global_prompt = sp_row["content"] if sp_row and sp_row["content"] else "" if rag_context: augmented_message = RAG_CONTEXT_TEMPLATE.format(context=rag_context, question=msg.message) cfg_dict["system_prompt"] = global_prompt or RAG_DEFAULT_SYSTEM_PROMPT else: augmented_message = msg.message if global_prompt: cfg_dict["system_prompt"] = global_prompt # ── Inject all config context into system prompt so model auto-resolves IDs ── ctx_parts = [] active_oci_id = cfg_dict.get("oci_config_id") or (msg.oci_config_id if msg.oci_config_id else None) with db() as c: oci_cfgs = c.execute("SELECT id,tenancy_name,region,compartment_id FROM oci_configs WHERE user_id=?", (user["id"],)).fetchall() genai_cfgs = c.execute("SELECT id,name,model_id,genai_region,compartment_id,oci_config_id FROM genai_configs WHERE user_id=?", (user["id"],)).fetchall() adb_cfgs = c.execute("SELECT id,config_name,dsn,table_name,is_active,genai_config_id,embedding_model_id FROM adb_vector_configs WHERE user_id=? AND is_active=1", (user["id"],)).fetchall() mcp_srvs = c.execute("SELECT id,name,description,is_active FROM mcp_servers WHERE is_active=1 AND (user_id=? OR EXISTS (SELECT 1 FROM users WHERE id=? AND role='admin'))", (user["id"], user["id"])).fetchall() # ── Active OCI config (selected by user in this session) ── if active_oci_id: for oc in oci_cfgs: if oc["id"] == active_oci_id: comp = _safe_dec(oc["compartment_id"]) if oc["compartment_id"] else "N/A" ctx_parts.append( f"⚡ CONFIG OCI ATIVA (usar como config_id em TODAS as tools que precisarem): " f"config_id=\"{oc['id']}\" tenancy=\"{oc['tenancy_name']}\" region=\"{oc['region']}\" compartment_id=\"{comp}\"" ) break if oci_cfgs: ctx_parts.append("\nTodas as configurações OCI disponíveis (use 'id' como config_id):") for oc in oci_cfgs: comp = _safe_dec(oc["compartment_id"]) if oc["compartment_id"] else "N/A" active_tag = " ← ATIVA" if oc["id"] == active_oci_id else "" ctx_parts.append(f" - id=\"{oc['id']}\" tenancy=\"{oc['tenancy_name']}\" region=\"{oc['region']}\" compartment_id=\"{comp}\"{active_tag}") if genai_cfgs: ctx_parts.append("\nConfigurações GenAI disponíveis (use 'id' como genai_config_id):") for gc in genai_cfgs: comp = _safe_dec(gc["compartment_id"]) if gc["compartment_id"] else "N/A" ctx_parts.append(f" - id=\"{gc['id']}\" name=\"{gc['name']}\" model=\"{gc['model_id']}\" region=\"{gc['genai_region']}\" compartment_id=\"{comp}\"") if adb_cfgs: ctx_parts.append("\nConfigurações ADB Vector Store disponíveis (use 'id' como adb_config_id):") for ac in adb_cfgs: emb = ac["embedding_model_id"] if ac["embedding_model_id"] else "N/A" ctx_parts.append(f" - id=\"{ac['id']}\" name=\"{ac['config_name']}\" table=\"{ac['table_name']}\" embedding_model=\"{emb}\"") if mcp_srvs: ctx_parts.append("\nMCP Servers ativos (tools disponíveis para uso):") for ms in mcp_srvs: desc = ms["description"] or "" ctx_parts.append(f" - id=\"{ms['id']}\" name=\"{ms['name']}\" desc=\"{desc}\"") if ctx_parts: ctx_parts.append("\nIMPORTANTE: Use automaticamente a config OCI ATIVA como config_id em todas as tools. NUNCA peça config_id, tenancy ou IDs ao usuário — já estão definidos acima.") config_hint = "\n".join(ctx_parts) base_prompt = cfg_dict.get("system_prompt", "") cfg_dict["system_prompt"] = f"{base_prompt}\n\n{config_hint}" if base_prompt else config_hint # ── Terraform agent: boost max_tokens (capped at 65K to avoid context overflow) + force reasoning_effort=high ── if agent_type == "terraform": tf_model_info = GENAI_MODELS.get(cfg_dict.get("model_id", ""), {}) tf_model_max = tf_model_info.get("max_tokens", 32768) cfg_dict["max_tokens"] = min(tf_model_max, 65000) if tf_model_info.get("reasoning") and not cfg_dict.get("reasoning_effort"): cfg_dict["reasoning_effort"] = "HIGH" # ── Inject existing OCI resources for terraform agent ── if agent_type == "terraform" and active_oci_id: try: # Determine compartment: from msg or from OCI config tf_compartment = getattr(msg, 'compartment_id', None) or None if not tf_compartment: for oc in oci_cfgs: if oc["id"] == active_oci_id: tf_compartment = _safe_dec(oc["compartment_id"]) if oc["compartment_id"] else None break if tf_compartment: tf_region = None for oc in oci_cfgs: if oc["id"] == active_oci_id: tf_region = oc["region"] break loop = asyncio.get_event_loop() resources = await loop.run_in_executor( _chat_executor, partial(_fetch_compartment_resources, active_oci_id, tf_compartment, tf_region)) resource_ctx = _build_resource_context(resources) cfg_dict["system_prompt"] = cfg_dict.get("system_prompt", "") + "\n\n" + resource_ctx log.info(f"Terraform: injected resource context for compartment {tf_compartment[:20]}...") except Exception as e: log.warning(f"Failed to inject terraform resource context: {e}") # ── Inject OCI Terraform resource reference for correct resource names ── if agent_type == "terraform": tf_ref = _load_tf_resource_reference() if tf_ref: # Extract only the categorized sections (not the full 900+ resource list) # to keep prompt size manageable sections = [] for line in tf_ref.split('\n'): if line.startswith('## All Resource Types'): break sections.append(line) ref_compact = '\n'.join(sections).strip() if ref_compact: cfg_dict["system_prompt"] = cfg_dict.get("system_prompt", "") + \ "\n\n### Referência de Recursos OCI Terraform (gerado do provider schema)\n" + \ "Use EXATAMENTE estes nomes de resource types. Se o recurso não estiver nesta lista, ele NÃO EXISTE no provider.\n\n" + \ ref_compact log.info(f"Terraform: injected resource reference ({len(ref_compact)} chars)") # ── Inject official Terraform resource docs (Example Usage + Arguments) ── if agent_type == "terraform": try: # Detect resource types from user message + recent conversation history detect_text = msg.message if hasattr(msg, 'message') else "" if history: # Include last 3 messages for context for h in history[-3:]: detect_text += "\n" + (h.get("content", "") or "") resource_types = _detect_tf_resource_types(detect_text) if resource_types: loop = asyncio.get_event_loop() docs_ctx = await loop.run_in_executor( _chat_executor, partial(_get_tf_docs_for_resources, resource_types)) if docs_ctx: cfg_dict["system_prompt"] = cfg_dict.get("system_prompt", "") + "\n\n" + docs_ctx log.info(f"Terraform: injected {len(resource_types)} resource docs ({len(docs_ctx)} chars)") except Exception as e: log.warning(f"Failed to inject terraform resource docs: {e}") # Log total prompt sizes for debugging if agent_type == "terraform": sp_len = len(cfg_dict.get("system_prompt", "")) msg_len = len(msg.message) if hasattr(msg, 'message') else 0 log.info(f"Terraform prompt sizes: system_prompt={sp_len} chars (~{sp_len//4} tokens), user_msg={msg_len} chars (~{msg_len//4} tokens), max_tokens={cfg_dict.get('max_tokens')}") mcp_tools = [] tool_defs = None if msg.use_tools: mcp_tools = _get_active_mcp_tools(user["id"]) if mcp_tools: tool_defs = [t["tool"] for t in mcp_tools] log.info(f"Chat with {len(tool_defs)} MCP tools available") if history and _should_compact(history): log.info(f"Compaction triggered: {len(history)} msgs, ~{_estimate_history_tokens(history)} est tokens") history = _compact_history(sid, user["id"], cfg_dict, history) log.info(f"Post-compaction: {len(history)} msgs, ~{_estimate_history_tokens(history)} est tokens") hist = history[:-1] if len(history) > 1 else None loop = asyncio.get_event_loop() try: resp_text, tool_calls, tool_calls_raw = await asyncio.wait_for( loop.run_in_executor( _chat_executor, partial(_call_genai, cfg_dict, augmented_message, hist, tool_defs, None, None, attachments)), timeout=300) except asyncio.TimeoutError: log.error(f"GenAI call timed out after 300s for session {sid}") raise HTTPException(504, "O modelo de IA demorou demais para responder. Tente novamente.") all_tool_results = [] accumulated_msgs = [] iterations = 0 api_format = GENAI_MODELS.get(genai_cfg.get("model_id", ""), {}).get("api_format", "GENERIC") while tool_calls and iterations < 5: iterations += 1 log.info(f"Tool use iteration {iterations}: {len(tool_calls)} tool call(s)") iteration_results = [] for tc in tool_calls: mcp_match = next((m for m in mcp_tools if m["tool"]["name"] == tc["name"]), None) if mcp_match: try: result = await _execute_mcp_tool(mcp_match["server"], tc["name"], tc["arguments"]) log.info(f"Tool {tc['name']} executed successfully ({len(result)} chars)") _chat_log(sid, mid, user["id"], "info", "tool", "tool_success", f"{tc['name']} ({len(result)} chars)") except Exception as te: result = f"Erro ao executar tool {tc['name']}: {str(te)[:300]}" log.warning(f"Tool {tc['name']} failed: {te}") _chat_log(sid, mid, user["id"], "error", "tool", "tool_error", f"{tc['name']}: {te}") else: result = f"Tool {tc['name']} não encontrada nos MCP servers ativos" _chat_log(sid, mid, user["id"], "error", "tool", "tool_not_found", tc["name"]) iteration_results.append({"tool_call_id": tc["id"], "name": tc["name"], "content": result}) all_tool_results.extend(iteration_results) if api_format == "COHERE": import oci cohere_results = [] for tr in iteration_results: tc_obj = oci.generative_ai_inference.models.CohereToolCall() tc_obj.name = tr["name"] tc_obj.parameters = {} tr_obj = oci.generative_ai_inference.models.CohereToolResult() tr_obj.call = tc_obj tr_obj.outputs = [{"result": tr["content"]}] cohere_results.append(tr_obj) try: resp_text, tool_calls, tool_calls_raw = await asyncio.wait_for( loop.run_in_executor( _chat_executor, partial(_call_genai, cfg_dict, augmented_message, hist, tool_defs, cohere_results)), timeout=300) except asyncio.TimeoutError: log.error(f"GenAI tool-use call timed out after 300s (Cohere, iteration {iterations})") raise HTTPException(504, "O modelo de IA demorou demais para responder. Tente novamente.") else: import oci assistant_msg = oci.generative_ai_inference.models.AssistantMessage() assistant_msg.tool_calls = tool_calls_raw accumulated_msgs.append(assistant_msg) for tr in iteration_results: tool_msg = oci.generative_ai_inference.models.ToolMessage() tool_msg.tool_call_id = tr["tool_call_id"] tool_content = oci.generative_ai_inference.models.TextContent() tool_content.text = tr["content"] tool_msg.content = [tool_content] accumulated_msgs.append(tool_msg) try: resp_text, tool_calls, tool_calls_raw = await asyncio.wait_for( loop.run_in_executor( _chat_executor, partial(_call_genai, cfg_dict, augmented_message, hist, tool_defs, None, accumulated_msgs)), timeout=300) except asyncio.TimeoutError: log.error(f"GenAI tool-use call timed out after 300s (Generic, iteration {iterations})") raise HTTPException(504, "O modelo de IA demorou demais para responder. Tente novamente.") resp = resp_text if all_tool_results: tools_info = '\n\n🔧 **Tools utilizadas:** ' + ', '.join(tr["name"] for tr in all_tool_results) resp += tools_info if not resp or not resp.strip(): model_id = genai_cfg.get("model_id", "unknown") resp = f"⚠️ O modelo **{model_id}** retornou uma resposta vazia. Isso pode ocorrer com alguns modelos. Tente novamente ou selecione outro modelo (ex: Gemini, Llama)." log.warning(f"Chat {mid}: empty response from model {model_id}, returning fallback message") # Validate terraform resource types against DB if agent_type == "terraform" and resp and '```' in resp: try: tf_errors = _validate_tf_resource_types(resp) if tf_errors: warning_lines = ["\n\n⚠️ **Validação automática — Resource types inválidos detectados:**"] for err in tf_errors: line = f"- `{err['type']}` — NÃO EXISTE no provider oracle/oci." if err['suggestions']: line += f" Sugestões: {', '.join('`' + s + '`' for s in err['suggestions'])}" warning_lines.append(line) warning_lines.append("\n**Clique em 'Re-plan' após o modelo corrigir, ou peça correção no chat.**") resp += '\n'.join(warning_lines) log.warning(f"Chat {mid}: {len(tf_errors)} invalid TF resource types detected: {[e['type'] for e in tf_errors]}") except Exception as e: log.warning(f"TF resource type validation failed: {e}") with db() as c: c.execute("UPDATE chat_messages SET content=?, status='done' WHERE id=?", (resp, mid)) log.info(f"Chat {mid} completed successfully") except Exception as e: log.error(f"Chat {mid} failed: {e}") _chat_log(sid, mid, user["id"], "error", "genai", "genai_failed", str(e)) with db() as c: c.execute("UPDATE chat_messages SET content=?, status='failed' WHERE id=?", (f"❌ Erro GenAI: {str(e)[:400]}", mid)) MAX_UPLOAD_FILES = 5 MAX_UPLOAD_SIZE = 10 * 1024 * 1024 # 10MB IMAGE_MIMES = {"image/png", "image/jpeg", "image/gif", "image/webp", "image/bmp"} DOC_MIMES = {"application/pdf"} @app.post("/api/chat/upload") async def chat_with_files( bg: BackgroundTasks, message: str = Form(""), session_id: Optional[str] = Form(None), genai_config_id: Optional[str] = Form(None), model_id: Optional[str] = Form(None), oci_config_id: Optional[str] = Form(None), genai_region: Optional[str] = Form(None), compartment_id: Optional[str] = Form(None), use_tools: Optional[bool] = Form(True), temperature: Optional[float] = Form(None), max_tokens: Optional[int] = Form(None), top_p: Optional[float] = Form(None), top_k: Optional[int] = Form(None), frequency_penalty: Optional[float] = Form(None), presence_penalty: Optional[float] = Form(None), reasoning_effort: Optional[str] = Form(None), files: List[UploadFile] = File(default=[]), u=Depends(current_user), ): if len(files) > MAX_UPLOAD_FILES: raise HTTPException(400, f"Máximo {MAX_UPLOAD_FILES} arquivos por mensagem") attachments = [] file_names = [] for f in files: data = await f.read() if len(data) > MAX_UPLOAD_SIZE: raise HTTPException(400, f"Arquivo {f.filename} excede 10MB") mime = f.content_type or "application/octet-stream" b64 = base64.b64encode(data).decode() data_uri = f"data:{mime};base64,{b64}" if mime in IMAGE_MIMES: attachments.append({"type": "image", "mime": mime, "data_uri": data_uri}) elif mime in DOC_MIMES: attachments.append({"type": "document", "mime": mime, "data_uri": data_uri}) else: try: text = data.decode("utf-8", errors="replace") except Exception as e: log.warning(f"UTF-8 decode failed for file attachment, falling back to latin-1: {e}") text = data.decode("latin-1", errors="replace") message = f"{message}\n\n--- Conteúdo de {f.filename} ---\n{text[:50000]}" file_names.append(f.filename) msg = ChatMsg( message=message or "Analise os arquivos anexados.", session_id=session_id, genai_config_id=genai_config_id, model_id=model_id, oci_config_id=oci_config_id, genai_region=genai_region, compartment_id=compartment_id, use_tools=use_tools if use_tools is not None else True, temperature=temperature, max_tokens=max_tokens, top_p=top_p, top_k=top_k, frequency_penalty=frequency_penalty, presence_penalty=presence_penalty, reasoning_effort=reasoning_effort, ) sid, mid_or_result, genai_cfg = _chat_start(msg, u, attachments=attachments if attachments else None) if genai_cfg is None: return mid_or_result # immediate fallback response if file_names: with db() as c: c.execute("UPDATE chat_messages SET content = content || ? WHERE session_id=? AND role='user' AND status='done' ORDER BY created_at DESC LIMIT 1", (f"\n[📎 {', '.join(file_names)}]", sid)) bg.add_task(_chat_background, mid_or_result, sid, msg, dict(u), genai_cfg, attachments if attachments else None) return {"message_id": mid_or_result, "session_id": sid, "status": "processing"} @app.post("/api/chat") async def chat(msg: ChatMsg, bg: BackgroundTasks, u=Depends(current_user)): sid, mid_or_result, genai_cfg = _chat_start(msg, u) if genai_cfg is None: return mid_or_result # immediate fallback response bg.add_task(_chat_background, mid_or_result, sid, msg, dict(u), genai_cfg) return {"message_id": mid_or_result, "session_id": sid, "status": "processing"} @app.get("/api/chat/{mid}/status") async def chat_message_status(mid: str, u=Depends(current_user)): with db() as c: r = c.execute("SELECT id, session_id, role, content, model_id, status FROM chat_messages WHERE id=?", (mid,)).fetchone() if not r: raise HTTPException(404, "Message not found") return dict(r) def _agent_respond(msg, user): m = msg.lower().strip() if any(k in m for k in ["status","health","como está","saúde"]): cli = "✅ Instalado" if shutil.which("oci") else "❌ Não instalado" with db() as c: nc=c.execute("SELECT COUNT(*) n FROM oci_configs").fetchone()["n"] nr=c.execute("SELECT COUNT(*) n FROM reports").fetchone()["n"] nu=c.execute("SELECT COUNT(*) n FROM users WHERE is_active=1").fetchone()["n"] nm=c.execute("SELECT COUNT(*) n FROM mcp_servers WHERE is_active=1").fetchone()["n"] ng=c.execute("SELECT COUNT(*) n FROM genai_configs").fetchone()["n"] return (f"📊 **Status do Sistema**\n\n• OCI CLI: {cli}\n• Configs OCI: {nc}\n• GenAI Models: {ng}\n• MCP Servers: {nm}\n• Relatórios: {nr}\n• Usuários ativos: {nu}\n• Servidor: ✅ Online\n• Versão: v{VERSION}") if any(k in m for k in ["ajuda","help","comandos"]): return ("🤖 **Comandos disponíveis:**\n\n• `status` — Status do sistema\n• `listar configs` — Configurações OCI\n• `verificar cis` — Checks CIS 3.0\n• `modelos` — Modelos GenAI disponíveis\n• `ajuda` — Esta mensagem\n\n💡 Configure um modelo GenAI para chat com IA.") if any(k in m for k in ["modelo","modelos","genai"]): lines = ["🧠 **Modelos OCI Generative AI disponíveis:**\n"] for mid, info in GENAI_MODELS.items(): lines.append(f"• `{mid}` — {info['name']} ({info['provider']})") lines.append("\nConfigure em **GenAI Config** para usar no chat.") return "\n".join(lines) if any(k in m for k in ["cis","benchmark","checks","verificar"]): return ("🔒 **CIS OCI Foundations Benchmark 3.0**\n\n54 controles em 8 domínios:\n• IAM: 17 controles\n• Networking: 8 controles\n• Compute: 3 controles\n• Logging & Monitoring: 18 controles\n• Storage: 6 controles\n• Asset Management: 2 controles\n\nConfigure OCI e execute um relatório na aba **Report**.") return ("Sou o **AI Agent v" + VERSION + "** — Infrastructure & Security Engineer. Sem modelo GenAI configurado, uso respostas locais.\n\n" "Para chat com IA:\n1. Configure **OCI Credentials**\n2. Configure **GenAI** com modelo e região\n3. Selecione o modelo no chat\n\nDigite **ajuda** para ver os comandos.") @app.get("/api/chat/sessions") async def list_chat_sessions(agent_type: str = "chat", limit: int = 50, u=Depends(current_user)): with db() as c: rows = c.execute( """SELECT cs.id, cs.title, cs.agent_type, cs.created_at, cs.updated_at, (SELECT COUNT(*) FROM chat_messages cm WHERE cm.session_id=cs.id) as message_count FROM chat_sessions cs WHERE cs.user_id=? AND cs.agent_type=? ORDER BY cs.updated_at DESC LIMIT ?""", (u["id"], agent_type, limit)).fetchall() return [dict(r) for r in rows] @app.get("/api/chat/sessions/{sid}/messages") async def get_session_messages(sid: str, skip: int = Query(0, ge=0), limit: int = Query(100, ge=1, le=500), u=Depends(current_user)): with db() as c: session = c.execute("SELECT * FROM chat_sessions WHERE id=? AND user_id=?", (sid, u["id"])).fetchone() if not session: raise HTTPException(404, "Session not found") total = c.execute( "SELECT COUNT(*) as cnt FROM chat_messages " "WHERE session_id=? AND status='done'", (sid,)).fetchone()["cnt"] msgs = c.execute( "SELECT id, role, content, model_id, status, created_at FROM chat_messages " "WHERE session_id=? AND status='done' ORDER BY created_at ASC LIMIT ? OFFSET ?", (sid, limit, skip)).fetchall() return {"session": dict(session), "messages": [dict(m) for m in msgs], "total": total} @app.put("/api/chat/sessions/{sid}/title") async def rename_session(sid: str, req: dict = Body(...), u=Depends(current_user)): with db() as c: c.execute("UPDATE chat_sessions SET title=? WHERE id=? AND user_id=?", (req.get("title", "")[:120], sid, u["id"])) return {"ok": True} @app.delete("/api/chat/{sid}") async def clear_chat(sid, u=Depends(current_user)): with db() as c: # Delete associated terraform workspaces and their files ws_rows = c.execute("SELECT id FROM terraform_workspaces WHERE session_id=? AND user_id=?", (sid, u["id"])).fetchall() for ws in ws_rows: wdir = TERRAFORM_DIR / ws["id"] if wdir.exists(): shutil.rmtree(wdir, ignore_errors=True) c.execute("DELETE FROM terraform_workspaces WHERE session_id=? AND user_id=?", (sid, u["id"])) c.execute("DELETE FROM chat_messages WHERE session_id=? AND user_id=?", (sid, u["id"])) c.execute("DELETE FROM chat_sessions WHERE id=? AND user_id=?", (sid, u["id"])) return {"ok": True} # ── Terraform Agent ─────────────────────────────────────────────────────────── def _fetch_compartment_resources(oci_config_id: str, compartment_id: str, region: str = None) -> dict: """Fetch existing OCI resources in a compartment for terraform context.""" import oci config = _get_oci_config(oci_config_id) if region: config["region"] = region resources = {} try: vn = oci.core.VirtualNetworkClient(config) vcns = vn.list_vcns(compartment_id).data resources["vcns"] = [{"id": v.id, "display_name": v.display_name, "cidr_blocks": v.cidr_blocks, "state": v.lifecycle_state} for v in vcns if v.lifecycle_state == "AVAILABLE"] subs = vn.list_subnets(compartment_id).data resources["subnets"] = [{"id": s.id, "display_name": s.display_name, "cidr_block": s.cidr_block, "vcn_id": s.vcn_id, "public": not s.prohibit_public_ip_on_vnic, "state": s.lifecycle_state} for s in subs if s.lifecycle_state == "AVAILABLE"] igws = vn.list_internet_gateways(compartment_id).data resources["internet_gateways"] = [{"id": g.id, "display_name": g.display_name, "vcn_id": g.vcn_id, "enabled": g.is_enabled} for g in igws if g.lifecycle_state == "AVAILABLE"] ngws = vn.list_nat_gateways(compartment_id).data resources["nat_gateways"] = [{"id": g.id, "display_name": g.display_name, "vcn_id": g.vcn_id} for g in ngws if g.lifecycle_state == "AVAILABLE"] rts = vn.list_route_tables(compartment_id).data resources["route_tables"] = [{"id": r.id, "display_name": r.display_name, "vcn_id": r.vcn_id} for r in rts if r.lifecycle_state == "AVAILABLE"] sls = vn.list_security_lists(compartment_id).data resources["security_lists"] = [{"id": s.id, "display_name": s.display_name, "vcn_id": s.vcn_id} for s in sls if s.lifecycle_state == "AVAILABLE"] except Exception as e: log.warning(f"Failed to fetch networking resources: {e}") try: compute = oci.core.ComputeClient(config) insts = compute.list_instances(compartment_id).data resources["instances"] = [{"id": i.id, "display_name": i.display_name, "shape": i.shape, "state": i.lifecycle_state} for i in insts if i.lifecycle_state in ("RUNNING", "STOPPED")] except Exception as e: log.warning(f"Failed to fetch compute resources: {e}") try: db_client = oci.database.DatabaseClient(config) adbs = db_client.list_autonomous_databases(compartment_id).data resources["autonomous_databases"] = [{"id": a.id, "display_name": a.display_name, "db_name": a.db_name, "state": a.lifecycle_state, "is_free_tier": a.is_free_tier} for a in adbs if a.lifecycle_state in ("AVAILABLE", "STOPPED")] except Exception as e: log.warning(f"Failed to fetch database resources: {e}") try: bs = oci.core.BlockstorageClient(config) vols = bs.list_volumes(compartment_id).data resources["block_volumes"] = [{"id": v.id, "display_name": v.display_name, "size_in_gbs": v.size_in_gbs, "state": v.lifecycle_state} for v in vols if v.lifecycle_state == "AVAILABLE"] except Exception as e: log.warning(f"Failed to fetch block storage resources: {e}") try: lb_client = oci.load_balancer.LoadBalancerClient(config) lbs = lb_client.list_load_balancers(compartment_id).data resources["load_balancers"] = [{"id": l.id, "display_name": l.display_name, "shape_name": l.shape_name, "state": l.lifecycle_state} for l in lbs if l.lifecycle_state == "ACTIVE"] except Exception as e: log.warning(f"Failed to fetch load balancer resources: {e}") return resources def _build_resource_context(resources: dict) -> str: """Build a text summary of existing resources for injection into system prompt.""" lines = ["## RECURSOS OCI EXISTENTES NO COMPARTMENT"] total = 0 resource_labels = { "vcns": ("VCNs", lambda r: f" - {r['display_name']} (CIDR: {','.join(r.get('cidr_blocks') or [])}) [OCID: {r['id'][:30]}...]"), "subnets": ("Subnets", lambda r: f" - {r['display_name']} (CIDR: {r['cidr_block']}, {'pública' if r.get('public') else 'privada'}, VCN: {r['vcn_id'][:30]}...)"), "internet_gateways": ("Internet Gateways", lambda r: f" - {r['display_name']} (VCN: {r['vcn_id'][:30]}...)"), "nat_gateways": ("NAT Gateways", lambda r: f" - {r['display_name']} (VCN: {r['vcn_id'][:30]}...)"), "route_tables": ("Route Tables", lambda r: f" - {r['display_name']} (VCN: {r['vcn_id'][:30]}...)"), "security_lists": ("Security Lists", lambda r: f" - {r['display_name']} (VCN: {r['vcn_id'][:30]}...)"), "instances": ("Compute Instances", lambda r: f" - {r['display_name']} (Shape: {r['shape']}, Estado: {r['state']})"), "autonomous_databases": ("Autonomous Databases", lambda r: f" - {r['display_name']} (DB: {r['db_name']}, Free Tier: {r.get('is_free_tier', False)})"), "block_volumes": ("Block Volumes", lambda r: f" - {r['display_name']} ({r.get('size_in_gbs', '?')} GB)"), "load_balancers": ("Load Balancers", lambda r: f" - {r['display_name']} (Shape: {r.get('shape_name', '?')})"), } for key, (label, formatter) in resource_labels.items(): items = resources.get(key, []) if items: total += len(items) lines.append(f"\n**{label}** ({len(items)}):") for item in items[:15]: # limit to 15 per category lines.append(formatter(item)) if len(items) > 15: lines.append(f" ... e mais {len(items) - 15}") if total == 0: lines.append("\nNenhum recurso encontrado neste compartment. Todos os recursos serão novos.") else: lines.append(f"\n**Total: {total} recursos existentes.** REUTILIZE-OS sempre que possível usando data sources.") return "\n".join(lines) @app.post("/api/oci/instances/{instance_id}/action") async def oci_instance_action(instance_id: str, req: dict, u=Depends(current_user)): action = req.get("action", "").upper() oci_config_id = req.get("oci_config_id", "") region = req.get("region") if action not in ("START", "STOP"): raise HTTPException(400, "Ação inválida. Use START ou STOP.") if not oci_config_id: raise HTTPException(400, "oci_config_id obrigatório") _verify_config_access("oci", oci_config_id, u) try: import oci config = _get_oci_config(oci_config_id) if region: config["region"] = region compute = oci.core.ComputeClient(config) compute.instance_action(instance_id, action) _audit(u["id"], u["username"], f"instance_{action.lower()}", instance_id) return {"ok": True, "action": action, "instance_id": instance_id} except Exception as e: raise HTTPException(500, str(e)[:500]) @app.post("/api/oci/autonomous-databases/{adb_id}/action") async def oci_adb_action(adb_id: str, req: dict, u=Depends(current_user)): action = req.get("action", "").lower() oci_config_id = req.get("oci_config_id", "") region = req.get("region") if action not in ("start", "stop"): raise HTTPException(400, "Ação inválida. Use start ou stop.") if not oci_config_id: raise HTTPException(400, "oci_config_id obrigatório") _verify_config_access("oci", oci_config_id, u) try: import oci config = _get_oci_config(oci_config_id) if region: config["region"] = region db_client = oci.database.DatabaseClient(config) if action == "start": db_client.start_autonomous_database(adb_id) else: db_client.stop_autonomous_database(adb_id) _audit(u["id"], u["username"], f"adb_{action}", adb_id) return {"ok": True, "action": action, "adb_id": adb_id} except Exception as e: raise HTTPException(500, str(e)[:500]) @app.post("/api/oci/autonomous-databases/{adb_id}/update-network-access") async def oci_adb_update_network(adb_id: str, req: dict, u=Depends(current_user)): oci_config_id = req.get("oci_config_id", "") ip = req.get("ip", "") region = req.get("region") if not oci_config_id: raise HTTPException(400, "oci_config_id obrigatório") _verify_config_access("oci", oci_config_id, u) if not ip: raise HTTPException(400, "ip obrigatório") try: import oci config = _get_oci_config(oci_config_id) if region: config["region"] = region db_client = oci.database.DatabaseClient(config) adb = db_client.get_autonomous_database(adb_id).data current_acl = adb.whitelisted_ips or [] # Build new ACL: keep existing entries, add new IP if not present ip_cidr = ip if "/" in ip else ip + "/32" new_acl = list(set(current_acl) | {ip_cidr}) db_client.update_autonomous_database( adb_id, oci.database.models.UpdateAutonomousDatabaseDetails(whitelisted_ips=new_acl)) _audit(u["id"], u["username"], "adb_update_network", adb_id, f"ip={ip_cidr}") return {"ok": True, "adb_id": adb_id, "ip_added": ip_cidr, "acl": new_acl} except Exception as e: raise HTTPException(500, str(e)[:500]) @app.post("/api/oci/db-systems/{db_system_id}/action") async def oci_db_system_action(db_system_id: str, req: dict, u=Depends(current_user)): action = req.get("action", "").upper() oci_config_id = req.get("oci_config_id", "") region = req.get("region") if action not in ("START", "STOP"): raise HTTPException(400, "Ação inválida. Use START ou STOP.") if not oci_config_id: raise HTTPException(400, "oci_config_id obrigatório") _verify_config_access("oci", oci_config_id, u) try: import oci config = _get_oci_config(oci_config_id) if region: config["region"] = region db_client = oci.database.DatabaseClient(config) if action == "START": db_client.start_db_system(db_system_id) else: db_client.stop_db_system(db_system_id) _audit(u["id"], u["username"], f"db_system_{action.lower()}", db_system_id) return {"ok": True, "action": action, "db_system_id": db_system_id} except Exception as e: raise HTTPException(500, str(e)[:500]) @app.post("/api/oci/mysql-db-systems/{db_system_id}/action") async def oci_mysql_action(db_system_id: str, req: dict, u=Depends(current_user)): action = req.get("action", "").lower() oci_config_id = req.get("oci_config_id", "") region = req.get("region") if action not in ("start", "stop"): raise HTTPException(400, "Ação inválida. Use start ou stop.") if not oci_config_id: raise HTTPException(400, "oci_config_id obrigatório") _verify_config_access("oci", oci_config_id, u) try: import oci config = _get_oci_config(oci_config_id) if region: config["region"] = region mysql = oci.mysql.DbSystemClient(config) if action == "start": mysql.start_db_system(db_system_id) else: shutdown_type = req.get("shutdown_type", "SLOW") mysql.stop_db_system(db_system_id, oci.mysql.models.StopDbSystemDetails(shutdown_type=shutdown_type)) _audit(u["id"], u["username"], f"mysql_{action}", db_system_id) return {"ok": True, "action": action, "db_system_id": db_system_id} except Exception as e: raise HTTPException(500, str(e)[:500]) @app.post("/api/oci/container-instances/{ci_id}/action") async def oci_container_instance_action(ci_id: str, req: dict, u=Depends(current_user)): action = req.get("action", "").lower() oci_config_id = req.get("oci_config_id", "") region = req.get("region") if action not in ("start", "stop"): raise HTTPException(400, "Ação inválida. Use start ou stop.") if not oci_config_id: raise HTTPException(400, "oci_config_id obrigatório") _verify_config_access("oci", oci_config_id, u) try: import oci config = _get_oci_config(oci_config_id) if region: config["region"] = region ci = oci.container_instances.ContainerInstanceClient(config) if action == "start": ci.start_container_instance(ci_id) else: ci.stop_container_instance(ci_id) _audit(u["id"], u["username"], f"container_instance_{action}", ci_id) return {"ok": True, "action": action, "container_instance_id": ci_id} except Exception as e: raise HTTPException(500, str(e)[:500]) @app.get("/api/oci/my-ip") async def get_my_ip(request: Request): """Return the caller's public IP address.""" forwarded = request.headers.get("x-forwarded-for", "") ip = forwarded.split(",")[0].strip() if forwarded else (request.client.host if request.client else "") return {"ip": ip} @app.get("/api/terraform/resources") async def tf_list_resources(oci_config_id: str = Query(...), compartment_id: str = Query(...), region: str = Query(None), u=Depends(current_user)): """List existing OCI resources in a compartment for terraform context.""" _verify_config_access("oci", oci_config_id, u) try: loop = asyncio.get_event_loop() resources = await loop.run_in_executor( _chat_executor, partial(_fetch_compartment_resources, oci_config_id, compartment_id, region)) return resources except Exception as e: raise HTTPException(500, str(e)[:500]) TFP_SYSTEM_PROMPT = """Você é um arquiteto sênior de infraestrutura OCI com profundo conhecimento do Terraform OCI Provider. O usuário vai descrever a infraestrutura desejada em linguagem natural. Sua tarefa é gerar um **prompt detalhado e estruturado** que será enviado a um agente Terraform para criar o código HCL. ### Formato do Prompt Gerado Retorne APENAS o prompt estruturado (não gere código Terraform). Use markdown: 1. **Título**: Uma linha descrevendo o projeto (ex: "## Infraestrutura Multi-Region HA para Aplicação Web") 2. **Arquitetura**: Diagrama textual ou descrição da topologia 3. **Recursos por Categoria**: Organizados em seções ## (Networking, Compute, Storage, Database, Security, Observability) - Para cada recurso: nome, especificações técnicas, relacionamentos 4. **Networking**: Sempre inclua CIDRs (10.0.0.0/16 para VCN, /24 para subnets), gateways necessários, route tables, security lists/NSGs 5. **Dependências**: Liste dependências entre recursos (ex: "Compute depende de Subnet, que depende de VCN") 6. **Requisitos Técnicos**: Seção final com regras para o agente Terraform: - Separação em arquivos (provider.tf, variables.tf, vcn.tf, subnets.tf, compute.tf, outputs.tf) - Variáveis obrigatórias (compartment_id, region, ssh_public_key, etc.) - Naming convention (ex: proj-env-resource) - Outputs necessários (IPs, OCIDs, endpoints) - Tags padrão (Environment, Project, ManagedBy=Terraform) ### Regras de Inferência - Se pedir VCN, inclua automaticamente: subnets (pública + privada), internet gateway, NAT gateway, service gateway, route tables, security lists - Se pedir Compute, inclua: boot volume, NSG, cloud-init básico, shape e imagem - Se pedir Database, inclua: subnet privada dedicada, NSG com porta do DB, backup policy - Se pedir OKE, inclua: VCN com topologia para K8s (API endpoint subnet, workers subnet, pods subnet, services LB subnet) - Se mencionar multi-region, use providers aliased e detalhe RPC/DRG - Se mencionar HA/DR, inclua redundância cross-AD ou cross-region - Se mencionar segurança/CIS, inclua Vault, Cloud Guard, WAF, logging, encryption at rest ### Restrições do OCI Provider - RPC (Remote Peering Connection): NÃO usar oci_core_drg_attachment para RPC. O attachment é criado automaticamente pelo recurso oci_core_remote_peering_connection - Cada VCN suporta apenas 1 DRG attachment — não criar duplicados - DRG attachment type para VCN é "VCN", não "REMOTE_PEERING_CONNECTION" ### Restrições do Workspace - NUNCA sugira o uso de `module` blocks. O workspace é flat (diretório único, sem subdiretórios). Toda infraestrutura deve ser definida com `resource` e `data` blocks diretamente. - Variáveis devem ser declaradas SOMENTE em `variables.tf`. Nunca duplicar declarações de variáveis entre arquivos. ### Tom - Seja técnico e preciso - Use português brasileiro - Não gere código, apenas o prompt estruturado""" class TfPromptReq(BaseModel): message: str genai_config: Optional[dict] = None history: Optional[list] = None session_id: Optional[str] = None @app.post("/api/terraform/generate-prompt") async def tf_generate_prompt(req: TfPromptReq, bg: BackgroundTasks, u=Depends(current_user)): gc = None if req.genai_config: if req.genai_config.get("genai_config_id"): with db() as c: row = c.execute("SELECT * FROM genai_configs WHERE id=?", (req.genai_config["genai_config_id"],)).fetchone() if row: gc = dict(row) elif req.genai_config.get("oci_config_id"): oci_id = req.genai_config["oci_config_id"] with db() as c: oc = c.execute("SELECT * FROM oci_configs WHERE id=?", (oci_id,)).fetchone() if oc: region = req.genai_config.get("genai_region") or oc["region"] compartment = _safe_dec(dict(oc).get("compartment_id") or oc["tenancy_ocid"]) gc = { "oci_config_id": oci_id, "model_id": req.genai_config.get("model_id", "openai.gpt-4.1"), "model_ocid": None, "compartment_id": compartment, "genai_region": region, "endpoint": f"https://inference.generativeai.{region}.oci.oraclecloud.com", "serving_type": "ON_DEMAND", "dedicated_endpoint_id": None, "temperature": 0.7, "max_tokens": 4000, "top_p": 0.9, } if not gc: with db() as c: row = c.execute("SELECT * FROM genai_configs WHERE user_id=? AND is_default=1", (u["id"],)).fetchone() if not row: row = c.execute("SELECT * FROM genai_configs WHERE user_id=? LIMIT 1", (u["id"],)).fetchone() if row: gc = dict(row) if not gc: raise HTTPException(400, "Nenhuma configuração GenAI disponível") # Session persistence is_new = not req.session_id sid = req.session_id or str(uuid.uuid4()) with db() as c: c.execute("INSERT INTO chat_messages (id,session_id,user_id,role,content,model_id,status) VALUES (?,?,?,?,?,?,?)", (str(uuid.uuid4()), sid, u["id"], "user", req.message, gc.get("model_id"), "done")) if is_new: title = (req.message or "Prompt Generator")[:80].strip() c.execute("INSERT OR IGNORE INTO chat_sessions (id,user_id,agent_type,title) VALUES (?,?,?,?)", (sid, u["id"], "tf-prompt", title)) else: c.execute("UPDATE chat_sessions SET updated_at=datetime('now') WHERE id=?", (sid,)) # Load TF resource reference (compact — same as Terraform Agent) tf_ref = _load_tf_resource_reference() system_with_ref = TFP_SYSTEM_PROMPT if tf_ref: sections = [] for line in tf_ref.split('\n'): if line.startswith('## All Resource Types'): break sections.append(line) ref_compact = '\n'.join(sections).strip() if ref_compact: system_with_ref += "\n\n### Referência de Recursos OCI Terraform (gerado do provider schema)\n" + \ "Use EXATAMENTE estes nomes de resource types. Se o recurso não estiver nesta lista, ele NÃO EXISTE no provider.\n\n" + \ ref_compact # Detect resource types from user message + history and fetch official docs detect_text = req.message if req.history: for h in req.history[-3:]: detect_text += "\n" + (h.get("content", "") or "") resource_types = _detect_tf_resource_types(detect_text) if resource_types: docs_ctx = _get_tf_docs_for_resources(resource_types) if docs_ctx: system_with_ref += "\n\n" + docs_ctx log.info(f"TF Prompt Generator: injected {len(resource_types)} resource docs ({len(docs_ctx)} chars)") gc["system_prompt"] = system_with_ref gc["temperature"] = 0.7 gc["max_tokens"] = 4000 # Build history from conversation (normalize roles to lowercase for _call_genai) hist = None if req.history: hist = [{"role": "user" if m.get("role","").upper() in ("USER","user") else "assistant", "content": m.get("content", "")} for m in req.history] # Async background processing — same pattern as Chat/Terraform agents mid = str(uuid.uuid4()) with db() as c: c.execute("INSERT INTO chat_messages (id,session_id,user_id,role,content,model_id,status) VALUES (?,?,?,?,?,?,?)", (mid, sid, u["id"], "assistant", "", gc.get("model_id"), "processing")) def _tfp_background(): try: answer, _, _ = _call_genai(gc, req.message, history=hist) with db() as c: c.execute("UPDATE chat_messages SET content=?, status='done' WHERE id=?", (answer, mid)) log.info(f"TF Prompt Generator completed: mid={mid} len={len(answer)}") except Exception as e: log.error(f"tf_generate_prompt error: {e}") with db() as c: c.execute("UPDATE chat_messages SET content=?, status='failed' WHERE id=?", (str(e)[:500], mid)) bg.add_task(_tfp_background) return {"message_id": mid, "session_id": sid, "status": "processing"} @app.post("/api/terraform/chat") async def terraform_chat(msg: ChatMsg, bg: BackgroundTasks, u=Depends(current_user)): sid, mid_or_result, genai_cfg = _chat_start(msg, u, agent_type="terraform") if genai_cfg is None: return mid_or_result bg.add_task(_chat_background, mid_or_result, sid, msg, dict(u), genai_cfg, None, "terraform") return {"message_id": mid_or_result, "session_id": sid, "status": "processing"} class TfWorkspaceReq(BaseModel): session_id: str oci_config_id: str compartment_id: Optional[str] = None name: str = "workspace" tf_code: str @app.post("/api/terraform/workspaces") async def tf_create_workspace(req: TfWorkspaceReq, u=Depends(current_user)): wid = str(uuid.uuid4()) with db() as c: c.execute( "INSERT INTO terraform_workspaces (id,user_id,session_id,oci_config_id,compartment_id,name,tf_code,status) VALUES (?,?,?,?,?,?,?,?)", (wid, u["id"], req.session_id, req.oci_config_id, req.compartment_id, req.name, req.tf_code, "draft")) return {"id": wid, "status": "draft"} @app.get("/api/terraform/workspaces") async def tf_list_workspaces(u=Depends(current_user)): with db() as c: rows = c.execute( """SELECT tw.id,tw.name,tw.session_id,tw.oci_config_id,tw.compartment_id,tw.status, tw.plan_output,tw.apply_output,tw.destroy_output,tw.error,tw.created_at,tw.updated_at, cs.title as session_title FROM terraform_workspaces tw INNER JOIN chat_sessions cs ON cs.id = tw.session_id AND cs.user_id = tw.user_id WHERE tw.user_id=? ORDER BY tw.updated_at DESC""", (u["id"],)).fetchall() return [dict(r) for r in rows] @app.get("/api/terraform/workspaces/{wid}") async def tf_get_workspace(wid: str, u=Depends(current_user)): with db() as c: r = c.execute("SELECT * FROM terraform_workspaces WHERE id=? AND user_id=?", (wid, u["id"])).fetchone() if not r: raise HTTPException(404) return dict(r) @app.put("/api/terraform/workspaces/{wid}/code") async def tf_update_code(wid: str, req: dict, u=Depends(current_user)): code = req.get("tf_code", "") with db() as c: r = c.execute("SELECT id FROM terraform_workspaces WHERE id=? AND user_id=?", (wid, u["id"])).fetchone() if not r: raise HTTPException(404) c.execute("UPDATE terraform_workspaces SET tf_code=?, status='draft', updated_at=datetime('now') WHERE id=?", (code, wid)) return {"ok": True} @app.post("/api/terraform/workspaces/{wid}/plan") async def tf_run_plan(wid: str, bg: BackgroundTasks, u=Depends(current_user)): with db() as c: ws = c.execute("SELECT * FROM terraform_workspaces WHERE id=? AND user_id=?", (wid, u["id"])).fetchone() if not ws: raise HTTPException(404) if ws["status"] in ("planning", "applying", "destroying"): raise HTTPException(400, f"Workspace is {ws['status']}") with db() as c: c.execute("UPDATE terraform_workspaces SET status='planning', plan_output='', apply_output='', destroy_output='', error=NULL, updated_at=datetime('now') WHERE id=?", (wid,)) bg.add_task(_terraform_exec, wid, "plan", dict(u)) return {"status": "planning"} @app.post("/api/terraform/workspaces/{wid}/apply") async def tf_run_apply(wid: str, bg: BackgroundTasks, u=Depends(current_user)): with db() as c: ws = c.execute("SELECT * FROM terraform_workspaces WHERE id=? AND user_id=?", (wid, u["id"])).fetchone() if not ws: raise HTTPException(404) if ws["status"] not in ("planned", "applied", "destroyed", "failed"): raise HTTPException(400, f"Cannot apply in status {ws['status']}. Run plan first.") with db() as c: c.execute("UPDATE terraform_workspaces SET status='applying', apply_output='', error=NULL, updated_at=datetime('now') WHERE id=?", (wid,)) bg.add_task(_terraform_exec, wid, "apply", dict(u)) return {"status": "applying"} @app.post("/api/terraform/workspaces/{wid}/destroy") async def tf_run_destroy(wid: str, req: dict, bg: BackgroundTasks, u=Depends(current_user)): if req.get("confirm") != "DESTROY": raise HTTPException(400, "Confirmação obrigatória: envie {\"confirm\": \"DESTROY\"}") with db() as c: ws = c.execute("SELECT * FROM terraform_workspaces WHERE id=? AND user_id=?", (wid, u["id"])).fetchone() if not ws: raise HTTPException(404) if ws["status"] in ("planning", "applying", "destroying"): raise HTTPException(400, f"Workspace is {ws['status']}") with db() as c: c.execute("UPDATE terraform_workspaces SET status='destroying', destroy_output='', error=NULL, updated_at=datetime('now') WHERE id=?", (wid,)) bg.add_task(_terraform_exec, wid, "destroy", dict(u)) return {"status": "destroying"} @app.get("/api/terraform/workspaces/{wid}/status") async def tf_workspace_status(wid: str, u=Depends(current_user)): with db() as c: r = c.execute("SELECT status, plan_output, apply_output, destroy_output, error, updated_at FROM terraform_workspaces WHERE id=? AND user_id=?", (wid, u["id"])).fetchone() if not r: raise HTTPException(404) return dict(r) @app.get("/api/terraform/workspaces/{wid}/download") async def tf_download(wid: str, token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))): u = _user_from_token(token) if token else (await current_user(cred) if cred else None) if not u: raise HTTPException(401, "Not authenticated") with db() as c: r = c.execute("SELECT tf_code, name FROM terraform_workspaces WHERE id=? AND user_id=?", (wid, u["id"])).fetchone() if not r or not r["tf_code"]: raise HTTPException(404, "No code found") from starlette.responses import Response import re as _re parts = _re.split(r'^//\s*filename:\s*(\S+)\s*$', r["tf_code"], flags=_re.MULTILINE) if len(parts) >= 3: import io, zipfile buf = io.BytesIO() with zipfile.ZipFile(buf, 'w', zipfile.ZIP_DEFLATED) as zf: if parts[0].strip(): zf.writestr("main.tf", parts[0].strip()) for i in range(1, len(parts), 2): fname = parts[i].strip().replace("/", "_").replace("..", "_") content = parts[i + 1].strip() if i + 1 < len(parts) else "" if fname and content: zf.writestr(fname, content) buf.seek(0) ws_name = r["name"] or "terraform" return Response(content=buf.read(), media_type="application/zip", headers={"Content-Disposition": f'attachment; filename="{ws_name}.zip"'}) return Response(content=r["tf_code"], media_type="text/plain", headers={"Content-Disposition": f'attachment; filename="main.tf"'}) @app.post("/api/terraform/workspaces/{wid}/cancel") async def tf_cancel(wid: str, u=Depends(current_user)): with db() as c: ws = c.execute("SELECT user_id FROM terraform_workspaces WHERE id=?", (wid,)).fetchone() if not ws or ws["user_id"] != u["id"]: raise HTTPException(404) proc = _running_terraform.get(wid) if proc: proc.terminate() try: await asyncio.wait_for(proc.wait(), timeout=5) except asyncio.TimeoutError: proc.kill() with db() as c: c.execute("UPDATE terraform_workspaces SET status='failed', error='Cancelado pelo usuário', updated_at=datetime('now') WHERE id=?", (wid,)) _running_terraform.pop(wid, None) return {"ok": True} @app.delete("/api/terraform/workspaces/{wid}") async def tf_delete_workspace(wid: str, u=Depends(current_user)): with db() as c: c.execute("DELETE FROM terraform_workspaces WHERE id=? AND user_id=?", (wid, u["id"])) wdir = TERRAFORM_DIR / wid if wdir.exists(): import shutil as _sh _sh.rmtree(wdir, ignore_errors=True) return {"ok": True} @app.post("/api/terraform/refresh-reference") async def tf_refresh_reference(u=Depends(current_user)): """Regenerate the OCI Terraform resource reference from provider schema (internal, triggered by UI button).""" loop = asyncio.get_event_loop() try: result = await asyncio.wait_for( loop.run_in_executor(_chat_executor, _regenerate_tf_reference), timeout=300) except asyncio.TimeoutError: log.error("TF reference regeneration timed out after 300s") raise HTTPException(504, "Geração de referência Terraform demorou demais. Tente novamente.") # Add status info p = Path(_TF_RESOURCE_REF_PATH) if p.exists(): content = _load_tf_resource_reference() result["lines"] = content.count('\n') if content else 0 result["updated_at"] = datetime.fromtimestamp(p.stat().st_mtime).strftime('%d/%m/%Y %H:%M') return result def _fix_single_line_blocks(content: str) -> str: """Expand ANY single-line HCL block with multiple args into multi-line format. Matches both top-level (variable, resource) AND nested blocks (ingress_security_rules, route_rules, etc.) Does NOT match map assignments (tags = { ... }) because of the '=' before '{'.""" import re as _re def _expand(m): header, body = m.group(1), m.group(2) # Split by comma first args = [a.strip() for a in body.split(',') if a.strip()] if len(args) < 2: # Try space-separated key=value args = _re.findall(r'\w+\s*=\s*(?:"[^"]*"|\S+)', body) if len(args) < 2: return m.group(0) indent = _re.match(r'^(\s*)', header).group(1) return header.rstrip() + ' {\n' + '\n'.join(indent + ' ' + a for a in args) + '\n' + indent + '}' return _re.sub( r'^(\s*[a-z]\w*(?:\s+"[^"]*")*)\s*\{\s*(.+?)\s*\}\s*$', _expand, content, flags=_re.MULTILINE) def _split_tf_monolith(content: str) -> dict: """Split a monolithic HCL string into multiple logical files by resource type. Returns dict of {filename: content} or None if splitting not needed.""" import re as _re content = _fix_single_line_blocks(content) lines = content.split('\n') top_blocks = [] preamble = [] accum = [] in_block = False b_depth = 0 cur_header = '' for line in lines: stripped = _re.sub(r'"[^"]*"', '', line) opens = stripped.count('{') closes = stripped.count('}') if not in_block: hdr = _re.match(r'^\s*(variable|output|resource|data|provider|module)\s+"', line) or \ _re.match(r'^\s*(locals|terraform)\s*\{', line) if hdr: in_block = True cur_header = line accum = preamble + [line] preamble = [] b_depth = opens - closes if b_depth <= 0: b_depth = 0 if b_depth == 0 and opens > 0: top_blocks.append((cur_header, '\n'.join(accum))) in_block = False accum = [] else: preamble.append(line) else: accum.append(line) b_depth += opens - closes if b_depth <= 0: b_depth = 0 top_blocks.append((cur_header, '\n'.join(accum))) in_block = False accum = [] cur_header = '' if accum: top_blocks.append((cur_header or '', '\n'.join(accum))) if len(top_blocks) < 3: return None cats = {'variables': [], 'outputs': [], 'networking': [], 'compute': [], 'database': [], 'firewall': [], 'loadbalancer': [], 'storage': [], 'iam': [], 'drg': [], 'data': [], 'providers': [], 'other': []} for header, body in top_blocks: h = header.strip() if h.startswith('variable '): cats['variables'].append(body) elif h.startswith('output '): cats['outputs'].append(body) elif h.startswith('provider '): cats['providers'].append(body) elif _re.search(r'resource\s+"oci_core_(drg|remote_peering|drg_route|drg_attachment)', h): cats['drg'].append(body) elif _re.search(r'resource\s+"oci_core_(vcn|subnet|internet_gateway|nat_gateway|service_gateway|route_table|security_list|network_security_group|dhcp_options|local_peering_gateway|virtual_circuit)', h): cats['networking'].append(body) elif _re.search(r'resource\s+"oci_core_instance', h): cats['compute'].append(body) elif _re.search(r'resource\s+"oci_(database|nosql|mysql|psql)', h) or _re.search(r'resource\s+"oci_core_autonomous', h): cats['database'].append(body) elif _re.search(r'resource\s+"oci_network_firewall', h): cats['firewall'].append(body) elif _re.search(r'resource\s+"oci_(load_balancer|network_load_balancer)', h): cats['loadbalancer'].append(body) elif _re.search(r'resource\s+"oci_(objectstorage|file_storage)', h): cats['storage'].append(body) elif _re.search(r'resource\s+"oci_identity', h): cats['iam'].append(body) elif h.startswith('data '): cats['data'].append(body) else: cats['other'].append(body) result = {} mapping = [('variables.tf', 'variables'), ('providers.tf', 'providers'), ('networking.tf', 'networking'), ('drg.tf', 'drg'), ('compute.tf', 'compute'), ('database.tf', 'database'), ('firewall.tf', 'firewall'), ('loadbalancer.tf', 'loadbalancer'), ('storage.tf', 'storage'), ('iam.tf', 'iam'), ('data.tf', 'data'), ('main.tf', 'other'), ('outputs.tf', 'outputs')] for fname, key in mapping: if cats[key]: result[fname] = '\n\n'.join(cats[key]) return result if len(result) >= 3 else None def _write_tf_files(wdir: Path, tf_code: str): """Parse tf_code for '// filename: xxx.tf' markers and write separate files. If only 1-2 large files, auto-split into multiple files by resource type.""" import re as _re parts = _re.split(r'^//\s*filename:\s*(\S+)\s*$', tf_code, flags=_re.MULTILINE) files = {} if len(parts) >= 3: if parts[0].strip(): files['main.tf'] = parts[0].strip() for i in range(1, len(parts), 2): fname = parts[i].strip().replace("/", "_").replace("..", "_") content = parts[i + 1].strip() if i + 1 < len(parts) else "" if fname and content: files[fname] = content else: files['main.tf'] = tf_code # Auto-split: if 1-2 large files, split by resource type if len(files) <= 2: all_content = '\n\n'.join(files.values()) if len(all_content) > 2000: split = _split_tf_monolith(all_content) if split: files = split log.info(f"Backend auto-split monolithic TF into {len(files)} files: {list(files.keys())}") # Deduplicate: if a resource appears in multiple files, keep only in the split file # This handles edge cases where monolithic + split files both exist if len(files) > 2: resource_locations = {} # "type.name" -> [(filename, line)] dupes_in = set() for fname, content in files.items(): for m in _re.finditer(r'^resource\s+"(\S+)"\s+"(\S+)"', content, _re.MULTILINE): key = f'{m.group(1)}.{m.group(2)}' resource_locations.setdefault(key, []).append(fname) # Find files that contain ONLY duplicate resources (= monolithic leftovers) for key, fnames in resource_locations.items(): if len(fnames) > 1: # The largest file is likely the monolithic one largest = max(fnames, key=lambda f: len(files.get(f, ''))) dupes_in.add(largest) # Remove monolithic files that are fully duplicated for fname in dupes_in: if all( any(f2 != fname for f2 in resource_locations.get(key, [])) for key, flist in resource_locations.items() if fname in flist ) and len(files) - 1 >= 2: log.info(f"Removing duplicate monolithic file: {fname}") del files[fname] # Fix single-line blocks in all files for fname in files: files[fname] = _fix_single_line_blocks(files[fname]) # Write files to disk for fname, content in files.items(): (wdir / fname).write_text(content) async def _terraform_exec(wid: str, action: str, user: dict): """Background: run terraform init + plan/apply/destroy in workspace dir.""" log.info(f"Terraform {action} started: wid={wid}") status_col = f"{action}_output" final_ok = {"plan": "planned", "apply": "applied", "destroy": "destroyed"}[action] try: with db() as c: ws = c.execute("SELECT * FROM terraform_workspaces WHERE id=?", (wid,)).fetchone() if not ws: return wdir = TERRAFORM_DIR / wid wdir.mkdir(parents=True, exist_ok=True) # Clean old .tf files (keep .terraform/, state, lock) for old_tf in wdir.glob("*.tf"): old_tf.unlink() # Write .tf files — parse // filename: comments to split into separate files _write_tf_files(wdir, ws["tf_code"] or "") # Auto-generate provider.tf from OCI config # Detect provider aliases declared by the model in generated files import re as _re2 oci_cfg = _get_oci_config(ws["oci_config_id"]) alias_blocks = [] # list of (alias_name, region_ref) # Scan all .tf files for provider "oci" { alias = "..." ... region = ... } provider_block_re = _re2.compile(r'provider\s+"oci"\s*\{', _re2.MULTILINE) for tf_file in sorted(wdir.glob("*.tf")): if tf_file.name == "provider.tf": continue content = tf_file.read_text() # Find each provider "oci" block and extract alias + region blocks_to_remove = [] for m in provider_block_re.finditer(content): start = m.start() # Find matching closing brace depth = 0 end = start for ci in range(m.end() - 1, len(content)): if content[ci] == '{': depth += 1 elif content[ci] == '}': depth -= 1 if depth == 0: end = ci + 1 break block = content[start:end] alias_m = _re2.search(r'alias\s*=\s*"([^"]+)"', block) region_m = _re2.search(r'region\s*=\s*(.+)', block) if alias_m: alias_name = alias_m.group(1) region_ref = region_m.group(1).strip().rstrip("}").strip() if region_m else '"unknown"' alias_blocks.append((alias_name, region_ref)) blocks_to_remove.append((start, end)) # Remove model-generated provider blocks (we centralize in provider.tf) if blocks_to_remove: new_content = content for s, e in reversed(blocks_to_remove): new_content = new_content[:s] + new_content[e:] # Also remove leading comments right before removed blocks new_content = _re2.sub(r'\n(//[^\n]*\n){1,5}\s*\n', '\n\n', new_content) new_content = new_content.strip() if new_content: tf_file.write_text(new_content + "\n") else: tf_file.unlink() # Remove empty file # Scan all .tf files for variable declarations with region-like defaults # so we can map alias providers to the correct variable reference var_region_map = {} # variable_name -> default_value var_re = _re2.compile(r'variable\s+"([^"]*region[^"]*)"\s*\{', _re2.IGNORECASE) for tf_file in sorted(wdir.glob("*.tf")): if tf_file.name in ("provider.tf", "terraform.tfvars"): continue content = tf_file.read_text() for vm in var_re.finditer(content): var_region_map[vm.group(1)] = f'var.{vm.group(1)}' # Check if variable "region" is declared — use it for primary provider has_region_var = "region" in var_region_map # Also scan for provider refs in resource blocks (provider = oci.xxx) for tf_file in sorted(wdir.glob("*.tf")): if tf_file.name == "provider.tf": continue content = tf_file.read_text() for ref_m in _re2.finditer(r'provider\s*=\s*oci\.(\w+)', content): ref_alias = ref_m.group(1) if ref_alias not in [a[0] for a in alias_blocks]: # Try to find a matching region variable for this alias # e.g. alias "mad_3" might match var.region_secondary region_ref = 'var.region' # fallback to primary region var for vname, vref in var_region_map.items(): if vname != "region": # prefer non-primary region vars for aliases region_ref = vref break alias_blocks.append((ref_alias, region_ref)) passphrase = oci_cfg.get('pass_phrase', '') cred_block = f''' tenancy_ocid = "{oci_cfg.get('tenancy', '')}" user_ocid = "{oci_cfg.get('user', '')}" fingerprint = "{oci_cfg.get('fingerprint', '')}" private_key_path = "{oci_cfg.get('key_file', '')}"''' if passphrase: cred_block += f'\n private_key_password = "{passphrase}"' # Primary region: MUST use var.region — model is required to declare it with db() as c: oci_row = c.execute("SELECT region FROM oci_configs WHERE id=?", (ws["oci_config_id"],)).fetchone() primary_region = oci_row["region"] if oci_row else oci_cfg.get("region", "") if not has_region_var: error_msg = ( f'ERRO: O código Terraform não declarou variable "region". ' f'Isso é obrigatório para garantir que os recursos sejam provisionados na região correta. ' f'Adicione no variables.tf: variable "region" {{ type = string; default = "{primary_region}" }}' ) log.warning(f"Workspace {wid}: missing variable 'region' — blocking plan") with db() as c: c.execute("UPDATE terraform_workspaces SET status='failed', plan_output=?, error=?, updated_at=datetime('now') WHERE id=?", (error_msg, error_msg, wid)) return region_value = 'var.region' provider_tf = f'''terraform {{ required_providers {{ oci = {{ source = "oracle/oci" }} }} }} provider "oci" {{ {cred_block} region = {region_value} }} ''' # Add alias providers with same credentials seen_aliases = set() for alias_name, region_ref in alias_blocks: if alias_name in seen_aliases: continue seen_aliases.add(alias_name) provider_tf += f''' provider "oci" {{ alias = "{alias_name}" {cred_block} region = {region_ref} }} ''' (wdir / "provider.tf").write_text(provider_tf) # Auto-generate terraform.tfvars — scan declared variables and provide values from OCI config with db() as c: oci_row = c.execute("SELECT compartment_id, region, ssh_public_key FROM oci_configs WHERE id=?", (ws["oci_config_id"],)).fetchone() comp_id = ws["compartment_id"] if ws["compartment_id"] else (_safe_dec(oci_row["compartment_id"]) if oci_row and oci_row["compartment_id"] else "") try: ssh_pub_key = oci_row["ssh_public_key"] if oci_row else "" except (IndexError, KeyError): ssh_pub_key = "" ssh_pub_key = ssh_pub_key or "" # Scan all declared variables in .tf files declared_vars = set() for tf_file in sorted(wdir.glob("*.tf")): if tf_file.name in ("provider.tf", "terraform.tfvars"): continue content = tf_file.read_text() for vm in _re2.finditer(r'variable\s+"([^"]+)"', content): declared_vars.add(vm.group(1)) # Map known variable names to OCI config values var_values = {"compartment_id": comp_id} oci_var_map = { "tenancy_ocid": oci_cfg.get("tenancy", ""), "tenancy_id": oci_cfg.get("tenancy", ""), "user_ocid": oci_cfg.get("user", ""), "fingerprint": oci_cfg.get("fingerprint", ""), "private_key_path": oci_cfg.get("key_file", ""), "ssh_public_key": ssh_pub_key, "ssh_authorized_keys": ssh_pub_key, } # NOTE: "region" is intentionally NOT included — Terraform uses the default from variable declarations for vname, vval in oci_var_map.items(): if vname in declared_vars and vval: var_values[vname] = vval tfvars_lines = [f'{k} = "{v}"' for k, v in var_values.items()] (wdir / "terraform.tfvars").write_text("\n".join(tfvars_lines) + "\n") output_lines = [] def _update_output(text): output_lines.append(text) with db() as c: c.execute(f"UPDATE terraform_workspaces SET {status_col}=?, updated_at=datetime('now') WHERE id=?", ("\n".join(output_lines[-200:]), wid)) # terraform init _update_output("$ terraform init ...") proc_init = await asyncio.create_subprocess_exec( "terraform", f"-chdir={wdir}", "init", "-no-color", "-input=false", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT) while True: line = await proc_init.stdout.readline() if not line: break _update_output(line.decode(errors="replace").rstrip()) await proc_init.wait() if proc_init.returncode != 0: _update_output(f"\n❌ terraform init failed (exit {proc_init.returncode})") with db() as c: c.execute("UPDATE terraform_workspaces SET status='failed', error=?, updated_at=datetime('now') WHERE id=?", ("terraform init failed", wid)) return # terraform action _update_output(f"\n$ terraform {action} ...") cmd = ["terraform", f"-chdir={wdir}", action, "-no-color", "-input=false"] if action in ("apply", "destroy"): cmd.append("-auto-approve") proc = await asyncio.create_subprocess_exec( *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT) _running_terraform[wid] = proc while True: line = await proc.stdout.readline() if not line: break _update_output(line.decode(errors="replace").rstrip()) await proc.wait() _running_terraform.pop(wid, None) if proc.returncode == 0: output_lines.append(f"\n✅ terraform {action} completed successfully") full_output = "\n".join(output_lines) with db() as c: c.execute(f"UPDATE terraform_workspaces SET {status_col}=?, status=?, updated_at=datetime('now') WHERE id=?", (full_output, final_ok, wid)) _audit(user["id"], user["username"], f"terraform_{action}", wid, f"status={final_ok}") else: output_lines.append(f"\n❌ terraform {action} failed (exit {proc.returncode})") full_output = "\n".join(output_lines) with db() as c: c.execute(f"UPDATE terraform_workspaces SET {status_col}=?, status='failed', error=?, updated_at=datetime('now') WHERE id=?", (full_output, f"terraform {action} failed (exit {proc.returncode})", wid)) except Exception as e: log.error(f"Terraform exec error: {e}") with db() as c: c.execute("UPDATE terraform_workspaces SET status='failed', error=?, updated_at=datetime('now') WHERE id=?", (str(e)[:500], wid)) # ── Audit ───────────────────────────────────────────────────────────────────── @app.get("/api/audit-log") async def audit_log(skip: int = Query(0, ge=0), limit: int = Query(100, ge=1, le=500), u=Depends(current_user)): with db() as c: if u["role"] == "admin": rows = c.execute("SELECT * FROM audit_log ORDER BY created_at DESC LIMIT ? OFFSET ?", (limit, skip)).fetchall() else: rows = c.execute("SELECT * FROM audit_log WHERE user_id=? ORDER BY created_at DESC LIMIT ? OFFSET ?", (u["id"], limit, skip)).fetchall() return [dict(r) for r in rows] # ── Config Logs ─────────────────────────────────────────────────────────────── @app.get("/api/config-logs") async def get_config_logs( config_type: str = Query(None), config_id: str = Query(None), severity: str = Query(None), limit: int = Query(50, le=200), u=Depends(current_user) ): query = "SELECT * FROM config_logs WHERE 1=1" params = [] if config_type: query += " AND config_type=?"; params.append(config_type) if config_id: query += " AND config_id=?"; params.append(config_id) if severity: query += " AND severity=?"; params.append(severity) if u["role"] != "admin": query += " AND user_id=?"; params.append(u["id"]) query += " ORDER BY created_at DESC LIMIT ?" params.append(limit) with db() as c: rows = c.execute(query, params).fetchall() return [dict(r) for r in rows] # ── Chat Logs ───────────────────────────────────────────────────────────────── @app.get("/api/chat-logs") async def get_chat_logs( severity: str = Query(None), session_id: str = Query(None), limit: int = Query(50, le=200), u=Depends(current_user) ): query = "SELECT * FROM chat_logs WHERE 1=1" params = [] if severity: query += " AND severity=?"; params.append(severity) if session_id: query += " AND session_id=?"; params.append(session_id) if u["role"] != "admin": query += " AND user_id=?"; params.append(u["id"]) query += " ORDER BY created_at DESC LIMIT ?" params.append(limit) with db() as c: rows = c.execute(query, params).fetchall() return [dict(r) for r in rows] # ── App Settings ────────────────────────────────────────────────────────────── _SENSITIVE_SETTING_PREFIXES = ("oidc_", "secret_", "app_secret", "encryption_") @app.get("/api/settings/{key}") async def get_setting(key: str, u=Depends(current_user)): if any(key.startswith(p) or key == p for p in _SENSITIVE_SETTING_PREFIXES) and u["role"] != "admin": raise HTTPException(403, "Acesso negado a configuração sensível") with db() as c: row = c.execute("SELECT value FROM app_settings WHERE key=?", (key,)).fetchone() val = row["value"] if row else "" if key == "oidc_client_secret" and val: val = _mask(_safe_dec(val)) return {"key": key, "value": val} @app.put("/api/settings/{key}") async def put_setting(key: str, body: dict, u=Depends(require("admin"))): value = body.get("value", "") if key == "oidc_client_secret" and value and "\u2022" in value: return {"key": key, "value": value} if key == "oidc_client_secret" and value: value = _enc(value) with db() as c: c.execute("INSERT INTO app_settings (key,value,updated_at) VALUES (?,?,datetime('now')) ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at", (key, value)) return {"key": key, "value": value} @app.get("/api/settings/timezone/options") async def timezone_options(u=Depends(current_user)): """List common timezone options.""" return {"timezones": [ "America/Sao_Paulo", "America/New_York", "America/Chicago", "America/Denver", "America/Los_Angeles", "America/Toronto", "America/Mexico_City", "America/Bogota", "America/Argentina/Buenos_Aires", "America/Santiago", "Europe/London", "Europe/Paris", "Europe/Berlin", "Europe/Madrid", "Europe/Lisbon", "Asia/Tokyo", "Asia/Shanghai", "Asia/Singapore", "Asia/Kolkata", "Asia/Dubai", "Australia/Sydney", "Pacific/Auckland", "UTC", ]} @app.put("/api/users/me/timezone") async def set_user_timezone(body: dict, u=Depends(current_user)): """Set the current user's timezone.""" tz = body.get("timezone", "").strip() if not tz: raise HTTPException(400, "timezone is required") try: import zoneinfo zoneinfo.ZoneInfo(tz) except Exception: raise HTTPException(400, f"Invalid timezone: {tz}") with db() as c: c.execute("UPDATE users SET timezone=? WHERE id=?", (tz, u["id"])) _audit(u["id"], u["username"], "set_timezone", tz) return {"ok": True, "timezone": tz} @app.get("/api/users/me/timezone") async def get_user_timezone(u=Depends(current_user)): """Get the current user's timezone.""" return {"timezone": u.get("timezone", "America/Sao_Paulo")} # ── System Prompts ──────────────────────────────────────────────────────────── @app.get("/api/prompts/{agent}") async def list_prompts(agent: str, u=Depends(current_user)): with db() as c: rows = c.execute("SELECT * FROM system_prompts WHERE agent=? ORDER BY is_active DESC, created_at DESC", (agent,)).fetchall() return [dict(r) for r in rows] @app.post("/api/prompts") async def save_prompt(body: dict, u=Depends(require("admin"))): agent = body.get("agent", "chat") with db() as c: count = c.execute("SELECT COUNT(*) FROM system_prompts WHERE agent=?", (agent,)).fetchone()[0] if count >= 10: raise HTTPException(400, "Limite de 10 prompts atingido. Exclua um antes de criar outro.") pid = str(uuid.uuid4()) name = body.get("name", "Sem nome") content = body.get("content", "") is_active = body.get("is_active", False) with db() as c: if is_active: c.execute("UPDATE system_prompts SET is_active=0 WHERE agent=?", (agent,)) c.execute("INSERT INTO system_prompts (id,name,agent,content,is_active) VALUES (?,?,?,?,?)", (pid, name, agent, content, int(is_active))) return {"id": pid} @app.put("/api/prompts/{pid}") async def update_prompt(pid: str, body: dict, u=Depends(require("admin"))): with db() as c: existing = c.execute("SELECT * FROM system_prompts WHERE id=?", (pid,)).fetchone() if not existing: raise HTTPException(404) name = body.get("name", existing["name"]) content = body.get("content", existing["content"]) is_active = body.get("is_active", existing["is_active"]) with db() as c: if is_active: c.execute("UPDATE system_prompts SET is_active=0 WHERE agent=?", (existing["agent"],)) c.execute("UPDATE system_prompts SET name=?,content=?,is_active=? WHERE id=?", (name, content, int(is_active), pid)) return {"id": pid} @app.delete("/api/prompts/{pid}") async def delete_prompt(pid: str, u=Depends(require("admin"))): with db() as c: c.execute("DELETE FROM system_prompts WHERE id=?", (pid,)) return {"ok": True} # ── Export / Import Configuration ───────────────────────────────────────────── @app.get("/api/config/export") async def export_config(u=Depends(require("admin"))): """Export configurations as JSON. Excludes sensitive credentials (private keys, passwords, secrets).""" _OCI_SAFE_COLS = "id,user_id,tenancy_name,region,compartment_id,auth_type,ssh_public_key,created_at" _ADB_SAFE_COLS = "id,user_id,config_name,dsn,username,table_name,use_mtls,is_active,is_global,genai_config_id,embedding_model_id,created_at" data = {"version": VERSION, "exported_at": datetime.now().isoformat()} with db() as c: # OCI configs (no private keys, no encrypted OCIDs, no passphrases) data["oci_configs"] = [dict(r) for r in c.execute(f"SELECT {_OCI_SAFE_COLS} FROM oci_configs").fetchall()] # GenAI configs (no secrets — model info only) data["genai_configs"] = [dict(r) for r in c.execute("SELECT * FROM genai_configs").fetchall()] # MCP servers data["mcp_servers"] = [dict(r) for r in c.execute("SELECT * FROM mcp_servers").fetchall()] # System prompts data["system_prompts"] = [dict(r) for r in c.execute("SELECT * FROM system_prompts").fetchall()] # App settings (exclude sensitive keys) data["app_settings"] = [dict(r) for r in c.execute("SELECT * FROM app_settings").fetchall() if not any(r["key"].startswith(p) for p in _SENSITIVE_SETTING_PREFIXES)] # ADB vector configs (no passwords, no wallet passwords) data["adb_vector_configs"] = [dict(r) for r in c.execute(f"SELECT {_ADB_SAFE_COLS} FROM adb_vector_configs").fetchall()] # ADB vector tables data["adb_vector_tables"] = [dict(r) for r in c.execute("SELECT * FROM adb_vector_tables").fetchall()] return StreamingResponse( iter([json.dumps(data, indent=2, ensure_ascii=False)]), media_type="application/json", headers={"Content-Disposition": f'attachment; filename="oci-cis-agent-config-{datetime.now().strftime("%Y%m%d-%H%M%S")}.json"'} ) @app.post("/api/config/import") async def import_config(file: UploadFile = File(...), u=Depends(require("admin"))): """Import configurations from exported JSON. Merges by ID (skip existing).""" await _validate_upload(file, {".json"}) file.file.seek(0) content = await file.read() try: data = json.loads(content) except json.JSONDecodeError: raise HTTPException(400, "Arquivo JSON inválido") counts = {"oci_configs": 0, "genai_configs": 0, "mcp_servers": 0, "system_prompts": 0, "app_settings": 0, "adb_vector_configs": 0, "adb_vector_tables": 0, "skipped": 0} with db() as c: # OCI configs for cfg in data.get("oci_configs", []): if c.execute("SELECT 1 FROM oci_configs WHERE id=?", (cfg["id"],)).fetchone(): counts["skipped"] += 1; continue # Import metadata only — credentials must be reconfigured manually cdir = OCI_DIR / cfg["id"]; cdir.mkdir(parents=True, exist_ok=True) kp = cdir / "oci_api_key.pem" # Restore private key only if present in legacy export format if cfg.get("_private_key_b64"): kp.write_bytes(base64.b64decode(cfg["_private_key_b64"])); kp.chmod(0o600) cfg_file = cdir / "config" cfg_content = (f"[DEFAULT]\nuser={_dec(cfg['user_ocid']) if cfg.get('user_ocid') else ''}\n" f"fingerprint={_dec(cfg['fingerprint']) if cfg.get('fingerprint') else ''}\n" f"tenancy={_dec(cfg['tenancy_ocid']) if cfg.get('tenancy_ocid') else ''}\n" f"region={cfg.get('region','')}\nkey_file={kp}\n") if cfg.get("_key_passphrase"): cfg_content += f"pass_phrase={cfg['_key_passphrase']}\n" cfg_file.write_text(cfg_content); cfg_file.chmod(0o600) c.execute("INSERT INTO oci_configs (id,user_id,tenancy_name,tenancy_ocid,user_ocid,fingerprint,region,key_file_path,compartment_id,created_at) VALUES (?,?,?,?,?,?,?,?,?,?)", (cfg["id"], cfg.get("user_id", u["id"]), cfg.get("tenancy_name",""), _enc(cfg["tenancy_ocid"]) if cfg.get("tenancy_ocid") else "", _enc(cfg["user_ocid"]) if cfg.get("user_ocid") else "", _enc(cfg["fingerprint"]) if cfg.get("fingerprint") else "", cfg.get("region",""), str(kp), _enc(cfg["compartment_id"]) if cfg.get("compartment_id") else None, cfg.get("created_at"))) counts["oci_configs"] += 1 # GenAI configs for cfg in data.get("genai_configs", []): if c.execute("SELECT 1 FROM genai_configs WHERE id=?", (cfg["id"],)).fetchone(): counts["skipped"] += 1; continue c.execute("INSERT INTO genai_configs (id,user_id,name,oci_config_id,model_id,model_ocid,compartment_id,genai_region,endpoint,serving_type,dedicated_endpoint_id,temperature,max_tokens,top_p,top_k,frequency_penalty,presence_penalty,reasoning_effort,is_default,system_prompt,created_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", (cfg["id"], cfg.get("user_id", u["id"]), cfg.get("name",""), cfg.get("oci_config_id",""), cfg.get("model_id",""), cfg.get("model_ocid"), cfg.get("compartment_id",""), cfg.get("genai_region",""), cfg.get("endpoint",""), cfg.get("serving_type","ON_DEMAND"), cfg.get("dedicated_endpoint_id"), cfg.get("temperature",1), cfg.get("max_tokens",6000), cfg.get("top_p",0.95), cfg.get("top_k",1), cfg.get("frequency_penalty",0), cfg.get("presence_penalty",0), cfg.get("reasoning_effort"), cfg.get("is_default",0), cfg.get("system_prompt",""), cfg.get("created_at"))) counts["genai_configs"] += 1 # MCP servers for srv in data.get("mcp_servers", []): if c.execute("SELECT 1 FROM mcp_servers WHERE id=?", (srv["id"],)).fetchone(): counts["skipped"] += 1; continue c.execute("INSERT INTO mcp_servers (id,user_id,name,description,server_type,command,args,env_vars,url,module_path,tools,linked_adb_id,is_active,is_global,created_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", (srv["id"], srv.get("user_id", u["id"]), srv.get("name",""), srv.get("description"), srv.get("server_type","stdio"), srv.get("command"), srv.get("args"), srv.get("env_vars"), srv.get("url"), srv.get("module_path"), srv.get("tools"), srv.get("linked_adb_id"), srv.get("is_active",1), srv.get("is_global",0), srv.get("created_at"))) counts["mcp_servers"] += 1 # System prompts for p in data.get("system_prompts", []): if c.execute("SELECT 1 FROM system_prompts WHERE id=?", (p["id"],)).fetchone(): counts["skipped"] += 1; continue c.execute("INSERT INTO system_prompts (id,name,agent,content,is_active,created_at) VALUES (?,?,?,?,?,?)", (p["id"], p.get("name",""), p.get("agent","chat"), p.get("content",""), p.get("is_active",0), p.get("created_at"))) counts["system_prompts"] += 1 # App settings for s in data.get("app_settings", []): if any(s["key"].startswith(p) for p in _SENSITIVE_SETTING_PREFIXES): counts["skipped"] += 1; continue c.execute("INSERT INTO app_settings (key,value,updated_at) VALUES (?,?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at", (s["key"], s.get("value",""), s.get("updated_at"))) counts["app_settings"] += 1 # ADB vector configs for cfg in data.get("adb_vector_configs", []): if c.execute("SELECT 1 FROM adb_vector_configs WHERE id=?", (cfg["id"],)).fetchone(): counts["skipped"] += 1; continue c.execute("INSERT INTO adb_vector_configs (id,user_id,config_name,dsn,username,password_enc,wallet_dir,wallet_password_enc,table_name,use_mtls,is_active,is_global,created_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", (cfg["id"], cfg.get("user_id", u["id"]), cfg.get("config_name",""), cfg.get("dsn",""), cfg.get("username",""), cfg.get("password_enc",""), cfg.get("wallet_dir"), cfg.get("wallet_password_enc"), cfg.get("table_name","CIS_EMBEDDINGS"), cfg.get("use_mtls",1), cfg.get("is_active",1), cfg.get("is_global",0), cfg.get("created_at"))) counts["adb_vector_configs"] += 1 # ADB vector tables for t in data.get("adb_vector_tables", []): if c.execute("SELECT 1 FROM adb_vector_tables WHERE id=?", (t["id"],)).fetchone(): counts["skipped"] += 1; continue c.execute("INSERT INTO adb_vector_tables (id,adb_config_id,table_name,description,is_active,created_at) VALUES (?,?,?,?,?,?)", (t["id"], t.get("adb_config_id",""), t.get("table_name",""), t.get("description",""), t.get("is_active",1), t.get("created_at"))) counts["adb_vector_tables"] += 1 _audit(u["id"], u["username"], "import_config", "bulk", json.dumps(counts)) return counts # ── CIS Engine Version Management ───────────────────────────────────────────── CIS_REPORTS_PATH = Path("/app/cis_reports.py") CIS_GITHUB_RAW = "https://raw.githubusercontent.com/oci-landing-zones/oci-cis-landingzone-quickstart/main/scripts/cis_reports.py" CIS_PATCHES = [ {"name": "region_none_check_1", "original": "elif region.region_name in self.__regions_to_run_in or self.__run_in_all_regions:", "patched": "elif self.__run_in_all_regions or (self.__regions_to_run_in and region.region_name in self.__regions_to_run_in):"}, {"name": "region_none_check_2", "original": "if self.__home_region not in self.__regions_to_run_in:", "patched": "if not self.__run_in_all_regions and self.__regions_to_run_in and self.__home_region not in self.__regions_to_run_in:"}, ] def _read_cis_version(content: str = None) -> dict: if content is None: if not CIS_REPORTS_PATH.exists(): return {"version": "unknown", "date": "unknown"} content = CIS_REPORTS_PATH.read_text(encoding="utf-8", errors="replace") ver = re.search(r'RELEASE_VERSION\s*=\s*["\']([^"\']+)["\']', content) dt = re.search(r'UPDATED_DATE\s*=\s*["\']([^"\']+)["\']', content) return {"version": ver.group(1) if ver else "unknown", "date": dt.group(1) if dt else "unknown"} @app.get("/api/cis-engine/version") async def cis_engine_version(u=Depends(current_user)): info = _read_cis_version() return {"version": info["version"], "updated_date": info["date"], "file_path": str(CIS_REPORTS_PATH)} @app.get("/api/cis-engine/check-update") async def cis_engine_check_update(u=Depends(require("admin"))): import requests as req local = _read_cis_version() try: resp = req.get(CIS_GITHUB_RAW, timeout=30) resp.raise_for_status() remote = _read_cis_version(resp.text) except Exception as e: raise HTTPException(502, f"Failed to check GitHub: {str(e)[:300]}") return { "local_version": local["version"], "local_date": local["date"], "remote_version": remote["version"], "remote_date": remote["date"], "update_available": remote["version"] != local["version"] } @app.post("/api/cis-engine/update") async def cis_engine_update(u=Depends(require("admin"))): import requests as req old = _read_cis_version() try: resp = req.get(CIS_GITHUB_RAW, timeout=60) resp.raise_for_status() content = resp.text except Exception as e: raise HTTPException(502, f"Failed to download from GitHub: {str(e)[:300]}") new = _read_cis_version(content) # Apply patches patches_applied = [] for p in CIS_PATCHES: if p["original"] in content: content = content.replace(p["original"], p["patched"]) patches_applied.append(p["name"]) CIS_REPORTS_PATH.write_text(content, encoding="utf-8") # Save version info with db() as c: c.execute("INSERT INTO app_settings (key,value,updated_at) VALUES ('cis_engine_version',?,datetime('now')) ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at", (new["version"],)) log.info(f"CIS engine updated: {old['version']} → {new['version']}, patches: {patches_applied}") _audit(u["id"], u["username"], "cis_engine_update", None, f"{old['version']} → {new['version']}") return {"ok": True, "old_version": old["version"], "new_version": new["version"], "patches_applied": patches_applied} @app.get("/api/health") async def health(): db_ok = False try: with db() as c: c.execute("SELECT 1") db_ok = True except Exception: pass return { "status": "ok", "version": VERSION, "uptime_seconds": int(time.time() - _START_TIME), "db_ok": db_ok, "ts": datetime.utcnow().isoformat(), } @app.on_event("shutdown") async def shutdown(): """Graceful shutdown: terminate running subprocesses and thread pool.""" for rid, proc in list(_running_reports.items()): try: proc.terminate() log.info(f"Terminated report process {rid}") except Exception as e: log.warning(f"Failed to terminate report process {rid}: {e}") _running_reports.clear() for wid, proc in list(_running_terraform.items()): try: proc.terminate() log.info(f"Terminated terraform process {wid}") except Exception as e: log.warning(f"Failed to terminate terraform process {wid}: {e}") _running_terraform.clear() _chat_executor.shutdown(wait=False) log.info("Graceful shutdown completed") @app.on_event("startup") async def startup(): init_db() # Apply timezone from app_settings (overrides TZ env var) try: with db() as c: tz_row = c.execute("SELECT value FROM app_settings WHERE key='timezone'").fetchone() if tz_row and tz_row["value"]: os.environ["TZ"] = tz_row["value"] time.tzset() log.info(f"Timezone set to {tz_row['value']} (from app_settings)") except Exception as e: log.warning(f"Failed to set timezone from settings: {e}") # Mark orphaned "running" reports as failed (e.g. after container restart) with db() as c: orphaned = c.execute("UPDATE reports SET status='failed', error_msg='Interrompido: container reiniciado' WHERE status='running'").rowcount if orphaned: log.warning(f"Marked {orphaned} orphaned running report(s) as failed") # Clean up orphaned .compliance_generating markers (left after container restart) for marker in REPORTS.glob("*/.compliance_generating"): marker.unlink(missing_ok=True) log.info(f"Cleaned orphaned compliance marker: {marker.parent.name}") # Auto-register CIS Compliance Scanner MCP server if not present (deduplicate on startup) with db() as c: dupes = c.execute("SELECT id FROM mcp_servers WHERE name='CIS Compliance Scanner' ORDER BY created_at ASC").fetchall() if len(dupes) > 1: keep = dupes[0]["id"] for d in dupes[1:]: c.execute("DELETE FROM mcp_servers WHERE id=?", (d["id"],)) log.info(f"Removed {len(dupes)-1} duplicate CIS Compliance Scanner entries, kept {keep}") if not dupes: mid = str(uuid.uuid4()) c.execute( "INSERT INTO mcp_servers (id, user_id, name, description, server_type, command, args) VALUES (?,?,?,?,?,?,?)", (mid, "system", "CIS Compliance Scanner", "OCI CIS Foundations Benchmark 3.0 - 48 CIS checks + 11 OBP checks via MCP", "stdio", "python3", json.dumps(["/app/mcp_cis_server.py"])) ) log.info(f"Auto-registered CIS Compliance Scanner MCP server (id={mid})") # Save CIS engine version to app_settings try: vi = _read_cis_version() with db() as c: c.execute("INSERT INTO app_settings (key,value,updated_at) VALUES ('cis_engine_version',?,datetime('now')) ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at", (vi["version"],)) log.info(f"CIS engine version: {vi['version']} ({vi['date']})") except Exception as e: log.warning(f"Could not read CIS engine version: {e}") # Generate TF resource reference if not present (volume mount may hide build-time file) ref_path = Path(_TF_RESOURCE_REF_PATH) if not ref_path.exists(): try: loop = asyncio.get_event_loop() result = await loop.run_in_executor(None, partial( subprocess.run, ["python", "/app/gen_tf_reference.py"], capture_output=True, text=True, timeout=300)) if result.returncode == 0 and ref_path.exists(): log.info(f"Generated TF resource reference at startup: {ref_path.stat().st_size} bytes") else: log.warning(f"TF reference generation failed at startup: {result.stderr[:200] if result.stderr else 'unknown'}") except Exception as e: log.warning(f"Could not generate TF reference at startup: {e}") # Populate tf_valid_types table from reference file if ref_path.exists(): try: count = _populate_tf_valid_types() if count: log.info(f"TF valid types DB populated: {count} entries") except Exception as e: log.warning(f"Could not populate tf_valid_types: {e}") log.info(f"AI Agent v{VERSION} API started") if __name__ == "__main__": import uvicorn uvicorn.run(app, host="0.0.0.0", port=8000)