refactor: decompose monolith — app.py 10,600→143 lines, 30+ modules
Fase 0 complete: extract all business logic, auth, database, and 176 endpoints from monolithic app.py into dedicated modules. Structure: - app.py: FastAPI hub (CORS, routers, startup/shutdown) - models.py: 13 Pydantic request models - utils.py: shared utilities (embed status, upload validation, process registries) - config.py: constants, env vars, model catalogs - auth/: crypto, jwt_auth, oidc, rate_limit - database/: db(), init_db(), schema DDL - services/: genai, compliance, chat, terraform, embeddings, cis_reports, mcp - routes/: 13 APIRouter modules (auth, users, oci_config, oci_explorer, genai, mcp, adb, embeddings, reports, chat, terraform, settings, cis_engine) Also: README updated with OCIR auth instructions for manual docker run. 86 tests passing.
This commit is contained in:
nogueiraguh
0
backend/routes/__init__.py
Normal file
0
backend/routes/__init__.py
Normal file
321
backend/routes/adb.py
Normal file
321
backend/routes/adb.py
Normal file
@@ -0,0 +1,321 @@
|
||||
"""ADB (Autonomous Database) routes — parse-wallet, config CRUD, wallet upload, test, tables, ingest."""
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import uuid
|
||||
import shutil
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends, UploadFile, File, Form, BackgroundTasks
|
||||
|
||||
from database import db
|
||||
from auth.jwt_auth import current_user, require, _audit, _config_log, _verify_config_access
|
||||
from auth.crypto import _enc
|
||||
from config import WALLET_DIR, log
|
||||
from models import IngestDocReq
|
||||
from utils import validate_upload
|
||||
from services.genai import (
|
||||
_get_adb_connection,
|
||||
_get_tables_for_config,
|
||||
)
|
||||
from services.embeddings import _ingest_documents_task
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
_TABLE_NAME_RE = re.compile(r'^[A-Za-z][A-Za-z0-9_]{0,127}$')
|
||||
|
||||
|
||||
@router.post("/api/adb/parse-wallet")
|
||||
async def parse_wallet(wallet: UploadFile = File(...), u=Depends(require("admin","user"))):
|
||||
"""Extract DSN names from tnsnames.ora inside wallet ZIP (temporary parse, no save)."""
|
||||
await validate_upload(wallet, {".zip"})
|
||||
wallet.file.seek(0)
|
||||
import zipfile, tempfile
|
||||
try:
|
||||
data = await wallet.read()
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
zp = Path(tmp) / "wallet.zip"
|
||||
zp.write_bytes(data)
|
||||
with zipfile.ZipFile(str(zp), 'r') as z:
|
||||
z.extractall(tmp)
|
||||
tns = Path(tmp) / "tnsnames.ora"
|
||||
if not tns.exists():
|
||||
raise HTTPException(400, "Wallet ZIP nao contem tnsnames.ora")
|
||||
tns_text = tns.read_text(errors="ignore")
|
||||
dsn_names = re.findall(r'^(\w[\w\-.]*)\s*=', tns_text, re.MULTILINE)
|
||||
if not dsn_names:
|
||||
raise HTTPException(400, "Nenhum DSN encontrado no tnsnames.ora")
|
||||
return {"dsn_names": dsn_names, "files": [n for n in os.listdir(tmp) if n != "wallet.zip"]}
|
||||
except zipfile.BadZipFile:
|
||||
raise HTTPException(400, "Arquivo nao e um ZIP valido")
|
||||
|
||||
@router.post("/api/adb/config")
|
||||
async def save_adb(
|
||||
config_name: str = Form(...), dsn: str = Form(...), username: str = Form(...),
|
||||
password: str = Form(...), wallet_password: str = Form(""),
|
||||
table_name: str = Form(""), use_mtls: str = Form("true"),
|
||||
genai_config_id: str = Form(""), embedding_model_id: str = Form("cohere.embed-v4.0"),
|
||||
is_global: str = Form("false"),
|
||||
wallet: Optional[UploadFile] = File(None),
|
||||
u=Depends(require("admin","user"))
|
||||
):
|
||||
if wallet and wallet.filename:
|
||||
await validate_upload(wallet, {".zip"})
|
||||
wallet.file.seek(0)
|
||||
vid = str(uuid.uuid4())
|
||||
use_mtls_bool = use_mtls.lower() in ("true", "1", "yes")
|
||||
is_global_val = 1 if (is_global.lower() in ("true", "1", "yes") and u["role"] == "admin") else 0
|
||||
with db() as c:
|
||||
c.execute(
|
||||
"INSERT INTO adb_vector_configs (id,user_id,config_name,dsn,username,password_enc,wallet_password_enc,table_name,use_mtls,genai_config_id,embedding_model_id,is_global) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(vid, u["id"], config_name, dsn, username, _enc(password),
|
||||
_enc(wallet_password) if wallet_password else None, table_name, int(use_mtls_bool),
|
||||
genai_config_id or None, embedding_model_id, is_global_val))
|
||||
# Auto-save wallet if provided
|
||||
if wallet and wallet.filename:
|
||||
import zipfile
|
||||
wdir = WALLET_DIR / vid; wdir.mkdir(parents=True, exist_ok=True)
|
||||
zp = wdir / "wallet.zip"
|
||||
zp.write_bytes(await wallet.read())
|
||||
with zipfile.ZipFile(str(zp), 'r') as z: z.extractall(str(wdir))
|
||||
with db() as c: c.execute("UPDATE adb_vector_configs SET wallet_dir=? WHERE id=?", (str(wdir), vid))
|
||||
_config_log("adb", vid, config_name, "success", "upload", f"Wallet enviada com a conexao", u["id"], u["username"])
|
||||
_audit(u["id"], u["username"], "save_adb_config", vid, config_name)
|
||||
_config_log("adb", vid, config_name, "success", "save", f"Conexao salva: {config_name} ({dsn})", u["id"], u["username"])
|
||||
return {"id": vid, "config_name": config_name}
|
||||
|
||||
@router.post("/api/adb/{vid}/upload-wallet")
|
||||
async def upload_wallet(vid: str, wallet: UploadFile = File(...), u=Depends(require("admin","user"))):
|
||||
_verify_config_access("adb", vid, u, require_owner=True)
|
||||
await validate_upload(wallet, {".zip"})
|
||||
wallet.file.seek(0)
|
||||
cname = None
|
||||
with db() as c:
|
||||
row = c.execute("SELECT config_name FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
||||
if row: cname = row["config_name"]
|
||||
try:
|
||||
wdir = WALLET_DIR / vid; wdir.mkdir(parents=True, exist_ok=True)
|
||||
zp = wdir / "wallet.zip"
|
||||
zp.write_bytes(await wallet.read())
|
||||
import zipfile
|
||||
with zipfile.ZipFile(str(zp), 'r') as z: z.extractall(str(wdir))
|
||||
with db() as c: c.execute("UPDATE adb_vector_configs SET wallet_dir=? WHERE id=?", (str(wdir), vid))
|
||||
files = os.listdir(str(wdir))
|
||||
_config_log("adb", vid, cname, "success", "upload", f"Wallet enviada: {', '.join(files)}", u["id"], u["username"])
|
||||
return {"ok": True, "wallet_dir": str(wdir), "files": files}
|
||||
except Exception as e:
|
||||
_config_log("adb", vid, cname, "error", "upload", str(e)[:500], u["id"], u["username"])
|
||||
raise HTTPException(500, str(e)[:500])
|
||||
|
||||
@router.get("/api/adb/configs")
|
||||
async def list_adb(u=Depends(current_user)):
|
||||
with db() as c:
|
||||
cols = "id,user_id,config_name,dsn,username,table_name,use_mtls,is_active,is_global,wallet_dir,genai_config_id,embedding_model_id,created_at"
|
||||
if u["role"]=="admin": rows=c.execute(f"SELECT {cols} FROM adb_vector_configs").fetchall()
|
||||
else: rows=c.execute(f"SELECT {cols} FROM adb_vector_configs WHERE user_id=? OR is_global=1",(u["id"],)).fetchall()
|
||||
result = []
|
||||
for r in rows:
|
||||
d = dict(r)
|
||||
d["tables"] = _get_tables_for_config(d["id"], active_only=False)
|
||||
result.append(d)
|
||||
return result
|
||||
|
||||
@router.post("/api/adb/test/{vid}")
|
||||
async def test_adb(vid: str, u=Depends(require("admin","user"))):
|
||||
_verify_config_access("adb", vid, u)
|
||||
with db() as c:
|
||||
cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?",(vid,)).fetchone()
|
||||
if not cfg: raise HTTPException(404)
|
||||
cname = cfg["config_name"]
|
||||
try:
|
||||
conn = _get_adb_connection(dict(cfg))
|
||||
cur = conn.cursor(); cur.execute("SELECT 1 FROM DUAL"); cur.close(); conn.close()
|
||||
_config_log("adb", vid, cname, "success", "test", "Conexao Autonomous DB OK!", u["id"], u["username"])
|
||||
return {"status":"success","message":"Conexao Autonomous DB OK!"}
|
||||
except ImportError:
|
||||
_config_log("adb", vid, cname, "error", "test", "python-oracledb nao disponivel no container.", u["id"], u["username"])
|
||||
return {"status":"error","message":"python-oracledb nao disponivel no container."}
|
||||
except Exception as e:
|
||||
msg = str(e)[:500]
|
||||
_config_log("adb", vid, cname, "error", "test", msg, u["id"], u["username"])
|
||||
return {"status":"error","message":msg}
|
||||
|
||||
@router.post("/api/adb/{vid}/tables/check")
|
||||
async def check_adb_tables(vid: str, u=Depends(require("admin","user"))):
|
||||
"""Check which registered tables exist in ADB and list all user tables."""
|
||||
_verify_config_access("adb", vid, u)
|
||||
with db() as c:
|
||||
cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
||||
if not cfg: raise HTTPException(404)
|
||||
registered = _get_tables_for_config(vid, active_only=False)
|
||||
reg_names_upper = {t["table_name"].upper() for t in registered}
|
||||
try:
|
||||
conn = _get_adb_connection(dict(cfg))
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT USER FROM DUAL")
|
||||
current_user_name = cur.fetchone()[0]
|
||||
cur.execute("SELECT table_name FROM user_tables ORDER BY table_name")
|
||||
adb_tables = [r[0] for r in cur.fetchall()]
|
||||
adb_lookup = {t.upper(): t for t in adb_tables}
|
||||
results = []
|
||||
for t in registered:
|
||||
tname_reg = t["table_name"]
|
||||
tname_upper = tname_reg.upper()
|
||||
real_name = adb_lookup.get(tname_upper)
|
||||
if real_name:
|
||||
try:
|
||||
cur.execute(f'SELECT COUNT(*) FROM "{real_name}"')
|
||||
cnt = cur.fetchone()[0]
|
||||
case_note = f" (ADB: {real_name})" if real_name != tname_reg else ""
|
||||
results.append({"table": tname_reg, "adb_name": real_name, "status": "ok", "rows": cnt, "message": f"{cnt} registros{case_note}"})
|
||||
except Exception as e:
|
||||
results.append({"table": tname_reg, "status": "error", "message": str(e)[:200]})
|
||||
else:
|
||||
results.append({"table": tname_reg, "status": "not_found", "message": f"Nao existe no schema {current_user_name}"})
|
||||
unregistered = [t for t in adb_tables if t.upper() not in reg_names_upper]
|
||||
cur.close()
|
||||
conn.close()
|
||||
return {
|
||||
"status": "success",
|
||||
"schema_user": current_user_name,
|
||||
"adb_tables_total": len(adb_tables),
|
||||
"registered": results,
|
||||
"unregistered_in_adb": unregistered[:50]
|
||||
}
|
||||
except Exception as e:
|
||||
return {"status": "error", "message": f"Erro de conexao: {str(e)[:500]}"}
|
||||
|
||||
@router.delete("/api/adb/configs/{vid}")
|
||||
async def del_adb(vid: str, u=Depends(require("admin","user"))):
|
||||
_verify_config_access("adb", vid, u, require_owner=True)
|
||||
with db() as c: c.execute("DELETE FROM adb_vector_configs WHERE id=?", (vid,))
|
||||
d = WALLET_DIR / vid
|
||||
if d.exists(): shutil.rmtree(d)
|
||||
return {"ok": True}
|
||||
|
||||
@router.put("/api/adb/configs/{vid}")
|
||||
async def update_adb(
|
||||
vid: str,
|
||||
config_name: str = Form(...), dsn: str = Form(...), username: str = Form(...),
|
||||
password: str = Form(""), wallet_password: str = Form(""),
|
||||
table_name: str = Form(""), use_mtls: str = Form("true"),
|
||||
genai_config_id: str = Form(""), embedding_model_id: str = Form("cohere.embed-v4.0"),
|
||||
is_global: str = Form(""),
|
||||
wallet: Optional[UploadFile] = File(None),
|
||||
u=Depends(require("admin","user"))
|
||||
):
|
||||
if wallet and wallet.filename:
|
||||
await validate_upload(wallet, {".zip"})
|
||||
wallet.file.seek(0)
|
||||
with db() as c:
|
||||
existing = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
||||
if not existing: raise HTTPException(404)
|
||||
if u["role"] != "admin" and existing["user_id"] != u["id"]: raise HTTPException(403)
|
||||
if u["role"] != "admin" and existing.get("is_global"): raise HTTPException(403, "Nao e possivel editar configuracao global")
|
||||
use_mtls_bool = use_mtls.lower() in ("true", "1", "yes")
|
||||
sets = "config_name=?,dsn=?,username=?,table_name=?,use_mtls=?,genai_config_id=?,embedding_model_id=?"
|
||||
vals = [config_name, dsn, username, table_name, int(use_mtls_bool), genai_config_id or None, embedding_model_id]
|
||||
if u["role"] == "admin" and is_global:
|
||||
is_global_val = 1 if is_global.lower() in ("true", "1", "yes") else 0
|
||||
sets += ",is_global=?"
|
||||
vals.append(is_global_val)
|
||||
if password:
|
||||
sets += ",password_enc=?"
|
||||
vals.append(_enc(password))
|
||||
if wallet_password:
|
||||
sets += ",wallet_password_enc=?"
|
||||
vals.append(_enc(wallet_password))
|
||||
vals.append(vid)
|
||||
with db() as c:
|
||||
c.execute(f"UPDATE adb_vector_configs SET {sets} WHERE id=?", vals)
|
||||
if wallet and wallet.filename:
|
||||
import zipfile
|
||||
wdir = WALLET_DIR / vid; wdir.mkdir(parents=True, exist_ok=True)
|
||||
zp = wdir / "wallet.zip"
|
||||
zp.write_bytes(await wallet.read())
|
||||
with zipfile.ZipFile(str(zp), 'r') as z: z.extractall(str(wdir))
|
||||
with db() as c: c.execute("UPDATE adb_vector_configs SET wallet_dir=? WHERE id=?", (str(wdir), vid))
|
||||
_config_log("adb", vid, config_name, "success", "upload", "Wallet atualizado", u["id"], u["username"])
|
||||
_audit(u["id"], u["username"], "update_adb_config", vid, config_name)
|
||||
_config_log("adb", vid, config_name, "success", "save", f"Conexao atualizada: {config_name} ({dsn})", u["id"], u["username"])
|
||||
return {"id": vid, "config_name": config_name}
|
||||
|
||||
@router.get("/api/adb/{vid}/tables")
|
||||
async def list_adb_tables(vid: str, u=Depends(current_user)):
|
||||
_verify_config_access("adb", vid, u)
|
||||
with db() as c:
|
||||
cfg = c.execute("SELECT id FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
||||
if not cfg: raise HTTPException(404)
|
||||
return _get_tables_for_config(vid, active_only=False)
|
||||
|
||||
@router.post("/api/adb/{vid}/tables")
|
||||
async def add_adb_table(vid: str, req: dict, u=Depends(require("admin","user"))):
|
||||
_verify_config_access("adb", vid, u, require_owner=True)
|
||||
table_name = req.get("table_name", "").strip()
|
||||
description = req.get("description", "").strip()
|
||||
if not table_name: raise HTTPException(400, "table_name e obrigatorio")
|
||||
if not _TABLE_NAME_RE.match(table_name): raise HTTPException(400, "Nome da tabela invalido. Use letras maiusculas, numeros e underscores.")
|
||||
with db() as c:
|
||||
cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
||||
if not cfg: raise HTTPException(404, "ADB config not found")
|
||||
tid = str(uuid.uuid4())
|
||||
try:
|
||||
with db() as c:
|
||||
c.execute("INSERT INTO adb_vector_tables (id, adb_config_id, table_name, description) VALUES (?,?,?,?)",
|
||||
(tid, vid, table_name, description))
|
||||
except sqlite3.IntegrityError:
|
||||
raise HTTPException(409, f"Tabela '{table_name}' ja registrada nesta conexao")
|
||||
_config_log("adb", vid, cfg["config_name"], "success", "add_table", f"Tabela '{table_name}' registrada", u["id"], u["username"])
|
||||
return {"id": tid, "table_name": table_name}
|
||||
|
||||
@router.delete("/api/adb/{vid}/tables/{tid}")
|
||||
async def remove_adb_table(vid: str, tid: str, u=Depends(require("admin","user"))):
|
||||
_verify_config_access("adb", vid, u, require_owner=True)
|
||||
with db() as c:
|
||||
row = c.execute("SELECT * FROM adb_vector_tables WHERE id=? AND adb_config_id=?", (tid, vid)).fetchone()
|
||||
if not row: raise HTTPException(404, "Table entry not found")
|
||||
with db() as c:
|
||||
c.execute("DELETE FROM adb_vector_tables WHERE id=?", (tid,))
|
||||
return {"ok": True}
|
||||
|
||||
@router.put("/api/adb/{vid}/tables/{tid}")
|
||||
async def update_adb_table(vid: str, tid: str, req: dict, u=Depends(require("admin","user"))):
|
||||
_verify_config_access("adb", vid, u, require_owner=True)
|
||||
with db() as c:
|
||||
row = c.execute("SELECT * FROM adb_vector_tables WHERE id=? AND adb_config_id=?", (tid, vid)).fetchone()
|
||||
if not row: raise HTTPException(404, "Table entry not found")
|
||||
sets, vals = [], []
|
||||
if "table_name" in req:
|
||||
tn = req["table_name"].strip()
|
||||
if not tn: raise HTTPException(400, "table_name e obrigatorio")
|
||||
if not _TABLE_NAME_RE.match(tn): raise HTTPException(400, "Nome da tabela invalido")
|
||||
sets.append("table_name=?"); vals.append(tn)
|
||||
if "description" in req:
|
||||
sets.append("description=?"); vals.append(req["description"])
|
||||
if "is_active" in req:
|
||||
sets.append("is_active=?"); vals.append(int(req["is_active"]))
|
||||
if not sets: return {"ok": True}
|
||||
vals.append(tid)
|
||||
try:
|
||||
with db() as c:
|
||||
c.execute(f"UPDATE adb_vector_tables SET {','.join(sets)} WHERE id=?", vals)
|
||||
except sqlite3.IntegrityError:
|
||||
raise HTTPException(409, "Tabela com este nome ja existe nesta conexao")
|
||||
return {"ok": True}
|
||||
|
||||
@router.post("/api/adb/{vid}/ingest")
|
||||
async def ingest_documents(vid: str, req: IngestDocReq, bg: BackgroundTasks, u=Depends(require("admin","user"))):
|
||||
_verify_config_access("adb", vid, u)
|
||||
with db() as c:
|
||||
cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
||||
if not cfg: raise HTTPException(404, "ADB config not found")
|
||||
cfg = dict(cfg)
|
||||
if not cfg.get("genai_config_id"):
|
||||
raise HTTPException(400, "GenAI config not linked to this ADB connection")
|
||||
with db() as c:
|
||||
gc = c.execute("SELECT * FROM genai_configs WHERE id=?", (cfg["genai_config_id"],)).fetchone()
|
||||
if not gc: raise HTTPException(400, "Linked GenAI config not found")
|
||||
bg.add_task(_ingest_documents_task, cfg, dict(gc), req.documents, u["id"], u["username"], table_name=req.table_name)
|
||||
return {"ok": True, "message": f"Ingestao iniciada para {len(req.documents)} documentos", "config_id": vid}
|
||||
227
backend/routes/auth.py
Normal file
227
backend/routes/auth.py
Normal file
@@ -0,0 +1,227 @@
|
||||
"""Auth routes — login, logout, register, change-password, OIDC, MFA."""
|
||||
import json
|
||||
import re
|
||||
import secrets
|
||||
import uuid
|
||||
from datetime import datetime, timedelta
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends, Request, Query
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse
|
||||
|
||||
from config import JWT_EXP_H
|
||||
from database import db
|
||||
from auth.crypto import _hash_pw, _verify_pw, _totp_secret, _totp_verify, _totp_uri, _make_token
|
||||
from auth.oidc import _get_oidc_config, _oidc_discover, _oidc_store_state, _oidc_pop_state, _validate_id_token
|
||||
from auth.jwt_auth import current_user, require, _audit
|
||||
from auth.rate_limit import _check_rate_limit
|
||||
from models import LoginReq, RegisterReq, TOTPVerify, ChangePwReq
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# ── Auth endpoints ────────────────────────────────────────────────────────────
|
||||
@router.post("/api/auth/login")
|
||||
async def login(req: LoginReq, request: Request):
|
||||
_check_rate_limit(request.client.host if request.client else "unknown")
|
||||
with db() as c:
|
||||
mode_row = c.execute("SELECT value FROM app_settings WHERE key='auth_mode'").fetchone()
|
||||
if mode_row and mode_row["value"] == "oidc":
|
||||
raise HTTPException(400, "Login local desabilitado. Use Oracle IAM.")
|
||||
with db() as c:
|
||||
u = c.execute("SELECT * FROM users WHERE username=? AND is_active=1", (req.username,)).fetchone()
|
||||
if not u or not _verify_pw(req.password, u["password_hash"]): raise HTTPException(401, "Credenciais inválidas")
|
||||
u = dict(u)
|
||||
if u["mfa_enabled"]:
|
||||
if not req.totp_code: return {"mfa_required": True, "message": "Código MFA necessário"}
|
||||
if not _totp_verify(u["mfa_secret"], req.totp_code): raise HTTPException(401, "Código MFA inválido")
|
||||
sid = str(uuid.uuid4()); exp = (datetime.utcnow()+timedelta(hours=JWT_EXP_H)).isoformat()
|
||||
with db() as c:
|
||||
c.execute("INSERT INTO sessions (id,user_id,expires_at) VALUES (?,?,?)", (sid, u["id"], exp))
|
||||
c.execute("UPDATE users SET last_login=datetime('now') WHERE id=?", (u["id"],))
|
||||
_audit(u["id"], u["username"], "login", ip=request.client.host if request.client else None)
|
||||
return {"token":_make_token(u["id"],u["role"],sid),
|
||||
"user":{"id":u["id"],"first_name":u["first_name"],"last_name":u["last_name"],"username":u["username"],"email":u["email"],"role":u["role"],"mfa_enabled":bool(u["mfa_enabled"]),"timezone":u.get("timezone","America/Sao_Paulo")},
|
||||
"must_change_password": bool(u.get("must_change_password", 0))}
|
||||
|
||||
@router.post("/api/auth/logout")
|
||||
async def logout_ep(u=Depends(current_user)):
|
||||
with db() as c: c.execute("UPDATE sessions SET is_active=0 WHERE user_id=?", (u["id"],))
|
||||
return {"ok": True}
|
||||
|
||||
@router.post("/api/auth/register")
|
||||
async def register(req: RegisterReq, adm=Depends(require("admin"))):
|
||||
if req.role not in ("admin","user","viewer"): raise HTTPException(400, "Role inválida")
|
||||
uid = str(uuid.uuid4())
|
||||
with db() as c:
|
||||
if c.execute("SELECT 1 FROM users WHERE username=?", (req.username,)).fetchone(): raise HTTPException(400, "Usuário já existe")
|
||||
c.execute("INSERT INTO users (id,first_name,last_name,username,email,password_hash,role) VALUES (?,?,?,?,?,?,?)",
|
||||
(uid, req.first_name, req.last_name, req.username, req.email, _hash_pw(req.password), req.role))
|
||||
_audit(adm["id"], adm["username"], "create_user", uid, f"user={req.username} role={req.role}")
|
||||
return {"id": uid, "username": req.username, "role": req.role}
|
||||
|
||||
@router.post("/api/auth/change-password")
|
||||
async def change_pw(req: ChangePwReq, u=Depends(current_user)):
|
||||
if u.get("auth_provider") == "oidc":
|
||||
raise HTTPException(400, "Usuários OIDC não podem alterar senha localmente")
|
||||
if not _verify_pw(req.current_password, u["password_hash"]): raise HTTPException(400, "Senha atual incorreta")
|
||||
pw = req.new_password
|
||||
if len(pw) < 8: raise HTTPException(400, "Senha deve ter no mínimo 8 caracteres")
|
||||
if not re.search(r'[A-Z]', pw): raise HTTPException(400, "Senha deve conter ao menos uma letra maiúscula")
|
||||
if not re.search(r'[a-z]', pw): raise HTTPException(400, "Senha deve conter ao menos uma letra minúscula")
|
||||
if not re.search(r'\d', pw): raise HTTPException(400, "Senha deve conter ao menos um número")
|
||||
if not re.search(r'[^A-Za-z0-9]', pw): raise HTTPException(400, "Senha deve conter ao menos um caractere especial")
|
||||
if pw == req.current_password: raise HTTPException(400, "A nova senha deve ser diferente da atual")
|
||||
with db() as c: c.execute("UPDATE users SET password_hash=?, must_change_password=0 WHERE id=?", (_hash_pw(pw), u["id"]))
|
||||
return {"ok": True}
|
||||
|
||||
# ── OIDC endpoints ───────────────────────────────────────────────────────────
|
||||
@router.get("/api/auth/oidc/config")
|
||||
async def oidc_public_config():
|
||||
with db() as c:
|
||||
row = c.execute("SELECT value FROM app_settings WHERE key='auth_mode'").fetchone()
|
||||
return {"auth_mode": row["value"] if row else "local"}
|
||||
|
||||
@router.get("/api/auth/oidc/login")
|
||||
async def oidc_login(request: Request):
|
||||
_check_rate_limit(request.client.host if request.client else "unknown")
|
||||
cfg = _get_oidc_config()
|
||||
if cfg.get("auth_mode") not in ("oidc", "both"):
|
||||
raise HTTPException(400, "OIDC login não está habilitado")
|
||||
if not cfg.get("oidc_issuer") or not cfg.get("oidc_client_id"):
|
||||
raise HTTPException(500, "OIDC não configurado (issuer/client_id ausente)")
|
||||
disco = _oidc_discover(cfg["oidc_issuer"])
|
||||
state = secrets.token_urlsafe(32)
|
||||
nonce = secrets.token_urlsafe(32)
|
||||
_oidc_store_state(state, nonce)
|
||||
params = {"response_type": "code", "client_id": cfg["oidc_client_id"],
|
||||
"redirect_uri": cfg["oidc_redirect_uri"], "scope": "openid profile email groups",
|
||||
"state": state, "nonce": nonce}
|
||||
auth_url = disco["authorization_endpoint"] + "?" + urlencode(params)
|
||||
return RedirectResponse(url=auth_url, status_code=302)
|
||||
|
||||
@router.get("/api/auth/oidc/callback")
|
||||
async def oidc_callback(request: Request, code: str = Query(...), state: str = Query(...)):
|
||||
ip = request.client.host if request.client else "unknown"
|
||||
_check_rate_limit(ip)
|
||||
pending = _oidc_pop_state(state)
|
||||
if not pending:
|
||||
raise HTTPException(400, "OIDC: state inválido ou expirado")
|
||||
nonce = pending["nonce"]
|
||||
cfg = _get_oidc_config()
|
||||
disco = _oidc_discover(cfg["oidc_issuer"])
|
||||
import requests as req
|
||||
token_resp = req.post(disco["token_endpoint"], data={
|
||||
"grant_type": "authorization_code", "code": code,
|
||||
"redirect_uri": cfg["oidc_redirect_uri"],
|
||||
"client_id": cfg["oidc_client_id"], "client_secret": cfg["oidc_client_secret"],
|
||||
}, timeout=15)
|
||||
if token_resp.status_code != 200:
|
||||
from config import log
|
||||
log.error(f"OIDC token exchange failed: {token_resp.status_code} {token_resp.text[:500]}")
|
||||
raise HTTPException(502, "OIDC: falha ao trocar código por token")
|
||||
tokens = token_resp.json()
|
||||
id_token_raw = tokens.get("id_token")
|
||||
if not id_token_raw:
|
||||
raise HTTPException(502, "OIDC: id_token ausente na resposta")
|
||||
claims = _validate_id_token(id_token_raw, cfg["oidc_issuer"], cfg["oidc_client_id"], nonce)
|
||||
sub = claims.get("sub")
|
||||
email = claims.get("email", "")
|
||||
given_name = claims.get("given_name") or claims.get("name", "").split(" ")[0] or "OIDC"
|
||||
family_name = claims.get("family_name") or " ".join(claims.get("name", "User").split(" ")[1:]) or "User"
|
||||
groups = claims.get("groups", [])
|
||||
if not groups:
|
||||
groups = claims.get("http://www.oracle.com/groups", [])
|
||||
role = "viewer"
|
||||
if cfg.get("oidc_group_admin") and cfg["oidc_group_admin"] in groups:
|
||||
role = "admin"
|
||||
elif cfg.get("oidc_group_user") and cfg["oidc_group_user"] in groups:
|
||||
role = "user"
|
||||
with db() as c:
|
||||
u = c.execute("SELECT * FROM users WHERE oidc_subject=? AND is_active=1", (sub,)).fetchone()
|
||||
if u:
|
||||
u = dict(u)
|
||||
c.execute("UPDATE users SET role=?, last_login=datetime('now'), email=? WHERE id=?",
|
||||
(role, email or u.get("email"), u["id"]))
|
||||
u["role"] = role
|
||||
_audit(u["id"], u["username"], "oidc_login", details=f"sub={sub}", ip=ip)
|
||||
else:
|
||||
uid = str(uuid.uuid4())
|
||||
username = email.split("@")[0] if email else f"oidc_{sub[:8]}"
|
||||
base_username = username
|
||||
counter = 1
|
||||
while c.execute("SELECT 1 FROM users WHERE username=?", (username,)).fetchone():
|
||||
username = f"{base_username}_{counter}"
|
||||
counter += 1
|
||||
fake_hash = _hash_pw(secrets.token_hex(32))
|
||||
c.execute(
|
||||
"INSERT INTO users (id,first_name,last_name,username,email,password_hash,role,auth_provider,oidc_subject) "
|
||||
"VALUES (?,?,?,?,?,?,?,?,?)",
|
||||
(uid, given_name, family_name, username, email, fake_hash, role, "oidc", sub))
|
||||
u = {"id": uid, "first_name": given_name, "last_name": family_name,
|
||||
"username": username, "email": email, "role": role}
|
||||
_audit(uid, username, "oidc_jit_provision", details=f"sub={sub} email={email} role={role}", ip=ip)
|
||||
c.execute("UPDATE users SET last_login=datetime('now') WHERE id=?", (uid,))
|
||||
sid = str(uuid.uuid4())
|
||||
exp = (datetime.utcnow() + timedelta(hours=JWT_EXP_H)).isoformat()
|
||||
with db() as c:
|
||||
c.execute("INSERT INTO sessions (id,user_id,expires_at) VALUES (?,?,?)", (sid, u["id"], exp))
|
||||
token = _make_token(u["id"], u["role"], sid)
|
||||
html = f"""<!DOCTYPE html><html><head><title>OIDC Login</title></head><body>
|
||||
<script>localStorage.setItem('t',{json.dumps(token)});window.location.href='/chat';</script>
|
||||
<noscript><p>Login OK. <a href="/chat">Continuar</a></p></noscript></body></html>"""
|
||||
return HTMLResponse(content=html)
|
||||
|
||||
@router.post("/api/auth/oidc/logout")
|
||||
async def oidc_logout(u=Depends(current_user)):
|
||||
with db() as c:
|
||||
c.execute("UPDATE sessions SET is_active=0 WHERE user_id=?", (u["id"],))
|
||||
_audit(u["id"], u["username"], "oidc_logout")
|
||||
idp_logout_url = None
|
||||
try:
|
||||
cfg = _get_oidc_config()
|
||||
if cfg.get("oidc_issuer"):
|
||||
disco = _oidc_discover(cfg["oidc_issuer"])
|
||||
idp_logout_url = disco.get("end_session_endpoint")
|
||||
except Exception:
|
||||
pass
|
||||
return {"ok": True, "idp_logout_url": idp_logout_url}
|
||||
|
||||
@router.post("/api/settings/oidc/test")
|
||||
async def test_oidc(u=Depends(require("admin"))):
|
||||
cfg = _get_oidc_config()
|
||||
if not cfg.get("oidc_issuer"):
|
||||
raise HTTPException(400, "Issuer URL não configurado")
|
||||
import requests as req
|
||||
try:
|
||||
url = cfg["oidc_issuer"].rstrip("/") + "/.well-known/openid-configuration"
|
||||
resp = req.get(url, timeout=10)
|
||||
resp.raise_for_status()
|
||||
disco = resp.json()
|
||||
has_auth = bool(disco.get("authorization_endpoint"))
|
||||
has_token = bool(disco.get("token_endpoint"))
|
||||
has_jwks = bool(disco.get("jwks_uri"))
|
||||
return {"ok": has_auth and has_token and has_jwks, "issuer": disco.get("issuer"),
|
||||
"authorization_endpoint": has_auth, "token_endpoint": has_token,
|
||||
"jwks_uri": has_jwks, "end_session_endpoint": bool(disco.get("end_session_endpoint"))}
|
||||
except Exception as e:
|
||||
raise HTTPException(502, f"Falha ao conectar ao discovery: {str(e)}")
|
||||
|
||||
# ── MFA ───────────────────────────────────────────────────────────────────────
|
||||
@router.post("/api/mfa/setup")
|
||||
async def mfa_setup(u=Depends(current_user)):
|
||||
sec = _totp_secret()
|
||||
with db() as c: c.execute("UPDATE users SET mfa_secret=? WHERE id=?", (sec, u["id"]))
|
||||
return {"secret": sec, "uri": _totp_uri(sec, u["username"])}
|
||||
|
||||
@router.post("/api/mfa/verify")
|
||||
async def mfa_verify(req: TOTPVerify, u=Depends(current_user)):
|
||||
if not u.get("mfa_secret"): raise HTTPException(400, "Chame /api/mfa/setup primeiro")
|
||||
if not _totp_verify(u["mfa_secret"], req.totp_code): raise HTTPException(400, "Código inválido")
|
||||
with db() as c: c.execute("UPDATE users SET mfa_enabled=1 WHERE id=?", (u["id"],))
|
||||
return {"ok": True, "message": "MFA ativado"}
|
||||
|
||||
@router.post("/api/mfa/disable/{user_id}")
|
||||
async def mfa_disable(user_id: str, adm=Depends(require("admin"))):
|
||||
with db() as c: c.execute("UPDATE users SET mfa_enabled=0,mfa_secret=NULL WHERE id=?", (user_id,))
|
||||
_audit(adm["id"], adm["username"], "disable_mfa", user_id)
|
||||
return {"ok": True}
|
||||
200
backend/routes/chat.py
Normal file
200
backend/routes/chat.py
Normal file
@@ -0,0 +1,200 @@
|
||||
"""Chat routes — chat upload, chat JSON, message status, sessions CRUD."""
|
||||
import base64
|
||||
import shutil
|
||||
from typing import Optional, List
|
||||
|
||||
from fastapi import (
|
||||
APIRouter, HTTPException, Depends, Query, BackgroundTasks,
|
||||
UploadFile, File, Form, Body,
|
||||
)
|
||||
from fastapi.security import HTTPBearer
|
||||
|
||||
from config import VERSION, GENAI_MODELS, TERRAFORM_DIR, log
|
||||
from database import db
|
||||
from auth.jwt_auth import current_user, _audit
|
||||
from models import ChatMsg
|
||||
from services.chat_background import _chat_start, _chat_background
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# ── Upload constants ──────────────────────────────────────────────────────────
|
||||
MAX_UPLOAD_FILES = 5
|
||||
MAX_UPLOAD_SIZE = 10 * 1024 * 1024 # 10MB
|
||||
IMAGE_MIMES = {"image/png", "image/jpeg", "image/gif", "image/webp", "image/bmp"}
|
||||
DOC_MIMES = {"application/pdf"}
|
||||
|
||||
|
||||
# ── Chat endpoints ────────────────────────────────────────────────────────────
|
||||
|
||||
@router.post("/api/chat/upload")
|
||||
async def chat_with_files(
|
||||
bg: BackgroundTasks,
|
||||
message: str = Form(""),
|
||||
session_id: Optional[str] = Form(None),
|
||||
genai_config_id: Optional[str] = Form(None),
|
||||
model_id: Optional[str] = Form(None),
|
||||
oci_config_id: Optional[str] = Form(None),
|
||||
genai_region: Optional[str] = Form(None),
|
||||
compartment_id: Optional[str] = Form(None),
|
||||
use_tools: Optional[bool] = Form(True),
|
||||
temperature: Optional[float] = Form(None),
|
||||
max_tokens: Optional[int] = Form(None),
|
||||
top_p: Optional[float] = Form(None),
|
||||
top_k: Optional[int] = Form(None),
|
||||
frequency_penalty: Optional[float] = Form(None),
|
||||
presence_penalty: Optional[float] = Form(None),
|
||||
reasoning_effort: Optional[str] = Form(None),
|
||||
files: List[UploadFile] = File(default=[]),
|
||||
u=Depends(current_user),
|
||||
):
|
||||
if len(files) > MAX_UPLOAD_FILES:
|
||||
raise HTTPException(400, f"Máximo {MAX_UPLOAD_FILES} arquivos por mensagem")
|
||||
|
||||
attachments = []
|
||||
file_names = []
|
||||
for f in files:
|
||||
data = await f.read()
|
||||
if len(data) > MAX_UPLOAD_SIZE:
|
||||
raise HTTPException(400, f"Arquivo {f.filename} excede 10MB")
|
||||
mime = f.content_type or "application/octet-stream"
|
||||
b64 = base64.b64encode(data).decode()
|
||||
data_uri = f"data:{mime};base64,{b64}"
|
||||
if mime in IMAGE_MIMES:
|
||||
attachments.append({"type": "image", "mime": mime, "data_uri": data_uri})
|
||||
elif mime in DOC_MIMES:
|
||||
attachments.append({"type": "document", "mime": mime, "data_uri": data_uri})
|
||||
else:
|
||||
try:
|
||||
text = data.decode("utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
log.warning(f"UTF-8 decode failed for file attachment, falling back to latin-1: {e}")
|
||||
text = data.decode("latin-1", errors="replace")
|
||||
message = f"{message}\n\n--- Conteúdo de {f.filename} ---\n{text[:50000]}"
|
||||
file_names.append(f.filename)
|
||||
|
||||
msg = ChatMsg(
|
||||
message=message or "Analise os arquivos anexados.",
|
||||
session_id=session_id,
|
||||
genai_config_id=genai_config_id,
|
||||
model_id=model_id,
|
||||
oci_config_id=oci_config_id,
|
||||
genai_region=genai_region,
|
||||
compartment_id=compartment_id,
|
||||
use_tools=use_tools if use_tools is not None else True,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
frequency_penalty=frequency_penalty,
|
||||
presence_penalty=presence_penalty,
|
||||
reasoning_effort=reasoning_effort,
|
||||
)
|
||||
|
||||
sid, mid_or_result, genai_cfg = _chat_start(msg, u, attachments=attachments if attachments else None)
|
||||
if genai_cfg is None:
|
||||
return mid_or_result # immediate fallback response
|
||||
|
||||
if file_names:
|
||||
with db() as c:
|
||||
c.execute("UPDATE chat_messages SET content = content || ? WHERE session_id=? AND role='user' AND status='done' ORDER BY created_at DESC LIMIT 1",
|
||||
(f"\n[📎 {', '.join(file_names)}]", sid))
|
||||
|
||||
bg.add_task(_chat_background, mid_or_result, sid, msg, dict(u), genai_cfg, attachments if attachments else None)
|
||||
return {"message_id": mid_or_result, "session_id": sid, "status": "processing"}
|
||||
|
||||
|
||||
@router.post("/api/chat")
|
||||
async def chat(msg: ChatMsg, bg: BackgroundTasks, u=Depends(current_user)):
|
||||
sid, mid_or_result, genai_cfg = _chat_start(msg, u)
|
||||
if genai_cfg is None:
|
||||
return mid_or_result # immediate fallback response
|
||||
bg.add_task(_chat_background, mid_or_result, sid, msg, dict(u), genai_cfg)
|
||||
return {"message_id": mid_or_result, "session_id": sid, "status": "processing"}
|
||||
|
||||
|
||||
@router.get("/api/chat/{mid}/status")
|
||||
async def chat_message_status(mid: str, u=Depends(current_user)):
|
||||
with db() as c:
|
||||
r = c.execute("SELECT id, session_id, role, content, model_id, status FROM chat_messages WHERE id=?", (mid,)).fetchone()
|
||||
if not r:
|
||||
raise HTTPException(404, "Message not found")
|
||||
return dict(r)
|
||||
|
||||
|
||||
# ── Local fallback agent (no GenAI configured) ───────────────────────────────
|
||||
|
||||
def _agent_respond(msg, user):
|
||||
m = msg.lower().strip()
|
||||
if any(k in m for k in ["status","health","como está","saúde"]):
|
||||
cli = "✅ Instalado" if shutil.which("oci") else "❌ Não instalado"
|
||||
with db() as c:
|
||||
nc=c.execute("SELECT COUNT(*) n FROM oci_configs").fetchone()["n"]
|
||||
nr=c.execute("SELECT COUNT(*) n FROM reports").fetchone()["n"]
|
||||
nu=c.execute("SELECT COUNT(*) n FROM users WHERE is_active=1").fetchone()["n"]
|
||||
nm=c.execute("SELECT COUNT(*) n FROM mcp_servers WHERE is_active=1").fetchone()["n"]
|
||||
ng=c.execute("SELECT COUNT(*) n FROM genai_configs").fetchone()["n"]
|
||||
return (f"📊 **Status do Sistema**\n\n• OCI CLI: {cli}\n• Configs OCI: {nc}\n• GenAI Models: {ng}\n• MCP Servers: {nm}\n• Relatórios: {nr}\n• Usuários ativos: {nu}\n• Servidor: ✅ Online\n• Versão: v{VERSION}")
|
||||
if any(k in m for k in ["ajuda","help","comandos"]):
|
||||
return ("🤖 **Comandos disponíveis:**\n\n• `status` — Status do sistema\n• `listar configs` — Configurações OCI\n• `verificar cis` — Checks CIS 3.0\n• `modelos` — Modelos GenAI disponíveis\n• `ajuda` — Esta mensagem\n\n💡 Configure um modelo GenAI para chat com IA.")
|
||||
if any(k in m for k in ["modelo","modelos","genai"]):
|
||||
lines = ["🧠 **Modelos OCI Generative AI disponíveis:**\n"]
|
||||
for mid, info in GENAI_MODELS.items():
|
||||
lines.append(f"• `{mid}` — {info['name']} ({info['provider']})")
|
||||
lines.append("\nConfigure em **GenAI Config** para usar no chat.")
|
||||
return "\n".join(lines)
|
||||
if any(k in m for k in ["cis","benchmark","checks","verificar"]):
|
||||
return ("🔒 **CIS OCI Foundations Benchmark 3.0**\n\n54 controles em 8 domínios:\n• IAM: 17 controles\n• Networking: 8 controles\n• Compute: 3 controles\n• Logging & Monitoring: 18 controles\n• Storage: 6 controles\n• Asset Management: 2 controles\n\nConfigure OCI e execute um relatório na aba **Report**.")
|
||||
return ("Sou o **AI Agent v" + VERSION + "** — Infrastructure & Security Engineer. Sem modelo GenAI configurado, uso respostas locais.\n\n"
|
||||
"Para chat com IA:\n1. Configure **OCI Credentials**\n2. Configure **GenAI** com modelo e região\n3. Selecione o modelo no chat\n\nDigite **ajuda** para ver os comandos.")
|
||||
|
||||
|
||||
# ── Chat sessions ─────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/api/chat/sessions")
|
||||
async def list_chat_sessions(agent_type: str = "chat", limit: int = 50, u=Depends(current_user)):
|
||||
with db() as c:
|
||||
rows = c.execute(
|
||||
"""SELECT cs.id, cs.title, cs.agent_type, cs.created_at, cs.updated_at,
|
||||
(SELECT COUNT(*) FROM chat_messages cm WHERE cm.session_id=cs.id) as message_count
|
||||
FROM chat_sessions cs WHERE cs.user_id=? AND cs.agent_type=?
|
||||
ORDER BY cs.updated_at DESC LIMIT ?""",
|
||||
(u["id"], agent_type, limit)).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
@router.get("/api/chat/sessions/{sid}/messages")
|
||||
async def get_session_messages(sid: str, skip: int = Query(0, ge=0), limit: int = Query(100, ge=1, le=500), u=Depends(current_user)):
|
||||
with db() as c:
|
||||
session = c.execute("SELECT * FROM chat_sessions WHERE id=? AND user_id=?", (sid, u["id"])).fetchone()
|
||||
if not session:
|
||||
raise HTTPException(404, "Session not found")
|
||||
total = c.execute(
|
||||
"SELECT COUNT(*) as cnt FROM chat_messages "
|
||||
"WHERE session_id=? AND status='done'", (sid,)).fetchone()["cnt"]
|
||||
msgs = c.execute(
|
||||
"SELECT id, role, content, model_id, status, created_at FROM chat_messages "
|
||||
"WHERE session_id=? AND status='done' ORDER BY created_at ASC LIMIT ? OFFSET ?", (sid, limit, skip)).fetchall()
|
||||
return {"session": dict(session), "messages": [dict(m) for m in msgs], "total": total}
|
||||
|
||||
|
||||
@router.put("/api/chat/sessions/{sid}/title")
|
||||
async def rename_session(sid: str, req: dict = Body(...), u=Depends(current_user)):
|
||||
with db() as c:
|
||||
c.execute("UPDATE chat_sessions SET title=? WHERE id=? AND user_id=?",
|
||||
(req.get("title", "")[:120], sid, u["id"]))
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.delete("/api/chat/{sid}")
|
||||
async def clear_chat(sid, u=Depends(current_user)):
|
||||
with db() as c:
|
||||
# Delete associated terraform workspaces and their files
|
||||
ws_rows = c.execute("SELECT id FROM terraform_workspaces WHERE session_id=? AND user_id=?", (sid, u["id"])).fetchall()
|
||||
for ws in ws_rows:
|
||||
wdir = TERRAFORM_DIR / ws["id"]
|
||||
if wdir.exists():
|
||||
shutil.rmtree(wdir, ignore_errors=True)
|
||||
c.execute("DELETE FROM terraform_workspaces WHERE session_id=? AND user_id=?", (sid, u["id"]))
|
||||
c.execute("DELETE FROM chat_messages WHERE session_id=? AND user_id=?", (sid, u["id"]))
|
||||
c.execute("DELETE FROM chat_sessions WHERE id=? AND user_id=?", (sid, u["id"]))
|
||||
return {"ok": True}
|
||||
105
backend/routes/cis_engine.py
Normal file
105
backend/routes/cis_engine.py
Normal file
@@ -0,0 +1,105 @@
|
||||
"""CIS Engine routes — version management, check-update, update, health check."""
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends
|
||||
|
||||
from config import VERSION, _START_TIME, log
|
||||
from database import db
|
||||
from auth.jwt_auth import current_user, require, _audit
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# ── Constants ────────────────────────────────────────────────────────────────
|
||||
|
||||
CIS_REPORTS_PATH = Path("/app/cis_reports.py")
|
||||
CIS_GITHUB_RAW = "https://raw.githubusercontent.com/oci-landing-zones/oci-cis-landingzone-quickstart/main/scripts/cis_reports.py"
|
||||
CIS_PATCHES = [
|
||||
{"name": "region_none_check_1",
|
||||
"original": "elif region.region_name in self.__regions_to_run_in or self.__run_in_all_regions:",
|
||||
"patched": "elif self.__run_in_all_regions or (self.__regions_to_run_in and region.region_name in self.__regions_to_run_in):"},
|
||||
{"name": "region_none_check_2",
|
||||
"original": "if self.__home_region not in self.__regions_to_run_in:",
|
||||
"patched": "if not self.__run_in_all_regions and self.__regions_to_run_in and self.__home_region not in self.__regions_to_run_in:"},
|
||||
]
|
||||
|
||||
|
||||
# ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def _read_cis_version(content: str = None) -> dict:
|
||||
if content is None:
|
||||
if not CIS_REPORTS_PATH.exists(): return {"version": "unknown", "date": "unknown"}
|
||||
content = CIS_REPORTS_PATH.read_text(encoding="utf-8", errors="replace")
|
||||
ver = re.search(r'RELEASE_VERSION\s*=\s*["\']([^"\']+)["\']', content)
|
||||
dt = re.search(r'UPDATED_DATE\s*=\s*["\']([^"\']+)["\']', content)
|
||||
return {"version": ver.group(1) if ver else "unknown", "date": dt.group(1) if dt else "unknown"}
|
||||
|
||||
|
||||
# ── Endpoints ────────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/api/cis-engine/version")
|
||||
async def cis_engine_version(u=Depends(current_user)):
|
||||
info = _read_cis_version()
|
||||
return {"version": info["version"], "updated_date": info["date"], "file_path": str(CIS_REPORTS_PATH)}
|
||||
|
||||
@router.get("/api/cis-engine/check-update")
|
||||
async def cis_engine_check_update(u=Depends(require("admin"))):
|
||||
import requests as req
|
||||
local = _read_cis_version()
|
||||
try:
|
||||
resp = req.get(CIS_GITHUB_RAW, timeout=30)
|
||||
resp.raise_for_status()
|
||||
remote = _read_cis_version(resp.text)
|
||||
except Exception as e:
|
||||
raise HTTPException(502, f"Failed to check GitHub: {str(e)[:300]}")
|
||||
return {
|
||||
"local_version": local["version"], "local_date": local["date"],
|
||||
"remote_version": remote["version"], "remote_date": remote["date"],
|
||||
"update_available": remote["version"] != local["version"]
|
||||
}
|
||||
|
||||
@router.post("/api/cis-engine/update")
|
||||
async def cis_engine_update(u=Depends(require("admin"))):
|
||||
import requests as req
|
||||
old = _read_cis_version()
|
||||
try:
|
||||
resp = req.get(CIS_GITHUB_RAW, timeout=60)
|
||||
resp.raise_for_status()
|
||||
content = resp.text
|
||||
except Exception as e:
|
||||
raise HTTPException(502, f"Failed to download from GitHub: {str(e)[:300]}")
|
||||
new = _read_cis_version(content)
|
||||
# Apply patches
|
||||
patches_applied = []
|
||||
for p in CIS_PATCHES:
|
||||
if p["original"] in content:
|
||||
content = content.replace(p["original"], p["patched"])
|
||||
patches_applied.append(p["name"])
|
||||
CIS_REPORTS_PATH.write_text(content, encoding="utf-8")
|
||||
# Save version info
|
||||
with db() as c:
|
||||
c.execute("INSERT INTO app_settings (key,value,updated_at) VALUES ('cis_engine_version',?,datetime('now')) ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at",
|
||||
(new["version"],))
|
||||
log.info(f"CIS engine updated: {old['version']} → {new['version']}, patches: {patches_applied}")
|
||||
_audit(u["id"], u["username"], "cis_engine_update", None, f"{old['version']} → {new['version']}")
|
||||
return {"ok": True, "old_version": old["version"], "new_version": new["version"], "patches_applied": patches_applied}
|
||||
|
||||
@router.get("/api/health")
|
||||
async def health():
|
||||
db_ok = False
|
||||
try:
|
||||
with db() as c:
|
||||
c.execute("SELECT 1")
|
||||
db_ok = True
|
||||
except Exception:
|
||||
pass
|
||||
return {
|
||||
"status": "ok",
|
||||
"version": VERSION,
|
||||
"uptime_seconds": int(time.time() - _START_TIME),
|
||||
"db_ok": db_ok,
|
||||
"ts": datetime.utcnow().isoformat(),
|
||||
}
|
||||
732
backend/routes/embeddings.py
Normal file
732
backend/routes/embeddings.py
Normal file
@@ -0,0 +1,732 @@
|
||||
"""Embedding routes — preview, embed report, status, section, file, upload, upload-url, consult, list, delete, purge."""
|
||||
import json
|
||||
import uuid
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends, UploadFile, File, Form, Query, BackgroundTasks
|
||||
|
||||
from database import db
|
||||
from auth.jwt_auth import current_user, require, _audit, _config_log, _verify_config_access, _verify_report_access
|
||||
from config import REPORTS, log, _chat_executor
|
||||
from models import ConsultQuery
|
||||
from utils import validate_upload, set_embed_status, get_embed_status, update_embed_status
|
||||
from services.genai import (
|
||||
_call_genai,
|
||||
_get_adb_connection,
|
||||
_resolve_embed_config,
|
||||
_embed_text,
|
||||
_DIM_TO_MODEL,
|
||||
_get_table_embedding_dim,
|
||||
_vector_search_multi,
|
||||
_relevant_tables,
|
||||
_build_rag_context,
|
||||
_get_active_adb_configs,
|
||||
_get_tables_for_config,
|
||||
RAG_CONTEXT_TEMPLATE,
|
||||
CONSULT_SYSTEM_PROMPT,
|
||||
)
|
||||
from services.embeddings import (
|
||||
_build_metadata_json,
|
||||
_auto_register_table,
|
||||
_ingest_documents_task,
|
||||
_chunk_report_by_section,
|
||||
_chunk_cis_pdf,
|
||||
_chunk_text_file,
|
||||
_get_adb_and_genai,
|
||||
_chunk_summary_csv,
|
||||
_purge_table_by_tenancy,
|
||||
_resolve_table_for_csv,
|
||||
_chunk_findings_csv,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# ── Module-local helpers ─────────────────────────────────────────────────────
|
||||
|
||||
def _extract_pdf_text(file_bytes: bytes) -> str:
|
||||
"""Extract text from a PDF file using PyPDF2 or pdfplumber."""
|
||||
import io
|
||||
try:
|
||||
import PyPDF2
|
||||
reader = PyPDF2.PdfReader(io.BytesIO(file_bytes))
|
||||
pages = []
|
||||
for page in reader.pages:
|
||||
text = page.extract_text()
|
||||
if text:
|
||||
pages.append(text.strip())
|
||||
return "\n\n".join(pages)
|
||||
except ImportError:
|
||||
pass
|
||||
try:
|
||||
import pdfplumber
|
||||
pages = []
|
||||
with pdfplumber.open(io.BytesIO(file_bytes)) as pdf:
|
||||
for page in pdf.pages:
|
||||
text = page.extract_text()
|
||||
if text:
|
||||
pages.append(text.strip())
|
||||
return "\n\n".join(pages)
|
||||
except ImportError:
|
||||
raise HTTPException(400, "PDF support requires PyPDF2 or pdfplumber. Install: pip install PyPDF2")
|
||||
|
||||
|
||||
def _extract_text_from_html(html: str) -> str:
|
||||
"""Extract readable text from HTML, stripping tags and scripts."""
|
||||
import re as _re
|
||||
text = _re.sub(r'<script[^>]*>[\s\S]*?</script>', ' ', html, flags=_re.IGNORECASE)
|
||||
text = _re.sub(r'<style[^>]*>[\s\S]*?</style>', ' ', text, flags=_re.IGNORECASE)
|
||||
text = _re.sub(r'<[^>]+>', ' ', text)
|
||||
text = _re.sub(r'&[a-zA-Z]+;', ' ', text)
|
||||
text = _re.sub(r'&#\d+;', ' ', text)
|
||||
text = _re.sub(r'\s+', ' ', text).strip()
|
||||
return text
|
||||
|
||||
|
||||
def _classify_report_file(fname: str) -> str:
|
||||
"""Classify a report file into a category based on its filename."""
|
||||
fl = fname.lower()
|
||||
if "summary_report" in fl: return "summary"
|
||||
if "error_report" in fl or "error" in fl and fl.endswith(".csv"): return "error"
|
||||
if fl.startswith("obp_") and "findings" in fl: return "obp_finding"
|
||||
if fl.startswith("obp_") and "best_practices" in fl: return "obp_best_practice"
|
||||
if fl.startswith("obp_"): return "obp_finding"
|
||||
if fl.startswith("raw_data_"): return "raw_data"
|
||||
if fl.startswith("cis_"): return "cis_finding"
|
||||
if "consolidated_report" in fl: return "consolidated"
|
||||
if fl.endswith(".png"): return "diagram"
|
||||
return "other"
|
||||
|
||||
|
||||
# ── Routes ───────────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/api/embeddings/preview/{rid}")
|
||||
async def preview_report_chunks(rid: str, u=Depends(current_user)):
|
||||
"""Preview the chunks that will be generated from a CIS report before embedding."""
|
||||
_verify_report_access(rid, u)
|
||||
with db() as c:
|
||||
r = c.execute("SELECT json_path,tenancy_name FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone()
|
||||
if not r: raise HTTPException(404, "Report not found or not completed")
|
||||
json_path = r["json_path"]
|
||||
if not json_path or not Path(json_path).exists():
|
||||
raise HTTPException(400, "Report JSON file not found")
|
||||
try:
|
||||
report_data = json.loads(Path(json_path).read_text())
|
||||
except Exception:
|
||||
raise HTTPException(400, "Invalid report data")
|
||||
documents = _chunk_report_by_section(report_data)
|
||||
rd = report_data if isinstance(report_data, dict) else {}
|
||||
return {"tenancy": rd.get("tenancy", "unknown"),
|
||||
"regions": rd.get("regions", []),
|
||||
"compartments": rd.get("compartments", []),
|
||||
"total_chunks": len(documents),
|
||||
"chunks": documents}
|
||||
|
||||
@router.post("/api/embeddings/report/{rid}")
|
||||
async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(require("admin","user"))):
|
||||
"""Auto-embed all CIS report CSVs into their mapped ADB vector tables."""
|
||||
_verify_report_access(rid, u)
|
||||
vid = req.get("adb_config_id")
|
||||
if not vid: raise HTTPException(400, "adb_config_id is required")
|
||||
_verify_config_access("adb", vid, u)
|
||||
with db() as c:
|
||||
r = c.execute("SELECT tenancy_name, config_id FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone()
|
||||
if not r: raise HTTPException(404, "Report not found or not completed")
|
||||
|
||||
rdir = REPORTS / rid
|
||||
tenancy = r["tenancy_name"] or "unknown"
|
||||
|
||||
# Read extract_date from summary CSV
|
||||
import csv as csvmod
|
||||
summary_csv = rdir / "cis_summary_report.csv"
|
||||
extract_date = ""
|
||||
if summary_csv.exists():
|
||||
with open(summary_csv, "r", encoding="utf-8") as f:
|
||||
first = next(csvmod.DictReader(f), {})
|
||||
extract_date = first.get("extract_date", "")
|
||||
|
||||
cfg, gc = _get_adb_and_genai(vid, oci_config_id=r["config_id"] if r["config_id"] else None, user_id=u["id"])
|
||||
|
||||
# Load registered tables for validation
|
||||
with db() as c:
|
||||
registered = {t["table_name"].lower() for t in
|
||||
c.execute("SELECT table_name FROM adb_vector_tables WHERE adb_config_id=? AND is_active=1", (vid,)).fetchall()}
|
||||
|
||||
# Scan all CSVs and map to tables
|
||||
task_id = str(uuid.uuid4())
|
||||
table_docs: dict[str, list] = {}
|
||||
missing_tables: list[str] = []
|
||||
skipped_tables: list[str] = []
|
||||
|
||||
for csv_file in sorted(rdir.glob("cis_*.csv")):
|
||||
table = _resolve_table_for_csv(csv_file.name)
|
||||
if not table:
|
||||
continue
|
||||
if table.lower() not in registered:
|
||||
if table not in skipped_tables:
|
||||
skipped_tables.append(table)
|
||||
continue
|
||||
if csv_file.name == "cis_summary_report.csv":
|
||||
docs = _chunk_summary_csv(str(csv_file), tenancy, extract_date)
|
||||
else:
|
||||
docs = _chunk_findings_csv(str(csv_file), tenancy, extract_date)
|
||||
if docs:
|
||||
table_docs.setdefault(table, []).extend(docs)
|
||||
|
||||
if not table_docs:
|
||||
if skipped_tables:
|
||||
raise HTTPException(400, f"Tabela(s) nao registrada(s) no ADB: {', '.join(skipped_tables)}. Registre em Configuracoes > ADB Vector.")
|
||||
raise HTTPException(400, "No CSV files found to embed")
|
||||
|
||||
# Calculate totals for status tracking
|
||||
total_docs = sum(len(d) for d in table_docs.values())
|
||||
tables_used = list(table_docs.keys())
|
||||
set_embed_status(task_id, {
|
||||
"status": "running", "table": ", ".join(tables_used), "tenancy": tenancy,
|
||||
"inserted": 0, "total": total_docs, "user_id": u["id"],
|
||||
"message": f"Embedding {total_docs} documentos em {len(tables_used)} tabelas..."
|
||||
})
|
||||
|
||||
# Build queue info: [(table, doc_count), ...]
|
||||
table_queue = [(tbl, len(docs)) for tbl, docs in table_docs.items()]
|
||||
|
||||
def _bg_embed_all():
|
||||
"""Background: embed documents into their respective tables sequentially."""
|
||||
import array
|
||||
default_model = cfg.get("embedding_model_id", "cohere.embed-v4.0")
|
||||
inserted_total = 0
|
||||
skipped_total = 0
|
||||
processed_total = 0
|
||||
errors = []
|
||||
for tbl_idx, (tbl, docs) in enumerate(table_docs.items()):
|
||||
remaining = table_queue[tbl_idx + 1:] if tbl_idx + 1 < len(table_queue) else []
|
||||
_auto_register_table(cfg["id"], tbl)
|
||||
purged = _purge_table_by_tenancy(cfg, tbl, tenancy, extract_date)
|
||||
if purged:
|
||||
update_embed_status(task_id, {"message": f"Purged {purged} old docs from {tbl}...",
|
||||
"current_table": tbl, "current_inserted": 0, "current_total": len(docs),
|
||||
"queue": [{"table": t, "docs": n} for t, n in remaining]})
|
||||
emb_model = default_model
|
||||
try:
|
||||
actual_dim = _get_table_embedding_dim(cfg, tbl)
|
||||
if actual_dim and actual_dim in _DIM_TO_MODEL:
|
||||
emb_model = _DIM_TO_MODEL[actual_dim]
|
||||
except Exception:
|
||||
pass
|
||||
current_inserted = 0
|
||||
current_skipped = 0
|
||||
try:
|
||||
conn = _get_adb_connection(cfg)
|
||||
cur = conn.cursor()
|
||||
for doc in docs:
|
||||
try:
|
||||
content = doc.get("content", "")
|
||||
if not content:
|
||||
skipped_total += 1
|
||||
current_skipped += 1
|
||||
processed_total += 1
|
||||
continue
|
||||
embedding = _embed_text(content, gc, emb_model)
|
||||
vec = array.array('f', [float(x) for x in embedding])
|
||||
metadata = _build_metadata_json(
|
||||
tenancy=doc.get("tenancy", tenancy),
|
||||
section=doc.get("section", ""),
|
||||
report_date=extract_date,
|
||||
user_id=u["id"],
|
||||
)
|
||||
cur.execute(f'INSERT INTO "{tbl}" (ID, TEXT, EMBEDDING, METADATA) VALUES (HEXTORAW(:1), :2, :3, :4)',
|
||||
[uuid.uuid4().hex.upper(), content, vec, metadata])
|
||||
inserted_total += 1
|
||||
current_inserted += 1
|
||||
except Exception as e:
|
||||
err_str = str(e)
|
||||
if "message" in err_str:
|
||||
import re as _re
|
||||
m = _re.search(r'"message"\s*:\s*"(.*?)"', err_str)
|
||||
log.warning(f"Embed skip in {tbl}: {m.group(1)[:200] if m else err_str[:200]}")
|
||||
else:
|
||||
log.warning(f"Embed skip in {tbl}: {err_str[:200]}")
|
||||
skipped_total += 1
|
||||
current_skipped += 1
|
||||
processed_total += 1
|
||||
update_embed_status(task_id, {
|
||||
"inserted": inserted_total, "skipped": skipped_total, "processed": processed_total,
|
||||
"current_table": tbl, "current_inserted": current_inserted, "current_skipped": current_skipped,
|
||||
"current_total": len(docs),
|
||||
"queue": [{"table": t, "docs": n} for t, n in remaining],
|
||||
"message": f"{tbl}: {current_inserted}/{len(docs)} — global: {inserted_total}/{total_docs}"
|
||||
})
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
log.info(f"Embedded {current_inserted}/{len(docs)} docs into {tbl} (tenancy={tenancy})")
|
||||
except Exception as e:
|
||||
log.error(f"Failed to connect/embed to {tbl}: {e}")
|
||||
errors.append(f"{tbl}: {str(e)[:100]}")
|
||||
|
||||
msg = f"{inserted_total}/{total_docs} documentos em {len(tables_used)} tabelas ({', '.join(tables_used)})"
|
||||
if tenancy: msg += f" — tenancy: {tenancy}"
|
||||
if errors: msg += f" | Erros: {'; '.join(errors)}"
|
||||
update_embed_status(task_id, {
|
||||
"status": "done" if not errors else "done",
|
||||
"inserted": inserted_total, "message": msg
|
||||
})
|
||||
_audit(u["id"], u["username"], "embed_report_auto", rid, msg)
|
||||
_config_log("adb", cfg["id"], cfg.get("config_name"),
|
||||
"success" if not errors else "error", "ingest", msg, u["id"], u["username"])
|
||||
|
||||
_chat_executor.submit(_bg_embed_all)
|
||||
msg = f"Embedding iniciado — {total_docs} documentos em {len(tables_used)} tabelas ({', '.join(tables_used)})"
|
||||
if skipped_tables:
|
||||
msg += f". Ignoradas (nao registradas): {', '.join(skipped_tables)}"
|
||||
return {
|
||||
"ok": True, "task_id": task_id, "message": msg,
|
||||
"tables": tables_used, "skipped": skipped_tables, "total_documents": total_docs, "tenancy": tenancy
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/embeddings/status/{task_id}")
|
||||
async def embedding_status(task_id: str, u=Depends(current_user)):
|
||||
"""Check embedding task progress."""
|
||||
st = get_embed_status(task_id)
|
||||
if not st:
|
||||
return {"status": "unknown", "message": "Task not found"}
|
||||
if st.get("user_id") and st["user_id"] != u["id"] and u["role"] != "admin":
|
||||
return {"status": "unknown", "message": "Task not found"}
|
||||
return {k: v for k, v in st.items() if k != "user_id"}
|
||||
|
||||
|
||||
@router.post("/api/embeddings/report/{rid}/section")
|
||||
async def embed_report_section(rid: str, req: dict, bg: BackgroundTasks, u=Depends(require("admin","user"))):
|
||||
"""Embed all CSV files from a specific report section into the mapped ADB table."""
|
||||
_verify_report_access(rid, u)
|
||||
import csv as csvmod
|
||||
vid = req.get("adb_config_id")
|
||||
file_names: list = req.get("file_names", [])
|
||||
if not vid:
|
||||
# Auto-detect first active ADB
|
||||
with db() as c:
|
||||
adb = c.execute("SELECT id FROM adb_vector_configs WHERE is_active=1 LIMIT 1").fetchone()
|
||||
if not adb: raise HTTPException(400, "No active ADB config found")
|
||||
vid = adb["id"]
|
||||
if not file_names: raise HTTPException(400, "file_names is required")
|
||||
|
||||
with db() as c:
|
||||
r = c.execute("SELECT tenancy_name, config_id FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone()
|
||||
if not r: raise HTTPException(404)
|
||||
rdir = REPORTS / rid
|
||||
tenancy = r["tenancy_name"] or "unknown"
|
||||
|
||||
# Read extract_date
|
||||
summary = rdir / "cis_summary_report.csv"
|
||||
extract_date = ""
|
||||
if summary.exists():
|
||||
with open(summary, "r", encoding="utf-8") as f:
|
||||
first = next(csvmod.DictReader(f), {})
|
||||
extract_date = first.get("extract_date", "")
|
||||
|
||||
cfg, gc = _get_adb_and_genai(vid, oci_config_id=r["config_id"] if r["config_id"] else None, user_id=u["id"])
|
||||
|
||||
# Load registered tables for validation
|
||||
with db() as c:
|
||||
registered = {t["table_name"].lower() for t in
|
||||
c.execute("SELECT table_name FROM adb_vector_tables WHERE adb_config_id=? AND is_active=1", (vid,)).fetchall()}
|
||||
|
||||
# Group files by target table and validate
|
||||
import array
|
||||
table_docs: dict[str, list] = {}
|
||||
missing_tables: list[str] = []
|
||||
for fname in file_names:
|
||||
csv_path = rdir / fname
|
||||
if not csv_path.exists(): continue
|
||||
table = _resolve_table_for_csv(fname)
|
||||
if not table: continue
|
||||
if table.lower() not in registered:
|
||||
if table not in missing_tables:
|
||||
missing_tables.append(table)
|
||||
continue
|
||||
if fname == "cis_summary_report.csv":
|
||||
docs = _chunk_summary_csv(str(csv_path), tenancy, extract_date)
|
||||
else:
|
||||
docs = _chunk_findings_csv(str(csv_path), tenancy, extract_date)
|
||||
if docs:
|
||||
table_docs.setdefault(table, []).extend(docs)
|
||||
|
||||
if missing_tables and not table_docs:
|
||||
raise HTTPException(400, f"Tabela(s) nao registrada(s) no ADB: {', '.join(missing_tables)}. Registre em Configuracoes > ADB Vector.")
|
||||
if not table_docs:
|
||||
raise HTTPException(400, "No embeddable content found in the selected files")
|
||||
|
||||
total_docs = sum(len(d) for d in table_docs.values())
|
||||
tables_used = list(table_docs.keys())
|
||||
task_id = str(uuid.uuid4())
|
||||
set_embed_status(task_id, {
|
||||
"status": "running", "table": ", ".join(tables_used), "tenancy": tenancy,
|
||||
"inserted": 0, "total": total_docs, "user_id": u["id"], "message": f"Embedding {total_docs} docs em {', '.join(tables_used)}..."
|
||||
})
|
||||
|
||||
def _bg():
|
||||
default_model = cfg.get("embedding_model_id", "cohere.embed-v4.0")
|
||||
inserted = 0
|
||||
skipped = 0
|
||||
processed = 0
|
||||
for tbl, docs in table_docs.items():
|
||||
purged = _purge_table_by_tenancy(cfg, tbl, tenancy, extract_date)
|
||||
if purged:
|
||||
update_embed_status(task_id, {"message": f"Purged {purged} old docs from {tbl}..."})
|
||||
_auto_register_table(cfg["id"], tbl)
|
||||
emb_model = default_model
|
||||
try:
|
||||
actual_dim = _get_table_embedding_dim(cfg, tbl)
|
||||
if actual_dim and actual_dim in _DIM_TO_MODEL:
|
||||
emb_model = _DIM_TO_MODEL[actual_dim]
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
conn = _get_adb_connection(cfg)
|
||||
cur = conn.cursor()
|
||||
for doc in docs:
|
||||
try:
|
||||
content = doc.get("content", "")
|
||||
if not content:
|
||||
skipped += 1
|
||||
processed += 1
|
||||
continue
|
||||
embedding = _embed_text(content, gc, emb_model)
|
||||
vec = array.array('f', [float(x) for x in embedding])
|
||||
metadata = _build_metadata_json(tenancy=doc.get("tenancy", tenancy), section=doc.get("section", ""), report_date=extract_date, user_id=u["id"])
|
||||
cur.execute(f'INSERT INTO "{tbl}" (ID, TEXT, EMBEDDING, METADATA) VALUES (HEXTORAW(:1), :2, :3, :4)',
|
||||
[uuid.uuid4().hex.upper(), content, vec, metadata])
|
||||
inserted += 1
|
||||
except Exception as e:
|
||||
skipped += 1
|
||||
err_str = str(e)
|
||||
if "message" in err_str:
|
||||
import re as _re
|
||||
m = _re.search(r'"message"\s*:\s*"(.*?)"', err_str)
|
||||
log.warning(f"Embed skip in {tbl}: {m.group(1)[:200] if m else err_str[:200]}")
|
||||
else:
|
||||
log.warning(f"Embed skip in {tbl}: {err_str[:200]}")
|
||||
processed += 1
|
||||
update_embed_status(task_id, {
|
||||
"inserted": inserted, "skipped": skipped, "processed": processed, "total": total_docs,
|
||||
"message": f"{tbl}: {inserted} OK, {skipped} falhas ({processed}/{total_docs})"
|
||||
})
|
||||
conn.commit(); cur.close(); conn.close()
|
||||
except Exception as e:
|
||||
log.error(f"Embed connection error {tbl}: {e}")
|
||||
msg = f"{inserted}/{total_docs} embeddings OK"
|
||||
if skipped: msg += f", {skipped} falhas"
|
||||
msg += f" em {', '.join(tables_used)} (tenancy: {tenancy})"
|
||||
update_embed_status(task_id, {"status": "done", "inserted": inserted, "skipped": skipped, "processed": processed, "message": msg})
|
||||
_audit(u["id"], u["username"], "embed_section", rid, msg)
|
||||
|
||||
_chat_executor.submit(_bg)
|
||||
return {"ok": True, "task_id": task_id, "tables": tables_used, "total": total_docs,
|
||||
"message": f"Embedding de {total_docs} docs iniciado ({', '.join(tables_used)})"}
|
||||
|
||||
|
||||
@router.post("/api/embeddings/report/{rid}/file/{fid}")
|
||||
async def embed_report_file(rid: str, fid: str, req: dict, bg: BackgroundTasks, u=Depends(require("admin","user"))):
|
||||
"""Embed a specific report file (CSV, JSON, TXT, etc.) into ADB vector store."""
|
||||
_verify_report_access(rid, u)
|
||||
vid = req.get("adb_config_id")
|
||||
if not vid: raise HTTPException(400, "adb_config_id is required")
|
||||
_verify_config_access("adb", vid, u)
|
||||
with db() as c:
|
||||
r = c.execute("SELECT tenancy_name, config_id FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone()
|
||||
if not r: raise HTTPException(404, "Report not found or not completed")
|
||||
f = c.execute("SELECT * FROM report_files WHERE id=? AND report_id=?", (fid, rid)).fetchone()
|
||||
if not f: raise HTTPException(404, "File not found")
|
||||
p = Path(f["file_path"])
|
||||
if not p.exists(): raise HTTPException(404, "File not found on disk")
|
||||
fname = f["file_name"].lower()
|
||||
allowed = ('.txt', '.csv', '.json', '.md', '.pdf')
|
||||
if not any(fname.endswith(ext) for ext in allowed):
|
||||
raise HTTPException(400, f"Formatos aceitos para embedding: {', '.join(allowed)}")
|
||||
raw = p.read_bytes()
|
||||
if fname.endswith('.pdf'):
|
||||
content = _extract_pdf_text(raw)
|
||||
else:
|
||||
content = raw.decode("utf-8", errors="replace")
|
||||
if not content.strip(): raise HTTPException(400, "File is empty")
|
||||
documents = _chunk_text_file(content, f["file_name"])
|
||||
if not documents: raise HTTPException(400, "No content chunks found")
|
||||
cfg, gc = _get_adb_and_genai(vid, oci_config_id=r["config_id"] if r["config_id"] else None, user_id=u["id"])
|
||||
target_table = req.get("table_name") or None
|
||||
tenancy = r["tenancy_name"] or "unknown"
|
||||
task_id = str(uuid.uuid4())
|
||||
bg.add_task(_ingest_documents_task, cfg, gc, documents, u["id"], u["username"],
|
||||
table_name=target_table, tenancy=tenancy, task_id=task_id)
|
||||
_audit(u["id"], u["username"], "embed_report_file", f"{rid}/{fid}", f"{f['file_name']}, {len(documents)} chunks, tenancy={tenancy}")
|
||||
return {"ok": True, "task_id": task_id, "message": f"Embedding de {f['file_name']} iniciado ({len(documents)} chunks)", "chunks": len(documents)}
|
||||
|
||||
@router.post("/api/embeddings/upload")
|
||||
async def embed_upload(adb_config_id: str = Form(...), table_name: str = Form(""), file: UploadFile = File(...), bg: BackgroundTasks = None, u=Depends(require("admin","user"))):
|
||||
_verify_config_access("adb", adb_config_id, u)
|
||||
fname = file.filename.lower()
|
||||
allowed = ('.txt', '.pdf', '.csv', '.json', '.md')
|
||||
if not any(fname.endswith(ext) for ext in allowed):
|
||||
raise HTTPException(400, f"Formatos aceitos: {', '.join(allowed)}")
|
||||
await validate_upload(file, allowed)
|
||||
file.file.seek(0)
|
||||
raw = await file.read()
|
||||
if fname.endswith('.pdf'):
|
||||
content = _extract_pdf_text(raw)
|
||||
else:
|
||||
content = raw.decode("utf-8", errors="replace")
|
||||
if not content.strip(): raise HTTPException(400, "File is empty")
|
||||
target_table = table_name.strip() or None
|
||||
# Use CIS-specific chunking for cisrecom table (segments by recommendation number with overlap)
|
||||
if target_table and 'cisrecom' in target_table.lower():
|
||||
documents = _chunk_cis_pdf(content, file.filename)
|
||||
if not documents:
|
||||
documents = _chunk_text_file(content, file.filename) # fallback
|
||||
else:
|
||||
documents = _chunk_text_file(content, file.filename)
|
||||
if not documents: raise HTTPException(400, "No content chunks found")
|
||||
cfg, gc = _get_adb_and_genai(adb_config_id, user_id=u["id"])
|
||||
bg.add_task(_ingest_documents_task, cfg, gc, documents, u["id"], u["username"], table_name=target_table)
|
||||
_audit(u["id"], u["username"], "embed_upload", file.filename, f"{len(documents)} chunks")
|
||||
return {"ok": True, "message": f"Embedding de {len(documents)} chunks iniciado", "chunks": len(documents), "filename": file.filename}
|
||||
|
||||
@router.post("/api/embeddings/upload-url")
|
||||
async def embed_upload_url(
|
||||
adb_config_id: str = Form(...),
|
||||
table_name: str = Form(""),
|
||||
url: str = Form(...),
|
||||
bg: BackgroundTasks = None,
|
||||
u=Depends(require("admin", "user"))
|
||||
):
|
||||
_verify_config_access("adb", adb_config_id, u)
|
||||
import requests as req
|
||||
url = url.strip()
|
||||
if not url.startswith(("http://", "https://")):
|
||||
raise HTTPException(400, "URL invalida — deve comecar com http:// ou https://")
|
||||
try:
|
||||
resp = req.get(url, timeout=30, headers={"User-Agent": "Mozilla/5.0 AI-Agent/1.0"})
|
||||
resp.raise_for_status()
|
||||
except Exception as e:
|
||||
raise HTTPException(400, f"Erro ao acessar URL: {str(e)[:300]}")
|
||||
ct = resp.headers.get("content-type", "")
|
||||
if "pdf" in ct:
|
||||
content = _extract_pdf_text(resp.content)
|
||||
elif "html" in ct or "text" in ct:
|
||||
content = _extract_text_from_html(resp.text)
|
||||
else:
|
||||
content = resp.text
|
||||
if not content or not content.strip():
|
||||
raise HTTPException(400, "Nenhum conteudo extraido da URL")
|
||||
documents = _chunk_text_file(content, url)
|
||||
if not documents:
|
||||
raise HTTPException(400, "Nenhum chunk gerado do conteudo")
|
||||
cfg, gc = _get_adb_and_genai(adb_config_id, user_id=u["id"])
|
||||
target_table = table_name.strip() or None
|
||||
bg.add_task(_ingest_documents_task, cfg, gc, documents, u["id"], u["username"], table_name=target_table)
|
||||
_audit(u["id"], u["username"], "embed_url", url, f"{len(documents)} chunks")
|
||||
return {"ok": True, "message": f"Embedding de {len(documents)} chunks iniciado", "chunks": len(documents), "url": url}
|
||||
|
||||
@router.post("/api/embeddings/consult")
|
||||
async def consult_embeddings(req: ConsultQuery, u=Depends(current_user)):
|
||||
"""Query embeddings via vector search + GenAI to get a formatted answer."""
|
||||
if not req.query.strip():
|
||||
raise HTTPException(400, "Query nao pode ser vazia")
|
||||
adb_configs = _get_active_adb_configs(u["id"])
|
||||
if not adb_configs:
|
||||
raise HTTPException(400, "Nenhuma conexao ADB ativa configurada")
|
||||
# Resolve tenancy for filtered search
|
||||
rag_tenancy = None
|
||||
if req.oci_config_id:
|
||||
with db() as c:
|
||||
oci_row = c.execute("SELECT tenancy_name FROM oci_configs WHERE id=?", (req.oci_config_id,)).fetchone()
|
||||
if oci_row:
|
||||
rag_tenancy = oci_row["tenancy_name"]
|
||||
log.info(f"Consult: filtering by tenancy '{rag_tenancy}'")
|
||||
|
||||
# Detect CIS recommendation number in query for exact text filtering
|
||||
import re as _re
|
||||
cis_match = _re.search(r'(?:cis|recommendation)\s*(\d+\.\d+)', req.query, _re.IGNORECASE)
|
||||
cis_text_filter = f"CIS Recommendation: {cis_match.group(1)}" if cis_match else None
|
||||
if cis_text_filter:
|
||||
log.info(f"Consult: detected CIS filter '{cis_text_filter}'")
|
||||
|
||||
# Collect results from all active ADB configs + tables
|
||||
all_docs = []
|
||||
rag_errors = []
|
||||
for adb_cfg in adb_configs:
|
||||
try:
|
||||
emb_genai = _resolve_embed_config(oci_config_id=adb_cfg.get("oci_config_id"), user_id=u["id"])
|
||||
except Exception as e:
|
||||
log.warning(f"Consult: resolve config failed for {adb_cfg['id']}: {e}")
|
||||
continue
|
||||
tables = _get_tables_for_config(adb_cfg["id"], active_only=True)
|
||||
if req.table_name:
|
||||
tables = [t for t in tables if t["table_name"] == req.table_name]
|
||||
all_table_names = [t["table_name"] for t in tables if t["table_name"]]
|
||||
# Smart skip
|
||||
relevant = _relevant_tables(req.query, all_table_names) if not req.table_name else all_table_names
|
||||
skipped = set(all_table_names) - set(relevant)
|
||||
if skipped:
|
||||
log.info(f"Consult: skipped {', '.join(skipped)}")
|
||||
# Auto-detect model
|
||||
emb_model = adb_cfg.get("embedding_model_id", "cohere.embed-v4.0")
|
||||
for tbl_name in relevant:
|
||||
try:
|
||||
dim = _get_table_embedding_dim(adb_cfg, tbl_name)
|
||||
if dim and dim in _DIM_TO_MODEL:
|
||||
emb_model = _DIM_TO_MODEL[dim]
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
query_embedding = _embed_text(req.query, emb_genai, emb_model)
|
||||
tbl_top_k = 10 if cis_text_filter else 3
|
||||
docs = _vector_search_multi(adb_cfg, query_embedding, relevant,
|
||||
top_k_per_table=tbl_top_k, tenancy=rag_tenancy,
|
||||
text_filter=cis_text_filter, user_id=u["id"])
|
||||
all_docs.extend(docs)
|
||||
if docs:
|
||||
sources = {}
|
||||
for d in docs:
|
||||
sources[d["source"]] = sources.get(d["source"], 0) + 1
|
||||
log.info(f"Consult: {len(docs)} docs — {', '.join(f'{k}:{v}' for k,v in sources.items())}")
|
||||
except Exception as e:
|
||||
err = str(e)[:150]
|
||||
log.warning(f"Consult: search failed: {err}")
|
||||
if "DPY-6001" in str(e) or "DPY-6005" in str(e) or "timeout" in str(e).lower():
|
||||
rag_errors.append(f"ADB offline ou timeout ({adb_cfg.get('config_name','?')})")
|
||||
if not all_docs:
|
||||
if rag_errors:
|
||||
return {"answer": "\u26a0\ufe0f " + "; ".join(set(rag_errors)) + ". A base de conhecimento nao esta disponivel no momento.", "documents": [], "total": 0}
|
||||
return {"answer": "Nenhum resultado encontrado nas bases vetoriais.", "documents": [], "total": 0}
|
||||
# Sort by distance and take top results
|
||||
all_docs.sort(key=lambda d: d.get("distance", 999))
|
||||
top_limit = 15 if cis_text_filter else 8
|
||||
top_docs = all_docs[:top_limit]
|
||||
# Build context with dates and sources
|
||||
rag_context = _build_rag_context(top_docs, max_total_chars=16000 if cis_text_filter else 12000)
|
||||
if rag_errors:
|
||||
rag_context += "\n\n\u26a0\ufe0f Algumas bases nao puderam ser consultadas: " + "; ".join(set(rag_errors))
|
||||
augmented = RAG_CONTEXT_TEMPLATE.format(context=rag_context, question=req.query)
|
||||
# Get GenAI config for answering — try saved config first, then auto-resolve from OCI
|
||||
gc = None
|
||||
with db() as c:
|
||||
gc_row = c.execute("SELECT * FROM genai_configs WHERE user_id=? AND is_default=1 ORDER BY created_at DESC", (u["id"],)).fetchone()
|
||||
if not gc_row:
|
||||
gc_row = c.execute("SELECT * FROM genai_configs WHERE user_id=? ORDER BY created_at DESC", (u["id"],)).fetchone()
|
||||
if gc_row:
|
||||
gc = dict(gc_row)
|
||||
else:
|
||||
# Auto-resolve: build GenAI config from OCI credentials + default model
|
||||
try:
|
||||
resolved = _resolve_embed_config(user_id=u["id"])
|
||||
gc = {
|
||||
"oci_config_id": resolved["oci_config_id"],
|
||||
"endpoint": resolved.get("endpoint", f"https://inference.generativeai.{resolved.get('genai_region','us-ashburn-1')}.oci.oraclecloud.com"),
|
||||
"compartment_id": resolved.get("compartment_id", ""),
|
||||
"genai_region": resolved.get("genai_region", "us-ashburn-1"),
|
||||
"model_id": "openai.gpt-5.2",
|
||||
"model_ocid": "",
|
||||
"serving_type": "ON_DEMAND",
|
||||
"temperature": 0.3,
|
||||
"max_tokens": 8000,
|
||||
"top_p": 0.9,
|
||||
"top_k": 1,
|
||||
"frequency_penalty": 0,
|
||||
"presence_penalty": 0,
|
||||
}
|
||||
log.info(f"Consult: auto-resolved GenAI config from OCI, model=openai.gpt-4.1")
|
||||
except Exception as e:
|
||||
log.warning(f"Consult: no GenAI config available: {e}")
|
||||
doc_list = [{"content": d.get("content", "")[:500], "source": d.get("source", ""), "distance": round(d.get("distance", 0), 4), "metadata": d.get("metadata", "")} for d in top_docs]
|
||||
parts = []
|
||||
for i, d in enumerate(top_docs, 1):
|
||||
content = d.get("content", "")
|
||||
if len(content) > 800: content = content[:800] + "..."
|
||||
parts.append(f"**Documento {i}** — `{d.get('source', '?')}` (distancia: {d.get('distance', 0):.4f})\n\n{content}")
|
||||
return {"answer": "\n\n---\n\n".join(parts), "documents": doc_list, "total": len(all_docs)}
|
||||
try:
|
||||
gc["system_prompt"] = CONSULT_SYSTEM_PROMPT
|
||||
answer, _, _ = _call_genai(gc, augmented)
|
||||
except Exception as e:
|
||||
log.error(f"Consult GenAI error: {e}")
|
||||
answer = f"Erro ao consultar GenAI: {str(e)[:300]}"
|
||||
doc_list = [{"content": d.get("content", "")[:500], "source": d.get("source", ""), "distance": round(d.get("distance", 0), 4), "metadata": d.get("metadata", "")} for d in top_docs]
|
||||
return {"answer": answer, "documents": doc_list, "total": len(all_docs)}
|
||||
|
||||
@router.get("/api/embeddings/{vid}/list")
|
||||
async def list_embeddings(vid: str, table_name: str = Query(""), limit: int = Query(50), offset: int = Query(0), u=Depends(current_user)):
|
||||
_verify_config_access("adb", vid, u)
|
||||
with db() as c:
|
||||
cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
||||
if not cfg: raise HTTPException(404)
|
||||
try:
|
||||
conn = _get_adb_connection(dict(cfg))
|
||||
cur = conn.cursor()
|
||||
table_name = table_name.strip() or cfg["table_name"]
|
||||
if not table_name: raise HTTPException(400, "Nenhuma tabela selecionada")
|
||||
cur.execute(f'SELECT COUNT(*) FROM "{table_name}"')
|
||||
total = cur.fetchone()[0]
|
||||
cur.execute(f"""
|
||||
SELECT ID, METADATA FROM "{table_name}"
|
||||
OFFSET :1 ROWS FETCH NEXT :2 ROWS ONLY
|
||||
""", [offset, limit])
|
||||
rows = []
|
||||
for row in cur:
|
||||
rid = row[0].hex() if isinstance(row[0], bytes) else str(row[0])
|
||||
meta = row[1]
|
||||
if hasattr(meta, 'read'): meta = meta.read()
|
||||
rows.append({"id": rid, "metadata": meta})
|
||||
cur.close(); conn.close()
|
||||
return {"total": total, "offset": offset, "limit": limit, "documents": rows}
|
||||
except Exception as e:
|
||||
raise HTTPException(500, f"Erro ao listar embeddings: {str(e)[:500]}")
|
||||
|
||||
@router.delete("/api/embeddings/{vid}/{doc_id}")
|
||||
async def delete_embedding(vid: str, doc_id: str, table_name: str = Query(""), u=Depends(require("admin","user"))):
|
||||
_verify_config_access("adb", vid, u, require_owner=True)
|
||||
with db() as c:
|
||||
cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
||||
if not cfg: raise HTTPException(404)
|
||||
try:
|
||||
conn = _get_adb_connection(dict(cfg))
|
||||
cur = conn.cursor()
|
||||
table_name = table_name.strip() or cfg["table_name"]
|
||||
if not table_name: raise HTTPException(400, "Nenhuma tabela selecionada")
|
||||
cur.execute(f'DELETE FROM "{table_name}" WHERE ID = :1', [doc_id])
|
||||
conn.commit()
|
||||
cur.close(); conn.close()
|
||||
return {"ok": True}
|
||||
except Exception as e:
|
||||
raise HTTPException(500, f"Erro ao deletar: {str(e)[:500]}")
|
||||
|
||||
@router.post("/api/embeddings/{vid}/purge")
|
||||
async def purge_embeddings(vid: str, req: dict, u=Depends(require("admin","user"))):
|
||||
"""Delete old embeddings from a table, optionally filtered by tenancy."""
|
||||
_verify_config_access("adb", vid, u, require_owner=True)
|
||||
table_name = req.get("table_name", "").strip()
|
||||
tenancy = req.get("tenancy", "").strip()
|
||||
if not table_name: raise HTTPException(400, "table_name e obrigatorio")
|
||||
with db() as c:
|
||||
cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
||||
if not cfg: raise HTTPException(404)
|
||||
try:
|
||||
conn = _get_adb_connection(dict(cfg))
|
||||
cur = conn.cursor()
|
||||
if tenancy:
|
||||
cur.execute(f'SELECT COUNT(*) FROM "{table_name}" WHERE METADATA LIKE :1',
|
||||
[f'%"tenancy":"{tenancy}"%'])
|
||||
count = cur.fetchone()[0]
|
||||
cur.execute(f'DELETE FROM "{table_name}" WHERE METADATA LIKE :1',
|
||||
[f'%"tenancy":"{tenancy}"%'])
|
||||
else:
|
||||
cur.execute(f'SELECT COUNT(*) FROM "{table_name}"')
|
||||
count = cur.fetchone()[0]
|
||||
cur.execute(f'DELETE FROM "{table_name}"')
|
||||
conn.commit()
|
||||
cur.close(); conn.close()
|
||||
_audit(u["id"], u["username"], "purge_embeddings", vid, f"table={table_name}, tenancy={tenancy or 'ALL'}, deleted={count}")
|
||||
return {"ok": True, "deleted": count, "table": table_name, "tenancy": tenancy or "ALL"}
|
||||
except Exception as e:
|
||||
raise HTTPException(500, f"Erro ao limpar: {str(e)[:500]}")
|
||||
91
backend/routes/genai.py
Normal file
91
backend/routes/genai.py
Normal file
@@ -0,0 +1,91 @@
|
||||
"""GenAI configuration routes — model catalog, CRUD, test."""
|
||||
import uuid
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from database import db
|
||||
from auth.jwt_auth import current_user, require, _audit, _config_log, _verify_config_access
|
||||
from config import GENAI_MODELS, GENAI_REGIONS, EMBEDDING_MODELS, OCI_REGIONS
|
||||
from models import GenAIConfigReq
|
||||
from services.genai import _call_genai
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/api/genai/models")
|
||||
async def list_genai_models(u=Depends(current_user)):
|
||||
return JSONResponse(
|
||||
content={"models": GENAI_MODELS, "regions": GENAI_REGIONS, "embedding_models": EMBEDDING_MODELS, "oci_regions": OCI_REGIONS},
|
||||
headers={"Cache-Control": "max-age=3600"},
|
||||
)
|
||||
|
||||
@router.post("/api/genai/config")
|
||||
async def save_genai(req: GenAIConfigReq, u=Depends(require("admin","user"))):
|
||||
gid = str(uuid.uuid4())
|
||||
ep = req.endpoint or f"https://inference.generativeai.{req.genai_region}.oci.oraclecloud.com"
|
||||
with db() as c:
|
||||
if req.is_default:
|
||||
c.execute("UPDATE genai_configs SET is_default=0 WHERE user_id=?", (u["id"],))
|
||||
c.execute(
|
||||
"""INSERT INTO genai_configs (id,user_id,name,oci_config_id,model_id,model_ocid,compartment_id,
|
||||
genai_region,endpoint,serving_type,dedicated_endpoint_id,temperature,max_tokens,top_p,top_k,
|
||||
frequency_penalty,presence_penalty,reasoning_effort,is_default) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
|
||||
(gid, u["id"], req.name, req.oci_config_id, req.model_id, req.model_ocid,
|
||||
req.compartment_id, req.genai_region, ep, req.serving_type, req.dedicated_endpoint_id,
|
||||
req.temperature, req.max_tokens, req.top_p, req.top_k,
|
||||
req.frequency_penalty, req.presence_penalty, req.reasoning_effort, int(req.is_default)))
|
||||
_audit(u["id"], u["username"], "save_genai_config", gid, req.model_id)
|
||||
_config_log("genai", gid, req.name, "success", "save", f"Modelo salvo: {req.model_id} ({req.genai_region})", u["id"], u["username"])
|
||||
return {"id": gid, "model_id": req.model_id, "endpoint": ep}
|
||||
|
||||
@router.get("/api/genai/configs")
|
||||
async def list_genai(u=Depends(current_user)):
|
||||
with db() as c:
|
||||
if u["role"]=="admin": rows=c.execute("SELECT * FROM genai_configs").fetchall()
|
||||
else: rows=c.execute("SELECT * FROM genai_configs WHERE user_id=?",(u["id"],)).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
@router.delete("/api/genai/configs/{gid}")
|
||||
async def del_genai(gid: str, u=Depends(require("admin","user"))):
|
||||
_verify_config_access("genai", gid, u, require_owner=True)
|
||||
with db() as c: c.execute("DELETE FROM genai_configs WHERE id=?", (gid,))
|
||||
return {"ok": True}
|
||||
|
||||
@router.put("/api/genai/configs/{gid}")
|
||||
async def update_genai(gid: str, req: GenAIConfigReq, u=Depends(require("admin","user"))):
|
||||
with db() as c:
|
||||
existing = c.execute("SELECT * FROM genai_configs WHERE id=?", (gid,)).fetchone()
|
||||
if not existing: raise HTTPException(404)
|
||||
if u["role"] != "admin" and existing["user_id"] != u["id"]: raise HTTPException(403)
|
||||
ep = req.endpoint or f"https://inference.generativeai.{req.genai_region}.oci.oraclecloud.com"
|
||||
with db() as c:
|
||||
if req.is_default:
|
||||
c.execute("UPDATE genai_configs SET is_default=0 WHERE user_id=?", (u["id"],))
|
||||
c.execute(
|
||||
"""UPDATE genai_configs SET name=?,oci_config_id=?,model_id=?,model_ocid=?,compartment_id=?,
|
||||
genai_region=?,endpoint=?,serving_type=?,dedicated_endpoint_id=?,temperature=?,max_tokens=?,
|
||||
top_p=?,top_k=?,frequency_penalty=?,presence_penalty=?,reasoning_effort=?,is_default=? WHERE id=?""",
|
||||
(req.name, req.oci_config_id, req.model_id, req.model_ocid,
|
||||
req.compartment_id, req.genai_region, ep, req.serving_type, req.dedicated_endpoint_id,
|
||||
req.temperature, req.max_tokens, req.top_p, req.top_k,
|
||||
req.frequency_penalty, req.presence_penalty, req.reasoning_effort, int(req.is_default), gid))
|
||||
_audit(u["id"], u["username"], "update_genai_config", gid, req.model_id)
|
||||
_config_log("genai", gid, req.name, "success", "save", f"Modelo atualizado: {req.model_id} ({req.genai_region})", u["id"], u["username"])
|
||||
return {"id": gid, "model_id": req.model_id, "endpoint": ep}
|
||||
|
||||
@router.post("/api/genai/test/{gid}")
|
||||
async def test_genai(gid: str, u=Depends(require("admin","user"))):
|
||||
_verify_config_access("genai", gid, u)
|
||||
with db() as c:
|
||||
gc = c.execute("SELECT * FROM genai_configs WHERE id=?",(gid,)).fetchone()
|
||||
if not gc: raise HTTPException(404)
|
||||
gname = gc["name"]
|
||||
try:
|
||||
resp, _, _ = _call_genai(dict(gc), "Say 'connection successful' in one short sentence.")
|
||||
_config_log("genai", gid, gname, "success", "test", f"GenAI OK: {resp[:200]}", u["id"], u["username"])
|
||||
return {"status":"success","message":"GenAI OK","response":resp[:300]}
|
||||
except Exception as e:
|
||||
msg = str(e)[:500]
|
||||
_config_log("genai", gid, gname, "error", "test", msg, u["id"], u["username"])
|
||||
return {"status":"error","message":msg}
|
||||
139
backend/routes/mcp.py
Normal file
139
backend/routes/mcp.py
Normal file
@@ -0,0 +1,139 @@
|
||||
"""MCP server routes — CRUD, toggle, upload, link-adb, discover-tools, update tools."""
|
||||
import json
|
||||
import uuid
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends, UploadFile, File, Query
|
||||
|
||||
from database import db
|
||||
from auth.jwt_auth import current_user, require, _audit, _config_log, _verify_config_access
|
||||
from config import MCP_DIR, log
|
||||
from models import MCPServerReq
|
||||
from utils import validate_upload
|
||||
from services.mcp_client import _discover_mcp_tools
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/api/mcp/servers")
|
||||
async def register_mcp(req: MCPServerReq, u=Depends(require("admin","user"))):
|
||||
mid = str(uuid.uuid4())
|
||||
is_global_val = 1 if (req.is_global and u["role"] == "admin") else 0
|
||||
with db() as c:
|
||||
c.execute(
|
||||
"INSERT INTO mcp_servers (id,user_id,name,description,server_type,command,args,env_vars,url,module_path,tools,linked_adb_id,is_global) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(mid, u["id"], req.name, req.description, req.server_type,
|
||||
req.command, json.dumps(req.args) if req.args else None,
|
||||
json.dumps(req.env_vars) if req.env_vars else None, req.url, req.module_path,
|
||||
json.dumps(req.tools) if req.tools else None, req.linked_adb_id, is_global_val))
|
||||
_audit(u["id"], u["username"], "register_mcp", mid, req.name)
|
||||
_config_log("mcp", mid, req.name, "success", "save", f"MCP registrado: {req.name} ({req.server_type})", u["id"], u["username"])
|
||||
return {"id": mid, "name": req.name, "server_type": req.server_type}
|
||||
|
||||
@router.get("/api/mcp/servers")
|
||||
async def list_mcp(u=Depends(current_user)):
|
||||
with db() as c:
|
||||
if u["role"]=="admin": rows=c.execute("SELECT * FROM mcp_servers ORDER BY created_at DESC").fetchall()
|
||||
else: rows=c.execute("SELECT * FROM mcp_servers WHERE user_id=? OR is_global=1 ORDER BY created_at DESC",(u["id"],)).fetchall()
|
||||
res = []
|
||||
for r in rows:
|
||||
d = dict(r)
|
||||
for k in ("args","env_vars","tools"):
|
||||
if d.get(k):
|
||||
try: d[k] = json.loads(d[k])
|
||||
except Exception as e: log.warning(f"Failed to parse MCP server field '{k}': {e}")
|
||||
res.append(d)
|
||||
return res
|
||||
|
||||
@router.delete("/api/mcp/servers/{mid}")
|
||||
async def del_mcp(mid: str, u=Depends(require("admin","user"))):
|
||||
_verify_config_access("mcp", mid, u, require_owner=True)
|
||||
with db() as c: c.execute("DELETE FROM mcp_servers WHERE id=?", (mid,))
|
||||
return {"ok": True}
|
||||
|
||||
@router.put("/api/mcp/servers/{mid}")
|
||||
async def update_mcp(mid: str, req: MCPServerReq, u=Depends(require("admin","user"))):
|
||||
with db() as c:
|
||||
existing = c.execute("SELECT * FROM mcp_servers WHERE id=?", (mid,)).fetchone()
|
||||
if not existing: raise HTTPException(404)
|
||||
if u["role"] != "admin" and existing["user_id"] != u["id"]: raise HTTPException(403)
|
||||
is_global_val = 1 if (req.is_global and u["role"] == "admin") else (existing["is_global"] if u["role"] != "admin" else (1 if req.is_global else 0))
|
||||
with db() as c:
|
||||
c.execute(
|
||||
"UPDATE mcp_servers SET name=?,description=?,server_type=?,command=?,args=?,env_vars=?,url=?,tools=?,linked_adb_id=?,is_global=? WHERE id=?",
|
||||
(req.name, req.description, req.server_type, req.command,
|
||||
json.dumps(req.args) if req.args else None,
|
||||
json.dumps(req.env_vars) if req.env_vars else None, req.url,
|
||||
json.dumps(req.tools) if req.tools else None, req.linked_adb_id, is_global_val, mid))
|
||||
_audit(u["id"], u["username"], "update_mcp", mid, req.name)
|
||||
_config_log("mcp", mid, req.name, "success", "save", f"MCP atualizado: {req.name} ({req.server_type})", u["id"], u["username"])
|
||||
return {"id": mid, "name": req.name, "server_type": req.server_type}
|
||||
|
||||
@router.put("/api/mcp/servers/{mid}/toggle")
|
||||
async def toggle_mcp(mid: str, u=Depends(require("admin","user"))):
|
||||
_verify_config_access("mcp", mid, u, require_owner=True)
|
||||
with db() as c:
|
||||
cur = c.execute("SELECT is_active FROM mcp_servers WHERE id=?",(mid,)).fetchone()
|
||||
if not cur: raise HTTPException(404)
|
||||
c.execute("UPDATE mcp_servers SET is_active=? WHERE id=?",(0 if cur["is_active"] else 1, mid))
|
||||
return {"ok": True, "is_active": not cur["is_active"]}
|
||||
|
||||
@router.post("/api/mcp/servers/{mid}/upload")
|
||||
async def upload_mcp_file(mid: str, file: UploadFile = File(...), u=Depends(require("admin","user"))):
|
||||
await validate_upload(file, {".py", ".js", ".ts", ".json", ".yaml", ".yml", ".toml", ".sh"})
|
||||
file.file.seek(0)
|
||||
sdir = MCP_DIR / mid; sdir.mkdir(parents=True, exist_ok=True)
|
||||
fp = sdir / file.filename
|
||||
fp.write_bytes(await file.read())
|
||||
with db() as c: c.execute("UPDATE mcp_servers SET module_path=? WHERE id=?", (str(fp), mid))
|
||||
return {"ok": True, "path": str(fp)}
|
||||
|
||||
@router.put("/api/mcp/servers/{mid}/link-adb")
|
||||
async def link_mcp_adb(mid: str, adb_id: str = Query(...), u=Depends(require("admin","user"))):
|
||||
with db() as c: c.execute("UPDATE mcp_servers SET linked_adb_id=? WHERE id=?", (adb_id, mid))
|
||||
return {"ok": True, "mcp_id": mid, "linked_adb_id": adb_id}
|
||||
|
||||
@router.post("/api/mcp/servers/{mid}/discover-tools")
|
||||
async def discover_mcp_tools(mid: str, u=Depends(require("admin","user"))):
|
||||
"""Connect to MCP server and discover available tools."""
|
||||
with db() as c:
|
||||
row = c.execute("SELECT * FROM mcp_servers WHERE id=?", (mid,)).fetchone()
|
||||
if not row: raise HTTPException(404)
|
||||
mcp_srv = dict(row)
|
||||
try:
|
||||
discovered = await _discover_mcp_tools(mcp_srv)
|
||||
except Exception as e:
|
||||
raise HTTPException(500, f"Erro ao descobrir tools: {str(e)[:500]}")
|
||||
# Merge with existing tools (keep manually added ones)
|
||||
existing = []
|
||||
if mcp_srv.get("tools"):
|
||||
try: existing = json.loads(mcp_srv["tools"]) if isinstance(mcp_srv["tools"], str) else mcp_srv["tools"]
|
||||
except Exception as e: log.warning(f"Failed to parse existing MCP tools JSON: {e}")
|
||||
existing_names = {t["name"] for t in existing if isinstance(t, dict)}
|
||||
for dt in discovered:
|
||||
if dt["name"] not in existing_names:
|
||||
existing.append(dt)
|
||||
else:
|
||||
# Update existing tool with discovered schema
|
||||
for i, et in enumerate(existing):
|
||||
if isinstance(et, dict) and et["name"] == dt["name"]:
|
||||
existing[i] = dt
|
||||
break
|
||||
with db() as c:
|
||||
c.execute("UPDATE mcp_servers SET tools=? WHERE id=?", (json.dumps(existing), mid))
|
||||
_audit(u["id"], u["username"], "discover_mcp_tools", mid, f"{len(discovered)} tools found")
|
||||
return {"ok": True, "discovered": len(discovered), "total": len(existing), "tools": existing}
|
||||
|
||||
@router.put("/api/mcp/servers/{mid}/tools")
|
||||
async def update_mcp_tools(mid: str, req: dict, u=Depends(require("admin","user"))):
|
||||
"""Manually update tools for an MCP server."""
|
||||
with db() as c:
|
||||
row = c.execute("SELECT id FROM mcp_servers WHERE id=?", (mid,)).fetchone()
|
||||
if not row: raise HTTPException(404)
|
||||
tools = req.get("tools", [])
|
||||
if not isinstance(tools, list): raise HTTPException(400, "tools must be a list")
|
||||
for t in tools:
|
||||
if not isinstance(t, dict) or "name" not in t:
|
||||
raise HTTPException(400, "Each tool must have a 'name' field")
|
||||
with db() as c:
|
||||
c.execute("UPDATE mcp_servers SET tools=? WHERE id=?", (json.dumps(tools), mid))
|
||||
return {"ok": True, "tools": tools}
|
||||
503
backend/routes/oci_config.py
Normal file
503
backend/routes/oci_config.py
Normal file
@@ -0,0 +1,503 @@
|
||||
"""OCI config routes — CRUD, test, OCI CLI terminal (execute, history, completions)."""
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import uuid
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import (
|
||||
APIRouter, HTTPException, Depends, UploadFile, File, Form, Query,
|
||||
)
|
||||
|
||||
from config import OCI_DIR, _chat_executor, log
|
||||
from database import db
|
||||
from auth.crypto import _enc, _dec, _mask, _safe_dec
|
||||
from auth.jwt_auth import current_user, require, _audit, _config_log, _verify_config_access
|
||||
from utils import validate_upload
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# ── OCI Config ────────────────────────────────────────────────────────────────
|
||||
@router.post("/api/oci/config")
|
||||
async def save_oci(
|
||||
tenancy_name: str = Form(...), tenancy_ocid: str = Form(...),
|
||||
user_ocid: str = Form(""), fingerprint: str = Form(""),
|
||||
region: str = Form(...), compartment_id: str = Form(""),
|
||||
key_passphrase: str = Form(""),
|
||||
ssh_public_key: str = Form(""),
|
||||
auth_type: str = Form("api_key"),
|
||||
security_token: str = Form(""),
|
||||
private_key: Optional[UploadFile] = File(None), public_key: Optional[UploadFile] = File(None),
|
||||
u = Depends(require("admin","user"))
|
||||
):
|
||||
if auth_type not in ("api_key", "session_token"):
|
||||
raise HTTPException(400, "auth_type deve ser 'api_key' ou 'session_token'")
|
||||
_PEM_EXTS = {".pem", ".key"}
|
||||
if private_key and private_key.filename:
|
||||
await validate_upload(private_key, _PEM_EXTS)
|
||||
private_key.file.seek(0)
|
||||
if public_key and public_key.filename:
|
||||
await validate_upload(public_key, _PEM_EXTS)
|
||||
public_key.file.seek(0)
|
||||
cid = str(uuid.uuid4()); cdir = OCI_DIR / cid; cdir.mkdir(parents=True)
|
||||
kp = cdir / "oci_api_key.pem"
|
||||
|
||||
if auth_type == "session_token":
|
||||
# Session token auth: requires tenancy_ocid, region, token, and session key
|
||||
if not security_token.strip():
|
||||
shutil.rmtree(cdir, ignore_errors=True)
|
||||
raise HTTPException(400, "Session Token é obrigatório para autenticação por token.")
|
||||
if not private_key or not private_key.filename:
|
||||
shutil.rmtree(cdir, ignore_errors=True)
|
||||
raise HTTPException(400, "A chave de sessão (.pem) é obrigatória para autenticação por token.")
|
||||
key_bytes = await private_key.read()
|
||||
kp.write_bytes(key_bytes); kp.chmod(0o600)
|
||||
# Save security token file
|
||||
token_file = cdir / "token"
|
||||
token_file.write_text(security_token.strip())
|
||||
token_file.chmod(0o600)
|
||||
# Generate config
|
||||
cfg_content = (f"[DEFAULT]\ntenancy={tenancy_ocid}\nregion={region}\n"
|
||||
f"key_file={kp}\nsecurity_token_file={token_file}\n")
|
||||
if user_ocid:
|
||||
cfg_content = f"[DEFAULT]\nuser={user_ocid}\n" + cfg_content.replace("[DEFAULT]\n", "")
|
||||
cfg_file = cdir / "config"
|
||||
cfg_file.write_text(cfg_content); cfg_file.chmod(0o600)
|
||||
# For session_token, fingerprint/user may not be provided — use placeholders
|
||||
user_ocid = user_ocid or "session_token_user"
|
||||
fingerprint = fingerprint or "session_token"
|
||||
else:
|
||||
# API Key auth (original flow)
|
||||
if not user_ocid or not fingerprint:
|
||||
shutil.rmtree(cdir, ignore_errors=True)
|
||||
raise HTTPException(400, "OCID User e Fingerprint são obrigatórios para API Key.")
|
||||
if not private_key or not private_key.filename:
|
||||
shutil.rmtree(cdir, ignore_errors=True)
|
||||
raise HTTPException(400, "Selecione a chave privada (.pem).")
|
||||
key_bytes = await private_key.read()
|
||||
kp.write_bytes(key_bytes); kp.chmod(0o600)
|
||||
key_text = key_bytes.decode("utf-8", errors="ignore")
|
||||
key_is_encrypted = "ENCRYPTED" in key_text or "Proc-Type: 4,ENCRYPTED" in key_text
|
||||
if key_is_encrypted and not key_passphrase:
|
||||
shutil.rmtree(cdir, ignore_errors=True)
|
||||
raise HTTPException(400, "A chave privada está criptografada (ENCRYPTED). Informe a Key Passphrase.")
|
||||
if public_key and public_key.filename:
|
||||
(cdir / "oci_api_key_public.pem").write_bytes(await public_key.read())
|
||||
cfg_file = cdir / "config"
|
||||
cfg_content = (f"[DEFAULT]\nuser={user_ocid}\nfingerprint={fingerprint}\n"
|
||||
f"tenancy={tenancy_ocid}\nregion={region}\nkey_file={kp}\n")
|
||||
if key_passphrase:
|
||||
cfg_content += f"pass_phrase={key_passphrase}\n"
|
||||
cfg_file.write_text(cfg_content); cfg_file.chmod(0o600)
|
||||
|
||||
with db() as c:
|
||||
c.execute("INSERT INTO oci_configs (id,user_id,tenancy_name,tenancy_ocid,user_ocid,fingerprint,region,key_file_path,compartment_id,ssh_public_key,auth_type) VALUES (?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(cid, u["id"], tenancy_name, _enc(tenancy_ocid), _enc(user_ocid), _enc(fingerprint), region, str(kp), _enc(compartment_id) if compartment_id else None, ssh_public_key.strip() if ssh_public_key.strip() else None, auth_type))
|
||||
_audit(u["id"], u["username"], "save_oci_config", cid, f"tenancy={tenancy_name} auth={auth_type}")
|
||||
_config_log("oci", cid, tenancy_name, "success", "save", f"Credencial salva: {tenancy_name} ({region}) [{auth_type}]", u["id"], u["username"])
|
||||
return {"id": cid, "tenancy_name": tenancy_name, "region": region}
|
||||
|
||||
@router.get("/api/oci/configs")
|
||||
async def list_oci(u=Depends(current_user)):
|
||||
with db() as c:
|
||||
cols = "id,user_id,tenancy_name,tenancy_ocid,user_ocid,fingerprint,region,compartment_id,ssh_public_key,auth_type,created_at"
|
||||
if u["role"]=="admin": rows=c.execute(f"SELECT {cols} FROM oci_configs").fetchall()
|
||||
else: rows=c.execute(f"SELECT {cols} FROM oci_configs WHERE user_id=?",(u["id"],)).fetchall()
|
||||
result = []
|
||||
for r in rows:
|
||||
d = dict(r)
|
||||
# Decrypt and mask sensitive fields for display
|
||||
try:
|
||||
d["tenancy_ocid"] = _mask(_dec(d["tenancy_ocid"]))
|
||||
except Exception:
|
||||
d["tenancy_ocid"] = _mask(d["tenancy_ocid"])
|
||||
try:
|
||||
d["user_ocid"] = _mask(_dec(d["user_ocid"]))
|
||||
except Exception:
|
||||
d["user_ocid"] = _mask(d["user_ocid"])
|
||||
d["fingerprint"] = "\u2022" * 20 # fingerprint fully masked
|
||||
if d.get("compartment_id"):
|
||||
try:
|
||||
d["compartment_id"] = _mask(_dec(d["compartment_id"]))
|
||||
except Exception:
|
||||
d["compartment_id"] = _mask(d["compartment_id"])
|
||||
if d.get("ssh_public_key"):
|
||||
# Show just type + first/last few chars for identification
|
||||
parts = d["ssh_public_key"].split()
|
||||
d["ssh_public_key_preview"] = f"{parts[0]} ...{parts[1][-12:]}" if len(parts) >= 2 else "ssh-rsa ..."
|
||||
result.append(d)
|
||||
return result
|
||||
|
||||
@router.delete("/api/oci/configs/{cid}")
|
||||
async def del_oci(cid: str, u=Depends(require("admin","user"))):
|
||||
with db() as c:
|
||||
cfg=c.execute("SELECT * FROM oci_configs WHERE id=?",(cid,)).fetchone()
|
||||
if not cfg: raise HTTPException(404)
|
||||
if u["role"]!="admin" and cfg["user_id"]!=u["id"]: raise HTTPException(403)
|
||||
c.execute("DELETE FROM oci_configs WHERE id=?",(cid,))
|
||||
d=OCI_DIR/cid
|
||||
if d.exists(): shutil.rmtree(d)
|
||||
return {"ok": True}
|
||||
|
||||
@router.put("/api/oci/configs/{cid}")
|
||||
async def update_oci(
|
||||
cid: str,
|
||||
tenancy_name: str = Form(...), tenancy_ocid: str = Form(""),
|
||||
user_ocid: str = Form(""), fingerprint: str = Form(""),
|
||||
region: str = Form(...), compartment_id: str = Form(""),
|
||||
key_passphrase: str = Form(""),
|
||||
ssh_public_key: str = Form(""),
|
||||
auth_type: str = Form(""),
|
||||
security_token: str = Form(""),
|
||||
private_key: Optional[UploadFile] = File(None), public_key: Optional[UploadFile] = File(None),
|
||||
u = Depends(require("admin","user"))
|
||||
):
|
||||
_PEM_EXTS = {".pem", ".key"}
|
||||
if private_key and private_key.filename:
|
||||
await validate_upload(private_key, _PEM_EXTS)
|
||||
private_key.file.seek(0)
|
||||
if public_key and public_key.filename:
|
||||
await validate_upload(public_key, _PEM_EXTS)
|
||||
public_key.file.seek(0)
|
||||
with db() as c:
|
||||
existing = c.execute("SELECT * FROM oci_configs WHERE id=?", (cid,)).fetchone()
|
||||
if not existing: raise HTTPException(404)
|
||||
if u["role"] != "admin" and existing["user_id"] != u["id"]: raise HTTPException(403)
|
||||
# Keep existing encrypted values if not provided (empty = keep current)
|
||||
tenancy_ocid = tenancy_ocid or _safe_dec(existing["tenancy_ocid"])
|
||||
user_ocid = user_ocid or _safe_dec(existing["user_ocid"])
|
||||
fingerprint = fingerprint or _safe_dec(existing["fingerprint"])
|
||||
compartment_id = compartment_id or _safe_dec(existing["compartment_id"]) or ""
|
||||
try:
|
||||
existing_auth = existing["auth_type"] or "api_key"
|
||||
except (IndexError, KeyError):
|
||||
existing_auth = "api_key"
|
||||
auth_type = auth_type or existing_auth
|
||||
try:
|
||||
existing_ssh = existing["ssh_public_key"] or ""
|
||||
except (IndexError, KeyError):
|
||||
existing_ssh = ""
|
||||
ssh_public_key = ssh_public_key.strip() if ssh_public_key.strip() else existing_ssh
|
||||
cdir = OCI_DIR / cid; cdir.mkdir(parents=True, exist_ok=True)
|
||||
kp = cdir / "oci_api_key.pem"
|
||||
if private_key and private_key.filename:
|
||||
key_bytes = await private_key.read()
|
||||
kp.write_bytes(key_bytes); kp.chmod(0o600)
|
||||
key_text = key_bytes.decode("utf-8", errors="ignore")
|
||||
if auth_type == "api_key" and ("ENCRYPTED" in key_text or "Proc-Type: 4,ENCRYPTED" in key_text) and not key_passphrase:
|
||||
raise HTTPException(400, "A chave privada está criptografada (ENCRYPTED). Informe a Key Passphrase.")
|
||||
if public_key and public_key.filename:
|
||||
(cdir / "oci_api_key_public.pem").write_bytes(await public_key.read())
|
||||
# Update session token file if provided
|
||||
if security_token.strip():
|
||||
token_file = cdir / "token"
|
||||
token_file.write_text(security_token.strip()); token_file.chmod(0o600)
|
||||
cfg_file = cdir / "config"
|
||||
if auth_type == "session_token":
|
||||
token_path = cdir / "token"
|
||||
cfg_content = f"[DEFAULT]\ntenancy={tenancy_ocid}\nregion={region}\nkey_file={kp}\n"
|
||||
if user_ocid and user_ocid != "session_token_user":
|
||||
cfg_content = f"[DEFAULT]\nuser={user_ocid}\ntenancy={tenancy_ocid}\nregion={region}\nkey_file={kp}\n"
|
||||
cfg_content += f"security_token_file={token_path}\n"
|
||||
else:
|
||||
cfg_content = (f"[DEFAULT]\nuser={user_ocid}\nfingerprint={fingerprint}\n"
|
||||
f"tenancy={tenancy_ocid}\nregion={region}\nkey_file={kp}\n")
|
||||
if key_passphrase:
|
||||
cfg_content += f"pass_phrase={key_passphrase}\n"
|
||||
cfg_file.write_text(cfg_content); cfg_file.chmod(0o600)
|
||||
with db() as c:
|
||||
c.execute("UPDATE oci_configs SET tenancy_name=?,tenancy_ocid=?,user_ocid=?,fingerprint=?,region=?,compartment_id=?,ssh_public_key=?,auth_type=? WHERE id=?",
|
||||
(tenancy_name, _enc(tenancy_ocid), _enc(user_ocid), _enc(fingerprint), region, _enc(compartment_id) if compartment_id else None, ssh_public_key or None, auth_type, cid))
|
||||
_audit(u["id"], u["username"], "update_oci_config", cid, f"tenancy={tenancy_name} auth={auth_type}")
|
||||
_config_log("oci", cid, tenancy_name, "success", "save", f"Credencial atualizada: {tenancy_name} ({region}) [{auth_type}]", u["id"], u["username"])
|
||||
return {"id": cid, "tenancy_name": tenancy_name, "region": region}
|
||||
|
||||
@router.post("/api/oci/test/{cid}")
|
||||
async def test_oci(cid: str, u=Depends(require("admin","user"))):
|
||||
_verify_config_access("oci", cid, u)
|
||||
cp = OCI_DIR / cid / "config"
|
||||
cname = None
|
||||
with db() as c:
|
||||
row = c.execute("SELECT tenancy_name FROM oci_configs WHERE id=?", (cid,)).fetchone()
|
||||
if row: cname = row["tenancy_name"]
|
||||
if not cp.exists():
|
||||
_config_log("oci", cid, cname, "error", "test", "Config não encontrada", u["id"], u["username"])
|
||||
return {"status":"error","message":"Config não encontrada"}
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
r = await loop.run_in_executor(None, partial(
|
||||
subprocess.run, ["oci","iam","region","list","--config-file",str(cp),"--output","json"],
|
||||
capture_output=True, text=True, timeout=30, stdin=subprocess.DEVNULL))
|
||||
if r.returncode == 0:
|
||||
_config_log("oci", cid, cname, "success", "test", "Conexão OK", u["id"], u["username"])
|
||||
return {"status":"success","message":"Conexão OK","data":json.loads(r.stdout)}
|
||||
msg = r.stderr[:500]
|
||||
if "passphrase" in msg.lower() or "getpass" in msg.lower():
|
||||
msg = "A chave privada requer passphrase. Recadastre a credencial informando a Key Passphrase."
|
||||
_config_log("oci", cid, cname, "error", "test", msg, u["id"], u["username"])
|
||||
return {"status":"error","message":msg}
|
||||
except FileNotFoundError:
|
||||
_config_log("oci", cid, cname, "error", "test", "OCI CLI não disponível no container.", u["id"], u["username"])
|
||||
return {"status":"error","message":"OCI CLI não disponível no container."}
|
||||
except subprocess.TimeoutExpired:
|
||||
_config_log("oci", cid, cname, "error", "test", "Timeout na conexão", u["id"], u["username"])
|
||||
return {"status":"error","message":"Timeout na conexão"}
|
||||
except Exception as e:
|
||||
_config_log("oci", cid, cname, "error", "test", str(e)[:500], u["id"], u["username"])
|
||||
return {"status":"error","message":str(e)[:500]}
|
||||
|
||||
# ── OCI CLI Terminal ─────────────────────────────────────────────────────────
|
||||
# OCID resource type → OCI CLI get command mapping
|
||||
_OCID_CMD_MAP = {
|
||||
"instance": "oci compute instance get --instance-id",
|
||||
"image": "oci compute image get --image-id",
|
||||
"vcn": "oci network vcn get --vcn-id",
|
||||
"subnet": "oci network subnet get --subnet-id",
|
||||
"securitylist": "oci network security-list get --security-list-id",
|
||||
"routetable": "oci network route-table get --rt-id",
|
||||
"internetgateway": "oci network internet-gateway get --ig-id",
|
||||
"natgateway": "oci network nat-gateway get --nat-gateway-id",
|
||||
"servicegateway": "oci network service-gateway get --service-gateway-id",
|
||||
"drg": "oci network drg get --drg-id",
|
||||
"drgattachment": "oci network drg-attachment get --drg-attachment-id",
|
||||
"localpeeringgateway": "oci network local-peering-gateway get --local-peering-gateway-id",
|
||||
"networksecuritygroup": "oci network nsg get --nsg-id",
|
||||
"vnic": "oci network vnic get --vnic-id",
|
||||
"privateip": "oci network private-ip get --private-ip-id",
|
||||
"publicip": "oci network public-ip get --public-ip-id",
|
||||
"dhcpoptions": "oci network dhcp-options get --dhcp-id",
|
||||
"cpe": "oci network cpe get --cpe-id",
|
||||
"ipsecconnection": "oci network ip-sec-connection get --ipsc-id",
|
||||
"volume": "oci bv volume get --volume-id",
|
||||
"bootvolume": "oci bv boot-volume get --boot-volume-id",
|
||||
"volumebackup": "oci bv backup get --volume-backup-id",
|
||||
"bootvolumereplica": "oci bv boot-volume-replica get --boot-volume-replica-id",
|
||||
"volumegroup": "oci bv volume-group get --volume-group-id",
|
||||
"bucket": "oci os bucket get --bucket-name",
|
||||
"compartment": "oci iam compartment get --compartment-id",
|
||||
"tenancy": "oci iam tenancy get --tenancy-id",
|
||||
"user": "oci iam user get --user-id",
|
||||
"group": "oci iam group get --group-id",
|
||||
"policy": "oci iam policy get --policy-id",
|
||||
"dynamicgroup": "oci iam dynamic-group get --dynamic-group-id",
|
||||
"identityprovider": "oci iam identity-provider get --identity-provider-id",
|
||||
"autonomousdatabase": "oci db autonomous-database get --autonomous-database-id",
|
||||
"dbsystem": "oci db system get --db-system-id",
|
||||
"database": "oci db database get --database-id",
|
||||
"dbhome": "oci db db-home get --db-home-id",
|
||||
"dbnode": "oci db node get --db-node-id",
|
||||
"mysqldbsystem": "oci mysql db-system get --db-system-id",
|
||||
"loadbalancer": "oci lb load-balancer get --load-balancer-id",
|
||||
"networkloadbalancer": "oci nlb network-load-balancer get --network-load-balancer-id",
|
||||
"certificate": "oci certs-mgmt certificate get --certificate-id",
|
||||
"vault": "oci kms management vault get --vault-id",
|
||||
"key": "oci kms management key get --key-id",
|
||||
"secret": "oci vault secret get --secret-id",
|
||||
"fnapp": "oci fn application get --application-id",
|
||||
"fnfunc": "oci fn function get --function-id",
|
||||
"apigateway": "oci api-gateway gateway get --gateway-id",
|
||||
"containerinstance": "oci container-instances container-instance get --container-instance-id",
|
||||
"cluster": "oci ce cluster get --cluster-id",
|
||||
"nodepool": "oci ce node-pool get --node-pool-id",
|
||||
"filesystem": "oci fs file-system get --file-system-id",
|
||||
"mounttarget": "oci fs mount-target get --mount-target-id",
|
||||
"alarm": "oci monitoring alarm get --alarm-id",
|
||||
"loggroup": "oci logging log-group get --log-group-id",
|
||||
"log": "oci logging log get --log-id",
|
||||
"topic": "oci ons topic get --topic-id",
|
||||
"subscription": "oci ons subscription get --subscription-id",
|
||||
"stream": "oci streaming stream get --stream-id",
|
||||
"streampool": "oci streaming stream-pool get --stream-pool-id",
|
||||
"dnszone": "oci dns zone get --zone-name-or-id",
|
||||
"networkfirewall": "oci network-firewall network-firewall get --network-firewall-id",
|
||||
"networkfirewallpolicy": "oci network-firewall network-firewall-policy get --network-firewall-policy-id",
|
||||
"waaspolicy": "oci waas waas-policy get --waas-policy-id",
|
||||
"instancepool": "oci compute-management instance-pool get --instance-pool-id",
|
||||
"instanceconfiguration": "oci compute-management instance-configuration get --instance-configuration-id",
|
||||
"capacityreservation": "oci compute capacity-reservation get --capacity-reservation-id",
|
||||
"dedicatedvmhost": "oci compute dedicated-vm-host get --dedicated-vm-host-id",
|
||||
}
|
||||
|
||||
def _resolve_ocid_command(ocid: str) -> str | None:
|
||||
"""Parse an OCID and return the corresponding OCI CLI get command."""
|
||||
parts = ocid.strip().split(".")
|
||||
if len(parts) < 4 or parts[0] != "ocid1":
|
||||
return None
|
||||
resource_type = parts[1].lower()
|
||||
# Extract region from OCID if present (4th part, non-empty for regional resources)
|
||||
region = parts[3] if len(parts) > 3 and parts[3] and parts[3] != parts[2] else None
|
||||
cmd_template = _OCID_CMD_MAP.get(resource_type)
|
||||
if not cmd_template:
|
||||
return None
|
||||
cmd = f"{cmd_template} {ocid}"
|
||||
if region:
|
||||
cmd += f" --region {region}"
|
||||
return cmd
|
||||
|
||||
_TERMINAL_BLOCKED_PATTERNS = re.compile(
|
||||
r'(rm\s|mv\s|cp\s|chmod\s|chown\s|sudo\s|bash\s|sh\s|curl\s|wget\s|python|pip\s|apt\s|yum\s|>\s|>>\s|\|)',
|
||||
re.IGNORECASE
|
||||
)
|
||||
|
||||
@router.post("/api/terminal/execute")
|
||||
async def terminal_execute(req: dict, u=Depends(current_user)):
|
||||
"""Execute an OCI CLI command using the selected OCI config. Only 'oci' commands allowed."""
|
||||
oci_config_id = req.get("oci_config_id", "")
|
||||
command = (req.get("command", "") or "").strip()
|
||||
if not oci_config_id:
|
||||
raise HTTPException(400, "oci_config_id obrigatório")
|
||||
if not command:
|
||||
raise HTTPException(400, "Comando vazio")
|
||||
_verify_config_access("oci", oci_config_id, u)
|
||||
# Auto-lookup: OCID pasted directly → resolve to oci get command
|
||||
resolved_cmd = None
|
||||
if command.startswith("ocid1."):
|
||||
resolved_cmd = _resolve_ocid_command(command)
|
||||
if resolved_cmd:
|
||||
command = resolved_cmd
|
||||
else:
|
||||
raise HTTPException(400, f"Tipo de recurso não reconhecido no OCID: {command.split('.')[1]}")
|
||||
# find <query> → search resources by display name using OCI Search service
|
||||
elif command.startswith("find "):
|
||||
query = command[5:].strip()
|
||||
if not query:
|
||||
raise HTTPException(400, "Uso: find <nome-do-recurso> | find %partial% | find 10.0.1.5")
|
||||
# Detect IP address → search by IP
|
||||
if re.match(r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$', query):
|
||||
search_query = f"query privateip resources where ipAddress = '{query}'"
|
||||
elif '%' in query:
|
||||
# Partial match with LIKE
|
||||
search_query = f"query all resources where displayName =~ '{query}'"
|
||||
else:
|
||||
search_query = f"query all resources where displayName = '{query}'"
|
||||
resolved_cmd = f"oci search resource structured-search --query-text \"{search_query}\" --limit 20"
|
||||
command = resolved_cmd
|
||||
# Security: only allow 'oci' commands
|
||||
if command == "oci":
|
||||
command = "oci --help"
|
||||
if not command.startswith("oci "):
|
||||
raise HTTPException(400, "Apenas comandos 'oci' são permitidos. Ex: oci iam user list")
|
||||
if _TERMINAL_BLOCKED_PATTERNS.search(command):
|
||||
raise HTTPException(400, "Comando contém operações não permitidas")
|
||||
# Build config path
|
||||
cfg_path = OCI_DIR / oci_config_id / "config"
|
||||
if not cfg_path.exists():
|
||||
raise HTTPException(400, "OCI config file não encontrado. Reconfigure as credenciais.")
|
||||
# Execute with timeout
|
||||
cmd_args = shlex.split(command) + ["--config-file", str(cfg_path)]
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
result = await asyncio.wait_for(
|
||||
loop.run_in_executor(_chat_executor, lambda: subprocess.run(
|
||||
cmd_args, capture_output=True, text=True, timeout=60, cwd="/tmp",
|
||||
env={**os.environ, "OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING": "True", "HOME": "/tmp"}
|
||||
)), timeout=65)
|
||||
output = result.stdout or ""
|
||||
error = result.stderr or ""
|
||||
exit_code = result.returncode
|
||||
# Save to history
|
||||
with db() as c:
|
||||
c.execute("INSERT INTO terminal_history (id,user_id,oci_config_id,command,exit_code,output,error) VALUES (?,?,?,?,?,?,?)",
|
||||
(str(uuid.uuid4()), u["id"], oci_config_id, command, exit_code, output[:10000], error[:2000]))
|
||||
return {
|
||||
"exit_code": exit_code,
|
||||
"output": output[:50000],
|
||||
"error": error[:5000] if exit_code != 0 else "",
|
||||
"resolved_cmd": resolved_cmd,
|
||||
}
|
||||
except asyncio.TimeoutError:
|
||||
with db() as c:
|
||||
c.execute("INSERT INTO terminal_history (id,user_id,oci_config_id,command,exit_code,error) VALUES (?,?,?,?,?,?)",
|
||||
(str(uuid.uuid4()), u["id"], oci_config_id, command, -1, "Timeout: 60s"))
|
||||
return {"exit_code": -1, "output": "", "error": "Timeout: comando excedeu 60 segundos"}
|
||||
except Exception as e:
|
||||
return {"exit_code": -1, "output": "", "error": str(e)[:2000]}
|
||||
|
||||
@router.get("/api/terminal/history")
|
||||
async def terminal_history(oci_config_id: str = Query(""), limit: int = Query(50), u=Depends(current_user)):
|
||||
"""Get recent terminal commands for the user."""
|
||||
with db() as c:
|
||||
if oci_config_id:
|
||||
rows = c.execute(
|
||||
"SELECT * FROM terminal_history WHERE user_id=? AND oci_config_id=? ORDER BY created_at DESC LIMIT ?",
|
||||
(u["id"], oci_config_id, limit)).fetchall()
|
||||
else:
|
||||
rows = c.execute(
|
||||
"SELECT * FROM terminal_history WHERE user_id=? ORDER BY created_at DESC LIMIT ?",
|
||||
(u["id"], limit)).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
# OCI CLI autocomplete cache
|
||||
_oci_completions_cache: dict = {}
|
||||
|
||||
def _parse_oci_help(args: list[str]) -> list[str]:
|
||||
"""Run oci <args> --help and extract available commands/options."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["oci"] + args + ["--help"], capture_output=True, text=True, timeout=10,
|
||||
env={**os.environ, "OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING": "True", "HOME": "/tmp"})
|
||||
text = result.stdout + result.stderr
|
||||
commands = []
|
||||
in_commands = False
|
||||
for line in text.splitlines():
|
||||
stripped = line.strip()
|
||||
# Detect transition into command/service listing
|
||||
if not in_commands:
|
||||
if re.match(r'^(Commands|Options|Core|Others|Networking|Database|Identity|Storage|Compute|Security)', stripped):
|
||||
in_commands = True
|
||||
continue
|
||||
if not stripped:
|
||||
continue
|
||||
# Section headers (single word or capitalized phrase not indented) — skip
|
||||
if not line.startswith(" "):
|
||||
continue
|
||||
# Capture " command-name Description text" pattern
|
||||
m = re.match(r'^ ([a-z][a-z0-9\-_]+)\s', line)
|
||||
if m and len(m.group(1)) > 1:
|
||||
commands.append(m.group(1))
|
||||
continue
|
||||
# Capture 2-space indent commands (sub-command help)
|
||||
m = re.match(r'^ ([a-z][a-z0-9\-_]+)\s', line)
|
||||
if m and len(m.group(1)) > 2:
|
||||
commands.append(m.group(1))
|
||||
continue
|
||||
# Capture --options
|
||||
if stripped.startswith("--"):
|
||||
opt = stripped.split()[0].rstrip(",")
|
||||
if opt.startswith("--"): commands.append(opt)
|
||||
return sorted(set(commands))
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
@router.get("/api/terminal/completions")
|
||||
async def terminal_completions(prefix: str = Query(""), u=Depends(current_user)):
|
||||
"""Get OCI CLI autocomplete suggestions for a partial command."""
|
||||
parts = prefix.strip().split()
|
||||
if not parts or parts[0] != "oci":
|
||||
return {"suggestions": []}
|
||||
# Build cache key from command path (without final partial word)
|
||||
# e.g. "oci iam us" → lookup completions for ["oci", "iam"], filter by "us"
|
||||
if prefix.endswith(" "):
|
||||
cmd_parts = parts[1:]
|
||||
filter_prefix = ""
|
||||
else:
|
||||
cmd_parts = parts[1:-1]
|
||||
filter_prefix = parts[-1] if len(parts) > 1 else ""
|
||||
cache_key = " ".join(cmd_parts)
|
||||
if cache_key not in _oci_completions_cache:
|
||||
loop = asyncio.get_event_loop()
|
||||
completions = await loop.run_in_executor(_chat_executor, lambda: _parse_oci_help(cmd_parts))
|
||||
_oci_completions_cache[cache_key] = completions
|
||||
suggestions = _oci_completions_cache[cache_key]
|
||||
if filter_prefix:
|
||||
suggestions = [s for s in suggestions if s.startswith(filter_prefix)]
|
||||
# Commands first, then options
|
||||
cmds = [s for s in suggestions if not s.startswith("--")]
|
||||
opts = [s for s in suggestions if s.startswith("--")]
|
||||
return {"suggestions": (cmds + opts)[:50]}
|
||||
1014
backend/routes/oci_explorer.py
Normal file
1014
backend/routes/oci_explorer.py
Normal file
File diff suppressed because it is too large
Load Diff
569
backend/routes/reports.py
Normal file
569
backend/routes/reports.py
Normal file
@@ -0,0 +1,569 @@
|
||||
"""Report routes — CIS report CRUD, compliance report, OCI health/services."""
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
import zipfile
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import (
|
||||
APIRouter, HTTPException, Depends, Query, BackgroundTasks, Request,
|
||||
)
|
||||
from fastapi.responses import FileResponse, HTMLResponse, Response
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
|
||||
from config import REPORTS, VERSION, _chat_executor, log
|
||||
from database import db
|
||||
from auth.jwt_auth import (
|
||||
current_user, _user_from_token, require, _audit,
|
||||
_verify_config_access, _verify_report_access,
|
||||
)
|
||||
from models import RunReportReq
|
||||
from utils import running_reports
|
||||
from services.genai import _get_adb_connection
|
||||
from services.cis_reports import _exec_report
|
||||
from services.compliance_gen import (
|
||||
_extract_section,
|
||||
_extract_remediation_section,
|
||||
_extract_audit_section,
|
||||
_extract_rationale_section,
|
||||
_extract_description_section,
|
||||
_format_remediation_html,
|
||||
_search_cis_remediation,
|
||||
_build_findings_table,
|
||||
_check_oci_services,
|
||||
_OCI_SERVICE_DESCRIPTIONS,
|
||||
_build_compliance_html,
|
||||
_generate_compliance_report,
|
||||
)
|
||||
from services.compliance_docx import (
|
||||
_generate_camo_strip,
|
||||
_add_floating_image_to_header,
|
||||
_generate_compliance_docx,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# ── Helper ────────────────────────────────────────────────────────────────────
|
||||
|
||||
# ── Report CRUD ───────────────────────────────────────────────────────────────
|
||||
|
||||
@router.post("/api/reports/run")
|
||||
async def run_report(req: RunReportReq, bg: BackgroundTasks, u=Depends(require("admin","user"))):
|
||||
if req.level not in (1, 2): raise HTTPException(400, "Level must be 1 or 2")
|
||||
_verify_config_access("oci", req.config_id, u)
|
||||
with db() as c:
|
||||
cfg = c.execute("SELECT * FROM oci_configs WHERE id=?",(req.config_id,)).fetchone()
|
||||
if not cfg: raise HTTPException(404, "Config não encontrada")
|
||||
rid = str(uuid.uuid4())
|
||||
c.execute("INSERT INTO reports (id,user_id,tenancy_name,config_id,level,obp_checks,raw_data,redact_output,status) VALUES (?,?,?,?,?,?,?,?,?)",
|
||||
(rid, u["id"], cfg["tenancy_name"], req.config_id, req.level, int(req.obp), int(req.raw), int(req.redact_output), "running"))
|
||||
bg.add_task(_exec_report, rid, dict(cfg), req.regions, req.level, req.obp, req.raw, req.redact_output)
|
||||
_audit(u["id"], u["username"], "run_report", rid)
|
||||
return {"report_id": rid, "status": "running"}
|
||||
|
||||
|
||||
@router.get("/api/reports")
|
||||
async def list_reports(skip: int = Query(0, ge=0), limit: int = Query(50, ge=1, le=200), u=Depends(current_user)):
|
||||
with db() as c:
|
||||
q = "SELECT id,user_id,tenancy_name,config_id,status,progress,level,obp_checks,raw_data,redact_output,created_at,completed_at,error_msg FROM reports"
|
||||
rows = c.execute(q+" WHERE user_id=? ORDER BY created_at DESC LIMIT ? OFFSET ?",(u["id"], limit, skip)).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
@router.get("/api/reports/{rid}/progress")
|
||||
async def get_report_progress(rid: str, u=Depends(current_user)):
|
||||
_verify_report_access(rid, u)
|
||||
with db() as c:
|
||||
r = c.execute("SELECT status,progress,created_at,completed_at,error_msg FROM reports WHERE id=?", (rid,)).fetchone()
|
||||
if not r: raise HTTPException(404)
|
||||
return dict(r)
|
||||
|
||||
|
||||
@router.post("/api/reports/{rid}/cancel")
|
||||
async def cancel_report(rid: str, u=Depends(require("admin", "user"))):
|
||||
_verify_report_access(rid, u)
|
||||
with db() as c:
|
||||
r = c.execute("SELECT status, worker_pid FROM reports WHERE id=?", (rid,)).fetchone()
|
||||
if not r: raise HTTPException(404)
|
||||
if r["status"] != "running":
|
||||
raise HTTPException(400, "Report is not running")
|
||||
proc = running_reports.get(rid)
|
||||
if proc:
|
||||
try:
|
||||
proc.terminate()
|
||||
try:
|
||||
await asyncio.wait_for(proc.wait(), timeout=5)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
running_reports.pop(rid, None)
|
||||
elif r["worker_pid"]:
|
||||
import signal
|
||||
try:
|
||||
os.kill(r["worker_pid"], signal.SIGTERM)
|
||||
except (ProcessLookupError, OSError):
|
||||
pass
|
||||
with db() as c:
|
||||
c.execute("UPDATE reports SET status='cancelled',error_msg='Cancelled by user',completed_at=datetime('now') WHERE id=?", (rid,))
|
||||
_audit(u["id"], u["username"], "cancel_report", rid)
|
||||
return {"ok": True, "status": "cancelled"}
|
||||
|
||||
|
||||
@router.get("/api/reports/{rid}")
|
||||
async def get_report(rid, u=Depends(current_user)):
|
||||
with db() as c: r=c.execute("SELECT * FROM reports WHERE id=?",(rid,)).fetchone()
|
||||
if not r: raise HTTPException(404)
|
||||
if u["role"]!="admin" and r["user_id"]!=u["id"]: raise HTTPException(403)
|
||||
return dict(r)
|
||||
|
||||
|
||||
@router.delete("/api/reports/{rid}")
|
||||
async def delete_report(rid: str, u=Depends(current_user)):
|
||||
with db() as c:
|
||||
r = c.execute("SELECT id, user_id, status, json_path, html_path FROM reports WHERE id=?", (rid,)).fetchone()
|
||||
if not r: raise HTTPException(404)
|
||||
if u["role"] != "admin" and r["user_id"] != u["id"]: raise HTTPException(403)
|
||||
if r["status"] == "running": raise HTTPException(400, "Cannot delete a running report")
|
||||
# Remove associated files
|
||||
for path_col in ("json_path", "html_path"):
|
||||
p = r[path_col]
|
||||
if p and Path(p).exists():
|
||||
try: Path(p).unlink()
|
||||
except OSError: pass
|
||||
c.execute("DELETE FROM reports WHERE id=?", (rid,))
|
||||
_audit(u["id"], u["username"], "delete_report", rid)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.get("/api/reports/{rid}/summary")
|
||||
async def report_summary(rid: str, u=Depends(current_user)):
|
||||
with db() as c:
|
||||
r = c.execute("SELECT user_id,json_path,tenancy_name,level,created_at FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone()
|
||||
if not r: raise HTTPException(404)
|
||||
if u["role"] != "admin" and r["user_id"] != u["id"]: raise HTTPException(403)
|
||||
jp = r["json_path"]
|
||||
if not jp or not Path(jp).exists():
|
||||
raise HTTPException(400, "Report JSON not found")
|
||||
data = json.loads(Path(jp).read_text())
|
||||
total = passed = failed = 0
|
||||
sections = {}
|
||||
total_findings = 0
|
||||
total_compliant = 0
|
||||
# Format: list of checks with Compliant=Yes/No, Section, Findings, Compliant Items, Total
|
||||
checks = data if isinstance(data, list) else data.get("findings", [])
|
||||
if isinstance(checks, dict):
|
||||
checks = list(checks.values())
|
||||
for check in checks:
|
||||
total += 1
|
||||
compliant = str(check.get("Compliant", check.get("status", ""))).strip().lower()
|
||||
is_pass = compliant in ("yes", "pass")
|
||||
if is_pass:
|
||||
passed += 1
|
||||
else:
|
||||
failed += 1
|
||||
fi = str(check.get("Findings", "0")).strip()
|
||||
ci = str(check.get("Compliant Items", "0")).strip()
|
||||
total_findings += int(fi) if fi.isdigit() else 0
|
||||
total_compliant += int(ci) if ci.isdigit() else 0
|
||||
sec = check.get("Section", check.get("section", "Other"))
|
||||
s = sections.setdefault(sec, {"passed": 0, "failed": 0, "total": 0})
|
||||
s["total"] += 1
|
||||
if is_pass:
|
||||
s["passed"] += 1
|
||||
else:
|
||||
s["failed"] += 1
|
||||
score = round((passed / total * 100), 1) if total else 0
|
||||
return {
|
||||
"tenancy": data.get("tenancy", r["tenancy_name"]) if isinstance(data, dict) else r["tenancy_name"],
|
||||
"level": r["level"] or 2,
|
||||
"regions": data.get("regions", []) if isinstance(data, dict) else [],
|
||||
"generated_at": r["created_at"],
|
||||
"total": total, "passed": passed, "failed": failed,
|
||||
"total_findings": total_findings, "total_compliant": total_compliant,
|
||||
"score": score,
|
||||
"sections": sections
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/reports/{rid}/files")
|
||||
async def list_report_files(rid: str, u=Depends(current_user)):
|
||||
with db() as c:
|
||||
r = c.execute("SELECT user_id FROM reports WHERE id=?", (rid,)).fetchone()
|
||||
if not r: raise HTTPException(404)
|
||||
if u["role"] != "admin" and r["user_id"] != u["id"]: raise HTTPException(403)
|
||||
files = c.execute(
|
||||
"SELECT id,file_name,file_type,file_category,file_size FROM report_files WHERE report_id=? ORDER BY file_category,file_name",
|
||||
(rid,)
|
||||
).fetchall()
|
||||
return [dict(f) for f in files]
|
||||
|
||||
|
||||
@router.get("/api/reports/{rid}/files/{fid}/download")
|
||||
async def download_report_file(rid: str, fid: str, token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))):
|
||||
u = _user_from_token(token) if token else (await current_user(cred) if cred else None)
|
||||
if not u: raise HTTPException(401, "Not authenticated")
|
||||
with db() as c:
|
||||
r = c.execute("SELECT user_id FROM reports WHERE id=?", (rid,)).fetchone()
|
||||
if not r: raise HTTPException(404)
|
||||
if u["role"] != "admin" and r["user_id"] != u["id"]: raise HTTPException(403)
|
||||
f = c.execute("SELECT * FROM report_files WHERE id=? AND report_id=?", (fid, rid)).fetchone()
|
||||
if not f: raise HTTPException(404)
|
||||
p = Path(f["file_path"])
|
||||
if not p.exists(): raise HTTPException(404, "File not found on disk")
|
||||
return FileResponse(p, filename=f["file_name"])
|
||||
|
||||
|
||||
@router.api_route("/api/reports/{rid}/html", methods=["GET", "HEAD"])
|
||||
async def report_html(rid, token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))):
|
||||
u = _user_from_token(token) if token else (await current_user(cred) if cred else None)
|
||||
if not u: raise HTTPException(401, "Not authenticated")
|
||||
_verify_report_access(rid, u)
|
||||
with db() as c: r=c.execute("SELECT html_path FROM reports WHERE id=?",(rid,)).fetchone()
|
||||
if not r or not r["html_path"] or not Path(r["html_path"]).exists(): raise HTTPException(404)
|
||||
return FileResponse(r["html_path"], media_type="text/html")
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# OCI Health & Services
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
@router.get("/api/oci-health")
|
||||
async def oci_health(u=Depends(current_user)):
|
||||
"""Proxy Oracle Cloud Infrastructure public status API."""
|
||||
import httpx
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
r = await client.get("https://ocistatus.oraclecloud.com/api/v2/components.json")
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
except Exception as e:
|
||||
log.warning(f"OCI Health API failed: {e}")
|
||||
raise HTTPException(502, f"Failed to fetch OCI status: {e}")
|
||||
|
||||
|
||||
_OCI_SERVICE_NAMES = ["Vulnerability Scanning Service", "Data Safe", "Cloud Guard", "Bastion", "Security Zones", "Vault"]
|
||||
|
||||
|
||||
@router.get("/api/oci-services/{config_id}")
|
||||
async def get_oci_services_status(config_id: str, detect: int = Query(1), u=Depends(current_user)):
|
||||
"""Get OCI services status (auto-detected + user overrides). Use detect=0 to skip auto-detect."""
|
||||
_verify_config_access("oci", config_id, u)
|
||||
import json as _json
|
||||
# Load user overrides
|
||||
with db() as c:
|
||||
row = c.execute("SELECT value FROM app_settings WHERE key=?", (f"oci_services_{config_id}",)).fetchone()
|
||||
overrides = _json.loads(row["value"]) if row else {}
|
||||
# Auto-detect (skip if detect=0)
|
||||
auto = {}
|
||||
if detect:
|
||||
try:
|
||||
detected = _check_oci_services(config_id)
|
||||
for svc in detected:
|
||||
auto[svc["service"]] = {"status": svc["status"], "description": svc["description"]}
|
||||
except Exception as e:
|
||||
log.warning(f"OCI services auto-detect failed: {e}")
|
||||
# Merge: override wins
|
||||
result = []
|
||||
for name in _OCI_SERVICE_NAMES:
|
||||
ov = overrides.get(name, {})
|
||||
au = auto.get(name, {"status": "WARNING", "description": "Unable to verify"})
|
||||
result.append({
|
||||
"service": name,
|
||||
"auto_status": au["status"],
|
||||
"auto_description": au["description"],
|
||||
"status": ov.get("status", au["status"]),
|
||||
"description": ov.get("description", au["description"]),
|
||||
"overridden": bool(ov),
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
@router.put("/api/oci-services/{config_id}")
|
||||
async def set_oci_services_status(config_id: str, request: Request, u=Depends(current_user)):
|
||||
"""Save user overrides for OCI services status."""
|
||||
_verify_config_access("oci", config_id, u)
|
||||
import json as _json
|
||||
body = await request.json()
|
||||
overrides = {}
|
||||
for svc in body:
|
||||
name = svc.get("service", "")
|
||||
if name in _OCI_SERVICE_NAMES and svc.get("overridden"):
|
||||
overrides[name] = {"status": svc.get("status", "OK"), "description": svc.get("description", "")}
|
||||
with db() as c:
|
||||
c.execute("INSERT INTO app_settings (key,value,updated_at) VALUES (?,?,datetime('now')) ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at",
|
||||
(f"oci_services_{config_id}", _json.dumps(overrides)))
|
||||
_audit(u["id"], u["username"], "set_oci_services", config_id)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# LAD A-Team CIS Compliance Report
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
@router.api_route("/api/reports/{rid}/compliance-report", methods=["GET", "HEAD"])
|
||||
async def compliance_report(rid: str, regen: int = Query(0), token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))):
|
||||
"""Serve cached LAD A-Team CIS Compliance Report, or return 404 if not generated yet."""
|
||||
u = _user_from_token(token) if token else (await current_user(cred) if cred else None)
|
||||
if not u: raise HTTPException(401, "Not authenticated")
|
||||
_verify_report_access(rid, u)
|
||||
rdir = REPORTS / rid
|
||||
cache_path = rdir / "compliance_report.html"
|
||||
|
||||
if regen == 1 and cache_path.exists():
|
||||
cache_path.unlink()
|
||||
|
||||
if cache_path.exists():
|
||||
return FileResponse(cache_path, media_type="text/html",
|
||||
headers={"Cache-Control": "no-cache, no-store, must-revalidate",
|
||||
"Pragma": "no-cache", "Expires": "0"})
|
||||
|
||||
# Return a friendly HTML page instead of JSON 404 (shown in iframe)
|
||||
marker = rdir / ".compliance_generating"
|
||||
if marker.exists():
|
||||
msg = "Generating compliance report... Please wait."
|
||||
else:
|
||||
msg = "Compliance report not generated yet."
|
||||
return HTMLResponse(
|
||||
f'<html><body style="display:flex;align-items:center;justify-content:center;height:100vh;margin:0;font-family:Calibri,sans-serif;color:#666">'
|
||||
f'<div style="text-align:center"><p style="font-size:14px">{msg}</p></div></body></html>',
|
||||
status_code=200,
|
||||
headers={"Cache-Control": "no-cache"})
|
||||
|
||||
|
||||
@router.post("/api/reports/{rid}/compliance-report/generate")
|
||||
async def generate_compliance_report(rid: str, bg: BackgroundTasks, u=Depends(current_user)):
|
||||
"""Start generating the compliance report in background."""
|
||||
_verify_report_access(rid, u)
|
||||
with db() as c:
|
||||
report = c.execute("SELECT * FROM reports WHERE id=?", (rid,)).fetchone()
|
||||
if not report: raise HTTPException(404, "Report not found")
|
||||
|
||||
rdir = REPORTS / rid
|
||||
csv_path = rdir / "cis_summary_report.csv"
|
||||
if not csv_path.exists():
|
||||
raise HTTPException(404, "cis_summary_report.csv not found")
|
||||
|
||||
# Reject if already generating
|
||||
marker = rdir / ".compliance_generating"
|
||||
if marker.exists():
|
||||
age = time.time() - marker.stat().st_mtime
|
||||
if age < 600:
|
||||
return {"status": "already_generating", "has_rag": False}
|
||||
|
||||
# Mark as generating BEFORE deleting cache
|
||||
marker.write_text(str(uuid.uuid4()), encoding="utf-8")
|
||||
|
||||
# Delete existing cache
|
||||
cache_path = rdir / "compliance_report.html"
|
||||
if cache_path.exists():
|
||||
cache_path.unlink()
|
||||
|
||||
has_adb = False
|
||||
with db() as c:
|
||||
adb_cfgs = c.execute("SELECT * FROM adb_vector_configs WHERE user_id=? AND is_active=1", (u["id"],)).fetchall()
|
||||
if adb_cfgs:
|
||||
has_adb = True
|
||||
|
||||
# Pre-check ADB connection before starting (fail fast instead of timeout loop)
|
||||
if has_adb:
|
||||
adb_cfg = dict(adb_cfgs[0])
|
||||
try:
|
||||
conn = _get_adb_connection(adb_cfg)
|
||||
conn.ping()
|
||||
conn.close()
|
||||
log.info(f"ADB connection OK for compliance report {rid}")
|
||||
except Exception as e:
|
||||
marker.unlink(missing_ok=True)
|
||||
log.warning(f"ADB connection failed for compliance report {rid}: {e}")
|
||||
raise HTTPException(503, f"Conexao com ADB falhou. Verifique a whitelist e a configuracao do ADB. Erro: {e}")
|
||||
|
||||
user_id = u["id"]
|
||||
log.info(f"Compliance report generation started for {rid} (has_adb={has_adb})")
|
||||
|
||||
def _bg_generate():
|
||||
try:
|
||||
log.info(f"Compliance report thread running for {rid}")
|
||||
_generate_compliance_report(rid, user_id, force_rag=has_adb)
|
||||
log.info(f"Compliance report generation completed for {rid}")
|
||||
except Exception as e:
|
||||
log.error(f"Compliance report generation failed for {rid}: {e}", exc_info=True)
|
||||
finally:
|
||||
# Remove generating marker
|
||||
m = REPORTS / rid / ".compliance_generating"
|
||||
if m.exists():
|
||||
m.unlink(missing_ok=True)
|
||||
|
||||
_chat_executor.submit(_bg_generate)
|
||||
return {"status": "generating", "has_rag": has_adb}
|
||||
|
||||
|
||||
@router.get("/api/reports/{rid}/compliance-report/status")
|
||||
async def compliance_report_status(rid: str, u=Depends(current_user)):
|
||||
"""Check compliance report status: ready, generating, or not started."""
|
||||
_verify_report_access(rid, u)
|
||||
rdir = REPORTS / rid
|
||||
cache_path = rdir / "compliance_report.html"
|
||||
marker = rdir / ".compliance_generating"
|
||||
if cache_path.exists():
|
||||
# Clean stale marker if report is ready
|
||||
if marker.exists():
|
||||
marker.unlink(missing_ok=True)
|
||||
return {"ready": True, "generating": False}
|
||||
if marker.exists():
|
||||
age = time.time() - marker.stat().st_mtime
|
||||
if age > 600:
|
||||
marker.unlink(missing_ok=True)
|
||||
log.warning(f"Expired stale compliance marker for {rid} (age={age:.0f}s)")
|
||||
return {"ready": False, "generating": False}
|
||||
# Read progress from marker file
|
||||
try:
|
||||
import json as _json
|
||||
progress = _json.loads(marker.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
progress = {}
|
||||
return {"ready": False, "generating": True,
|
||||
"current": progress.get("current", 0),
|
||||
"total": progress.get("total", 0),
|
||||
"step": progress.get("step", ""),
|
||||
"rec": progress.get("rec", "")}
|
||||
return {"ready": False, "generating": False}
|
||||
|
||||
|
||||
@router.get("/api/reports/{rid}/compliance-report/docx")
|
||||
async def compliance_report_docx(rid: str, token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))):
|
||||
"""Download DOCX compliance report directly."""
|
||||
import re as _re
|
||||
u = _user_from_token(token) if token else (await current_user(cred) if cred else None)
|
||||
if not u: raise HTTPException(401, "Not authenticated")
|
||||
_verify_report_access(rid, u)
|
||||
with db() as c:
|
||||
report = c.execute("SELECT tenancy_name FROM reports WHERE id=?", (rid,)).fetchone()
|
||||
if not report: raise HTTPException(404)
|
||||
files = c.execute("SELECT file_name, file_path FROM report_files WHERE report_id=?", (rid,)).fetchall()
|
||||
tenancy = report["tenancy_name"] or "report"
|
||||
safe_name = _re.sub(r'[^\w\-]', '_', tenancy)
|
||||
try:
|
||||
file_map = {f["file_name"]: f["file_path"] for f in files}
|
||||
docx_bytes = _generate_compliance_docx(rid, tenancy, file_map)
|
||||
if not docx_bytes:
|
||||
raise HTTPException(500, "DOCX generation failed")
|
||||
log.info(f"DOCX direct download: {len(docx_bytes)} bytes")
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
log.error(f"DOCX generation failed: {e}", exc_info=True)
|
||||
raise HTTPException(500, f"DOCX generation failed: {e}")
|
||||
return Response(
|
||||
content=docx_bytes,
|
||||
media_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
headers={"Content-Disposition": f'attachment; filename="CIS_Compliance_Report_{safe_name}.docx"'}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/reports/{rid}/compliance-report/download")
|
||||
async def compliance_report_download(rid: str, token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))):
|
||||
"""Download ZIP with compliance report PDF (Chromium headless) + CSV files."""
|
||||
import re as _re
|
||||
u = _user_from_token(token) if token else (await current_user(cred) if cred else None)
|
||||
if not u: raise HTTPException(401, "Not authenticated")
|
||||
_verify_report_access(rid, u)
|
||||
|
||||
rdir = REPORTS / rid
|
||||
html_path = rdir / "compliance_report.html"
|
||||
if not html_path.exists():
|
||||
raise HTTPException(404, "Compliance report not generated yet")
|
||||
|
||||
with db() as c:
|
||||
report = c.execute("SELECT tenancy_name FROM reports WHERE id=?", (rid,)).fetchone()
|
||||
if not report: raise HTTPException(404)
|
||||
files = c.execute("SELECT file_name, file_path FROM report_files WHERE report_id=?", (rid,)).fetchall()
|
||||
|
||||
tenancy = report["tenancy_name"] or "report"
|
||||
safe_name = _re.sub(r'[^\w\-]', '_', tenancy)
|
||||
file_map = {f["file_name"]: f["file_path"] for f in files}
|
||||
|
||||
# Prepare HTML for PDF: rewrite CSV links to relative paths
|
||||
html = html_path.read_text(encoding="utf-8")
|
||||
html = _re.sub(r'<button class="print-btn[^"]*"[^>]*>.*?</button>', '', html, flags=_re.DOTALL)
|
||||
html = _re.sub(r'<script>.*?</script>', '', html, flags=_re.DOTALL)
|
||||
for fname in file_map:
|
||||
html = _re.sub(
|
||||
r'href="[^"]*?/download[^"]*?"([^>]*>)\s*\[CSV\]\s*' + _re.escape(fname),
|
||||
f'href="csv/{fname}"\\1[CSV] {fname}',
|
||||
html
|
||||
)
|
||||
|
||||
# Generate PDF via Chromium headless (same engine as browser print)
|
||||
pdf_bytes = None
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmp_html = Path(tmpdir) / "report.html"
|
||||
tmp_pdf = Path(tmpdir) / "report.pdf"
|
||||
tmp_html.write_text(html, encoding="utf-8")
|
||||
loop = asyncio.get_event_loop()
|
||||
result = await asyncio.wait_for(
|
||||
loop.run_in_executor(None, partial(
|
||||
subprocess.run, [
|
||||
"chromium", "--headless", "--no-sandbox", "--disable-gpu",
|
||||
"--disable-software-rasterizer", "--run-all-compositor-stages-before-draw",
|
||||
f"--print-to-pdf={tmp_pdf}", "--no-pdf-header-footer",
|
||||
f"file://{tmp_html}"
|
||||
], capture_output=True, timeout=120)),
|
||||
timeout=120)
|
||||
if tmp_pdf.exists():
|
||||
pdf_bytes = tmp_pdf.read_bytes()
|
||||
log.info(f"Chromium PDF generated: {len(pdf_bytes)} bytes")
|
||||
else:
|
||||
log.error(f"Chromium PDF failed: {result.stderr.decode()[:500]}")
|
||||
except Exception as e:
|
||||
log.error(f"PDF generation failed: {e}", exc_info=True)
|
||||
|
||||
if not pdf_bytes:
|
||||
raise HTTPException(500, "PDF generation failed")
|
||||
|
||||
# Generate DOCX
|
||||
docx_bytes = None
|
||||
try:
|
||||
docx_bytes = _generate_compliance_docx(rid, tenancy, file_map)
|
||||
if docx_bytes:
|
||||
log.info(f"DOCX generated: {len(docx_bytes)} bytes")
|
||||
except Exception as e:
|
||||
log.warning(f"DOCX generation failed (non-fatal): {e}")
|
||||
|
||||
# Build ZIP: PDF + DOCX + CSVs
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, 'w', zipfile.ZIP_DEFLATED) as zf:
|
||||
zf.writestr(f"CIS_Compliance_Report_{safe_name}.pdf", pdf_bytes)
|
||||
if docx_bytes:
|
||||
zf.writestr(f"CIS_Compliance_Report_{safe_name}.docx", docx_bytes)
|
||||
for fname, fpath in file_map.items():
|
||||
p = Path(fpath)
|
||||
if p.exists() and fname.lower().endswith('.csv'):
|
||||
zf.write(str(p), f"csv/{fname}")
|
||||
|
||||
buf.seek(0)
|
||||
return Response(
|
||||
content=buf.getvalue(),
|
||||
media_type="application/zip",
|
||||
headers={"Content-Disposition": f'attachment; filename="CIS_Compliance_Report_{safe_name}.zip"'}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/reports/{rid}/download")
|
||||
async def report_dl(rid, fmt: str = Query("json"), token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))):
|
||||
u = _user_from_token(token) if token else (await current_user(cred) if cred else None)
|
||||
if not u: raise HTTPException(401, "Not authenticated")
|
||||
_verify_report_access(rid, u)
|
||||
with db() as c: r=c.execute("SELECT * FROM reports WHERE id=?",(rid,)).fetchone()
|
||||
if not r: raise HTTPException(404)
|
||||
p = r["json_path"] if fmt=="json" else r["html_path"]
|
||||
if not p or not Path(p).exists(): raise HTTPException(404)
|
||||
return FileResponse(p, filename=f"cis_{r['tenancy_name']}_{rid[:8]}.{fmt}")
|
||||
300
backend/routes/settings.py
Normal file
300
backend/routes/settings.py
Normal file
@@ -0,0 +1,300 @@
|
||||
"""Settings routes — audit log, config logs, chat logs, app settings, system prompts, config export/import."""
|
||||
import uuid
|
||||
import json
|
||||
import base64
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends, Query, UploadFile, File
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from config import VERSION, OCI_DIR, log
|
||||
from database import db
|
||||
from auth.jwt_auth import current_user, require, _audit
|
||||
from auth.crypto import _enc, _dec, _mask, _safe_dec
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# ── Audit Log ────────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/api/audit-log")
|
||||
async def audit_log(skip: int = Query(0, ge=0), limit: int = Query(100, ge=1, le=500), u=Depends(current_user)):
|
||||
with db() as c:
|
||||
if u["role"] == "admin":
|
||||
rows = c.execute("SELECT * FROM audit_log ORDER BY created_at DESC LIMIT ? OFFSET ?", (limit, skip)).fetchall()
|
||||
else:
|
||||
rows = c.execute("SELECT * FROM audit_log WHERE user_id=? ORDER BY created_at DESC LIMIT ? OFFSET ?", (u["id"], limit, skip)).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
# ── Config Logs ──────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/api/config-logs")
|
||||
async def get_config_logs(
|
||||
config_type: str = Query(None), config_id: str = Query(None),
|
||||
severity: str = Query(None), limit: int = Query(50, le=200),
|
||||
u=Depends(current_user)
|
||||
):
|
||||
query = "SELECT * FROM config_logs WHERE 1=1"
|
||||
params = []
|
||||
if config_type: query += " AND config_type=?"; params.append(config_type)
|
||||
if config_id: query += " AND config_id=?"; params.append(config_id)
|
||||
if severity: query += " AND severity=?"; params.append(severity)
|
||||
if u["role"] != "admin": query += " AND user_id=?"; params.append(u["id"])
|
||||
query += " ORDER BY created_at DESC LIMIT ?"
|
||||
params.append(limit)
|
||||
with db() as c: rows = c.execute(query, params).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
# ── Chat Logs ────────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/api/chat-logs")
|
||||
async def get_chat_logs(
|
||||
severity: str = Query(None), session_id: str = Query(None),
|
||||
limit: int = Query(50, le=200), u=Depends(current_user)
|
||||
):
|
||||
query = "SELECT * FROM chat_logs WHERE 1=1"
|
||||
params = []
|
||||
if severity: query += " AND severity=?"; params.append(severity)
|
||||
if session_id: query += " AND session_id=?"; params.append(session_id)
|
||||
if u["role"] != "admin": query += " AND user_id=?"; params.append(u["id"])
|
||||
query += " ORDER BY created_at DESC LIMIT ?"
|
||||
params.append(limit)
|
||||
with db() as c: rows = c.execute(query, params).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
# ── App Settings ─────────────────────────────────────────────────────────────
|
||||
|
||||
_SENSITIVE_SETTING_PREFIXES = ("oidc_", "secret_", "app_secret", "encryption_")
|
||||
|
||||
@router.get("/api/settings/{key}")
|
||||
async def get_setting(key: str, u=Depends(current_user)):
|
||||
if any(key.startswith(p) or key == p for p in _SENSITIVE_SETTING_PREFIXES) and u["role"] != "admin":
|
||||
raise HTTPException(403, "Acesso negado a configuração sensível")
|
||||
with db() as c:
|
||||
row = c.execute("SELECT value FROM app_settings WHERE key=?", (key,)).fetchone()
|
||||
val = row["value"] if row else ""
|
||||
if key == "oidc_client_secret" and val:
|
||||
val = _mask(_safe_dec(val))
|
||||
return {"key": key, "value": val}
|
||||
|
||||
@router.put("/api/settings/{key}")
|
||||
async def put_setting(key: str, body: dict, u=Depends(require("admin"))):
|
||||
value = body.get("value", "")
|
||||
if key == "oidc_client_secret" and value and "\u2022" in value:
|
||||
return {"key": key, "value": value}
|
||||
if key == "oidc_client_secret" and value:
|
||||
value = _enc(value)
|
||||
with db() as c:
|
||||
c.execute("INSERT INTO app_settings (key,value,updated_at) VALUES (?,?,datetime('now')) ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at", (key, value))
|
||||
return {"key": key, "value": value}
|
||||
|
||||
@router.get("/api/settings/timezone/options")
|
||||
async def timezone_options(u=Depends(current_user)):
|
||||
"""List common timezone options."""
|
||||
return {"timezones": [
|
||||
"America/Sao_Paulo", "America/New_York", "America/Chicago", "America/Denver",
|
||||
"America/Los_Angeles", "America/Toronto", "America/Mexico_City", "America/Bogota",
|
||||
"America/Argentina/Buenos_Aires", "America/Santiago",
|
||||
"Europe/London", "Europe/Paris", "Europe/Berlin", "Europe/Madrid", "Europe/Lisbon",
|
||||
"Asia/Tokyo", "Asia/Shanghai", "Asia/Singapore", "Asia/Kolkata", "Asia/Dubai",
|
||||
"Australia/Sydney", "Pacific/Auckland", "UTC",
|
||||
]}
|
||||
|
||||
@router.put("/api/users/me/timezone")
|
||||
async def set_user_timezone(body: dict, u=Depends(current_user)):
|
||||
"""Set the current user's timezone."""
|
||||
tz = body.get("timezone", "").strip()
|
||||
if not tz:
|
||||
raise HTTPException(400, "timezone is required")
|
||||
try:
|
||||
import zoneinfo
|
||||
zoneinfo.ZoneInfo(tz)
|
||||
except Exception:
|
||||
raise HTTPException(400, f"Invalid timezone: {tz}")
|
||||
with db() as c:
|
||||
c.execute("UPDATE users SET timezone=? WHERE id=?", (tz, u["id"]))
|
||||
_audit(u["id"], u["username"], "set_timezone", tz)
|
||||
return {"ok": True, "timezone": tz}
|
||||
|
||||
@router.get("/api/users/me/timezone")
|
||||
async def get_user_timezone(u=Depends(current_user)):
|
||||
"""Get the current user's timezone."""
|
||||
return {"timezone": u.get("timezone", "America/Sao_Paulo")}
|
||||
|
||||
|
||||
# ── System Prompts ───────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/api/prompts/{agent}")
|
||||
async def list_prompts(agent: str, u=Depends(current_user)):
|
||||
with db() as c:
|
||||
rows = c.execute("SELECT * FROM system_prompts WHERE agent=? ORDER BY is_active DESC, created_at DESC", (agent,)).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
@router.post("/api/prompts")
|
||||
async def save_prompt(body: dict, u=Depends(require("admin"))):
|
||||
agent = body.get("agent", "chat")
|
||||
with db() as c:
|
||||
count = c.execute("SELECT COUNT(*) FROM system_prompts WHERE agent=?", (agent,)).fetchone()[0]
|
||||
if count >= 10:
|
||||
raise HTTPException(400, "Limite de 10 prompts atingido. Exclua um antes de criar outro.")
|
||||
pid = str(uuid.uuid4())
|
||||
name = body.get("name", "Sem nome")
|
||||
content = body.get("content", "")
|
||||
is_active = body.get("is_active", False)
|
||||
with db() as c:
|
||||
if is_active:
|
||||
c.execute("UPDATE system_prompts SET is_active=0 WHERE agent=?", (agent,))
|
||||
c.execute("INSERT INTO system_prompts (id,name,agent,content,is_active) VALUES (?,?,?,?,?)",
|
||||
(pid, name, agent, content, int(is_active)))
|
||||
return {"id": pid}
|
||||
|
||||
@router.put("/api/prompts/{pid}")
|
||||
async def update_prompt(pid: str, body: dict, u=Depends(require("admin"))):
|
||||
with db() as c:
|
||||
existing = c.execute("SELECT * FROM system_prompts WHERE id=?", (pid,)).fetchone()
|
||||
if not existing:
|
||||
raise HTTPException(404)
|
||||
name = body.get("name", existing["name"])
|
||||
content = body.get("content", existing["content"])
|
||||
is_active = body.get("is_active", existing["is_active"])
|
||||
with db() as c:
|
||||
if is_active:
|
||||
c.execute("UPDATE system_prompts SET is_active=0 WHERE agent=?", (existing["agent"],))
|
||||
c.execute("UPDATE system_prompts SET name=?,content=?,is_active=? WHERE id=?",
|
||||
(name, content, int(is_active), pid))
|
||||
return {"id": pid}
|
||||
|
||||
@router.delete("/api/prompts/{pid}")
|
||||
async def delete_prompt(pid: str, u=Depends(require("admin"))):
|
||||
with db() as c:
|
||||
c.execute("DELETE FROM system_prompts WHERE id=?", (pid,))
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
# ── Export / Import Configuration ────────────────────────────────────────────
|
||||
|
||||
@router.get("/api/config/export")
|
||||
async def export_config(u=Depends(require("admin"))):
|
||||
"""Export configurations as JSON. Excludes sensitive credentials (private keys, passwords, secrets)."""
|
||||
_OCI_SAFE_COLS = "id,user_id,tenancy_name,region,compartment_id,auth_type,ssh_public_key,created_at"
|
||||
_ADB_SAFE_COLS = "id,user_id,config_name,dsn,username,table_name,use_mtls,is_active,is_global,genai_config_id,embedding_model_id,created_at"
|
||||
data = {"version": VERSION, "exported_at": datetime.now().isoformat()}
|
||||
with db() as c:
|
||||
# OCI configs (no private keys, no encrypted OCIDs, no passphrases)
|
||||
data["oci_configs"] = [dict(r) for r in c.execute(f"SELECT {_OCI_SAFE_COLS} FROM oci_configs").fetchall()]
|
||||
# GenAI configs (no secrets — model info only)
|
||||
data["genai_configs"] = [dict(r) for r in c.execute("SELECT * FROM genai_configs").fetchall()]
|
||||
# MCP servers
|
||||
data["mcp_servers"] = [dict(r) for r in c.execute("SELECT * FROM mcp_servers").fetchall()]
|
||||
# System prompts
|
||||
data["system_prompts"] = [dict(r) for r in c.execute("SELECT * FROM system_prompts").fetchall()]
|
||||
# App settings (exclude sensitive keys)
|
||||
data["app_settings"] = [dict(r) for r in c.execute("SELECT * FROM app_settings").fetchall()
|
||||
if not any(r["key"].startswith(p) for p in _SENSITIVE_SETTING_PREFIXES)]
|
||||
# ADB vector configs (no passwords, no wallet passwords)
|
||||
data["adb_vector_configs"] = [dict(r) for r in c.execute(f"SELECT {_ADB_SAFE_COLS} FROM adb_vector_configs").fetchall()]
|
||||
# ADB vector tables
|
||||
data["adb_vector_tables"] = [dict(r) for r in c.execute("SELECT * FROM adb_vector_tables").fetchall()]
|
||||
return StreamingResponse(
|
||||
iter([json.dumps(data, indent=2, ensure_ascii=False)]),
|
||||
media_type="application/json",
|
||||
headers={"Content-Disposition": f'attachment; filename="oci-cis-agent-config-{datetime.now().strftime("%Y%m%d-%H%M%S")}.json"'}
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/config/import")
|
||||
async def import_config(file: UploadFile = File(...), u=Depends(require("admin"))):
|
||||
"""Import configurations from exported JSON. Merges by ID (skip existing)."""
|
||||
from utils import validate_upload
|
||||
await validate_upload(file, {".json"})
|
||||
file.file.seek(0)
|
||||
content = await file.read()
|
||||
try:
|
||||
data = json.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
raise HTTPException(400, "Arquivo JSON inválido")
|
||||
counts = {"oci_configs": 0, "genai_configs": 0, "mcp_servers": 0, "system_prompts": 0, "app_settings": 0, "adb_vector_configs": 0, "adb_vector_tables": 0, "skipped": 0}
|
||||
with db() as c:
|
||||
# OCI configs
|
||||
for cfg in data.get("oci_configs", []):
|
||||
if c.execute("SELECT 1 FROM oci_configs WHERE id=?", (cfg["id"],)).fetchone():
|
||||
counts["skipped"] += 1; continue
|
||||
# Import metadata only — credentials must be reconfigured manually
|
||||
cdir = OCI_DIR / cfg["id"]; cdir.mkdir(parents=True, exist_ok=True)
|
||||
kp = cdir / "oci_api_key.pem"
|
||||
# Restore private key only if present in legacy export format
|
||||
if cfg.get("_private_key_b64"):
|
||||
kp.write_bytes(base64.b64decode(cfg["_private_key_b64"])); kp.chmod(0o600)
|
||||
cfg_file = cdir / "config"
|
||||
cfg_content = (f"[DEFAULT]\nuser={_dec(cfg['user_ocid']) if cfg.get('user_ocid') else ''}\n"
|
||||
f"fingerprint={_dec(cfg['fingerprint']) if cfg.get('fingerprint') else ''}\n"
|
||||
f"tenancy={_dec(cfg['tenancy_ocid']) if cfg.get('tenancy_ocid') else ''}\n"
|
||||
f"region={cfg.get('region','')}\nkey_file={kp}\n")
|
||||
if cfg.get("_key_passphrase"):
|
||||
cfg_content += f"pass_phrase={cfg['_key_passphrase']}\n"
|
||||
cfg_file.write_text(cfg_content); cfg_file.chmod(0o600)
|
||||
c.execute("INSERT INTO oci_configs (id,user_id,tenancy_name,tenancy_ocid,user_ocid,fingerprint,region,key_file_path,compartment_id,created_at) VALUES (?,?,?,?,?,?,?,?,?,?)",
|
||||
(cfg["id"], cfg.get("user_id", u["id"]), cfg.get("tenancy_name",""),
|
||||
_enc(cfg["tenancy_ocid"]) if cfg.get("tenancy_ocid") else "",
|
||||
_enc(cfg["user_ocid"]) if cfg.get("user_ocid") else "",
|
||||
_enc(cfg["fingerprint"]) if cfg.get("fingerprint") else "",
|
||||
cfg.get("region",""), str(kp),
|
||||
_enc(cfg["compartment_id"]) if cfg.get("compartment_id") else None,
|
||||
cfg.get("created_at")))
|
||||
counts["oci_configs"] += 1
|
||||
# GenAI configs
|
||||
for cfg in data.get("genai_configs", []):
|
||||
if c.execute("SELECT 1 FROM genai_configs WHERE id=?", (cfg["id"],)).fetchone():
|
||||
counts["skipped"] += 1; continue
|
||||
c.execute("INSERT INTO genai_configs (id,user_id,name,oci_config_id,model_id,model_ocid,compartment_id,genai_region,endpoint,serving_type,dedicated_endpoint_id,temperature,max_tokens,top_p,top_k,frequency_penalty,presence_penalty,reasoning_effort,is_default,system_prompt,created_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(cfg["id"], cfg.get("user_id", u["id"]), cfg.get("name",""), cfg.get("oci_config_id",""), cfg.get("model_id",""),
|
||||
cfg.get("model_ocid"), cfg.get("compartment_id",""), cfg.get("genai_region",""), cfg.get("endpoint",""),
|
||||
cfg.get("serving_type","ON_DEMAND"), cfg.get("dedicated_endpoint_id"), cfg.get("temperature",1),
|
||||
cfg.get("max_tokens",6000), cfg.get("top_p",0.95), cfg.get("top_k",1), cfg.get("frequency_penalty",0),
|
||||
cfg.get("presence_penalty",0), cfg.get("reasoning_effort"), cfg.get("is_default",0), cfg.get("system_prompt",""), cfg.get("created_at")))
|
||||
counts["genai_configs"] += 1
|
||||
# MCP servers
|
||||
for srv in data.get("mcp_servers", []):
|
||||
if c.execute("SELECT 1 FROM mcp_servers WHERE id=?", (srv["id"],)).fetchone():
|
||||
counts["skipped"] += 1; continue
|
||||
c.execute("INSERT INTO mcp_servers (id,user_id,name,description,server_type,command,args,env_vars,url,module_path,tools,linked_adb_id,is_active,is_global,created_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(srv["id"], srv.get("user_id", u["id"]), srv.get("name",""), srv.get("description"), srv.get("server_type","stdio"),
|
||||
srv.get("command"), srv.get("args"), srv.get("env_vars"), srv.get("url"), srv.get("module_path"),
|
||||
srv.get("tools"), srv.get("linked_adb_id"), srv.get("is_active",1), srv.get("is_global",0), srv.get("created_at")))
|
||||
counts["mcp_servers"] += 1
|
||||
# System prompts
|
||||
for p in data.get("system_prompts", []):
|
||||
if c.execute("SELECT 1 FROM system_prompts WHERE id=?", (p["id"],)).fetchone():
|
||||
counts["skipped"] += 1; continue
|
||||
c.execute("INSERT INTO system_prompts (id,name,agent,content,is_active,created_at) VALUES (?,?,?,?,?,?)",
|
||||
(p["id"], p.get("name",""), p.get("agent","chat"), p.get("content",""), p.get("is_active",0), p.get("created_at")))
|
||||
counts["system_prompts"] += 1
|
||||
# App settings
|
||||
for s in data.get("app_settings", []):
|
||||
if any(s["key"].startswith(p) for p in _SENSITIVE_SETTING_PREFIXES):
|
||||
counts["skipped"] += 1; continue
|
||||
c.execute("INSERT INTO app_settings (key,value,updated_at) VALUES (?,?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at",
|
||||
(s["key"], s.get("value",""), s.get("updated_at")))
|
||||
counts["app_settings"] += 1
|
||||
# ADB vector configs
|
||||
for cfg in data.get("adb_vector_configs", []):
|
||||
if c.execute("SELECT 1 FROM adb_vector_configs WHERE id=?", (cfg["id"],)).fetchone():
|
||||
counts["skipped"] += 1; continue
|
||||
c.execute("INSERT INTO adb_vector_configs (id,user_id,config_name,dsn,username,password_enc,wallet_dir,wallet_password_enc,table_name,use_mtls,is_active,is_global,created_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(cfg["id"], cfg.get("user_id", u["id"]), cfg.get("config_name",""), cfg.get("dsn",""), cfg.get("username",""),
|
||||
cfg.get("password_enc",""), cfg.get("wallet_dir"), cfg.get("wallet_password_enc"), cfg.get("table_name","CIS_EMBEDDINGS"),
|
||||
cfg.get("use_mtls",1), cfg.get("is_active",1), cfg.get("is_global",0), cfg.get("created_at")))
|
||||
counts["adb_vector_configs"] += 1
|
||||
# ADB vector tables
|
||||
for t in data.get("adb_vector_tables", []):
|
||||
if c.execute("SELECT 1 FROM adb_vector_tables WHERE id=?", (t["id"],)).fetchone():
|
||||
counts["skipped"] += 1; continue
|
||||
c.execute("INSERT INTO adb_vector_tables (id,adb_config_id,table_name,description,is_active,created_at) VALUES (?,?,?,?,?,?)",
|
||||
(t["id"], t.get("adb_config_id",""), t.get("table_name",""), t.get("description",""), t.get("is_active",1), t.get("created_at")))
|
||||
counts["adb_vector_tables"] += 1
|
||||
_audit(u["id"], u["username"], "import_config", "bulk", json.dumps(counts))
|
||||
return counts
|
||||
641
backend/routes/terraform.py
Normal file
641
backend/routes/terraform.py
Normal file
@@ -0,0 +1,641 @@
|
||||
"""Terraform Agent routes — OCI resource actions, workspace CRUD, plan/apply/destroy."""
|
||||
import uuid
|
||||
import asyncio
|
||||
import shutil
|
||||
import re
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends, Query, BackgroundTasks, Request
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
from fastapi.responses import Response
|
||||
|
||||
from config import TERRAFORM_DIR, _chat_executor, log
|
||||
from database import db
|
||||
from auth.jwt_auth import current_user, _user_from_token, _audit, _verify_config_access
|
||||
from auth.crypto import _safe_dec
|
||||
from services.oci_helpers import _get_oci_config
|
||||
from services.terraform_exec import _split_tf_monolith, _write_tf_files, _terraform_exec
|
||||
from services.genai import (
|
||||
_call_genai,
|
||||
_TF_RESOURCE_REF_PATH,
|
||||
_load_tf_resource_reference,
|
||||
_regenerate_tf_reference,
|
||||
_get_tf_docs_for_resources,
|
||||
_detect_tf_resource_types,
|
||||
)
|
||||
from services.chat_background import _chat_start, _chat_background
|
||||
from utils import running_terraform
|
||||
from models import ChatMsg, TfPromptReq, TfWorkspaceReq
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def _fetch_compartment_resources(oci_config_id: str, compartment_id: str, region: str = None) -> dict:
|
||||
"""Fetch existing OCI resources in a compartment for terraform context."""
|
||||
import oci
|
||||
config = _get_oci_config(oci_config_id)
|
||||
if region:
|
||||
config["region"] = region
|
||||
resources = {}
|
||||
try:
|
||||
vn = oci.core.VirtualNetworkClient(config)
|
||||
vcns = vn.list_vcns(compartment_id).data
|
||||
resources["vcns"] = [{"id": v.id, "display_name": v.display_name, "cidr_blocks": v.cidr_blocks, "state": v.lifecycle_state} for v in vcns if v.lifecycle_state == "AVAILABLE"]
|
||||
subs = vn.list_subnets(compartment_id).data
|
||||
resources["subnets"] = [{"id": s.id, "display_name": s.display_name, "cidr_block": s.cidr_block, "vcn_id": s.vcn_id, "public": not s.prohibit_public_ip_on_vnic, "state": s.lifecycle_state} for s in subs if s.lifecycle_state == "AVAILABLE"]
|
||||
igws = vn.list_internet_gateways(compartment_id).data
|
||||
resources["internet_gateways"] = [{"id": g.id, "display_name": g.display_name, "vcn_id": g.vcn_id, "enabled": g.is_enabled} for g in igws if g.lifecycle_state == "AVAILABLE"]
|
||||
ngws = vn.list_nat_gateways(compartment_id).data
|
||||
resources["nat_gateways"] = [{"id": g.id, "display_name": g.display_name, "vcn_id": g.vcn_id} for g in ngws if g.lifecycle_state == "AVAILABLE"]
|
||||
rts = vn.list_route_tables(compartment_id).data
|
||||
resources["route_tables"] = [{"id": r.id, "display_name": r.display_name, "vcn_id": r.vcn_id} for r in rts if r.lifecycle_state == "AVAILABLE"]
|
||||
sls = vn.list_security_lists(compartment_id).data
|
||||
resources["security_lists"] = [{"id": s.id, "display_name": s.display_name, "vcn_id": s.vcn_id} for s in sls if s.lifecycle_state == "AVAILABLE"]
|
||||
except Exception as e:
|
||||
log.warning(f"Failed to fetch networking resources: {e}")
|
||||
try:
|
||||
compute = oci.core.ComputeClient(config)
|
||||
insts = compute.list_instances(compartment_id).data
|
||||
resources["instances"] = [{"id": i.id, "display_name": i.display_name, "shape": i.shape, "state": i.lifecycle_state} for i in insts if i.lifecycle_state in ("RUNNING", "STOPPED")]
|
||||
except Exception as e:
|
||||
log.warning(f"Failed to fetch compute resources: {e}")
|
||||
try:
|
||||
db_client = oci.database.DatabaseClient(config)
|
||||
adbs = db_client.list_autonomous_databases(compartment_id).data
|
||||
resources["autonomous_databases"] = [{"id": a.id, "display_name": a.display_name, "db_name": a.db_name, "state": a.lifecycle_state, "is_free_tier": a.is_free_tier} for a in adbs if a.lifecycle_state in ("AVAILABLE", "STOPPED")]
|
||||
except Exception as e:
|
||||
log.warning(f"Failed to fetch database resources: {e}")
|
||||
try:
|
||||
bs = oci.core.BlockstorageClient(config)
|
||||
vols = bs.list_volumes(compartment_id).data
|
||||
resources["block_volumes"] = [{"id": v.id, "display_name": v.display_name, "size_in_gbs": v.size_in_gbs, "state": v.lifecycle_state} for v in vols if v.lifecycle_state == "AVAILABLE"]
|
||||
except Exception as e:
|
||||
log.warning(f"Failed to fetch block storage resources: {e}")
|
||||
try:
|
||||
lb_client = oci.load_balancer.LoadBalancerClient(config)
|
||||
lbs = lb_client.list_load_balancers(compartment_id).data
|
||||
resources["load_balancers"] = [{"id": l.id, "display_name": l.display_name, "shape_name": l.shape_name, "state": l.lifecycle_state} for l in lbs if l.lifecycle_state == "ACTIVE"]
|
||||
except Exception as e:
|
||||
log.warning(f"Failed to fetch load balancer resources: {e}")
|
||||
return resources
|
||||
|
||||
|
||||
def _build_resource_context(resources: dict) -> str:
|
||||
"""Build a text summary of existing resources for injection into system prompt."""
|
||||
lines = ["## RECURSOS OCI EXISTENTES NO COMPARTMENT"]
|
||||
total = 0
|
||||
resource_labels = {
|
||||
"vcns": ("VCNs", lambda r: f" - {r['display_name']} (CIDR: {','.join(r.get('cidr_blocks') or [])}) [OCID: {r['id'][:30]}...]"),
|
||||
"subnets": ("Subnets", lambda r: f" - {r['display_name']} (CIDR: {r['cidr_block']}, {'pública' if r.get('public') else 'privada'}, VCN: {r['vcn_id'][:30]}...)"),
|
||||
"internet_gateways": ("Internet Gateways", lambda r: f" - {r['display_name']} (VCN: {r['vcn_id'][:30]}...)"),
|
||||
"nat_gateways": ("NAT Gateways", lambda r: f" - {r['display_name']} (VCN: {r['vcn_id'][:30]}...)"),
|
||||
"route_tables": ("Route Tables", lambda r: f" - {r['display_name']} (VCN: {r['vcn_id'][:30]}...)"),
|
||||
"security_lists": ("Security Lists", lambda r: f" - {r['display_name']} (VCN: {r['vcn_id'][:30]}...)"),
|
||||
"instances": ("Compute Instances", lambda r: f" - {r['display_name']} (Shape: {r['shape']}, Estado: {r['state']})"),
|
||||
"autonomous_databases": ("Autonomous Databases", lambda r: f" - {r['display_name']} (DB: {r['db_name']}, Free Tier: {r.get('is_free_tier', False)})"),
|
||||
"block_volumes": ("Block Volumes", lambda r: f" - {r['display_name']} ({r.get('size_in_gbs', '?')} GB)"),
|
||||
"load_balancers": ("Load Balancers", lambda r: f" - {r['display_name']} (Shape: {r.get('shape_name', '?')})"),
|
||||
}
|
||||
for key, (label, formatter) in resource_labels.items():
|
||||
items = resources.get(key, [])
|
||||
if items:
|
||||
total += len(items)
|
||||
lines.append(f"\n**{label}** ({len(items)}):")
|
||||
for item in items[:15]: # limit to 15 per category
|
||||
lines.append(formatter(item))
|
||||
if len(items) > 15:
|
||||
lines.append(f" ... e mais {len(items) - 15}")
|
||||
if total == 0:
|
||||
lines.append("\nNenhum recurso encontrado neste compartment. Todos os recursos serão novos.")
|
||||
else:
|
||||
lines.append(f"\n**Total: {total} recursos existentes.** REUTILIZE-OS sempre que possível usando data sources.")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _fix_single_line_blocks(content: str) -> str:
|
||||
"""Expand ANY single-line HCL block with multiple args into multi-line format.
|
||||
Matches both top-level (variable, resource) AND nested blocks (ingress_security_rules, route_rules, etc.)
|
||||
Does NOT match map assignments (tags = { ... }) because of the '=' before '{'."""
|
||||
import re as _re
|
||||
def _expand(m):
|
||||
header, body = m.group(1), m.group(2)
|
||||
# Split by comma first
|
||||
args = [a.strip() for a in body.split(',') if a.strip()]
|
||||
if len(args) < 2:
|
||||
# Try space-separated key=value
|
||||
args = _re.findall(r'\w+\s*=\s*(?:"[^"]*"|\S+)', body)
|
||||
if len(args) < 2:
|
||||
return m.group(0)
|
||||
indent = _re.match(r'^(\s*)', header).group(1)
|
||||
return header.rstrip() + ' {\n' + '\n'.join(indent + ' ' + a for a in args) + '\n' + indent + '}'
|
||||
return _re.sub(
|
||||
r'^(\s*[a-z]\w*(?:\s+"[^"]*")*)\s*\{\s*(.+?)\s*\}\s*$',
|
||||
_expand, content, flags=_re.MULTILINE)
|
||||
|
||||
|
||||
# ── Constants ────────────────────────────────────────────────────────────────
|
||||
|
||||
TFP_SYSTEM_PROMPT = """Você é um arquiteto sênior de infraestrutura OCI com profundo conhecimento do Terraform OCI Provider.
|
||||
O usuário vai descrever a infraestrutura desejada em linguagem natural. Sua tarefa é gerar um **prompt detalhado e estruturado** que será enviado a um agente Terraform para criar o código HCL.
|
||||
|
||||
### Formato do Prompt Gerado
|
||||
Retorne APENAS o prompt estruturado (não gere código Terraform). Use markdown:
|
||||
|
||||
1. **Título**: Uma linha descrevendo o projeto (ex: "## Infraestrutura Multi-Region HA para Aplicação Web")
|
||||
2. **Arquitetura**: Diagrama textual ou descrição da topologia
|
||||
3. **Recursos por Categoria**: Organizados em seções ## (Networking, Compute, Storage, Database, Security, Observability)
|
||||
- Para cada recurso: nome, especificações técnicas, relacionamentos
|
||||
4. **Networking**: Sempre inclua CIDRs (10.0.0.0/16 para VCN, /24 para subnets), gateways necessários, route tables, security lists/NSGs
|
||||
5. **Dependências**: Liste dependências entre recursos (ex: "Compute depende de Subnet, que depende de VCN")
|
||||
6. **Requisitos Técnicos**: Seção final com regras para o agente Terraform:
|
||||
- Separação em arquivos (provider.tf, variables.tf, vcn.tf, subnets.tf, compute.tf, outputs.tf)
|
||||
- Variáveis obrigatórias (compartment_id, region, ssh_public_key, etc.)
|
||||
- Naming convention (ex: proj-env-resource)
|
||||
- Outputs necessários (IPs, OCIDs, endpoints)
|
||||
- Tags padrão (Environment, Project, ManagedBy=Terraform)
|
||||
|
||||
### Regras de Inferência
|
||||
- Se pedir VCN, inclua automaticamente: subnets (pública + privada), internet gateway, NAT gateway, service gateway, route tables, security lists
|
||||
- Se pedir Compute, inclua: boot volume, NSG, cloud-init básico, shape e imagem
|
||||
- Se pedir Database, inclua: subnet privada dedicada, NSG com porta do DB, backup policy
|
||||
- Se pedir OKE, inclua: VCN com topologia para K8s (API endpoint subnet, workers subnet, pods subnet, services LB subnet)
|
||||
- Se mencionar multi-region, use providers aliased e detalhe RPC/DRG
|
||||
- Se mencionar HA/DR, inclua redundância cross-AD ou cross-region
|
||||
- Se mencionar segurança/CIS, inclua Vault, Cloud Guard, WAF, logging, encryption at rest
|
||||
|
||||
### Restrições do OCI Provider
|
||||
- RPC (Remote Peering Connection): NÃO usar oci_core_drg_attachment para RPC. O attachment é criado automaticamente pelo recurso oci_core_remote_peering_connection
|
||||
- Cada VCN suporta apenas 1 DRG attachment — não criar duplicados
|
||||
- DRG attachment type para VCN é "VCN", não "REMOTE_PEERING_CONNECTION"
|
||||
|
||||
### Restrições do Workspace
|
||||
- NUNCA sugira o uso de `module` blocks. O workspace é flat (diretório único, sem subdiretórios). Toda infraestrutura deve ser definida com `resource` e `data` blocks diretamente.
|
||||
- Variáveis devem ser declaradas SOMENTE em `variables.tf`. Nunca duplicar declarações de variáveis entre arquivos.
|
||||
|
||||
### Tom
|
||||
- Seja técnico e preciso
|
||||
- Use português brasileiro
|
||||
- Não gere código, apenas o prompt estruturado"""
|
||||
|
||||
|
||||
# ── OCI Resource Actions ─────────────────────────────────────────────────────
|
||||
|
||||
@router.post("/api/oci/instances/{instance_id}/action")
|
||||
async def oci_instance_action(instance_id: str, req: dict, u=Depends(current_user)):
|
||||
action = req.get("action", "").upper()
|
||||
oci_config_id = req.get("oci_config_id", "")
|
||||
region = req.get("region")
|
||||
if action not in ("START", "STOP"):
|
||||
raise HTTPException(400, "Ação inválida. Use START ou STOP.")
|
||||
if not oci_config_id:
|
||||
raise HTTPException(400, "oci_config_id obrigatório")
|
||||
_verify_config_access("oci", oci_config_id, u)
|
||||
try:
|
||||
import oci
|
||||
config = _get_oci_config(oci_config_id)
|
||||
if region:
|
||||
config["region"] = region
|
||||
compute = oci.core.ComputeClient(config)
|
||||
compute.instance_action(instance_id, action)
|
||||
_audit(u["id"], u["username"], f"instance_{action.lower()}", instance_id)
|
||||
return {"ok": True, "action": action, "instance_id": instance_id}
|
||||
except Exception as e:
|
||||
raise HTTPException(500, str(e)[:500])
|
||||
|
||||
|
||||
@router.post("/api/oci/autonomous-databases/{adb_id}/action")
|
||||
async def oci_adb_action(adb_id: str, req: dict, u=Depends(current_user)):
|
||||
action = req.get("action", "").lower()
|
||||
oci_config_id = req.get("oci_config_id", "")
|
||||
region = req.get("region")
|
||||
if action not in ("start", "stop"):
|
||||
raise HTTPException(400, "Ação inválida. Use start ou stop.")
|
||||
if not oci_config_id:
|
||||
raise HTTPException(400, "oci_config_id obrigatório")
|
||||
_verify_config_access("oci", oci_config_id, u)
|
||||
try:
|
||||
import oci
|
||||
config = _get_oci_config(oci_config_id)
|
||||
if region:
|
||||
config["region"] = region
|
||||
db_client = oci.database.DatabaseClient(config)
|
||||
if action == "start":
|
||||
db_client.start_autonomous_database(adb_id)
|
||||
else:
|
||||
db_client.stop_autonomous_database(adb_id)
|
||||
_audit(u["id"], u["username"], f"adb_{action}", adb_id)
|
||||
return {"ok": True, "action": action, "adb_id": adb_id}
|
||||
except Exception as e:
|
||||
raise HTTPException(500, str(e)[:500])
|
||||
|
||||
|
||||
@router.post("/api/oci/autonomous-databases/{adb_id}/update-network-access")
|
||||
async def oci_adb_update_network(adb_id: str, req: dict, u=Depends(current_user)):
|
||||
oci_config_id = req.get("oci_config_id", "")
|
||||
ip = req.get("ip", "")
|
||||
region = req.get("region")
|
||||
if not oci_config_id:
|
||||
raise HTTPException(400, "oci_config_id obrigatório")
|
||||
_verify_config_access("oci", oci_config_id, u)
|
||||
if not ip:
|
||||
raise HTTPException(400, "ip obrigatório")
|
||||
try:
|
||||
import oci
|
||||
config = _get_oci_config(oci_config_id)
|
||||
if region:
|
||||
config["region"] = region
|
||||
db_client = oci.database.DatabaseClient(config)
|
||||
adb = db_client.get_autonomous_database(adb_id).data
|
||||
current_acl = adb.whitelisted_ips or []
|
||||
# Build new ACL: keep existing entries, add new IP if not present
|
||||
ip_cidr = ip if "/" in ip else ip + "/32"
|
||||
new_acl = list(set(current_acl) | {ip_cidr})
|
||||
db_client.update_autonomous_database(
|
||||
adb_id,
|
||||
oci.database.models.UpdateAutonomousDatabaseDetails(whitelisted_ips=new_acl))
|
||||
_audit(u["id"], u["username"], "adb_update_network", adb_id, f"ip={ip_cidr}")
|
||||
return {"ok": True, "adb_id": adb_id, "ip_added": ip_cidr, "acl": new_acl}
|
||||
except Exception as e:
|
||||
raise HTTPException(500, str(e)[:500])
|
||||
|
||||
|
||||
@router.post("/api/oci/db-systems/{db_system_id}/action")
|
||||
async def oci_db_system_action(db_system_id: str, req: dict, u=Depends(current_user)):
|
||||
action = req.get("action", "").upper()
|
||||
oci_config_id = req.get("oci_config_id", "")
|
||||
region = req.get("region")
|
||||
if action not in ("START", "STOP"):
|
||||
raise HTTPException(400, "Ação inválida. Use START ou STOP.")
|
||||
if not oci_config_id:
|
||||
raise HTTPException(400, "oci_config_id obrigatório")
|
||||
_verify_config_access("oci", oci_config_id, u)
|
||||
try:
|
||||
import oci
|
||||
config = _get_oci_config(oci_config_id)
|
||||
if region: config["region"] = region
|
||||
db_client = oci.database.DatabaseClient(config)
|
||||
if action == "START":
|
||||
db_client.start_db_system(db_system_id)
|
||||
else:
|
||||
db_client.stop_db_system(db_system_id)
|
||||
_audit(u["id"], u["username"], f"db_system_{action.lower()}", db_system_id)
|
||||
return {"ok": True, "action": action, "db_system_id": db_system_id}
|
||||
except Exception as e:
|
||||
raise HTTPException(500, str(e)[:500])
|
||||
|
||||
|
||||
@router.post("/api/oci/mysql-db-systems/{db_system_id}/action")
|
||||
async def oci_mysql_action(db_system_id: str, req: dict, u=Depends(current_user)):
|
||||
action = req.get("action", "").lower()
|
||||
oci_config_id = req.get("oci_config_id", "")
|
||||
region = req.get("region")
|
||||
if action not in ("start", "stop"):
|
||||
raise HTTPException(400, "Ação inválida. Use start ou stop.")
|
||||
if not oci_config_id:
|
||||
raise HTTPException(400, "oci_config_id obrigatório")
|
||||
_verify_config_access("oci", oci_config_id, u)
|
||||
try:
|
||||
import oci
|
||||
config = _get_oci_config(oci_config_id)
|
||||
if region: config["region"] = region
|
||||
mysql = oci.mysql.DbSystemClient(config)
|
||||
if action == "start":
|
||||
mysql.start_db_system(db_system_id)
|
||||
else:
|
||||
shutdown_type = req.get("shutdown_type", "SLOW")
|
||||
mysql.stop_db_system(db_system_id, oci.mysql.models.StopDbSystemDetails(shutdown_type=shutdown_type))
|
||||
_audit(u["id"], u["username"], f"mysql_{action}", db_system_id)
|
||||
return {"ok": True, "action": action, "db_system_id": db_system_id}
|
||||
except Exception as e:
|
||||
raise HTTPException(500, str(e)[:500])
|
||||
|
||||
|
||||
@router.post("/api/oci/container-instances/{ci_id}/action")
|
||||
async def oci_container_instance_action(ci_id: str, req: dict, u=Depends(current_user)):
|
||||
action = req.get("action", "").lower()
|
||||
oci_config_id = req.get("oci_config_id", "")
|
||||
region = req.get("region")
|
||||
if action not in ("start", "stop"):
|
||||
raise HTTPException(400, "Ação inválida. Use start ou stop.")
|
||||
if not oci_config_id:
|
||||
raise HTTPException(400, "oci_config_id obrigatório")
|
||||
_verify_config_access("oci", oci_config_id, u)
|
||||
try:
|
||||
import oci
|
||||
config = _get_oci_config(oci_config_id)
|
||||
if region: config["region"] = region
|
||||
ci = oci.container_instances.ContainerInstanceClient(config)
|
||||
if action == "start":
|
||||
ci.start_container_instance(ci_id)
|
||||
else:
|
||||
ci.stop_container_instance(ci_id)
|
||||
_audit(u["id"], u["username"], f"container_instance_{action}", ci_id)
|
||||
return {"ok": True, "action": action, "container_instance_id": ci_id}
|
||||
except Exception as e:
|
||||
raise HTTPException(500, str(e)[:500])
|
||||
|
||||
|
||||
@router.get("/api/oci/my-ip")
|
||||
async def get_my_ip(request: Request):
|
||||
"""Return the caller's public IP address."""
|
||||
forwarded = request.headers.get("x-forwarded-for", "")
|
||||
ip = forwarded.split(",")[0].strip() if forwarded else (request.client.host if request.client else "")
|
||||
return {"ip": ip}
|
||||
|
||||
|
||||
# ── Terraform Resources ──────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/api/terraform/resources")
|
||||
async def tf_list_resources(oci_config_id: str = Query(...), compartment_id: str = Query(...), region: str = Query(None), u=Depends(current_user)):
|
||||
"""List existing OCI resources in a compartment for terraform context."""
|
||||
_verify_config_access("oci", oci_config_id, u)
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
resources = await loop.run_in_executor(
|
||||
_chat_executor, partial(_fetch_compartment_resources, oci_config_id, compartment_id, region))
|
||||
return resources
|
||||
except Exception as e:
|
||||
raise HTTPException(500, str(e)[:500])
|
||||
|
||||
|
||||
# ── Terraform Generate Prompt ────────────────────────────────────────────────
|
||||
|
||||
@router.post("/api/terraform/generate-prompt")
|
||||
async def tf_generate_prompt(req: TfPromptReq, bg: BackgroundTasks, u=Depends(current_user)):
|
||||
gc = None
|
||||
if req.genai_config:
|
||||
if req.genai_config.get("genai_config_id"):
|
||||
with db() as c:
|
||||
row = c.execute("SELECT * FROM genai_configs WHERE id=?", (req.genai_config["genai_config_id"],)).fetchone()
|
||||
if row: gc = dict(row)
|
||||
elif req.genai_config.get("oci_config_id"):
|
||||
oci_id = req.genai_config["oci_config_id"]
|
||||
with db() as c:
|
||||
oc = c.execute("SELECT * FROM oci_configs WHERE id=?", (oci_id,)).fetchone()
|
||||
if oc:
|
||||
region = req.genai_config.get("genai_region") or oc["region"]
|
||||
compartment = _safe_dec(dict(oc).get("compartment_id") or oc["tenancy_ocid"])
|
||||
gc = {
|
||||
"oci_config_id": oci_id, "model_id": req.genai_config.get("model_id", "openai.gpt-4.1"),
|
||||
"model_ocid": None, "compartment_id": compartment, "genai_region": region,
|
||||
"endpoint": f"https://inference.generativeai.{region}.oci.oraclecloud.com",
|
||||
"serving_type": "ON_DEMAND", "dedicated_endpoint_id": None,
|
||||
"temperature": 0.7, "max_tokens": 4000, "top_p": 0.9,
|
||||
}
|
||||
if not gc:
|
||||
with db() as c:
|
||||
row = c.execute("SELECT * FROM genai_configs WHERE user_id=? AND is_default=1", (u["id"],)).fetchone()
|
||||
if not row:
|
||||
row = c.execute("SELECT * FROM genai_configs WHERE user_id=? LIMIT 1", (u["id"],)).fetchone()
|
||||
if row: gc = dict(row)
|
||||
if not gc:
|
||||
raise HTTPException(400, "Nenhuma configuração GenAI disponível")
|
||||
# Session persistence
|
||||
is_new = not req.session_id
|
||||
sid = req.session_id or str(uuid.uuid4())
|
||||
with db() as c:
|
||||
c.execute("INSERT INTO chat_messages (id,session_id,user_id,role,content,model_id,status) VALUES (?,?,?,?,?,?,?)",
|
||||
(str(uuid.uuid4()), sid, u["id"], "user", req.message, gc.get("model_id"), "done"))
|
||||
if is_new:
|
||||
title = (req.message or "Prompt Generator")[:80].strip()
|
||||
c.execute("INSERT OR IGNORE INTO chat_sessions (id,user_id,agent_type,title) VALUES (?,?,?,?)",
|
||||
(sid, u["id"], "tf-prompt", title))
|
||||
else:
|
||||
c.execute("UPDATE chat_sessions SET updated_at=datetime('now') WHERE id=?", (sid,))
|
||||
# Load TF resource reference (compact — same as Terraform Agent)
|
||||
tf_ref = _load_tf_resource_reference()
|
||||
system_with_ref = TFP_SYSTEM_PROMPT
|
||||
if tf_ref:
|
||||
sections = []
|
||||
for line in tf_ref.split('\n'):
|
||||
if line.startswith('## All Resource Types'):
|
||||
break
|
||||
sections.append(line)
|
||||
ref_compact = '\n'.join(sections).strip()
|
||||
if ref_compact:
|
||||
system_with_ref += "\n\n### Referência de Recursos OCI Terraform (gerado do provider schema)\n" + \
|
||||
"Use EXATAMENTE estes nomes de resource types. Se o recurso não estiver nesta lista, ele NÃO EXISTE no provider.\n\n" + \
|
||||
ref_compact
|
||||
|
||||
# Detect resource types from user message + history and fetch official docs
|
||||
detect_text = req.message
|
||||
if req.history:
|
||||
for h in req.history[-3:]:
|
||||
detect_text += "\n" + (h.get("content", "") or "")
|
||||
resource_types = _detect_tf_resource_types(detect_text)
|
||||
if resource_types:
|
||||
docs_ctx = _get_tf_docs_for_resources(resource_types)
|
||||
if docs_ctx:
|
||||
system_with_ref += "\n\n" + docs_ctx
|
||||
log.info(f"TF Prompt Generator: injected {len(resource_types)} resource docs ({len(docs_ctx)} chars)")
|
||||
|
||||
gc["system_prompt"] = system_with_ref
|
||||
gc["temperature"] = 0.7
|
||||
gc["max_tokens"] = 4000
|
||||
# Build history from conversation (normalize roles to lowercase for _call_genai)
|
||||
hist = None
|
||||
if req.history:
|
||||
hist = [{"role": "user" if m.get("role","").upper() in ("USER","user") else "assistant",
|
||||
"content": m.get("content", "")} for m in req.history]
|
||||
# Async background processing — same pattern as Chat/Terraform agents
|
||||
mid = str(uuid.uuid4())
|
||||
with db() as c:
|
||||
c.execute("INSERT INTO chat_messages (id,session_id,user_id,role,content,model_id,status) VALUES (?,?,?,?,?,?,?)",
|
||||
(mid, sid, u["id"], "assistant", "", gc.get("model_id"), "processing"))
|
||||
def _tfp_background():
|
||||
try:
|
||||
answer, _, _ = _call_genai(gc, req.message, history=hist)
|
||||
with db() as c:
|
||||
c.execute("UPDATE chat_messages SET content=?, status='done' WHERE id=?", (answer, mid))
|
||||
log.info(f"TF Prompt Generator completed: mid={mid} len={len(answer)}")
|
||||
except Exception as e:
|
||||
log.error(f"tf_generate_prompt error: {e}")
|
||||
with db() as c:
|
||||
c.execute("UPDATE chat_messages SET content=?, status='failed' WHERE id=?", (str(e)[:500], mid))
|
||||
bg.add_task(_tfp_background)
|
||||
return {"message_id": mid, "session_id": sid, "status": "processing"}
|
||||
|
||||
|
||||
# ── Terraform Chat ───────────────────────────────────────────────────────────
|
||||
|
||||
@router.post("/api/terraform/chat")
|
||||
async def terraform_chat(msg: ChatMsg, bg: BackgroundTasks, u=Depends(current_user)):
|
||||
sid, mid_or_result, genai_cfg = _chat_start(msg, u, agent_type="terraform")
|
||||
if genai_cfg is None:
|
||||
return mid_or_result
|
||||
bg.add_task(_chat_background, mid_or_result, sid, msg, dict(u), genai_cfg, None, "terraform")
|
||||
return {"message_id": mid_or_result, "session_id": sid, "status": "processing"}
|
||||
|
||||
|
||||
# ── Terraform Workspaces CRUD ────────────────────────────────────────────────
|
||||
|
||||
@router.post("/api/terraform/workspaces")
|
||||
async def tf_create_workspace(req: TfWorkspaceReq, u=Depends(current_user)):
|
||||
wid = str(uuid.uuid4())
|
||||
with db() as c:
|
||||
c.execute(
|
||||
"INSERT INTO terraform_workspaces (id,user_id,session_id,oci_config_id,compartment_id,name,tf_code,status) VALUES (?,?,?,?,?,?,?,?)",
|
||||
(wid, u["id"], req.session_id, req.oci_config_id, req.compartment_id, req.name, req.tf_code, "draft"))
|
||||
return {"id": wid, "status": "draft"}
|
||||
|
||||
|
||||
@router.get("/api/terraform/workspaces")
|
||||
async def tf_list_workspaces(u=Depends(current_user)):
|
||||
with db() as c:
|
||||
rows = c.execute(
|
||||
"""SELECT tw.id,tw.name,tw.session_id,tw.oci_config_id,tw.compartment_id,tw.status,
|
||||
tw.plan_output,tw.apply_output,tw.destroy_output,tw.error,tw.created_at,tw.updated_at,
|
||||
cs.title as session_title
|
||||
FROM terraform_workspaces tw
|
||||
INNER JOIN chat_sessions cs ON cs.id = tw.session_id AND cs.user_id = tw.user_id
|
||||
WHERE tw.user_id=? ORDER BY tw.updated_at DESC""",
|
||||
(u["id"],)).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
@router.get("/api/terraform/workspaces/{wid}")
|
||||
async def tf_get_workspace(wid: str, u=Depends(current_user)):
|
||||
with db() as c:
|
||||
r = c.execute("SELECT * FROM terraform_workspaces WHERE id=? AND user_id=?", (wid, u["id"])).fetchone()
|
||||
if not r: raise HTTPException(404)
|
||||
return dict(r)
|
||||
|
||||
|
||||
@router.put("/api/terraform/workspaces/{wid}/code")
|
||||
async def tf_update_code(wid: str, req: dict, u=Depends(current_user)):
|
||||
code = req.get("tf_code", "")
|
||||
with db() as c:
|
||||
r = c.execute("SELECT id FROM terraform_workspaces WHERE id=? AND user_id=?", (wid, u["id"])).fetchone()
|
||||
if not r: raise HTTPException(404)
|
||||
c.execute("UPDATE terraform_workspaces SET tf_code=?, status='draft', updated_at=datetime('now') WHERE id=?", (code, wid))
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.post("/api/terraform/workspaces/{wid}/plan")
|
||||
async def tf_run_plan(wid: str, bg: BackgroundTasks, u=Depends(current_user)):
|
||||
with db() as c:
|
||||
ws = c.execute("SELECT * FROM terraform_workspaces WHERE id=? AND user_id=?", (wid, u["id"])).fetchone()
|
||||
if not ws: raise HTTPException(404)
|
||||
if ws["status"] in ("planning", "applying", "destroying"):
|
||||
raise HTTPException(400, f"Workspace is {ws['status']}")
|
||||
with db() as c:
|
||||
c.execute("UPDATE terraform_workspaces SET status='planning', plan_output='', apply_output='', destroy_output='', error=NULL, updated_at=datetime('now') WHERE id=?", (wid,))
|
||||
bg.add_task(_terraform_exec, wid, "plan", dict(u))
|
||||
return {"status": "planning"}
|
||||
|
||||
|
||||
@router.post("/api/terraform/workspaces/{wid}/apply")
|
||||
async def tf_run_apply(wid: str, bg: BackgroundTasks, u=Depends(current_user)):
|
||||
with db() as c:
|
||||
ws = c.execute("SELECT * FROM terraform_workspaces WHERE id=? AND user_id=?", (wid, u["id"])).fetchone()
|
||||
if not ws: raise HTTPException(404)
|
||||
if ws["status"] not in ("planned", "applied", "destroyed", "failed"):
|
||||
raise HTTPException(400, f"Cannot apply in status {ws['status']}. Run plan first.")
|
||||
with db() as c:
|
||||
c.execute("UPDATE terraform_workspaces SET status='applying', apply_output='', error=NULL, updated_at=datetime('now') WHERE id=?", (wid,))
|
||||
bg.add_task(_terraform_exec, wid, "apply", dict(u))
|
||||
return {"status": "applying"}
|
||||
|
||||
|
||||
@router.post("/api/terraform/workspaces/{wid}/destroy")
|
||||
async def tf_run_destroy(wid: str, req: dict, bg: BackgroundTasks, u=Depends(current_user)):
|
||||
if req.get("confirm") != "DESTROY":
|
||||
raise HTTPException(400, "Confirmação obrigatória: envie {\"confirm\": \"DESTROY\"}")
|
||||
with db() as c:
|
||||
ws = c.execute("SELECT * FROM terraform_workspaces WHERE id=? AND user_id=?", (wid, u["id"])).fetchone()
|
||||
if not ws: raise HTTPException(404)
|
||||
if ws["status"] in ("planning", "applying", "destroying"):
|
||||
raise HTTPException(400, f"Workspace is {ws['status']}")
|
||||
with db() as c:
|
||||
c.execute("UPDATE terraform_workspaces SET status='destroying', destroy_output='', error=NULL, updated_at=datetime('now') WHERE id=?", (wid,))
|
||||
bg.add_task(_terraform_exec, wid, "destroy", dict(u))
|
||||
return {"status": "destroying"}
|
||||
|
||||
|
||||
@router.get("/api/terraform/workspaces/{wid}/status")
|
||||
async def tf_workspace_status(wid: str, u=Depends(current_user)):
|
||||
with db() as c:
|
||||
r = c.execute("SELECT status, plan_output, apply_output, destroy_output, error, updated_at FROM terraform_workspaces WHERE id=? AND user_id=?",
|
||||
(wid, u["id"])).fetchone()
|
||||
if not r: raise HTTPException(404)
|
||||
return dict(r)
|
||||
|
||||
|
||||
@router.get("/api/terraform/workspaces/{wid}/download")
|
||||
async def tf_download(wid: str, token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))):
|
||||
u = _user_from_token(token) if token else (await current_user(cred) if cred else None)
|
||||
if not u: raise HTTPException(401, "Not authenticated")
|
||||
with db() as c:
|
||||
r = c.execute("SELECT tf_code, name FROM terraform_workspaces WHERE id=? AND user_id=?", (wid, u["id"])).fetchone()
|
||||
if not r or not r["tf_code"]: raise HTTPException(404, "No code found")
|
||||
import re as _re
|
||||
parts = _re.split(r'^//\s*filename:\s*(\S+)\s*$', r["tf_code"], flags=_re.MULTILINE)
|
||||
if len(parts) >= 3:
|
||||
import io, zipfile
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, 'w', zipfile.ZIP_DEFLATED) as zf:
|
||||
if parts[0].strip():
|
||||
zf.writestr("main.tf", parts[0].strip())
|
||||
for i in range(1, len(parts), 2):
|
||||
fname = parts[i].strip().replace("/", "_").replace("..", "_")
|
||||
content = parts[i + 1].strip() if i + 1 < len(parts) else ""
|
||||
if fname and content:
|
||||
zf.writestr(fname, content)
|
||||
buf.seek(0)
|
||||
ws_name = r["name"] or "terraform"
|
||||
return Response(content=buf.read(), media_type="application/zip",
|
||||
headers={"Content-Disposition": f'attachment; filename="{ws_name}.zip"'})
|
||||
return Response(content=r["tf_code"], media_type="text/plain",
|
||||
headers={"Content-Disposition": f'attachment; filename="main.tf"'})
|
||||
|
||||
|
||||
@router.post("/api/terraform/workspaces/{wid}/cancel")
|
||||
async def tf_cancel(wid: str, u=Depends(current_user)):
|
||||
with db() as c:
|
||||
ws = c.execute("SELECT user_id FROM terraform_workspaces WHERE id=?", (wid,)).fetchone()
|
||||
if not ws or ws["user_id"] != u["id"]: raise HTTPException(404)
|
||||
proc = running_terraform.get(wid)
|
||||
if proc:
|
||||
proc.terminate()
|
||||
try:
|
||||
await asyncio.wait_for(proc.wait(), timeout=5)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
with db() as c:
|
||||
c.execute("UPDATE terraform_workspaces SET status='failed', error='Cancelado pelo usuário', updated_at=datetime('now') WHERE id=?", (wid,))
|
||||
running_terraform.pop(wid, None)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.delete("/api/terraform/workspaces/{wid}")
|
||||
async def tf_delete_workspace(wid: str, u=Depends(current_user)):
|
||||
with db() as c:
|
||||
c.execute("DELETE FROM terraform_workspaces WHERE id=? AND user_id=?", (wid, u["id"]))
|
||||
wdir = TERRAFORM_DIR / wid
|
||||
if wdir.exists():
|
||||
shutil.rmtree(wdir, ignore_errors=True)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.post("/api/terraform/refresh-reference")
|
||||
async def tf_refresh_reference(u=Depends(current_user)):
|
||||
"""Regenerate the OCI Terraform resource reference from provider schema (internal, triggered by UI button)."""
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
loop.run_in_executor(_chat_executor, _regenerate_tf_reference),
|
||||
timeout=300)
|
||||
except asyncio.TimeoutError:
|
||||
log.error("TF reference regeneration timed out after 300s")
|
||||
raise HTTPException(504, "Geração de referência Terraform demorou demais. Tente novamente.")
|
||||
# Add status info
|
||||
p = Path(_TF_RESOURCE_REF_PATH)
|
||||
if p.exists():
|
||||
content = _load_tf_resource_reference()
|
||||
result["lines"] = content.count('\n') if content else 0
|
||||
result["updated_at"] = datetime.fromtimestamp(p.stat().st_mtime).strftime('%d/%m/%Y %H:%M')
|
||||
return result
|
||||
64
backend/routes/users.py
Normal file
64
backend/routes/users.py
Normal file
@@ -0,0 +1,64 @@
|
||||
"""User routes — list, me, update, delete, timezone."""
|
||||
from fastapi import APIRouter, HTTPException, Depends
|
||||
|
||||
from database import db
|
||||
from auth.jwt_auth import current_user, require, _audit
|
||||
from models import UserUpdateReq
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# ── Users ─────────────────────────────────────────────────────────────────────
|
||||
@router.get("/api/users")
|
||||
async def list_users(u=Depends(require("admin"))):
|
||||
with db() as c: rows = c.execute("SELECT id,first_name,last_name,username,email,role,mfa_enabled,timezone,is_active,created_at,last_login,auth_provider FROM users").fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
@router.get("/api/users/me")
|
||||
async def me(u=Depends(current_user)):
|
||||
fields = ("id","first_name","last_name","username","email","role","mfa_enabled","timezone","auth_provider","must_change_password")
|
||||
r = {k: u.get(k, "local" if k == "auth_provider" else None) for k in fields}
|
||||
r["must_change_password"] = bool(r.get("must_change_password", 0))
|
||||
return r
|
||||
|
||||
@router.put("/api/users/{uid}")
|
||||
async def update_user(uid: str, req: UserUpdateReq, adm=Depends(require("admin"))):
|
||||
sets, vals = [], []
|
||||
if req.email is not None: sets.append("email=?"); vals.append(req.email)
|
||||
if req.role is not None:
|
||||
if req.role not in ("admin","user","viewer"): raise HTTPException(400, "Role inválida")
|
||||
sets.append("role=?"); vals.append(req.role)
|
||||
if req.is_active is not None: sets.append("is_active=?"); vals.append(int(req.is_active))
|
||||
if req.timezone is not None: sets.append("timezone=?"); vals.append(req.timezone)
|
||||
if sets:
|
||||
vals.append(uid)
|
||||
with db() as c: c.execute(f"UPDATE users SET {','.join(sets)} WHERE id=?", vals)
|
||||
_audit(adm["id"], adm["username"], "update_user", uid)
|
||||
return {"ok": True}
|
||||
|
||||
@router.delete("/api/users/{uid}")
|
||||
async def del_user(uid: str, adm=Depends(require("admin"))):
|
||||
if uid == adm["id"]: raise HTTPException(400, "Não pode desativar a si mesmo")
|
||||
with db() as c: c.execute("UPDATE users SET is_active=0 WHERE id=?", (uid,))
|
||||
_audit(adm["id"], adm["username"], "deactivate_user", uid)
|
||||
return {"ok": True}
|
||||
|
||||
@router.put("/api/users/me/timezone")
|
||||
async def set_user_timezone(body: dict, u=Depends(current_user)):
|
||||
"""Set the current user's timezone."""
|
||||
tz = body.get("timezone", "").strip()
|
||||
if not tz:
|
||||
raise HTTPException(400, "timezone is required")
|
||||
try:
|
||||
import zoneinfo
|
||||
zoneinfo.ZoneInfo(tz)
|
||||
except Exception:
|
||||
raise HTTPException(400, f"Invalid timezone: {tz}")
|
||||
with db() as c:
|
||||
c.execute("UPDATE users SET timezone=? WHERE id=?", (tz, u["id"]))
|
||||
_audit(u["id"], u["username"], "set_timezone", tz)
|
||||
return {"ok": True, "timezone": tz}
|
||||
|
||||
@router.get("/api/users/me/timezone")
|
||||
async def get_user_timezone(u=Depends(current_user)):
|
||||
"""Get the current user's timezone."""
|
||||
return {"timezone": u.get("timezone", "America/Sao_Paulo")}
|
||||
Reference in New Issue
Block a user