1135e9d6a9
Fase 0 complete: extract all business logic, auth, database, and 176 endpoints from monolithic app.py into dedicated modules. Structure: - app.py: FastAPI hub (CORS, routers, startup/shutdown) - models.py: 13 Pydantic request models - utils.py: shared utilities (embed status, upload validation, process registries) - config.py: constants, env vars, model catalogs - auth/: crypto, jwt_auth, oidc, rate_limit - database/: db(), init_db(), schema DDL - services/: genai, compliance, chat, terraform, embeddings, cis_reports, mcp - routes/: 13 APIRouter modules (auth, users, oci_config, oci_explorer, genai, mcp, adb, embeddings, reports, chat, terraform, settings, cis_engine) Also: README updated with OCIR auth instructions for manual docker run. 86 tests passing.
322 lines
16 KiB
Python
322 lines
16 KiB
Python
"""ADB (Autonomous Database) routes — parse-wallet, config CRUD, wallet upload, test, tables, ingest."""
|
|
import os
|
|
import re
|
|
import json
|
|
import uuid
|
|
import shutil
|
|
import sqlite3
|
|
from typing import Optional
|
|
from pathlib import Path
|
|
|
|
from fastapi import APIRouter, HTTPException, Depends, UploadFile, File, Form, BackgroundTasks
|
|
|
|
from database import db
|
|
from auth.jwt_auth import current_user, require, _audit, _config_log, _verify_config_access
|
|
from auth.crypto import _enc
|
|
from config import WALLET_DIR, log
|
|
from models import IngestDocReq
|
|
from utils import validate_upload
|
|
from services.genai import (
|
|
_get_adb_connection,
|
|
_get_tables_for_config,
|
|
)
|
|
from services.embeddings import _ingest_documents_task
|
|
|
|
router = APIRouter()
|
|
|
|
_TABLE_NAME_RE = re.compile(r'^[A-Za-z][A-Za-z0-9_]{0,127}$')
|
|
|
|
|
|
@router.post("/api/adb/parse-wallet")
|
|
async def parse_wallet(wallet: UploadFile = File(...), u=Depends(require("admin","user"))):
|
|
"""Extract DSN names from tnsnames.ora inside wallet ZIP (temporary parse, no save)."""
|
|
await validate_upload(wallet, {".zip"})
|
|
wallet.file.seek(0)
|
|
import zipfile, tempfile
|
|
try:
|
|
data = await wallet.read()
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
zp = Path(tmp) / "wallet.zip"
|
|
zp.write_bytes(data)
|
|
with zipfile.ZipFile(str(zp), 'r') as z:
|
|
z.extractall(tmp)
|
|
tns = Path(tmp) / "tnsnames.ora"
|
|
if not tns.exists():
|
|
raise HTTPException(400, "Wallet ZIP nao contem tnsnames.ora")
|
|
tns_text = tns.read_text(errors="ignore")
|
|
dsn_names = re.findall(r'^(\w[\w\-.]*)\s*=', tns_text, re.MULTILINE)
|
|
if not dsn_names:
|
|
raise HTTPException(400, "Nenhum DSN encontrado no tnsnames.ora")
|
|
return {"dsn_names": dsn_names, "files": [n for n in os.listdir(tmp) if n != "wallet.zip"]}
|
|
except zipfile.BadZipFile:
|
|
raise HTTPException(400, "Arquivo nao e um ZIP valido")
|
|
|
|
@router.post("/api/adb/config")
|
|
async def save_adb(
|
|
config_name: str = Form(...), dsn: str = Form(...), username: str = Form(...),
|
|
password: str = Form(...), wallet_password: str = Form(""),
|
|
table_name: str = Form(""), use_mtls: str = Form("true"),
|
|
genai_config_id: str = Form(""), embedding_model_id: str = Form("cohere.embed-v4.0"),
|
|
is_global: str = Form("false"),
|
|
wallet: Optional[UploadFile] = File(None),
|
|
u=Depends(require("admin","user"))
|
|
):
|
|
if wallet and wallet.filename:
|
|
await validate_upload(wallet, {".zip"})
|
|
wallet.file.seek(0)
|
|
vid = str(uuid.uuid4())
|
|
use_mtls_bool = use_mtls.lower() in ("true", "1", "yes")
|
|
is_global_val = 1 if (is_global.lower() in ("true", "1", "yes") and u["role"] == "admin") else 0
|
|
with db() as c:
|
|
c.execute(
|
|
"INSERT INTO adb_vector_configs (id,user_id,config_name,dsn,username,password_enc,wallet_password_enc,table_name,use_mtls,genai_config_id,embedding_model_id,is_global) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
(vid, u["id"], config_name, dsn, username, _enc(password),
|
|
_enc(wallet_password) if wallet_password else None, table_name, int(use_mtls_bool),
|
|
genai_config_id or None, embedding_model_id, is_global_val))
|
|
# Auto-save wallet if provided
|
|
if wallet and wallet.filename:
|
|
import zipfile
|
|
wdir = WALLET_DIR / vid; wdir.mkdir(parents=True, exist_ok=True)
|
|
zp = wdir / "wallet.zip"
|
|
zp.write_bytes(await wallet.read())
|
|
with zipfile.ZipFile(str(zp), 'r') as z: z.extractall(str(wdir))
|
|
with db() as c: c.execute("UPDATE adb_vector_configs SET wallet_dir=? WHERE id=?", (str(wdir), vid))
|
|
_config_log("adb", vid, config_name, "success", "upload", f"Wallet enviada com a conexao", u["id"], u["username"])
|
|
_audit(u["id"], u["username"], "save_adb_config", vid, config_name)
|
|
_config_log("adb", vid, config_name, "success", "save", f"Conexao salva: {config_name} ({dsn})", u["id"], u["username"])
|
|
return {"id": vid, "config_name": config_name}
|
|
|
|
@router.post("/api/adb/{vid}/upload-wallet")
|
|
async def upload_wallet(vid: str, wallet: UploadFile = File(...), u=Depends(require("admin","user"))):
|
|
_verify_config_access("adb", vid, u, require_owner=True)
|
|
await validate_upload(wallet, {".zip"})
|
|
wallet.file.seek(0)
|
|
cname = None
|
|
with db() as c:
|
|
row = c.execute("SELECT config_name FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
|
if row: cname = row["config_name"]
|
|
try:
|
|
wdir = WALLET_DIR / vid; wdir.mkdir(parents=True, exist_ok=True)
|
|
zp = wdir / "wallet.zip"
|
|
zp.write_bytes(await wallet.read())
|
|
import zipfile
|
|
with zipfile.ZipFile(str(zp), 'r') as z: z.extractall(str(wdir))
|
|
with db() as c: c.execute("UPDATE adb_vector_configs SET wallet_dir=? WHERE id=?", (str(wdir), vid))
|
|
files = os.listdir(str(wdir))
|
|
_config_log("adb", vid, cname, "success", "upload", f"Wallet enviada: {', '.join(files)}", u["id"], u["username"])
|
|
return {"ok": True, "wallet_dir": str(wdir), "files": files}
|
|
except Exception as e:
|
|
_config_log("adb", vid, cname, "error", "upload", str(e)[:500], u["id"], u["username"])
|
|
raise HTTPException(500, str(e)[:500])
|
|
|
|
@router.get("/api/adb/configs")
|
|
async def list_adb(u=Depends(current_user)):
|
|
with db() as c:
|
|
cols = "id,user_id,config_name,dsn,username,table_name,use_mtls,is_active,is_global,wallet_dir,genai_config_id,embedding_model_id,created_at"
|
|
if u["role"]=="admin": rows=c.execute(f"SELECT {cols} FROM adb_vector_configs").fetchall()
|
|
else: rows=c.execute(f"SELECT {cols} FROM adb_vector_configs WHERE user_id=? OR is_global=1",(u["id"],)).fetchall()
|
|
result = []
|
|
for r in rows:
|
|
d = dict(r)
|
|
d["tables"] = _get_tables_for_config(d["id"], active_only=False)
|
|
result.append(d)
|
|
return result
|
|
|
|
@router.post("/api/adb/test/{vid}")
|
|
async def test_adb(vid: str, u=Depends(require("admin","user"))):
|
|
_verify_config_access("adb", vid, u)
|
|
with db() as c:
|
|
cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?",(vid,)).fetchone()
|
|
if not cfg: raise HTTPException(404)
|
|
cname = cfg["config_name"]
|
|
try:
|
|
conn = _get_adb_connection(dict(cfg))
|
|
cur = conn.cursor(); cur.execute("SELECT 1 FROM DUAL"); cur.close(); conn.close()
|
|
_config_log("adb", vid, cname, "success", "test", "Conexao Autonomous DB OK!", u["id"], u["username"])
|
|
return {"status":"success","message":"Conexao Autonomous DB OK!"}
|
|
except ImportError:
|
|
_config_log("adb", vid, cname, "error", "test", "python-oracledb nao disponivel no container.", u["id"], u["username"])
|
|
return {"status":"error","message":"python-oracledb nao disponivel no container."}
|
|
except Exception as e:
|
|
msg = str(e)[:500]
|
|
_config_log("adb", vid, cname, "error", "test", msg, u["id"], u["username"])
|
|
return {"status":"error","message":msg}
|
|
|
|
@router.post("/api/adb/{vid}/tables/check")
|
|
async def check_adb_tables(vid: str, u=Depends(require("admin","user"))):
|
|
"""Check which registered tables exist in ADB and list all user tables."""
|
|
_verify_config_access("adb", vid, u)
|
|
with db() as c:
|
|
cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
|
if not cfg: raise HTTPException(404)
|
|
registered = _get_tables_for_config(vid, active_only=False)
|
|
reg_names_upper = {t["table_name"].upper() for t in registered}
|
|
try:
|
|
conn = _get_adb_connection(dict(cfg))
|
|
cur = conn.cursor()
|
|
cur.execute("SELECT USER FROM DUAL")
|
|
current_user_name = cur.fetchone()[0]
|
|
cur.execute("SELECT table_name FROM user_tables ORDER BY table_name")
|
|
adb_tables = [r[0] for r in cur.fetchall()]
|
|
adb_lookup = {t.upper(): t for t in adb_tables}
|
|
results = []
|
|
for t in registered:
|
|
tname_reg = t["table_name"]
|
|
tname_upper = tname_reg.upper()
|
|
real_name = adb_lookup.get(tname_upper)
|
|
if real_name:
|
|
try:
|
|
cur.execute(f'SELECT COUNT(*) FROM "{real_name}"')
|
|
cnt = cur.fetchone()[0]
|
|
case_note = f" (ADB: {real_name})" if real_name != tname_reg else ""
|
|
results.append({"table": tname_reg, "adb_name": real_name, "status": "ok", "rows": cnt, "message": f"{cnt} registros{case_note}"})
|
|
except Exception as e:
|
|
results.append({"table": tname_reg, "status": "error", "message": str(e)[:200]})
|
|
else:
|
|
results.append({"table": tname_reg, "status": "not_found", "message": f"Nao existe no schema {current_user_name}"})
|
|
unregistered = [t for t in adb_tables if t.upper() not in reg_names_upper]
|
|
cur.close()
|
|
conn.close()
|
|
return {
|
|
"status": "success",
|
|
"schema_user": current_user_name,
|
|
"adb_tables_total": len(adb_tables),
|
|
"registered": results,
|
|
"unregistered_in_adb": unregistered[:50]
|
|
}
|
|
except Exception as e:
|
|
return {"status": "error", "message": f"Erro de conexao: {str(e)[:500]}"}
|
|
|
|
@router.delete("/api/adb/configs/{vid}")
|
|
async def del_adb(vid: str, u=Depends(require("admin","user"))):
|
|
_verify_config_access("adb", vid, u, require_owner=True)
|
|
with db() as c: c.execute("DELETE FROM adb_vector_configs WHERE id=?", (vid,))
|
|
d = WALLET_DIR / vid
|
|
if d.exists(): shutil.rmtree(d)
|
|
return {"ok": True}
|
|
|
|
@router.put("/api/adb/configs/{vid}")
|
|
async def update_adb(
|
|
vid: str,
|
|
config_name: str = Form(...), dsn: str = Form(...), username: str = Form(...),
|
|
password: str = Form(""), wallet_password: str = Form(""),
|
|
table_name: str = Form(""), use_mtls: str = Form("true"),
|
|
genai_config_id: str = Form(""), embedding_model_id: str = Form("cohere.embed-v4.0"),
|
|
is_global: str = Form(""),
|
|
wallet: Optional[UploadFile] = File(None),
|
|
u=Depends(require("admin","user"))
|
|
):
|
|
if wallet and wallet.filename:
|
|
await validate_upload(wallet, {".zip"})
|
|
wallet.file.seek(0)
|
|
with db() as c:
|
|
existing = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
|
if not existing: raise HTTPException(404)
|
|
if u["role"] != "admin" and existing["user_id"] != u["id"]: raise HTTPException(403)
|
|
if u["role"] != "admin" and existing.get("is_global"): raise HTTPException(403, "Nao e possivel editar configuracao global")
|
|
use_mtls_bool = use_mtls.lower() in ("true", "1", "yes")
|
|
sets = "config_name=?,dsn=?,username=?,table_name=?,use_mtls=?,genai_config_id=?,embedding_model_id=?"
|
|
vals = [config_name, dsn, username, table_name, int(use_mtls_bool), genai_config_id or None, embedding_model_id]
|
|
if u["role"] == "admin" and is_global:
|
|
is_global_val = 1 if is_global.lower() in ("true", "1", "yes") else 0
|
|
sets += ",is_global=?"
|
|
vals.append(is_global_val)
|
|
if password:
|
|
sets += ",password_enc=?"
|
|
vals.append(_enc(password))
|
|
if wallet_password:
|
|
sets += ",wallet_password_enc=?"
|
|
vals.append(_enc(wallet_password))
|
|
vals.append(vid)
|
|
with db() as c:
|
|
c.execute(f"UPDATE adb_vector_configs SET {sets} WHERE id=?", vals)
|
|
if wallet and wallet.filename:
|
|
import zipfile
|
|
wdir = WALLET_DIR / vid; wdir.mkdir(parents=True, exist_ok=True)
|
|
zp = wdir / "wallet.zip"
|
|
zp.write_bytes(await wallet.read())
|
|
with zipfile.ZipFile(str(zp), 'r') as z: z.extractall(str(wdir))
|
|
with db() as c: c.execute("UPDATE adb_vector_configs SET wallet_dir=? WHERE id=?", (str(wdir), vid))
|
|
_config_log("adb", vid, config_name, "success", "upload", "Wallet atualizado", u["id"], u["username"])
|
|
_audit(u["id"], u["username"], "update_adb_config", vid, config_name)
|
|
_config_log("adb", vid, config_name, "success", "save", f"Conexao atualizada: {config_name} ({dsn})", u["id"], u["username"])
|
|
return {"id": vid, "config_name": config_name}
|
|
|
|
@router.get("/api/adb/{vid}/tables")
|
|
async def list_adb_tables(vid: str, u=Depends(current_user)):
|
|
_verify_config_access("adb", vid, u)
|
|
with db() as c:
|
|
cfg = c.execute("SELECT id FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
|
if not cfg: raise HTTPException(404)
|
|
return _get_tables_for_config(vid, active_only=False)
|
|
|
|
@router.post("/api/adb/{vid}/tables")
|
|
async def add_adb_table(vid: str, req: dict, u=Depends(require("admin","user"))):
|
|
_verify_config_access("adb", vid, u, require_owner=True)
|
|
table_name = req.get("table_name", "").strip()
|
|
description = req.get("description", "").strip()
|
|
if not table_name: raise HTTPException(400, "table_name e obrigatorio")
|
|
if not _TABLE_NAME_RE.match(table_name): raise HTTPException(400, "Nome da tabela invalido. Use letras maiusculas, numeros e underscores.")
|
|
with db() as c:
|
|
cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
|
if not cfg: raise HTTPException(404, "ADB config not found")
|
|
tid = str(uuid.uuid4())
|
|
try:
|
|
with db() as c:
|
|
c.execute("INSERT INTO adb_vector_tables (id, adb_config_id, table_name, description) VALUES (?,?,?,?)",
|
|
(tid, vid, table_name, description))
|
|
except sqlite3.IntegrityError:
|
|
raise HTTPException(409, f"Tabela '{table_name}' ja registrada nesta conexao")
|
|
_config_log("adb", vid, cfg["config_name"], "success", "add_table", f"Tabela '{table_name}' registrada", u["id"], u["username"])
|
|
return {"id": tid, "table_name": table_name}
|
|
|
|
@router.delete("/api/adb/{vid}/tables/{tid}")
|
|
async def remove_adb_table(vid: str, tid: str, u=Depends(require("admin","user"))):
|
|
_verify_config_access("adb", vid, u, require_owner=True)
|
|
with db() as c:
|
|
row = c.execute("SELECT * FROM adb_vector_tables WHERE id=? AND adb_config_id=?", (tid, vid)).fetchone()
|
|
if not row: raise HTTPException(404, "Table entry not found")
|
|
with db() as c:
|
|
c.execute("DELETE FROM adb_vector_tables WHERE id=?", (tid,))
|
|
return {"ok": True}
|
|
|
|
@router.put("/api/adb/{vid}/tables/{tid}")
|
|
async def update_adb_table(vid: str, tid: str, req: dict, u=Depends(require("admin","user"))):
|
|
_verify_config_access("adb", vid, u, require_owner=True)
|
|
with db() as c:
|
|
row = c.execute("SELECT * FROM adb_vector_tables WHERE id=? AND adb_config_id=?", (tid, vid)).fetchone()
|
|
if not row: raise HTTPException(404, "Table entry not found")
|
|
sets, vals = [], []
|
|
if "table_name" in req:
|
|
tn = req["table_name"].strip()
|
|
if not tn: raise HTTPException(400, "table_name e obrigatorio")
|
|
if not _TABLE_NAME_RE.match(tn): raise HTTPException(400, "Nome da tabela invalido")
|
|
sets.append("table_name=?"); vals.append(tn)
|
|
if "description" in req:
|
|
sets.append("description=?"); vals.append(req["description"])
|
|
if "is_active" in req:
|
|
sets.append("is_active=?"); vals.append(int(req["is_active"]))
|
|
if not sets: return {"ok": True}
|
|
vals.append(tid)
|
|
try:
|
|
with db() as c:
|
|
c.execute(f"UPDATE adb_vector_tables SET {','.join(sets)} WHERE id=?", vals)
|
|
except sqlite3.IntegrityError:
|
|
raise HTTPException(409, "Tabela com este nome ja existe nesta conexao")
|
|
return {"ok": True}
|
|
|
|
@router.post("/api/adb/{vid}/ingest")
|
|
async def ingest_documents(vid: str, req: IngestDocReq, bg: BackgroundTasks, u=Depends(require("admin","user"))):
|
|
_verify_config_access("adb", vid, u)
|
|
with db() as c:
|
|
cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
|
if not cfg: raise HTTPException(404, "ADB config not found")
|
|
cfg = dict(cfg)
|
|
if not cfg.get("genai_config_id"):
|
|
raise HTTPException(400, "GenAI config not linked to this ADB connection")
|
|
with db() as c:
|
|
gc = c.execute("SELECT * FROM genai_configs WHERE id=?", (cfg["genai_config_id"],)).fetchone()
|
|
if not gc: raise HTTPException(400, "Linked GenAI config not found")
|
|
bg.add_task(_ingest_documents_task, cfg, dict(gc), req.documents, u["id"], u["username"], table_name=req.table_name)
|
|
return {"ok": True, "message": f"Ingestao iniciada para {len(req.documents)} documentos", "config_id": vid}
|