refactor: decompose monolith — app.py 10,600→143 lines, 30+ modules
Fase 0 complete: extract all business logic, auth, database, and 176 endpoints from monolithic app.py into dedicated modules. Structure: - app.py: FastAPI hub (CORS, routers, startup/shutdown) - models.py: 13 Pydantic request models - utils.py: shared utilities (embed status, upload validation, process registries) - config.py: constants, env vars, model catalogs - auth/: crypto, jwt_auth, oidc, rate_limit - database/: db(), init_db(), schema DDL - services/: genai, compliance, chat, terraform, embeddings, cis_reports, mcp - routes/: 13 APIRouter modules (auth, users, oci_config, oci_explorer, genai, mcp, adb, embeddings, reports, chat, terraform, settings, cis_engine) Also: README updated with OCIR auth instructions for manual docker run. 86 tests passing.
This commit is contained in:
nogueiraguh
321
backend/routes/adb.py
Normal file
321
backend/routes/adb.py
Normal file
@@ -0,0 +1,321 @@
|
||||
"""ADB (Autonomous Database) routes — parse-wallet, config CRUD, wallet upload, test, tables, ingest."""
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import uuid
|
||||
import shutil
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends, UploadFile, File, Form, BackgroundTasks
|
||||
|
||||
from database import db
|
||||
from auth.jwt_auth import current_user, require, _audit, _config_log, _verify_config_access
|
||||
from auth.crypto import _enc
|
||||
from config import WALLET_DIR, log
|
||||
from models import IngestDocReq
|
||||
from utils import validate_upload
|
||||
from services.genai import (
|
||||
_get_adb_connection,
|
||||
_get_tables_for_config,
|
||||
)
|
||||
from services.embeddings import _ingest_documents_task
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
_TABLE_NAME_RE = re.compile(r'^[A-Za-z][A-Za-z0-9_]{0,127}$')
|
||||
|
||||
|
||||
@router.post("/api/adb/parse-wallet")
|
||||
async def parse_wallet(wallet: UploadFile = File(...), u=Depends(require("admin","user"))):
|
||||
"""Extract DSN names from tnsnames.ora inside wallet ZIP (temporary parse, no save)."""
|
||||
await validate_upload(wallet, {".zip"})
|
||||
wallet.file.seek(0)
|
||||
import zipfile, tempfile
|
||||
try:
|
||||
data = await wallet.read()
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
zp = Path(tmp) / "wallet.zip"
|
||||
zp.write_bytes(data)
|
||||
with zipfile.ZipFile(str(zp), 'r') as z:
|
||||
z.extractall(tmp)
|
||||
tns = Path(tmp) / "tnsnames.ora"
|
||||
if not tns.exists():
|
||||
raise HTTPException(400, "Wallet ZIP nao contem tnsnames.ora")
|
||||
tns_text = tns.read_text(errors="ignore")
|
||||
dsn_names = re.findall(r'^(\w[\w\-.]*)\s*=', tns_text, re.MULTILINE)
|
||||
if not dsn_names:
|
||||
raise HTTPException(400, "Nenhum DSN encontrado no tnsnames.ora")
|
||||
return {"dsn_names": dsn_names, "files": [n for n in os.listdir(tmp) if n != "wallet.zip"]}
|
||||
except zipfile.BadZipFile:
|
||||
raise HTTPException(400, "Arquivo nao e um ZIP valido")
|
||||
|
||||
@router.post("/api/adb/config")
|
||||
async def save_adb(
|
||||
config_name: str = Form(...), dsn: str = Form(...), username: str = Form(...),
|
||||
password: str = Form(...), wallet_password: str = Form(""),
|
||||
table_name: str = Form(""), use_mtls: str = Form("true"),
|
||||
genai_config_id: str = Form(""), embedding_model_id: str = Form("cohere.embed-v4.0"),
|
||||
is_global: str = Form("false"),
|
||||
wallet: Optional[UploadFile] = File(None),
|
||||
u=Depends(require("admin","user"))
|
||||
):
|
||||
if wallet and wallet.filename:
|
||||
await validate_upload(wallet, {".zip"})
|
||||
wallet.file.seek(0)
|
||||
vid = str(uuid.uuid4())
|
||||
use_mtls_bool = use_mtls.lower() in ("true", "1", "yes")
|
||||
is_global_val = 1 if (is_global.lower() in ("true", "1", "yes") and u["role"] == "admin") else 0
|
||||
with db() as c:
|
||||
c.execute(
|
||||
"INSERT INTO adb_vector_configs (id,user_id,config_name,dsn,username,password_enc,wallet_password_enc,table_name,use_mtls,genai_config_id,embedding_model_id,is_global) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(vid, u["id"], config_name, dsn, username, _enc(password),
|
||||
_enc(wallet_password) if wallet_password else None, table_name, int(use_mtls_bool),
|
||||
genai_config_id or None, embedding_model_id, is_global_val))
|
||||
# Auto-save wallet if provided
|
||||
if wallet and wallet.filename:
|
||||
import zipfile
|
||||
wdir = WALLET_DIR / vid; wdir.mkdir(parents=True, exist_ok=True)
|
||||
zp = wdir / "wallet.zip"
|
||||
zp.write_bytes(await wallet.read())
|
||||
with zipfile.ZipFile(str(zp), 'r') as z: z.extractall(str(wdir))
|
||||
with db() as c: c.execute("UPDATE adb_vector_configs SET wallet_dir=? WHERE id=?", (str(wdir), vid))
|
||||
_config_log("adb", vid, config_name, "success", "upload", f"Wallet enviada com a conexao", u["id"], u["username"])
|
||||
_audit(u["id"], u["username"], "save_adb_config", vid, config_name)
|
||||
_config_log("adb", vid, config_name, "success", "save", f"Conexao salva: {config_name} ({dsn})", u["id"], u["username"])
|
||||
return {"id": vid, "config_name": config_name}
|
||||
|
||||
@router.post("/api/adb/{vid}/upload-wallet")
|
||||
async def upload_wallet(vid: str, wallet: UploadFile = File(...), u=Depends(require("admin","user"))):
|
||||
_verify_config_access("adb", vid, u, require_owner=True)
|
||||
await validate_upload(wallet, {".zip"})
|
||||
wallet.file.seek(0)
|
||||
cname = None
|
||||
with db() as c:
|
||||
row = c.execute("SELECT config_name FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
||||
if row: cname = row["config_name"]
|
||||
try:
|
||||
wdir = WALLET_DIR / vid; wdir.mkdir(parents=True, exist_ok=True)
|
||||
zp = wdir / "wallet.zip"
|
||||
zp.write_bytes(await wallet.read())
|
||||
import zipfile
|
||||
with zipfile.ZipFile(str(zp), 'r') as z: z.extractall(str(wdir))
|
||||
with db() as c: c.execute("UPDATE adb_vector_configs SET wallet_dir=? WHERE id=?", (str(wdir), vid))
|
||||
files = os.listdir(str(wdir))
|
||||
_config_log("adb", vid, cname, "success", "upload", f"Wallet enviada: {', '.join(files)}", u["id"], u["username"])
|
||||
return {"ok": True, "wallet_dir": str(wdir), "files": files}
|
||||
except Exception as e:
|
||||
_config_log("adb", vid, cname, "error", "upload", str(e)[:500], u["id"], u["username"])
|
||||
raise HTTPException(500, str(e)[:500])
|
||||
|
||||
@router.get("/api/adb/configs")
|
||||
async def list_adb(u=Depends(current_user)):
|
||||
with db() as c:
|
||||
cols = "id,user_id,config_name,dsn,username,table_name,use_mtls,is_active,is_global,wallet_dir,genai_config_id,embedding_model_id,created_at"
|
||||
if u["role"]=="admin": rows=c.execute(f"SELECT {cols} FROM adb_vector_configs").fetchall()
|
||||
else: rows=c.execute(f"SELECT {cols} FROM adb_vector_configs WHERE user_id=? OR is_global=1",(u["id"],)).fetchall()
|
||||
result = []
|
||||
for r in rows:
|
||||
d = dict(r)
|
||||
d["tables"] = _get_tables_for_config(d["id"], active_only=False)
|
||||
result.append(d)
|
||||
return result
|
||||
|
||||
@router.post("/api/adb/test/{vid}")
|
||||
async def test_adb(vid: str, u=Depends(require("admin","user"))):
|
||||
_verify_config_access("adb", vid, u)
|
||||
with db() as c:
|
||||
cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?",(vid,)).fetchone()
|
||||
if not cfg: raise HTTPException(404)
|
||||
cname = cfg["config_name"]
|
||||
try:
|
||||
conn = _get_adb_connection(dict(cfg))
|
||||
cur = conn.cursor(); cur.execute("SELECT 1 FROM DUAL"); cur.close(); conn.close()
|
||||
_config_log("adb", vid, cname, "success", "test", "Conexao Autonomous DB OK!", u["id"], u["username"])
|
||||
return {"status":"success","message":"Conexao Autonomous DB OK!"}
|
||||
except ImportError:
|
||||
_config_log("adb", vid, cname, "error", "test", "python-oracledb nao disponivel no container.", u["id"], u["username"])
|
||||
return {"status":"error","message":"python-oracledb nao disponivel no container."}
|
||||
except Exception as e:
|
||||
msg = str(e)[:500]
|
||||
_config_log("adb", vid, cname, "error", "test", msg, u["id"], u["username"])
|
||||
return {"status":"error","message":msg}
|
||||
|
||||
@router.post("/api/adb/{vid}/tables/check")
|
||||
async def check_adb_tables(vid: str, u=Depends(require("admin","user"))):
|
||||
"""Check which registered tables exist in ADB and list all user tables."""
|
||||
_verify_config_access("adb", vid, u)
|
||||
with db() as c:
|
||||
cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
||||
if not cfg: raise HTTPException(404)
|
||||
registered = _get_tables_for_config(vid, active_only=False)
|
||||
reg_names_upper = {t["table_name"].upper() for t in registered}
|
||||
try:
|
||||
conn = _get_adb_connection(dict(cfg))
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT USER FROM DUAL")
|
||||
current_user_name = cur.fetchone()[0]
|
||||
cur.execute("SELECT table_name FROM user_tables ORDER BY table_name")
|
||||
adb_tables = [r[0] for r in cur.fetchall()]
|
||||
adb_lookup = {t.upper(): t for t in adb_tables}
|
||||
results = []
|
||||
for t in registered:
|
||||
tname_reg = t["table_name"]
|
||||
tname_upper = tname_reg.upper()
|
||||
real_name = adb_lookup.get(tname_upper)
|
||||
if real_name:
|
||||
try:
|
||||
cur.execute(f'SELECT COUNT(*) FROM "{real_name}"')
|
||||
cnt = cur.fetchone()[0]
|
||||
case_note = f" (ADB: {real_name})" if real_name != tname_reg else ""
|
||||
results.append({"table": tname_reg, "adb_name": real_name, "status": "ok", "rows": cnt, "message": f"{cnt} registros{case_note}"})
|
||||
except Exception as e:
|
||||
results.append({"table": tname_reg, "status": "error", "message": str(e)[:200]})
|
||||
else:
|
||||
results.append({"table": tname_reg, "status": "not_found", "message": f"Nao existe no schema {current_user_name}"})
|
||||
unregistered = [t for t in adb_tables if t.upper() not in reg_names_upper]
|
||||
cur.close()
|
||||
conn.close()
|
||||
return {
|
||||
"status": "success",
|
||||
"schema_user": current_user_name,
|
||||
"adb_tables_total": len(adb_tables),
|
||||
"registered": results,
|
||||
"unregistered_in_adb": unregistered[:50]
|
||||
}
|
||||
except Exception as e:
|
||||
return {"status": "error", "message": f"Erro de conexao: {str(e)[:500]}"}
|
||||
|
||||
@router.delete("/api/adb/configs/{vid}")
|
||||
async def del_adb(vid: str, u=Depends(require("admin","user"))):
|
||||
_verify_config_access("adb", vid, u, require_owner=True)
|
||||
with db() as c: c.execute("DELETE FROM adb_vector_configs WHERE id=?", (vid,))
|
||||
d = WALLET_DIR / vid
|
||||
if d.exists(): shutil.rmtree(d)
|
||||
return {"ok": True}
|
||||
|
||||
@router.put("/api/adb/configs/{vid}")
|
||||
async def update_adb(
|
||||
vid: str,
|
||||
config_name: str = Form(...), dsn: str = Form(...), username: str = Form(...),
|
||||
password: str = Form(""), wallet_password: str = Form(""),
|
||||
table_name: str = Form(""), use_mtls: str = Form("true"),
|
||||
genai_config_id: str = Form(""), embedding_model_id: str = Form("cohere.embed-v4.0"),
|
||||
is_global: str = Form(""),
|
||||
wallet: Optional[UploadFile] = File(None),
|
||||
u=Depends(require("admin","user"))
|
||||
):
|
||||
if wallet and wallet.filename:
|
||||
await validate_upload(wallet, {".zip"})
|
||||
wallet.file.seek(0)
|
||||
with db() as c:
|
||||
existing = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
||||
if not existing: raise HTTPException(404)
|
||||
if u["role"] != "admin" and existing["user_id"] != u["id"]: raise HTTPException(403)
|
||||
if u["role"] != "admin" and existing.get("is_global"): raise HTTPException(403, "Nao e possivel editar configuracao global")
|
||||
use_mtls_bool = use_mtls.lower() in ("true", "1", "yes")
|
||||
sets = "config_name=?,dsn=?,username=?,table_name=?,use_mtls=?,genai_config_id=?,embedding_model_id=?"
|
||||
vals = [config_name, dsn, username, table_name, int(use_mtls_bool), genai_config_id or None, embedding_model_id]
|
||||
if u["role"] == "admin" and is_global:
|
||||
is_global_val = 1 if is_global.lower() in ("true", "1", "yes") else 0
|
||||
sets += ",is_global=?"
|
||||
vals.append(is_global_val)
|
||||
if password:
|
||||
sets += ",password_enc=?"
|
||||
vals.append(_enc(password))
|
||||
if wallet_password:
|
||||
sets += ",wallet_password_enc=?"
|
||||
vals.append(_enc(wallet_password))
|
||||
vals.append(vid)
|
||||
with db() as c:
|
||||
c.execute(f"UPDATE adb_vector_configs SET {sets} WHERE id=?", vals)
|
||||
if wallet and wallet.filename:
|
||||
import zipfile
|
||||
wdir = WALLET_DIR / vid; wdir.mkdir(parents=True, exist_ok=True)
|
||||
zp = wdir / "wallet.zip"
|
||||
zp.write_bytes(await wallet.read())
|
||||
with zipfile.ZipFile(str(zp), 'r') as z: z.extractall(str(wdir))
|
||||
with db() as c: c.execute("UPDATE adb_vector_configs SET wallet_dir=? WHERE id=?", (str(wdir), vid))
|
||||
_config_log("adb", vid, config_name, "success", "upload", "Wallet atualizado", u["id"], u["username"])
|
||||
_audit(u["id"], u["username"], "update_adb_config", vid, config_name)
|
||||
_config_log("adb", vid, config_name, "success", "save", f"Conexao atualizada: {config_name} ({dsn})", u["id"], u["username"])
|
||||
return {"id": vid, "config_name": config_name}
|
||||
|
||||
@router.get("/api/adb/{vid}/tables")
|
||||
async def list_adb_tables(vid: str, u=Depends(current_user)):
|
||||
_verify_config_access("adb", vid, u)
|
||||
with db() as c:
|
||||
cfg = c.execute("SELECT id FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
||||
if not cfg: raise HTTPException(404)
|
||||
return _get_tables_for_config(vid, active_only=False)
|
||||
|
||||
@router.post("/api/adb/{vid}/tables")
|
||||
async def add_adb_table(vid: str, req: dict, u=Depends(require("admin","user"))):
|
||||
_verify_config_access("adb", vid, u, require_owner=True)
|
||||
table_name = req.get("table_name", "").strip()
|
||||
description = req.get("description", "").strip()
|
||||
if not table_name: raise HTTPException(400, "table_name e obrigatorio")
|
||||
if not _TABLE_NAME_RE.match(table_name): raise HTTPException(400, "Nome da tabela invalido. Use letras maiusculas, numeros e underscores.")
|
||||
with db() as c:
|
||||
cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
||||
if not cfg: raise HTTPException(404, "ADB config not found")
|
||||
tid = str(uuid.uuid4())
|
||||
try:
|
||||
with db() as c:
|
||||
c.execute("INSERT INTO adb_vector_tables (id, adb_config_id, table_name, description) VALUES (?,?,?,?)",
|
||||
(tid, vid, table_name, description))
|
||||
except sqlite3.IntegrityError:
|
||||
raise HTTPException(409, f"Tabela '{table_name}' ja registrada nesta conexao")
|
||||
_config_log("adb", vid, cfg["config_name"], "success", "add_table", f"Tabela '{table_name}' registrada", u["id"], u["username"])
|
||||
return {"id": tid, "table_name": table_name}
|
||||
|
||||
@router.delete("/api/adb/{vid}/tables/{tid}")
|
||||
async def remove_adb_table(vid: str, tid: str, u=Depends(require("admin","user"))):
|
||||
_verify_config_access("adb", vid, u, require_owner=True)
|
||||
with db() as c:
|
||||
row = c.execute("SELECT * FROM adb_vector_tables WHERE id=? AND adb_config_id=?", (tid, vid)).fetchone()
|
||||
if not row: raise HTTPException(404, "Table entry not found")
|
||||
with db() as c:
|
||||
c.execute("DELETE FROM adb_vector_tables WHERE id=?", (tid,))
|
||||
return {"ok": True}
|
||||
|
||||
@router.put("/api/adb/{vid}/tables/{tid}")
|
||||
async def update_adb_table(vid: str, tid: str, req: dict, u=Depends(require("admin","user"))):
|
||||
_verify_config_access("adb", vid, u, require_owner=True)
|
||||
with db() as c:
|
||||
row = c.execute("SELECT * FROM adb_vector_tables WHERE id=? AND adb_config_id=?", (tid, vid)).fetchone()
|
||||
if not row: raise HTTPException(404, "Table entry not found")
|
||||
sets, vals = [], []
|
||||
if "table_name" in req:
|
||||
tn = req["table_name"].strip()
|
||||
if not tn: raise HTTPException(400, "table_name e obrigatorio")
|
||||
if not _TABLE_NAME_RE.match(tn): raise HTTPException(400, "Nome da tabela invalido")
|
||||
sets.append("table_name=?"); vals.append(tn)
|
||||
if "description" in req:
|
||||
sets.append("description=?"); vals.append(req["description"])
|
||||
if "is_active" in req:
|
||||
sets.append("is_active=?"); vals.append(int(req["is_active"]))
|
||||
if not sets: return {"ok": True}
|
||||
vals.append(tid)
|
||||
try:
|
||||
with db() as c:
|
||||
c.execute(f"UPDATE adb_vector_tables SET {','.join(sets)} WHERE id=?", vals)
|
||||
except sqlite3.IntegrityError:
|
||||
raise HTTPException(409, "Tabela com este nome ja existe nesta conexao")
|
||||
return {"ok": True}
|
||||
|
||||
@router.post("/api/adb/{vid}/ingest")
|
||||
async def ingest_documents(vid: str, req: IngestDocReq, bg: BackgroundTasks, u=Depends(require("admin","user"))):
|
||||
_verify_config_access("adb", vid, u)
|
||||
with db() as c:
|
||||
cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
||||
if not cfg: raise HTTPException(404, "ADB config not found")
|
||||
cfg = dict(cfg)
|
||||
if not cfg.get("genai_config_id"):
|
||||
raise HTTPException(400, "GenAI config not linked to this ADB connection")
|
||||
with db() as c:
|
||||
gc = c.execute("SELECT * FROM genai_configs WHERE id=?", (cfg["genai_config_id"],)).fetchone()
|
||||
if not gc: raise HTTPException(400, "Linked GenAI config not found")
|
||||
bg.add_task(_ingest_documents_task, cfg, dict(gc), req.documents, u["id"], u["username"], table_name=req.table_name)
|
||||
return {"ok": True, "message": f"Ingestao iniciada para {len(req.documents)} documentos", "config_id": vid}
|
||||
Reference in New Issue
Block a user