A-Team-Security-Infra-Agent-Engineer/backend/routes/embeddings.py

"""Embedding routes — preview, embed report, status, section, file, upload, upload-url, consult, list, delete, purge."""
import json
import uuid
import re
from pathlib import Path

from fastapi import APIRouter, HTTPException, Depends, UploadFile, File, Form, Query, BackgroundTasks

from database import db
from auth.jwt_auth import current_user, require, _audit, _config_log, _verify_config_access, _verify_report_access
from config import REPORTS, log, _chat_executor
from models import ConsultQuery
from utils import validate_upload, set_embed_status, get_embed_status, update_embed_status
from services.genai import (
    _call_genai,
    _get_adb_connection,
    _resolve_embed_config,
    _embed_text,
    _DIM_TO_MODEL,
    _get_table_embedding_dim,
    _vector_search_multi,
    _relevant_tables,
    _build_rag_context,
    _get_active_adb_configs,
    _get_tables_for_config,
    RAG_CONTEXT_TEMPLATE,
    CONSULT_SYSTEM_PROMPT,
)
from services.embeddings import (
    _build_metadata_json,
    _auto_register_table,
    _ingest_documents_task,
    _chunk_report_by_section,
    _chunk_cis_pdf,
    _chunk_text_file,
    _get_adb_and_genai,
    _chunk_summary_csv,
    _purge_table_by_tenancy,
    _resolve_table_for_csv,
    _chunk_findings_csv,
)

router = APIRouter()


# ── Module-local helpers ─────────────────────────────────────────────────────

def _extract_pdf_text(file_bytes: bytes) -> str:
    """Extract text from a PDF file using PyPDF2 or pdfplumber."""
    import io
    try:
        import PyPDF2
        reader = PyPDF2.PdfReader(io.BytesIO(file_bytes))
        pages = []
        for page in reader.pages:
            text = page.extract_text()
            if text:
                pages.append(text.strip())
        return "\n\n".join(pages)
    except ImportError:
        pass
    try:
        import pdfplumber
        pages = []
        with pdfplumber.open(io.BytesIO(file_bytes)) as pdf:
            for page in pdf.pages:
                text = page.extract_text()
                if text:
                    pages.append(text.strip())
        return "\n\n".join(pages)
    except ImportError:
        raise HTTPException(400, "PDF support requires PyPDF2 or pdfplumber. Install: pip install PyPDF2")


def _extract_text_from_html(html: str) -> str:
    """Extract readable text from HTML, stripping tags and scripts."""
    import re as _re
    text = _re.sub(r'<script[^>]*>[\s\S]*?</script>', ' ', html, flags=_re.IGNORECASE)
    text = _re.sub(r'<style[^>]*>[\s\S]*?</style>', ' ', text, flags=_re.IGNORECASE)
    text = _re.sub(r'<[^>]+>', ' ', text)
    text = _re.sub(r'&[a-zA-Z]+;', ' ', text)
    text = _re.sub(r'&#\d+;', ' ', text)
    text = _re.sub(r'\s+', ' ', text).strip()
    return text


def _classify_report_file(fname: str) -> str:
    """Classify a report file into a category based on its filename."""
    fl = fname.lower()
    if "summary_report" in fl: return "summary"
    if "error_report" in fl or "error" in fl and fl.endswith(".csv"): return "error"
    if fl.startswith("obp_") and "findings" in fl: return "obp_finding"
    if fl.startswith("obp_") and "best_practices" in fl: return "obp_best_practice"
    if fl.startswith("obp_"): return "obp_finding"
    if fl.startswith("raw_data_"): return "raw_data"
    if fl.startswith("cis_"): return "cis_finding"
    if "consolidated_report" in fl: return "consolidated"
    if fl.endswith(".png"): return "diagram"
    return "other"


# ── Routes ───────────────────────────────────────────────────────────────────

@router.get("/api/embeddings/preview/{rid}")
async def preview_report_chunks(rid: str, u=Depends(current_user)):
    """Preview the chunks that will be generated from a CIS report before embedding."""
    _verify_report_access(rid, u)
    with db() as c:
        r = c.execute("SELECT json_path,tenancy_name FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone()
    if not r: raise HTTPException(404, "Report not found or not completed")
    json_path = r["json_path"]
    if not json_path or not Path(json_path).exists():
        raise HTTPException(400, "Report JSON file not found")
    try:
        report_data = json.loads(Path(json_path).read_text())
    except Exception:
        raise HTTPException(400, "Invalid report data")
    documents = _chunk_report_by_section(report_data)
    rd = report_data if isinstance(report_data, dict) else {}
    return {"tenancy": rd.get("tenancy", "unknown"),
            "regions": rd.get("regions", []),
            "compartments": rd.get("compartments", []),
            "total_chunks": len(documents),
            "chunks": documents}

@router.post("/api/embeddings/report/{rid}")
async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(require("admin","user"))):
    """Auto-embed all CIS report CSVs into their mapped ADB vector tables."""
    _verify_report_access(rid, u)
    vid = req.get("adb_config_id")
    if not vid: raise HTTPException(400, "adb_config_id is required")
    _verify_config_access("adb", vid, u)
    with db() as c:
        r = c.execute("SELECT tenancy_name, config_id FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone()
    if not r: raise HTTPException(404, "Report not found or not completed")

    rdir = REPORTS / rid
    tenancy = r["tenancy_name"] or "unknown"

    # Read extract_date from summary CSV
    import csv as csvmod
    summary_csv = rdir / "cis_summary_report.csv"
    extract_date = ""
    if summary_csv.exists():
        with open(summary_csv, "r", encoding="utf-8") as f:
            first = next(csvmod.DictReader(f), {})
        extract_date = first.get("extract_date", "")

    cfg, gc = _get_adb_and_genai(vid, oci_config_id=r["config_id"] if r["config_id"] else None, user_id=u["id"])

    # Load registered tables for validation
    with db() as c:
        registered = {t["table_name"].lower() for t in
                      c.execute("SELECT table_name FROM adb_vector_tables WHERE adb_config_id=? AND is_active=1", (vid,)).fetchall()}

    # Scan all CSVs and map to tables
    task_id = str(uuid.uuid4())
    table_docs: dict[str, list] = {}
    missing_tables: list[str] = []
    skipped_tables: list[str] = []

    for csv_file in sorted(rdir.glob("cis_*.csv")):
        table = _resolve_table_for_csv(csv_file.name)
        if not table:
            continue
        if table.lower() not in registered:
            if table not in skipped_tables:
                skipped_tables.append(table)
            continue
        if csv_file.name == "cis_summary_report.csv":
            docs = _chunk_summary_csv(str(csv_file), tenancy, extract_date)
        else:
            docs = _chunk_findings_csv(str(csv_file), tenancy, extract_date)
        if docs:
            table_docs.setdefault(table, []).extend(docs)

    if not table_docs:
        if skipped_tables:
            raise HTTPException(400, f"Tabela(s) nao registrada(s) no ADB: {', '.join(skipped_tables)}. Registre em Configuracoes > ADB Vector.")
        raise HTTPException(400, "No CSV files found to embed")

    # Calculate totals for status tracking
    total_docs = sum(len(d) for d in table_docs.values())
    tables_used = list(table_docs.keys())
    set_embed_status(task_id, {
        "status": "running", "table": ", ".join(tables_used), "tenancy": tenancy,
        "inserted": 0, "total": total_docs, "user_id": u["id"],
        "message": f"Embedding {total_docs} documentos em {len(tables_used)} tabelas..."
    })

    # Build queue info: [(table, doc_count), ...]
    table_queue = [(tbl, len(docs)) for tbl, docs in table_docs.items()]

    def _bg_embed_all():
        """Background: embed documents into their respective tables sequentially."""
        import array
        default_model = cfg.get("embedding_model_id", "cohere.embed-v4.0")
        inserted_total = 0
        skipped_total = 0
        processed_total = 0
        errors = []
        for tbl_idx, (tbl, docs) in enumerate(table_docs.items()):
            remaining = table_queue[tbl_idx + 1:] if tbl_idx + 1 < len(table_queue) else []
            _auto_register_table(cfg["id"], tbl)
            purged = _purge_table_by_tenancy(cfg, tbl, tenancy, extract_date)
            if purged:
                update_embed_status(task_id, {"message": f"Purged {purged} old docs from {tbl}...",
                                               "current_table": tbl, "current_inserted": 0, "current_total": len(docs),
                                               "queue": [{"table": t, "docs": n} for t, n in remaining]})
            emb_model = default_model
            try:
                actual_dim = _get_table_embedding_dim(cfg, tbl)
                if actual_dim and actual_dim in _DIM_TO_MODEL:
                    emb_model = _DIM_TO_MODEL[actual_dim]
            except Exception:
                pass
            current_inserted = 0
            current_skipped = 0
            try:
                conn = _get_adb_connection(cfg)
                cur = conn.cursor()
                for doc in docs:
                    try:
                        content = doc.get("content", "")
                        if not content:
                            skipped_total += 1
                            current_skipped += 1
                            processed_total += 1
                            continue
                        embedding = _embed_text(content, gc, emb_model)
                        vec = array.array('f', [float(x) for x in embedding])
                        metadata = _build_metadata_json(
                            tenancy=doc.get("tenancy", tenancy),
                            section=doc.get("section", ""),
                            report_date=extract_date,
                            user_id=u["id"],
                        )
                        cur.execute(f'INSERT INTO "{tbl}" (ID, TEXT, EMBEDDING, METADATA) VALUES (HEXTORAW(:1), :2, :3, :4)',
                                    [uuid.uuid4().hex.upper(), content, vec, metadata])
                        inserted_total += 1
                        current_inserted += 1
                    except Exception as e:
                        err_str = str(e)
                        if "message" in err_str:
                            import re as _re
                            m = _re.search(r'"message"\s*:\s*"(.*?)"', err_str)
                            log.warning(f"Embed skip in {tbl}: {m.group(1)[:200] if m else err_str[:200]}")
                        else:
                            log.warning(f"Embed skip in {tbl}: {err_str[:200]}")
                        skipped_total += 1
                        current_skipped += 1
                    processed_total += 1
                    update_embed_status(task_id, {
                        "inserted": inserted_total, "skipped": skipped_total, "processed": processed_total,
                        "current_table": tbl, "current_inserted": current_inserted, "current_skipped": current_skipped,
                        "current_total": len(docs),
                        "queue": [{"table": t, "docs": n} for t, n in remaining],
                        "message": f"{tbl}: {current_inserted}/{len(docs)} — global: {inserted_total}/{total_docs}"
                    })
                conn.commit()
                cur.close()
                conn.close()
                log.info(f"Embedded {current_inserted}/{len(docs)} docs into {tbl} (tenancy={tenancy})")
            except Exception as e:
                log.error(f"Failed to connect/embed to {tbl}: {e}")
                errors.append(f"{tbl}: {str(e)[:100]}")

        msg = f"{inserted_total}/{total_docs} documentos em {len(tables_used)} tabelas ({', '.join(tables_used)})"
        if tenancy: msg += f" — tenancy: {tenancy}"
        if errors: msg += f" | Erros: {'; '.join(errors)}"
        update_embed_status(task_id, {
            "status": "done" if not errors else "done",
            "inserted": inserted_total, "message": msg
        })
        _audit(u["id"], u["username"], "embed_report_auto", rid, msg)
        _config_log("adb", cfg["id"], cfg.get("config_name"),
                    "success" if not errors else "error", "ingest", msg, u["id"], u["username"])

    _chat_executor.submit(_bg_embed_all)
    msg = f"Embedding iniciado — {total_docs} documentos em {len(tables_used)} tabelas ({', '.join(tables_used)})"
    if skipped_tables:
        msg += f". Ignoradas (nao registradas): {', '.join(skipped_tables)}"
    return {
        "ok": True, "task_id": task_id, "message": msg,
        "tables": tables_used, "skipped": skipped_tables, "total_documents": total_docs, "tenancy": tenancy
    }


@router.get("/api/embeddings/status/{task_id}")
async def embedding_status(task_id: str, u=Depends(current_user)):
    """Check embedding task progress."""
    st = get_embed_status(task_id)
    if not st:
        return {"status": "unknown", "message": "Task not found"}
    if st.get("user_id") and st["user_id"] != u["id"] and u["role"] != "admin":
        return {"status": "unknown", "message": "Task not found"}
    return {k: v for k, v in st.items() if k != "user_id"}


@router.post("/api/embeddings/report/{rid}/section")
async def embed_report_section(rid: str, req: dict, bg: BackgroundTasks, u=Depends(require("admin","user"))):
    """Embed all CSV files from a specific report section into the mapped ADB table."""
    _verify_report_access(rid, u)
    import csv as csvmod
    vid = req.get("adb_config_id")
    file_names: list = req.get("file_names", [])
    if not vid:
        # Auto-detect first active ADB
        with db() as c:
            adb = c.execute("SELECT id FROM adb_vector_configs WHERE is_active=1 LIMIT 1").fetchone()
        if not adb: raise HTTPException(400, "No active ADB config found")
        vid = adb["id"]
    if not file_names: raise HTTPException(400, "file_names is required")

    with db() as c:
        r = c.execute("SELECT tenancy_name, config_id FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone()
    if not r: raise HTTPException(404)
    rdir = REPORTS / rid
    tenancy = r["tenancy_name"] or "unknown"

    # Read extract_date
    summary = rdir / "cis_summary_report.csv"
    extract_date = ""
    if summary.exists():
        with open(summary, "r", encoding="utf-8") as f:
            first = next(csvmod.DictReader(f), {})
        extract_date = first.get("extract_date", "")

    cfg, gc = _get_adb_and_genai(vid, oci_config_id=r["config_id"] if r["config_id"] else None, user_id=u["id"])

    # Load registered tables for validation
    with db() as c:
        registered = {t["table_name"].lower() for t in
                      c.execute("SELECT table_name FROM adb_vector_tables WHERE adb_config_id=? AND is_active=1", (vid,)).fetchall()}

    # Group files by target table and validate
    import array
    table_docs: dict[str, list] = {}
    missing_tables: list[str] = []
    for fname in file_names:
        csv_path = rdir / fname
        if not csv_path.exists(): continue
        table = _resolve_table_for_csv(fname)
        if not table: continue
        if table.lower() not in registered:
            if table not in missing_tables:
                missing_tables.append(table)
            continue
        if fname == "cis_summary_report.csv":
            docs = _chunk_summary_csv(str(csv_path), tenancy, extract_date)
        else:
            docs = _chunk_findings_csv(str(csv_path), tenancy, extract_date)
        if docs:
            table_docs.setdefault(table, []).extend(docs)

    if missing_tables and not table_docs:
        raise HTTPException(400, f"Tabela(s) nao registrada(s) no ADB: {', '.join(missing_tables)}. Registre em Configuracoes > ADB Vector.")
    if not table_docs:
        raise HTTPException(400, "No embeddable content found in the selected files")

    total_docs = sum(len(d) for d in table_docs.values())
    tables_used = list(table_docs.keys())
    task_id = str(uuid.uuid4())
    set_embed_status(task_id, {
        "status": "running", "table": ", ".join(tables_used), "tenancy": tenancy,
        "inserted": 0, "total": total_docs, "user_id": u["id"], "message": f"Embedding {total_docs} docs em {', '.join(tables_used)}..."
    })

    def _bg():
        default_model = cfg.get("embedding_model_id", "cohere.embed-v4.0")
        inserted = 0
        skipped = 0
        processed = 0
        for tbl, docs in table_docs.items():
            purged = _purge_table_by_tenancy(cfg, tbl, tenancy, extract_date)
            if purged:
                update_embed_status(task_id, {"message": f"Purged {purged} old docs from {tbl}..."})
            _auto_register_table(cfg["id"], tbl)
            emb_model = default_model
            try:
                actual_dim = _get_table_embedding_dim(cfg, tbl)
                if actual_dim and actual_dim in _DIM_TO_MODEL:
                    emb_model = _DIM_TO_MODEL[actual_dim]
            except Exception:
                pass
            try:
                conn = _get_adb_connection(cfg)
                cur = conn.cursor()
                for doc in docs:
                    try:
                        content = doc.get("content", "")
                        if not content:
                            skipped += 1
                            processed += 1
                            continue
                        embedding = _embed_text(content, gc, emb_model)
                        vec = array.array('f', [float(x) for x in embedding])
                        metadata = _build_metadata_json(tenancy=doc.get("tenancy", tenancy), section=doc.get("section", ""), report_date=extract_date, user_id=u["id"])
                        cur.execute(f'INSERT INTO "{tbl}" (ID, TEXT, EMBEDDING, METADATA) VALUES (HEXTORAW(:1), :2, :3, :4)',
                                    [uuid.uuid4().hex.upper(), content, vec, metadata])
                        inserted += 1
                    except Exception as e:
                        skipped += 1
                        err_str = str(e)
                        if "message" in err_str:
                            import re as _re
                            m = _re.search(r'"message"\s*:\s*"(.*?)"', err_str)
                            log.warning(f"Embed skip in {tbl}: {m.group(1)[:200] if m else err_str[:200]}")
                        else:
                            log.warning(f"Embed skip in {tbl}: {err_str[:200]}")
                    processed += 1
                    update_embed_status(task_id, {
                        "inserted": inserted, "skipped": skipped, "processed": processed, "total": total_docs,
                        "message": f"{tbl}: {inserted} OK, {skipped} falhas ({processed}/{total_docs})"
                    })
                conn.commit(); cur.close(); conn.close()
            except Exception as e:
                log.error(f"Embed connection error {tbl}: {e}")
        msg = f"{inserted}/{total_docs} embeddings OK"
        if skipped: msg += f", {skipped} falhas"
        msg += f" em {', '.join(tables_used)} (tenancy: {tenancy})"
        update_embed_status(task_id, {"status": "done", "inserted": inserted, "skipped": skipped, "processed": processed, "message": msg})
        _audit(u["id"], u["username"], "embed_section", rid, msg)

    _chat_executor.submit(_bg)
    return {"ok": True, "task_id": task_id, "tables": tables_used, "total": total_docs,
            "message": f"Embedding de {total_docs} docs iniciado ({', '.join(tables_used)})"}


@router.post("/api/embeddings/report/{rid}/file/{fid}")
async def embed_report_file(rid: str, fid: str, req: dict, bg: BackgroundTasks, u=Depends(require("admin","user"))):
    """Embed a specific report file (CSV, JSON, TXT, etc.) into ADB vector store."""
    _verify_report_access(rid, u)
    vid = req.get("adb_config_id")
    if not vid: raise HTTPException(400, "adb_config_id is required")
    _verify_config_access("adb", vid, u)
    with db() as c:
        r = c.execute("SELECT tenancy_name, config_id FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone()
        if not r: raise HTTPException(404, "Report not found or not completed")
        f = c.execute("SELECT * FROM report_files WHERE id=? AND report_id=?", (fid, rid)).fetchone()
        if not f: raise HTTPException(404, "File not found")
    p = Path(f["file_path"])
    if not p.exists(): raise HTTPException(404, "File not found on disk")
    fname = f["file_name"].lower()
    allowed = ('.txt', '.csv', '.json', '.md', '.pdf')
    if not any(fname.endswith(ext) for ext in allowed):
        raise HTTPException(400, f"Formatos aceitos para embedding: {', '.join(allowed)}")
    raw = p.read_bytes()
    if fname.endswith('.pdf'):
        content = _extract_pdf_text(raw)
    else:
        content = raw.decode("utf-8", errors="replace")
    if not content.strip(): raise HTTPException(400, "File is empty")
    documents = _chunk_text_file(content, f["file_name"])
    if not documents: raise HTTPException(400, "No content chunks found")
    cfg, gc = _get_adb_and_genai(vid, oci_config_id=r["config_id"] if r["config_id"] else None, user_id=u["id"])
    target_table = req.get("table_name") or None
    tenancy = r["tenancy_name"] or "unknown"
    task_id = str(uuid.uuid4())
    bg.add_task(_ingest_documents_task, cfg, gc, documents, u["id"], u["username"],
                table_name=target_table, tenancy=tenancy, task_id=task_id)
    _audit(u["id"], u["username"], "embed_report_file", f"{rid}/{fid}", f"{f['file_name']}, {len(documents)} chunks, tenancy={tenancy}")
    return {"ok": True, "task_id": task_id, "message": f"Embedding de {f['file_name']} iniciado ({len(documents)} chunks)", "chunks": len(documents)}

@router.post("/api/embeddings/upload")
async def embed_upload(adb_config_id: str = Form(...), table_name: str = Form(""), file: UploadFile = File(...), bg: BackgroundTasks = None, u=Depends(require("admin","user"))):
    _verify_config_access("adb", adb_config_id, u)
    fname = file.filename.lower()
    allowed = ('.txt', '.pdf', '.csv', '.json', '.md')
    if not any(fname.endswith(ext) for ext in allowed):
        raise HTTPException(400, f"Formatos aceitos: {', '.join(allowed)}")
    await validate_upload(file, allowed)
    file.file.seek(0)
    raw = await file.read()
    if fname.endswith('.pdf'):
        content = _extract_pdf_text(raw)
    else:
        content = raw.decode("utf-8", errors="replace")
    if not content.strip(): raise HTTPException(400, "File is empty")
    target_table = table_name.strip() or None
    # Use CIS-specific chunking for cisrecom table (segments by recommendation number with overlap)
    if target_table and 'cisrecom' in target_table.lower():
        documents = _chunk_cis_pdf(content, file.filename)
        if not documents:
            documents = _chunk_text_file(content, file.filename)  # fallback
    else:
        documents = _chunk_text_file(content, file.filename)
    if not documents: raise HTTPException(400, "No content chunks found")
    cfg, gc = _get_adb_and_genai(adb_config_id, user_id=u["id"])
    bg.add_task(_ingest_documents_task, cfg, gc, documents, u["id"], u["username"], table_name=target_table)
    _audit(u["id"], u["username"], "embed_upload", file.filename, f"{len(documents)} chunks")
    return {"ok": True, "message": f"Embedding de {len(documents)} chunks iniciado", "chunks": len(documents), "filename": file.filename}

@router.post("/api/embeddings/upload-url")
async def embed_upload_url(
    adb_config_id: str = Form(...),
    table_name: str = Form(""),
    url: str = Form(...),
    bg: BackgroundTasks = None,
    u=Depends(require("admin", "user"))
):
    _verify_config_access("adb", adb_config_id, u)
    import requests as req
    url = url.strip()
    if not url.startswith(("http://", "https://")):
        raise HTTPException(400, "URL invalida — deve comecar com http:// ou https://")
    try:
        resp = req.get(url, timeout=30, headers={"User-Agent": "Mozilla/5.0 AI-Agent/1.0"})
        resp.raise_for_status()
    except Exception as e:
        raise HTTPException(400, f"Erro ao acessar URL: {str(e)[:300]}")
    ct = resp.headers.get("content-type", "")
    if "pdf" in ct:
        content = _extract_pdf_text(resp.content)
    elif "html" in ct or "text" in ct:
        content = _extract_text_from_html(resp.text)
    else:
        content = resp.text
    if not content or not content.strip():
        raise HTTPException(400, "Nenhum conteudo extraido da URL")
    documents = _chunk_text_file(content, url)
    if not documents:
        raise HTTPException(400, "Nenhum chunk gerado do conteudo")
    cfg, gc = _get_adb_and_genai(adb_config_id, user_id=u["id"])
    target_table = table_name.strip() or None
    bg.add_task(_ingest_documents_task, cfg, gc, documents, u["id"], u["username"], table_name=target_table)
    _audit(u["id"], u["username"], "embed_url", url, f"{len(documents)} chunks")
    return {"ok": True, "message": f"Embedding de {len(documents)} chunks iniciado", "chunks": len(documents), "url": url}

@router.post("/api/embeddings/consult")
async def consult_embeddings(req: ConsultQuery, u=Depends(current_user)):
    """Query embeddings via vector search + GenAI to get a formatted answer."""
    if not req.query.strip():
        raise HTTPException(400, "Query nao pode ser vazia")
    adb_configs = _get_active_adb_configs(u["id"])
    if not adb_configs:
        raise HTTPException(400, "Nenhuma conexao ADB ativa configurada")
    # Resolve tenancy for filtered search
    rag_tenancy = None
    if req.oci_config_id:
        with db() as c:
            oci_row = c.execute("SELECT tenancy_name FROM oci_configs WHERE id=?", (req.oci_config_id,)).fetchone()
        if oci_row:
            rag_tenancy = oci_row["tenancy_name"]
            log.info(f"Consult: filtering by tenancy '{rag_tenancy}'")

    # Detect CIS recommendation number in query for exact text filtering
    import re as _re
    cis_match = _re.search(r'(?:cis|recommendation)\s*(\d+\.\d+)', req.query, _re.IGNORECASE)
    cis_text_filter = f"CIS Recommendation: {cis_match.group(1)}" if cis_match else None
    if cis_text_filter:
        log.info(f"Consult: detected CIS filter '{cis_text_filter}'")

    # Collect results from all active ADB configs + tables
    all_docs = []
    rag_errors = []
    for adb_cfg in adb_configs:
        try:
            emb_genai = _resolve_embed_config(oci_config_id=adb_cfg.get("oci_config_id"), user_id=u["id"])
        except Exception as e:
            log.warning(f"Consult: resolve config failed for {adb_cfg['id']}: {e}")
            continue
        tables = _get_tables_for_config(adb_cfg["id"], active_only=True)
        if req.table_name:
            tables = [t for t in tables if t["table_name"] == req.table_name]
        all_table_names = [t["table_name"] for t in tables if t["table_name"]]
        # Smart skip
        relevant = _relevant_tables(req.query, all_table_names) if not req.table_name else all_table_names
        skipped = set(all_table_names) - set(relevant)
        if skipped:
            log.info(f"Consult: skipped {', '.join(skipped)}")
        # Auto-detect model
        emb_model = adb_cfg.get("embedding_model_id", "cohere.embed-v4.0")
        for tbl_name in relevant:
            try:
                dim = _get_table_embedding_dim(adb_cfg, tbl_name)
                if dim and dim in _DIM_TO_MODEL:
                    emb_model = _DIM_TO_MODEL[dim]
                    break
            except Exception:
                pass
        try:
            query_embedding = _embed_text(req.query, emb_genai, emb_model)
            tbl_top_k = 10 if cis_text_filter else 3
            docs = _vector_search_multi(adb_cfg, query_embedding, relevant,
                                       top_k_per_table=tbl_top_k, tenancy=rag_tenancy,
                                       text_filter=cis_text_filter, user_id=u["id"])
            all_docs.extend(docs)
            if docs:
                sources = {}
                for d in docs:
                    sources[d["source"]] = sources.get(d["source"], 0) + 1
                log.info(f"Consult: {len(docs)} docs — {', '.join(f'{k}:{v}' for k,v in sources.items())}")
        except Exception as e:
            err = str(e)[:150]
            log.warning(f"Consult: search failed: {err}")
            if "DPY-6001" in str(e) or "DPY-6005" in str(e) or "timeout" in str(e).lower():
                rag_errors.append(f"ADB offline ou timeout ({adb_cfg.get('config_name','?')})")
    if not all_docs:
        if rag_errors:
            return {"answer": "\u26a0\ufe0f " + "; ".join(set(rag_errors)) + ". A base de conhecimento nao esta disponivel no momento.", "documents": [], "total": 0}
        return {"answer": "Nenhum resultado encontrado nas bases vetoriais.", "documents": [], "total": 0}
    # Sort by distance and take top results
    all_docs.sort(key=lambda d: d.get("distance", 999))
    top_limit = 15 if cis_text_filter else 8
    top_docs = all_docs[:top_limit]
    # Build context with dates and sources
    rag_context = _build_rag_context(top_docs, max_total_chars=16000 if cis_text_filter else 12000)
    if rag_errors:
        rag_context += "\n\n\u26a0\ufe0f Algumas bases nao puderam ser consultadas: " + "; ".join(set(rag_errors))
    augmented = RAG_CONTEXT_TEMPLATE.format(context=rag_context, question=req.query)
    # Get GenAI config for answering — try saved config first, then auto-resolve from OCI
    gc = None
    with db() as c:
        gc_row = c.execute("SELECT * FROM genai_configs WHERE user_id=? AND is_default=1 ORDER BY created_at DESC", (u["id"],)).fetchone()
        if not gc_row:
            gc_row = c.execute("SELECT * FROM genai_configs WHERE user_id=? ORDER BY created_at DESC", (u["id"],)).fetchone()
    if gc_row:
        gc = dict(gc_row)
    else:
        # Auto-resolve: build GenAI config from OCI credentials + default model
        try:
            resolved = _resolve_embed_config(user_id=u["id"])
            gc = {
                "oci_config_id": resolved["oci_config_id"],
                "endpoint": resolved.get("endpoint", f"https://inference.generativeai.{resolved.get('genai_region','us-ashburn-1')}.oci.oraclecloud.com"),
                "compartment_id": resolved.get("compartment_id", ""),
                "genai_region": resolved.get("genai_region", "us-ashburn-1"),
                "model_id": "openai.gpt-5.2",
                "model_ocid": "",
                "serving_type": "ON_DEMAND",
                "temperature": 0.3,
                "max_tokens": 8000,
                "top_p": 0.9,
                "top_k": 1,
                "frequency_penalty": 0,
                "presence_penalty": 0,
            }
            log.info(f"Consult: auto-resolved GenAI config from OCI, model=openai.gpt-4.1")
        except Exception as e:
            log.warning(f"Consult: no GenAI config available: {e}")
            doc_list = [{"content": d.get("content", "")[:500], "source": d.get("source", ""), "distance": round(d.get("distance", 0), 4), "metadata": d.get("metadata", "")} for d in top_docs]
            parts = []
            for i, d in enumerate(top_docs, 1):
                content = d.get("content", "")
                if len(content) > 800: content = content[:800] + "..."
                parts.append(f"**Documento {i}** — `{d.get('source', '?')}` (distancia: {d.get('distance', 0):.4f})\n\n{content}")
            return {"answer": "\n\n---\n\n".join(parts), "documents": doc_list, "total": len(all_docs)}
    try:
        gc["system_prompt"] = CONSULT_SYSTEM_PROMPT
        answer, _, _ = _call_genai(gc, augmented)
    except Exception as e:
        log.error(f"Consult GenAI error: {e}")
        answer = f"Erro ao consultar GenAI: {str(e)[:300]}"
    doc_list = [{"content": d.get("content", "")[:500], "source": d.get("source", ""), "distance": round(d.get("distance", 0), 4), "metadata": d.get("metadata", "")} for d in top_docs]
    return {"answer": answer, "documents": doc_list, "total": len(all_docs)}

@router.get("/api/embeddings/{vid}/list")
async def list_embeddings(vid: str, table_name: str = Query(""), limit: int = Query(50), offset: int = Query(0), u=Depends(current_user)):
    _verify_config_access("adb", vid, u)
    with db() as c:
        cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
    if not cfg: raise HTTPException(404)
    try:
        conn = _get_adb_connection(dict(cfg))
        cur = conn.cursor()
        table_name = table_name.strip() or cfg["table_name"]
        if not table_name: raise HTTPException(400, "Nenhuma tabela selecionada")
        cur.execute(f'SELECT COUNT(*) FROM "{table_name}"')
        total = cur.fetchone()[0]
        cur.execute(f"""
            SELECT ID, METADATA FROM "{table_name}"
            OFFSET :1 ROWS FETCH NEXT :2 ROWS ONLY
        """, [offset, limit])
        rows = []
        for row in cur:
            rid = row[0].hex() if isinstance(row[0], bytes) else str(row[0])
            meta = row[1]
            if hasattr(meta, 'read'): meta = meta.read()
            rows.append({"id": rid, "metadata": meta})
        cur.close(); conn.close()
        return {"total": total, "offset": offset, "limit": limit, "documents": rows}
    except Exception as e:
        raise HTTPException(500, f"Erro ao listar embeddings: {str(e)[:500]}")

@router.delete("/api/embeddings/{vid}/{doc_id}")
async def delete_embedding(vid: str, doc_id: str, table_name: str = Query(""), u=Depends(require("admin","user"))):
    _verify_config_access("adb", vid, u, require_owner=True)
    with db() as c:
        cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
    if not cfg: raise HTTPException(404)
    try:
        conn = _get_adb_connection(dict(cfg))
        cur = conn.cursor()
        table_name = table_name.strip() or cfg["table_name"]
        if not table_name: raise HTTPException(400, "Nenhuma tabela selecionada")
        cur.execute(f'DELETE FROM "{table_name}" WHERE ID = :1', [doc_id])
        conn.commit()
        cur.close(); conn.close()
        return {"ok": True}
    except Exception as e:
        raise HTTPException(500, f"Erro ao deletar: {str(e)[:500]}")

@router.post("/api/embeddings/{vid}/purge")
async def purge_embeddings(vid: str, req: dict, u=Depends(require("admin","user"))):
    """Delete old embeddings from a table, optionally filtered by tenancy."""
    _verify_config_access("adb", vid, u, require_owner=True)
    table_name = req.get("table_name", "").strip()
    tenancy = req.get("tenancy", "").strip()
    if not table_name: raise HTTPException(400, "table_name e obrigatorio")
    with db() as c:
        cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
    if not cfg: raise HTTPException(404)
    try:
        conn = _get_adb_connection(dict(cfg))
        cur = conn.cursor()
        if tenancy:
            cur.execute(f'SELECT COUNT(*) FROM "{table_name}" WHERE METADATA LIKE :1',
                        [f'%"tenancy":"{tenancy}"%'])
            count = cur.fetchone()[0]
            cur.execute(f'DELETE FROM "{table_name}" WHERE METADATA LIKE :1',
                        [f'%"tenancy":"{tenancy}"%'])
        else:
            cur.execute(f'SELECT COUNT(*) FROM "{table_name}"')
            count = cur.fetchone()[0]
            cur.execute(f'DELETE FROM "{table_name}"')
        conn.commit()
        cur.close(); conn.close()
        _audit(u["id"], u["username"], "purge_embeddings", vid, f"table={table_name}, tenancy={tenancy or 'ALL'}, deleted={count}")
        return {"ok": True, "deleted": count, "table": table_name, "tenancy": tenancy or "ALL"}
    except Exception as e:
        raise HTTPException(500, f"Erro ao limpar: {str(e)[:500]}")