feat: knowledge base with URL import, ADB case-sensitivity fix, and RAG enhancements

- Add Knowledge Base (Base de Conhecimento) with file upload + URL import - URL content extraction: HTML stripping, remote PDF parsing - Auto-resolve embedding config from OCI credentials - Case-insensitive ADB table matching with quoted identifiers - Table validation endpoint, report_date metadata, purge & re-embed flow - Embeddings tab cleanup, PyPDF2 dependency, README updated to v2.3
2026-03-11 02:02:31 -03:00
parent 851def30b6
commit 4a802c8e64
4 changed files with 848 additions and 153 deletions
--- a/backend/app.py
+++ b/backend/app.py
@@ -466,6 +466,16 @@ async def current_user(cred: HTTPAuthorizationCredentials = Depends(security)):
    if not u or not s: raise HTTPException(401, "Sessão inválida")
    return dict(u)

+def _user_from_token(token: str):
+    """Resolve user from a raw JWT token string (for query-param auth on downloads)."""
+    try: p = pyjwt.decode(token, APP_SECRET, algorithms=[JWT_ALG])
+    except Exception: raise HTTPException(401, "Token inválido")
+    with db() as c:
+        u = c.execute("SELECT * FROM users WHERE id=? AND is_active=1", (p["sub"],)).fetchone()
+        s = c.execute("SELECT * FROM sessions WHERE id=? AND is_active=1", (p["sid"],)).fetchone()
+    if not u or not s: raise HTTPException(401, "Sessão inválida")
+    return dict(u)
+
 def require(*roles):
    async def dep(u=Depends(current_user)):
        if u["role"] not in roles: raise HTTPException(403, f"Requer role: {', '.join(roles)}")
@@ -929,7 +939,7 @@ async def explore_security_lists(cid: str, compartment_id: str = Query(None), re
        _, default_comp = _explore_comp(cid)
        comp = compartment_id or default_comp
        sls = vn.list_security_lists(comp).data
-        def _fmt_rule(r, direction):
+        def _fmt_rule(r, direction, idx):
            proto_map = {"6":"TCP","17":"UDP","1":"ICMP","all":"ALL"}
            proto = proto_map.get(r.protocol, r.protocol)
            src_dst = r.source if direction == "ingress" else r.destination
@@ -941,17 +951,98 @@ async def explore_security_lists(cid: str, compartment_id: str = Query(None), re
                dp = r.udp_options.destination_port_range
                if dp: ports = f"{dp.min}-{dp.max}" if dp.min != dp.max else str(dp.min)
            return {"direction":direction,"protocol":proto,"source_dest":src_dst or "","ports":ports,
-                    "stateless":r.is_stateless,"description":getattr(r,'description','') or ""}
+                    "stateless":r.is_stateless,"description":getattr(r,'description','') or "","rule_index":idx}
        result = []
        for s in sls:
-            ingress = [_fmt_rule(r,"ingress") for r in (s.ingress_security_rules or [])]
-            egress = [_fmt_rule(r,"egress") for r in (s.egress_security_rules or [])]
+            ingress = [_fmt_rule(r,"ingress",i) for i,r in enumerate(s.ingress_security_rules or [])]
+            egress = [_fmt_rule(r,"egress",i) for i,r in enumerate(s.egress_security_rules or [])]
            result.append({"id":s.id,"display_name":s.display_name,"vcn_id":s.vcn_id,"lifecycle_state":s.lifecycle_state,
                           "ingress_count":len(ingress),"egress_count":len(egress),"rules":ingress+egress})
        return result
    except Exception as e:
        return {"error": str(e)[:500]}

+@app.post("/api/oci/explore/{cid}/security_lists/{sl_id}/rules")
+async def add_security_list_rule(cid: str, sl_id: str, req: dict, u=Depends(require("admin", "user"))):
+    """Add a rule to a Security List (appends to existing rules)."""
+    try:
+        import oci
+        config = _get_oci_config(cid)
+        region = req.get("region")
+        if region: config["region"] = region
+        vn = oci.core.VirtualNetworkClient(config)
+        sl = vn.get_security_list(sl_id).data
+        direction = req.get("direction", "ingress").lower()
+        protocol = str(req.get("protocol", "6"))
+        source_dest = req.get("source_dest", "0.0.0.0/0")
+        port_min = req.get("port_min")
+        port_max = req.get("port_max")
+        description = req.get("description", "").strip() or None
+        is_stateless = req.get("is_stateless", False)
+        tcp_opts = None
+        udp_opts = None
+        if protocol == "6" and port_min is not None:
+            tcp_opts = oci.core.models.TcpOptions(
+                destination_port_range=oci.core.models.PortRange(min=int(port_min), max=int(port_max or port_min)))
+        elif protocol == "17" and port_min is not None:
+            udp_opts = oci.core.models.UdpOptions(
+                destination_port_range=oci.core.models.PortRange(min=int(port_min), max=int(port_max or port_min)))
+        ingress_rules = list(sl.ingress_security_rules or [])
+        egress_rules = list(sl.egress_security_rules or [])
+        if direction == "ingress":
+            ingress_rules.append(oci.core.models.IngressSecurityRule(
+                protocol=protocol, source=source_dest, source_type="CIDR_BLOCK",
+                is_stateless=is_stateless, description=description,
+                tcp_options=tcp_opts, udp_options=udp_opts))
+        else:
+            egress_rules.append(oci.core.models.EgressSecurityRule(
+                protocol=protocol, destination=source_dest, destination_type="CIDR_BLOCK",
+                is_stateless=is_stateless, description=description,
+                tcp_options=tcp_opts, udp_options=udp_opts))
+        update = oci.core.models.UpdateSecurityListDetails(
+            ingress_security_rules=ingress_rules, egress_security_rules=egress_rules)
+        vn.update_security_list(sl_id, update)
+        _audit(u["id"], u["username"], "seclist_add_rule", sl_id,
+               f"{direction} {protocol} {source_dest} port={port_min or 'ALL'}")
+        return {"ok": True, "message": "Regra adicionada com sucesso"}
+    except Exception as e:
+        raise HTTPException(400, str(e)[:500])
+
+@app.delete("/api/oci/explore/{cid}/security_lists/{sl_id}/rules")
+async def remove_security_list_rule(cid: str, sl_id: str, req: dict, u=Depends(require("admin", "user"))):
+    """Remove a rule from a Security List by index."""
+    try:
+        import oci
+        config = _get_oci_config(cid)
+        region = req.get("region")
+        if region: config["region"] = region
+        vn = oci.core.VirtualNetworkClient(config)
+        sl = vn.get_security_list(sl_id).data
+        direction = req.get("direction", "ingress").lower()
+        rule_index = req.get("rule_index")
+        if rule_index is None: raise HTTPException(400, "rule_index obrigatório")
+        ingress_rules = list(sl.ingress_security_rules or [])
+        egress_rules = list(sl.egress_security_rules or [])
+        if direction == "ingress":
+            if 0 <= rule_index < len(ingress_rules):
+                removed = ingress_rules.pop(rule_index)
+                desc = f"ingress idx={rule_index} src={removed.source}"
+            else:
+                raise HTTPException(400, "Índice inválido")
+        else:
+            if 0 <= rule_index < len(egress_rules):
+                removed = egress_rules.pop(rule_index)
+                desc = f"egress idx={rule_index} dst={removed.destination}"
+            else:
+                raise HTTPException(400, "Índice inválido")
+        update = oci.core.models.UpdateSecurityListDetails(
+            ingress_security_rules=ingress_rules, egress_security_rules=egress_rules)
+        vn.update_security_list(sl_id, update)
+        _audit(u["id"], u["username"], "seclist_remove_rule", sl_id, desc)
+        return {"ok": True, "message": "Regra removida com sucesso"}
+    except Exception as e:
+        raise HTTPException(400, str(e)[:500])
+
@app.get("/api/oci/explore/{cid}/block_volumes")
 async def explore_block_volumes(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)):
    try:
@@ -1029,11 +1120,97 @@ async def explore_nsgs(cid: str, compartment_id: str = Query(None), region: str
        _, default_comp = _explore_comp(cid)
        comp = compartment_id or default_comp
        nsgs = vn.list_network_security_groups(compartment_id=comp).data
-        return [{"id":n.id,"display_name":n.display_name,"vcn_id":n.vcn_id,"lifecycle_state":n.lifecycle_state,
-                 "time_created":str(n.time_created)} for n in nsgs]
+        result = []
+        proto_map = {"1": "ICMP", "6": "TCP", "17": "UDP", "all": "ALL"}
+        for n in nsgs:
+            rules_data = oci.pagination.list_call_get_all_results(
+                vn.list_network_security_group_security_rules, n.id).data
+            rules = []
+            for r in rules_data:
+                ports = ""
+                if r.tcp_options and r.tcp_options.destination_port_range:
+                    pr = r.tcp_options.destination_port_range
+                    ports = str(pr.min) if pr.min == pr.max else f"{pr.min}-{pr.max}"
+                elif r.udp_options and r.udp_options.destination_port_range:
+                    pr = r.udp_options.destination_port_range
+                    ports = str(pr.min) if pr.min == pr.max else f"{pr.min}-{pr.max}"
+                src_dst = ""
+                src_dst_type = ""
+                if r.direction == "INGRESS":
+                    src_dst = r.source or ""
+                    src_dst_type = r.source_type or ""
+                else:
+                    src_dst = r.destination or ""
+                    src_dst_type = r.destination_type or ""
+                rules.append({
+                    "id": r.id, "direction": r.direction,
+                    "protocol": proto_map.get(r.protocol, r.protocol),
+                    "protocol_num": r.protocol,
+                    "source_dest": src_dst, "source_dest_type": src_dst_type,
+                    "ports": ports, "is_stateless": r.is_stateless or False,
+                    "description": r.description or ""
+                })
+            result.append({"id": n.id, "display_name": n.display_name, "vcn_id": n.vcn_id,
+                           "lifecycle_state": n.lifecycle_state, "time_created": str(n.time_created),
+                           "rule_count": len(rules), "rules": rules})
+        return result
    except Exception as e:
        return {"error": str(e)[:500]}

+@app.post("/api/oci/explore/{cid}/nsgs/{nsg_id}/rules")
+async def add_nsg_rule(cid: str, nsg_id: str, req: dict, u=Depends(require("admin", "user"))):
+    """Add a security rule to an NSG."""
+    try:
+        import oci
+        config = _get_oci_config(cid)
+        region = req.get("region")
+        if region: config["region"] = region
+        vn = oci.core.VirtualNetworkClient(config)
+        direction = req.get("direction", "INGRESS").upper()
+        protocol = str(req.get("protocol", "6"))
+        source_dest = req.get("source_dest", "0.0.0.0/0")
+        source_dest_type = req.get("source_dest_type", "CIDR_BLOCK")
+        port_min = req.get("port_min")
+        port_max = req.get("port_max")
+        description = req.get("description", "")
+        is_stateless = req.get("is_stateless", False)
+        rule = {"direction": direction, "protocol": protocol, "is_stateless": is_stateless, "description": description}
+        if direction == "INGRESS":
+            rule["source"] = source_dest
+            rule["source_type"] = source_dest_type
+        else:
+            rule["destination"] = source_dest
+            rule["destination_type"] = source_dest_type
+        if protocol == "6" and port_min is not None:
+            rule["tcp_options"] = oci.core.models.TcpOptions(
+                destination_port_range=oci.core.models.PortRange(min=int(port_min), max=int(port_max or port_min)))
+        elif protocol == "17" and port_min is not None:
+            rule["udp_options"] = oci.core.models.UdpOptions(
+                destination_port_range=oci.core.models.PortRange(min=int(port_min), max=int(port_max or port_min)))
+        details = oci.core.models.AddNetworkSecurityGroupSecurityRulesDetails(
+            security_rules=[oci.core.models.AddSecurityRuleDetails(**rule)])
+        resp = vn.add_network_security_group_security_rules(nsg_id, details)
+        _audit(u["id"], u["username"], "nsg_add_rule", nsg_id, f"{direction} {protocol} {source_dest} {port_min or 'ALL'}")
+        added = [{"id": r.id} for r in (resp.data.security_rules or [])]
+        return {"ok": True, "message": "Regra adicionada com sucesso", "rules_added": added}
+    except Exception as e:
+        raise HTTPException(400, str(e)[:500])
+
+@app.delete("/api/oci/explore/{cid}/nsgs/{nsg_id}/rules/{rule_id}")
+async def remove_nsg_rule(cid: str, nsg_id: str, rule_id: str, region: str = Query(None), u=Depends(require("admin", "user"))):
+    """Remove a security rule from an NSG."""
+    try:
+        import oci
+        config = _get_oci_config(cid)
+        if region: config["region"] = region
+        vn = oci.core.VirtualNetworkClient(config)
+        details = oci.core.models.RemoveNetworkSecurityGroupSecurityRulesDetails(security_rule_ids=[rule_id])
+        vn.remove_network_security_group_security_rules(nsg_id, details)
+        _audit(u["id"], u["username"], "nsg_remove_rule", nsg_id, f"rule_id={rule_id}")
+        return {"ok": True, "message": "Regra removida com sucesso"}
+    except Exception as e:
+        raise HTTPException(400, str(e)[:500])
+
@app.get("/api/oci/explore/{cid}/route_tables")
 async def explore_route_tables(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)):
    try:
@@ -2225,44 +2402,92 @@ def _get_adb_connection(cfg: dict):
        params["wallet_password"] = _dec(cfg["wallet_password_enc"]) if cfg.get("wallet_password_enc") else ""
    return oracledb.connect(**params)

+def _resolve_embed_config(oci_config_id: str = None, genai_cfg: dict = None) -> dict:
+    """Resolve embedding config from genai_cfg or oci_config. Returns dict with oci_config_id, endpoint, genai_region, compartment_id."""
+    if genai_cfg:
+        return genai_cfg
+    if oci_config_id:
+        with db() as c:
+            # Try linked genai config first
+            gc = c.execute("SELECT * FROM genai_configs WHERE oci_config_id=? ORDER BY is_default DESC, created_at DESC",
+                           (oci_config_id,)).fetchone()
+            if gc: return dict(gc)
+            # Fallback: build from oci_config directly
+            oc = c.execute("SELECT * FROM oci_configs WHERE id=?", (oci_config_id,)).fetchone()
+            if oc:
+                return {
+                    "oci_config_id": oc["id"],
+                    "genai_region": oc["region"],
+                    "endpoint": f"https://inference.generativeai.{oc['region']}.oci.oraclecloud.com",
+                    "compartment_id": oc.get("compartment_id") or oc["tenancy_ocid"],
+                }
+    # Last resort: any genai config
+    with db() as c:
+        gc = c.execute("SELECT * FROM genai_configs ORDER BY is_default DESC, created_at DESC").fetchone()
+        if gc: return dict(gc)
+        # Or any oci config
+        oc = c.execute("SELECT * FROM oci_configs ORDER BY created_at DESC").fetchone()
+        if oc:
+            return {
+                "oci_config_id": oc["id"],
+                "genai_region": oc["region"],
+                "endpoint": f"https://inference.generativeai.{oc['region']}.oci.oraclecloud.com",
+                "compartment_id": oc.get("compartment_id") or oc["tenancy_ocid"],
+            }
+    raise HTTPException(400, "Nenhuma credencial OCI configurada para gerar embeddings.")
+
 def _embed_text(text: str, genai_cfg: dict, embedding_model_id: str) -> list:
    """Generate embedding using OCI GenAI embed endpoint."""
    import oci
    config_path = str(OCI_DIR / genai_cfg["oci_config_id"] / "config")
    config = oci.config.from_file(config_path, "DEFAULT")
-    endpoint = genai_cfg["endpoint"]
+    endpoint = genai_cfg.get("endpoint") or f"https://inference.generativeai.{genai_cfg.get('genai_region','us-ashburn-1')}.oci.oraclecloud.com"
    client = oci.generative_ai_inference.GenerativeAiInferenceClient(
        config=config, service_endpoint=endpoint,
        retry_strategy=oci.retry.NoneRetryStrategy(), timeout=(10, 120)
    )
    embed_detail = oci.generative_ai_inference.models.EmbedTextDetails()
    embed_detail.inputs = [text]
-    # Resolve OCID for embedding model by region
    emb_info = EMBEDDING_MODELS.get(embedding_model_id, {})
    region = genai_cfg.get("genai_region", "")
    emb_ref = emb_info.get("ocids", {}).get(region) or embedding_model_id
    embed_detail.serving_mode = oci.generative_ai_inference.models.OnDemandServingMode(model_id=emb_ref)
-    embed_detail.compartment_id = genai_cfg["compartment_id"]
+    embed_detail.compartment_id = genai_cfg.get("compartment_id", "")
    embed_detail.truncate = "NONE"
    embed_detail.input_type = "SEARCH_QUERY"
    response = client.embed_text(embed_detail)
    return response.data.embeddings[0]

-def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name: str = None) -> list:
-    """Search ADB vector store using cosine similarity. Returns top-K documents."""
+def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name: str = None, tenancy: str = None) -> list:
+    """Search ADB vector store using cosine similarity. Returns top-K documents.
+    If tenancy is provided, filters METADATA JSON to match that tenancy only."""
    import array
    table_name = table_name or cfg.get("table_name", "")
    conn = _get_adb_connection(cfg)
    try:
        cur = conn.cursor()
        vec = array.array('d', query_embedding)
-        cur.execute(f"""
-            SELECT ID, CONTENT, METADATA, SOURCE,
-                   VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance
-            FROM {table_name}
-            ORDER BY distance ASC
-            FETCH FIRST :2 ROWS ONLY
-        """, [vec, top_k])
+        if tenancy:
+            # Filter by tenancy in METADATA JSON field using LIKE for broad compatibility
+            # Matches both structured JSON {"tenancy":"X",...} and legacy "tenancy: X, ..."
+            tenancy_filter = f'%"tenancy":"{tenancy}"%'
+            tenancy_filter_legacy = f'%tenancy: {tenancy}%'
+            cur.execute(f"""
+                SELECT ID, TEXT, METADATA,
+                       VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance
+                FROM "{table_name}"
+                WHERE (METADATA LIKE :3 OR METADATA LIKE :4)
+                ORDER BY distance ASC
+                FETCH FIRST :2 ROWS ONLY
+            """, [vec, top_k, tenancy_filter, tenancy_filter_legacy])
+        else:
+            cur.execute(f"""
+                SELECT ID, TEXT, METADATA,
+                       VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance
+                FROM "{table_name}"
+                ORDER BY distance ASC
+                FETCH FIRST :2 ROWS ONLY
+            """, [vec, top_k])
        results = []
        for row in cur:
            content = row[1]
@@ -2270,7 +2495,7 @@ def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name:
                content = content.read()
            results.append({
                "id": row[0], "content": content or "",
-                "metadata": row[2], "source": row[3], "distance": float(row[4])
+                "metadata": row[2], "distance": float(row[3])
            })
        cur.close()
        return results
@@ -2291,15 +2516,15 @@ def _build_rag_context(documents: list) -> str:
    return "\n\n---\n\n".join(parts)

 def _get_active_adb_configs(user_id: str) -> list[dict]:
-    """Get all active ADB vector configs with a linked GenAI config."""
+    """Get all active ADB vector configs. GenAI config is resolved automatically if not linked."""
    with db() as c:
        rows = c.execute(
-            "SELECT * FROM adb_vector_configs WHERE user_id=? AND is_active=1 AND genai_config_id IS NOT NULL ORDER BY created_at DESC",
+            "SELECT * FROM adb_vector_configs WHERE user_id=? AND is_active=1 ORDER BY created_at DESC",
            (user_id,)
        ).fetchall()
        if not rows:
            rows = c.execute(
-                "SELECT * FROM adb_vector_configs WHERE is_active=1 AND genai_config_id IS NOT NULL ORDER BY created_at DESC"
+                "SELECT * FROM adb_vector_configs WHERE is_active=1 ORDER BY created_at DESC"
            ).fetchall()
    return [dict(r) for r in rows]

@@ -2623,6 +2848,50 @@ async def test_adb(vid: str, u=Depends(require("admin","user"))):
        _config_log("adb", vid, cname, "error", "test", msg, u["id"], u["username"])
        return {"status":"error","message":msg}

+@app.post("/api/adb/{vid}/tables/check")
+async def check_adb_tables(vid: str, u=Depends(require("admin","user"))):
+    """Check which registered tables exist in ADB and list all user tables."""
+    with db() as c:
+        cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
+    if not cfg: raise HTTPException(404)
+    registered = _get_tables_for_config(vid, active_only=False)
+    reg_names_upper = {t["table_name"].upper() for t in registered}
+    try:
+        conn = _get_adb_connection(dict(cfg))
+        cur = conn.cursor()
+        cur.execute("SELECT USER FROM DUAL")
+        current_user_name = cur.fetchone()[0]
+        cur.execute("SELECT table_name FROM user_tables ORDER BY table_name")
+        adb_tables = [r[0] for r in cur.fetchall()]
+        adb_lookup = {t.upper(): t for t in adb_tables}
+        results = []
+        for t in registered:
+            tname_reg = t["table_name"]
+            tname_upper = tname_reg.upper()
+            real_name = adb_lookup.get(tname_upper)
+            if real_name:
+                try:
+                    cur.execute(f'SELECT COUNT(*) FROM "{real_name}"')
+                    cnt = cur.fetchone()[0]
+                    case_note = f" (ADB: {real_name})" if real_name != tname_reg else ""
+                    results.append({"table": tname_reg, "adb_name": real_name, "status": "ok", "rows": cnt, "message": f"{cnt} registros{case_note}"})
+                except Exception as e:
+                    results.append({"table": tname_reg, "status": "error", "message": str(e)[:200]})
+            else:
+                results.append({"table": tname_reg, "status": "not_found", "message": f"Não existe no schema {current_user_name}"})
+        unregistered = [t for t in adb_tables if t.upper() not in reg_names_upper]
+        cur.close()
+        conn.close()
+        return {
+            "status": "success",
+            "schema_user": current_user_name,
+            "adb_tables_total": len(adb_tables),
+            "registered": results,
+            "unregistered_in_adb": unregistered[:50]
+        }
+    except Exception as e:
+        return {"status": "error", "message": f"Erro de conexão: {str(e)[:500]}"}
+
@app.delete("/api/adb/configs/{vid}")
 async def del_adb(vid: str, u=Depends(require("admin","user"))):
    with db() as c: c.execute("DELETE FROM adb_vector_configs WHERE id=?", (vid,))
@@ -2668,7 +2937,7 @@ async def update_adb(
    _config_log("adb", vid, config_name, "success", "save", f"Conexão atualizada: {config_name} ({dsn})", u["id"], u["username"])
    return {"id": vid, "config_name": config_name}

-_TABLE_NAME_RE = re.compile(r'^[A-Z][A-Z0-9_]{0,127}$')
+_TABLE_NAME_RE = re.compile(r'^[A-Za-z][A-Za-z0-9_]{0,127}$')

@app.get("/api/adb/{vid}/tables")
 async def list_adb_tables(vid: str, u=Depends(current_user)):
@@ -2679,7 +2948,7 @@ async def list_adb_tables(vid: str, u=Depends(current_user)):

@app.post("/api/adb/{vid}/tables")
 async def add_adb_table(vid: str, req: dict, u=Depends(require("admin","user"))):
-    table_name = req.get("table_name", "").strip().upper()
+    table_name = req.get("table_name", "").strip()
    description = req.get("description", "").strip()
    if not table_name: raise HTTPException(400, "table_name é obrigatório")
    if not _TABLE_NAME_RE.match(table_name): raise HTTPException(400, "Nome da tabela inválido. Use letras maiúsculas, números e underscores.")
@@ -2712,7 +2981,7 @@ async def update_adb_table(vid: str, tid: str, req: dict, u=Depends(require("adm
    if not row: raise HTTPException(404, "Table entry not found")
    sets, vals = [], []
    if "table_name" in req:
-        tn = req["table_name"].strip().upper()
+        tn = req["table_name"].strip()
        if not tn: raise HTTPException(400, "table_name é obrigatório")
        if not _TABLE_NAME_RE.match(tn): raise HTTPException(400, "Nome da tabela inválido")
        sets.append("table_name=?"); vals.append(tn)
@@ -2743,11 +3012,45 @@ async def ingest_documents(vid: str, req: IngestDocReq, bg: BackgroundTasks, u=D
    bg.add_task(_ingest_documents_task, cfg, dict(gc), req.documents, u["id"], u["username"], table_name=req.table_name)
    return {"ok": True, "message": f"Ingestão iniciada para {len(req.documents)} documentos", "config_id": vid}

-def _ingest_documents_task(cfg: dict, genai_cfg: dict, documents: list, user_id: str, username: str, table_name: str = None):
-    """Background task: embed and insert documents into ADB via OCI GenAI."""
+def _build_metadata_json(tenancy: str = "", compartments: str = "", section: str = "",
+                         report_date: str = "", extra: dict = None) -> str:
+    """Build a structured JSON metadata string for vector embeddings."""
+    meta = {}
+    if tenancy:
+        meta["tenancy"] = tenancy
+    if compartments:
+        meta["compartments"] = compartments
+    if section:
+        meta["section"] = section
+    if report_date:
+        meta["report_date"] = report_date
+    if extra:
+        meta.update(extra)
+    return json.dumps(meta, ensure_ascii=False) if meta else ""
+
+
+def _auto_register_table(adb_config_id: str, table_name: str, description: str = ""):
+    """Auto-register a table in adb_vector_tables if not already present."""
+    if not table_name:
+        return
+    with db() as c:
+        exists = c.execute("SELECT 1 FROM adb_vector_tables WHERE adb_config_id=? AND table_name=? COLLATE NOCASE",
+                           (adb_config_id, table_name)).fetchone()
+        if not exists:
+            c.execute("INSERT INTO adb_vector_tables (id, adb_config_id, table_name, description) VALUES (?,?,?,?)",
+                      (str(uuid.uuid4()), adb_config_id, table_name, description))
+            log.info(f"Auto-registered table '{table_name}' for ADB config {adb_config_id}")
+
+def _ingest_documents_task(cfg: dict, genai_cfg: dict, documents: list, user_id: str, username: str,
+                           table_name: str = None, tenancy: str = None, compartments: str = None,
+                           report_date: str = None):
+    """Background task: embed and insert documents into ADB via OCI GenAI.
+    Tenancy and compartments are stored in METADATA as structured JSON for filtering."""
    import array
    emb_model = cfg.get("embedding_model_id", "cohere.embed-v4.0")
    table_name = table_name or cfg.get("table_name", "")
+    # Auto-register table so it appears in multi-table RAG search
+    _auto_register_table(cfg["id"], table_name)
    conn = _get_adb_connection(cfg)
    try:
        cur = conn.cursor()
@@ -2758,18 +3061,31 @@ def _ingest_documents_task(cfg: dict, genai_cfg: dict, documents: list, user_id:
                if not content: continue
                embedding = _embed_text(content, genai_cfg, emb_model)
                vec = array.array('d', embedding)
+                # Build structured metadata with tenancy isolation
+                doc_tenancy = tenancy or doc.get("tenancy", "")
+                doc_compartments = compartments or doc.get("compartments", "")
+                metadata = _build_metadata_json(
+                    tenancy=doc_tenancy,
+                    compartments=doc_compartments,
+                    section=doc.get("section", ""),
+                    report_date=report_date or "",
+                    extra={"legacy_metadata": doc.get("metadata", "")} if doc.get("metadata") else None
+                )
                cur.execute(f"""
-                    INSERT INTO {table_name} (ID, CONTENT, EMBEDDING, METADATA, SOURCE)
-                    VALUES (:1, :2, :3, :4, :5)
-                """, [str(uuid.uuid4()), content, vec, doc.get("metadata", ""), doc.get("source", "manual_upload")])
+                    INSERT INTO "{table_name}" (ID, TEXT, EMBEDDING, METADATA)
+                    VALUES (:1, :2, :3, :4)
+                """, [str(uuid.uuid4()), content, vec, metadata])
                inserted += 1
            except Exception as e:
                log.error(f"Failed to ingest document: {e}")
        conn.commit()
        cur.close()
-        log.info(f"Ingested {inserted}/{len(documents)} documents into {table_name}")
+        log.info(f"Ingested {inserted}/{len(documents)} documents into {table_name}" +
+                 (f" (tenancy={tenancy})" if tenancy else ""))
        _audit(user_id, username, "ingest_documents", cfg["id"], f"{inserted} documents")
-        _config_log("adb", cfg["id"], cfg.get("config_name"), "success", "ingest", f"{inserted}/{len(documents)} documentos ingeridos em {table_name}", user_id, username)
+        _config_log("adb", cfg["id"], cfg.get("config_name"), "success", "ingest",
+                    f"{inserted}/{len(documents)} documentos ingeridos em {table_name}" +
+                    (f" (tenancy: {tenancy})" if tenancy else ""), user_id, username)
    except Exception as e:
        log.error(f"Ingestion task failed: {e}")
        _config_log("adb", cfg["id"], cfg.get("config_name"), "error", "ingest", str(e)[:500], user_id, username)
@@ -2781,6 +3097,8 @@ def _chunk_report_by_section(report_data: dict) -> list:
    """Chunk a CIS report into documents grouped by section."""
    if isinstance(report_data, str):
        report_data = json.loads(report_data)
+    if isinstance(report_data, list):
+        report_data = {"findings": {str(i): item for i, item in enumerate(report_data)}, "tenancy": "unknown"}
    findings = report_data.get("findings", {})
    tenancy = report_data.get("tenancy", "unknown")
    generated_at = report_data.get("generated_at", "")
@@ -2813,6 +3131,9 @@ def _chunk_report_by_section(report_data: dict) -> list:
        documents.append({
            "content": "\n".join(lines),
            "source": f"CIS Report - {tenancy} - {generated_at}",
+            "section": section_name,
+            "tenancy": tenancy,
+            "compartments": ", ".join(compartments[:50]),
            "metadata": f"tenancy: {tenancy}, section: {section_name}, total: {len(checks)}, passed: {passed}, failed: {failed}, review: {review}"
        })
    return documents
@@ -2834,18 +3155,20 @@ def _chunk_text_file(text: str, filename: str, chunk_size: int = 1000) -> list:
        documents.append({"content": current_chunk, "source": filename, "metadata": f"chunk: {chunk_num}"})
    return documents

-def _get_adb_and_genai(vid: str):
-    """Load ADB config and its linked GenAI config. Returns (adb_cfg, genai_cfg) or raises."""
+def _get_adb_and_genai(vid: str, oci_config_id: str = None):
+    """Load ADB config and resolve embed config.
+    Priority: ADB.genai_config_id → genai by oci_config_id → oci_config directly → any available."""
    with db() as c:
        cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
    if not cfg: raise HTTPException(404, "ADB config not found")
    cfg = dict(cfg)
-    if not cfg.get("genai_config_id"):
-        raise HTTPException(400, "GenAI config not linked to this ADB connection")
-    with db() as c:
-        gc = c.execute("SELECT * FROM genai_configs WHERE id=?", (cfg["genai_config_id"],)).fetchone()
-    if not gc: raise HTTPException(400, "Linked GenAI config not found")
-    return cfg, dict(gc)
+    genai_cfg = None
+    if cfg.get("genai_config_id"):
+        with db() as c:
+            row = c.execute("SELECT * FROM genai_configs WHERE id=?", (cfg["genai_config_id"],)).fetchone()
+            if row: genai_cfg = dict(row)
+    gc = _resolve_embed_config(oci_config_id=oci_config_id, genai_cfg=genai_cfg)
+    return cfg, gc

@app.get("/api/embeddings/preview/{rid}")
 async def preview_report_chunks(rid: str, u=Depends(current_user)):
@@ -2861,9 +3184,10 @@ async def preview_report_chunks(rid: str, u=Depends(current_user)):
    except Exception:
        raise HTTPException(400, "Invalid report data")
    documents = _chunk_report_by_section(report_data)
-    return {"tenancy": report_data.get("tenancy", "unknown"),
-            "regions": report_data.get("regions", []),
-            "compartments": report_data.get("compartments", []),
+    rd = report_data if isinstance(report_data, dict) else {}
+    return {"tenancy": rd.get("tenancy", "unknown"),
+            "regions": rd.get("regions", []),
+            "compartments": rd.get("compartments", []),
            "total_chunks": len(documents),
            "chunks": documents}

@@ -2872,7 +3196,7 @@ async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(requi
    vid = req.get("adb_config_id")
    if not vid: raise HTTPException(400, "adb_config_id is required")
    with db() as c:
-        r = c.execute("SELECT json_path,tenancy_name FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone()
+        r = c.execute("SELECT json_path,tenancy_name,config_id FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone()
    if not r: raise HTTPException(404, "Report not found or not completed")
    json_path = r["json_path"]
    if not json_path or not Path(json_path).exists():
@@ -2883,17 +3207,63 @@ async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(requi
        raise HTTPException(400, "Invalid report data")
    documents = _chunk_report_by_section(report_data)
    if not documents: raise HTTPException(400, "No sections found in report")
-    cfg, gc = _get_adb_and_genai(vid)
+    # Optional section filter — embed only a specific section
+    section_filter = req.get("section")
+    if section_filter:
+        documents = [d for d in documents if d.get("section") == section_filter]
+        if not documents: raise HTTPException(400, f"Section '{section_filter}' not found in report")
+    cfg, gc = _get_adb_and_genai(vid, oci_config_id=r.get("config_id"))
    target_table = req.get("table_name") or None
-    bg.add_task(_ingest_documents_task, cfg, gc, documents, u["id"], u["username"], table_name=target_table)
-    _audit(u["id"], u["username"], "embed_report", rid, f"{len(documents)} sections")
-    return {"ok": True, "message": f"Embedding de {len(documents)} seções iniciado", "sections": len(documents)}
+    # Extract tenancy and compartments for isolation
+    tenancy = report_data.get("tenancy", r["tenancy_name"] or "unknown")
+    report_date = report_data.get("generated_at", "")
+    compartments_list = report_data.get("compartments", [])
+    compartments_str = json.dumps(compartments_list) if compartments_list else ""
+    bg.add_task(_ingest_documents_task, cfg, gc, documents, u["id"], u["username"],
+                table_name=target_table, tenancy=tenancy, compartments=compartments_str,
+                report_date=report_date)
+    label = f"section={section_filter}" if section_filter else f"{len(documents)} sections"
+    _audit(u["id"], u["username"], "embed_report", rid, f"{label}, tenancy={tenancy}")
+    return {"ok": True, "message": f"Embedding de {len(documents)} seção(ões) iniciado (tenancy: {tenancy})", "sections": len(documents), "tenancy": tenancy}
+
+def _extract_pdf_text(file_bytes: bytes) -> str:
+    """Extract text from a PDF file using PyPDF2 or pdfplumber."""
+    try:
+        import PyPDF2
+        import io
+        reader = PyPDF2.PdfReader(io.BytesIO(file_bytes))
+        pages = []
+        for page in reader.pages:
+            text = page.extract_text()
+            if text:
+                pages.append(text.strip())
+        return "\n\n".join(pages)
+    except ImportError:
+        pass
+    try:
+        import pdfplumber
+        import io
+        pages = []
+        with pdfplumber.open(io.BytesIO(file_bytes)) as pdf:
+            for page in pdf.pages:
+                text = page.extract_text()
+                if text:
+                    pages.append(text.strip())
+        return "\n\n".join(pages)
+    except ImportError:
+        raise HTTPException(400, "PDF support requires PyPDF2 or pdfplumber. Install: pip install PyPDF2")

@app.post("/api/embeddings/upload")
 async def embed_upload(adb_config_id: str = Form(...), table_name: str = Form(""), file: UploadFile = File(...), bg: BackgroundTasks = None, u=Depends(require("admin","user"))):
-    if not file.filename.lower().endswith(".txt"):
-        raise HTTPException(400, "Only .txt files are supported")
-    content = (await file.read()).decode("utf-8", errors="replace")
+    fname = file.filename.lower()
+    allowed = ('.txt', '.pdf', '.csv', '.json', '.md')
+    if not any(fname.endswith(ext) for ext in allowed):
+        raise HTTPException(400, f"Formatos aceitos: {', '.join(allowed)}")
+    raw = await file.read()
+    if fname.endswith('.pdf'):
+        content = _extract_pdf_text(raw)
+    else:
+        content = raw.decode("utf-8", errors="replace")
    if not content.strip(): raise HTTPException(400, "File is empty")
    documents = _chunk_text_file(content, file.filename)
    if not documents: raise HTTPException(400, "No content chunks found")
@@ -2903,6 +3273,52 @@ async def embed_upload(adb_config_id: str = Form(...), table_name: str = Form(""
    _audit(u["id"], u["username"], "embed_upload", file.filename, f"{len(documents)} chunks")
    return {"ok": True, "message": f"Embedding de {len(documents)} chunks iniciado", "chunks": len(documents), "filename": file.filename}

+def _extract_text_from_html(html: str) -> str:
+    """Extract readable text from HTML, stripping tags and scripts."""
+    import re as _re
+    text = _re.sub(r'<script[^>]*>[\s\S]*?</script>', ' ', html, flags=_re.IGNORECASE)
+    text = _re.sub(r'<style[^>]*>[\s\S]*?</style>', ' ', text, flags=_re.IGNORECASE)
+    text = _re.sub(r'<[^>]+>', ' ', text)
+    text = _re.sub(r'&[a-zA-Z]+;', ' ', text)
+    text = _re.sub(r'&#\d+;', ' ', text)
+    text = _re.sub(r'\s+', ' ', text).strip()
+    return text
+
+@app.post("/api/embeddings/upload-url")
+async def embed_upload_url(
+    adb_config_id: str = Form(...),
+    table_name: str = Form(""),
+    url: str = Form(...),
+    bg: BackgroundTasks = None,
+    u=Depends(require("admin", "user"))
+):
+    import requests as req
+    url = url.strip()
+    if not url.startswith(("http://", "https://")):
+        raise HTTPException(400, "URL inválida — deve começar com http:// ou https://")
+    try:
+        resp = req.get(url, timeout=30, headers={"User-Agent": "Mozilla/5.0 OCI-CIS-Agent/1.0"})
+        resp.raise_for_status()
+    except Exception as e:
+        raise HTTPException(400, f"Erro ao acessar URL: {str(e)[:300]}")
+    ct = resp.headers.get("content-type", "")
+    if "pdf" in ct:
+        content = _extract_pdf_text(resp.content)
+    elif "html" in ct or "text" in ct:
+        content = _extract_text_from_html(resp.text)
+    else:
+        content = resp.text
+    if not content or not content.strip():
+        raise HTTPException(400, "Nenhum conteúdo extraído da URL")
+    documents = _chunk_text_file(content, url)
+    if not documents:
+        raise HTTPException(400, "Nenhum chunk gerado do conteúdo")
+    cfg, gc = _get_adb_and_genai(adb_config_id)
+    target_table = table_name.strip() or None
+    bg.add_task(_ingest_documents_task, cfg, gc, documents, u["id"], u["username"], table_name=target_table)
+    _audit(u["id"], u["username"], "embed_url", url, f"{len(documents)} chunks")
+    return {"ok": True, "message": f"Embedding de {len(documents)} chunks iniciado", "chunks": len(documents), "url": url}
+
@app.get("/api/embeddings/{vid}/list")
 async def list_embeddings(vid: str, table_name: str = Query(""), limit: int = Query(50), offset: int = Query(0), u=Depends(current_user)):
    with db() as c:
@@ -2913,17 +3329,18 @@ async def list_embeddings(vid: str, table_name: str = Query(""), limit: int = Qu
        cur = conn.cursor()
        table_name = table_name.strip() or cfg["table_name"]
        if not table_name: raise HTTPException(400, "Nenhuma tabela selecionada")
-        cur.execute(f"SELECT COUNT(*) FROM {table_name}")
+        cur.execute(f'SELECT COUNT(*) FROM "{table_name}"')
        total = cur.fetchone()[0]
        cur.execute(f"""
-            SELECT ID, SOURCE, METADATA, CREATED_AT FROM {table_name}
-            ORDER BY CREATED_AT DESC
+            SELECT ID, METADATA FROM "{table_name}"
            OFFSET :1 ROWS FETCH NEXT :2 ROWS ONLY
        """, [offset, limit])
        rows = []
        for row in cur:
-            rows.append({"id": row[0], "source": row[1], "metadata": row[2],
-                         "created_at": str(row[3]) if row[3] else None})
+            rid = row[0].hex() if isinstance(row[0], bytes) else str(row[0])
+            meta = row[1]
+            if hasattr(meta, 'read'): meta = meta.read()
+            rows.append({"id": rid, "metadata": meta})
        cur.close(); conn.close()
        return {"total": total, "offset": offset, "limit": limit, "documents": rows}
    except Exception as e:
@@ -2939,13 +3356,42 @@ async def delete_embedding(vid: str, doc_id: str, table_name: str = Query(""), u
        cur = conn.cursor()
        table_name = table_name.strip() or cfg["table_name"]
        if not table_name: raise HTTPException(400, "Nenhuma tabela selecionada")
-        cur.execute(f"DELETE FROM {table_name} WHERE ID = :1", [doc_id])
+        cur.execute(f'DELETE FROM "{table_name}" WHERE ID = :1', [doc_id])
        conn.commit()
        cur.close(); conn.close()
        return {"ok": True}
    except Exception as e:
        raise HTTPException(500, f"Erro ao deletar: {str(e)[:500]}")

+@app.post("/api/embeddings/{vid}/purge")
+async def purge_embeddings(vid: str, req: dict, u=Depends(require("admin","user"))):
+    """Delete old embeddings from a table, optionally filtered by tenancy."""
+    table_name = req.get("table_name", "").strip()
+    tenancy = req.get("tenancy", "").strip()
+    if not table_name: raise HTTPException(400, "table_name é obrigatório")
+    with db() as c:
+        cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
+    if not cfg: raise HTTPException(404)
+    try:
+        conn = _get_adb_connection(dict(cfg))
+        cur = conn.cursor()
+        if tenancy:
+            cur.execute(f'SELECT COUNT(*) FROM "{table_name}" WHERE METADATA LIKE :1',
+                        [f'%"tenancy":"{tenancy}"%'])
+            count = cur.fetchone()[0]
+            cur.execute(f'DELETE FROM "{table_name}" WHERE METADATA LIKE :1',
+                        [f'%"tenancy":"{tenancy}"%'])
+        else:
+            cur.execute(f'SELECT COUNT(*) FROM "{table_name}"')
+            count = cur.fetchone()[0]
+            cur.execute(f'DELETE FROM "{table_name}"')
+        conn.commit()
+        cur.close(); conn.close()
+        _audit(u["id"], u["username"], "purge_embeddings", vid, f"table={table_name}, tenancy={tenancy or 'ALL'}, deleted={count}")
+        return {"ok": True, "deleted": count, "table": table_name, "tenancy": tenancy or "ALL"}
+    except Exception as e:
+        raise HTTPException(500, f"Erro ao limpar: {str(e)[:500]}")
+
 # ── Reports ───────────────────────────────────────────────────────────────────

 def _classify_report_file(fname: str) -> str:
@@ -3158,7 +3604,9 @@ async def list_report_files(rid: str, u=Depends(current_user)):
    return [dict(f) for f in files]

@app.get("/api/reports/{rid}/files/{fid}/download")
-async def download_report_file(rid: str, fid: str, u=Depends(current_user)):
+async def download_report_file(rid: str, fid: str, token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))):
+    u = _user_from_token(token) if token else (await current_user(cred) if cred else None)
+    if not u: raise HTTPException(401, "Not authenticated")
    with db() as c:
        r = c.execute("SELECT user_id FROM reports WHERE id=?", (rid,)).fetchone()
        if not r: raise HTTPException(404)
@@ -3176,7 +3624,9 @@ async def report_html(rid):
    return FileResponse(r["html_path"], media_type="text/html")

@app.get("/api/reports/{rid}/download")
-async def report_dl(rid, fmt: str = Query("json"), u=Depends(current_user)):
+async def report_dl(rid, fmt: str = Query("json"), token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))):
+    u = _user_from_token(token) if token else (await current_user(cred) if cred else None)
+    if not u: raise HTTPException(401, "Not authenticated")
    with db() as c: r=c.execute("SELECT * FROM reports WHERE id=?",(rid,)).fetchone()
    if not r: raise HTTPException(404)
    p = r["json_path"] if fmt=="json" else r["html_path"]
@@ -3265,23 +3715,37 @@ async def _chat_background(mid: str, sid: str, msg: ChatMsg, user: dict, genai_c
            history = [{"role":r["role"],"content":r["content"]} for r in prev]

        # ── RAG: augment with vector context from ALL active ADB configs ──
+        # Resolve active tenancy for filtered vector search
+        rag_tenancy = None
+        active_oci_for_rag = genai_cfg.get("oci_config_id") or (msg.oci_config_id if hasattr(msg, 'oci_config_id') and msg.oci_config_id else None)
+        if active_oci_for_rag:
+            with db() as c:
+                oci_for_rag = c.execute("SELECT tenancy_name FROM oci_configs WHERE id=?", (active_oci_for_rag,)).fetchone()
+            if oci_for_rag:
+                rag_tenancy = oci_for_rag["tenancy_name"]
+                log.info(f"RAG: filtering by tenancy '{rag_tenancy}'")
+
        rag_context = ""
        adb_cfgs = _get_active_adb_configs(user["id"])
        if adb_cfgs:
            all_documents = []
            for adb_cfg in adb_cfgs:
                try:
-                    with db() as c:
-                        emb_genai = c.execute("SELECT * FROM genai_configs WHERE id=?", (adb_cfg["genai_config_id"],)).fetchone()
+                    genai_linked = None
+                    if adb_cfg.get("genai_config_id"):
+                        with db() as c:
+                            row = c.execute("SELECT * FROM genai_configs WHERE id=?", (adb_cfg["genai_config_id"],)).fetchone()
+                            if row: genai_linked = dict(row)
+                    emb_genai = _resolve_embed_config(oci_config_id=active_oci_for_rag, genai_cfg=genai_linked)
                    if emb_genai:
                        emb_model = adb_cfg.get("embedding_model_id", "cohere.embed-v4.0")
-                        query_embedding = _embed_text(msg.message, dict(emb_genai), emb_model)
+                        query_embedding = _embed_text(msg.message, emb_genai, emb_model)
                        tables = _get_tables_for_config(adb_cfg["id"], active_only=True)
                        if not tables:
                            tables = [{"table_name": adb_cfg.get("table_name", "")}]
                        for tbl in tables:
                            try:
-                                documents = _vector_search(adb_cfg, query_embedding, top_k=5, table_name=tbl["table_name"])
+                                documents = _vector_search(adb_cfg, query_embedding, top_k=5, table_name=tbl["table_name"], tenancy=rag_tenancy)
                                if documents:
                                    for doc in documents:
                                        doc["source"] = f"{doc.get('source', 'unknown')} [{tbl['table_name']}]"
@@ -3993,7 +4457,9 @@ async def tf_workspace_status(wid: str, u=Depends(current_user)):


@app.get("/api/terraform/workspaces/{wid}/download")
-async def tf_download(wid: str, u=Depends(current_user)):
+async def tf_download(wid: str, token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))):
+    u = _user_from_token(token) if token else (await current_user(cred) if cred else None)
+    if not u: raise HTTPException(401, "Not authenticated")
    with db() as c:
        r = c.execute("SELECT tf_code, name FROM terraform_workspaces WHERE id=? AND user_id=?", (wid, u["id"])).fetchone()
    if not r or not r["tf_code"]: raise HTTPException(404, "No code found")
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -7,3 +7,4 @@ oci==2.133.0
 oracledb==2.4.1
 mcp>=1.0.0
 requests>=2.31.0
+PyPDF2>=3.0.0