feat: CIS number in embeddings, tenancy dropdown in consult, text filter for exact CIS search

- Chunk findings CSV: includes CIS Recommendation number, Section, Status in document header - Consult embeddings: tenancy dropdown (OCI config selector) for filtered search - CIS number detection: regex extracts "cis X.Y" from query → TEXT LIKE filter for exact match - Dynamic top_k: 10 per table when CIS filter active (vs 3 default), 15 global results - Vector search text_filter: combined vector similarity + TEXT LIKE for precise results - Purged 121174 legacy docs without tenancy metadata from all CIS tables - Re-embedded 4364 docs across 7 tables with full CIS metadata - GPT-5.2 for consult (was GPT-4.1), max_tokens 8000
2026-03-24 22:12:15 -03:00
parent a2a9fda6c7
commit c6b7cd75a9
3 changed files with 94 additions and 34 deletions
--- a/backend/app.py
+++ b/backend/app.py
@@ -2915,9 +2915,10 @@ def _get_table_embedding_dim(cfg: dict, table_name: str) -> int:

 _GLOBAL_TABLES = {"cisrecom", "engineerknowledgebase"}  # Tables without tenancy filter (generic knowledge)

-def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name: str = None, tenancy: str = None) -> list:
+def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name: str = None, tenancy: str = None, text_filter: str = None) -> list:
    """Search ADB vector store using cosine similarity. Returns top-K documents.
-    If tenancy is provided and table is not global, filters by tenancy."""
+    If tenancy is provided and table is not global, filters by tenancy.
+    If text_filter is provided, also filters TEXT content with LIKE."""
    import array
    table_name = table_name or cfg.get("table_name", "")
    conn = _get_adb_connection(cfg)
@@ -2927,23 +2928,27 @@ def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name:
        vec = array.array('f', query_embedding)
        limit = int(top_k)
        use_tenancy_filter = tenancy and table_name.lower() not in _GLOBAL_TABLES
+        # Build WHERE clauses
+        conditions = []
+        params = [vec]
+        param_idx = 2
        if use_tenancy_filter:
-            cur.execute(f"""
-                SELECT ID, TEXT, METADATA,
-                       VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance
-                FROM "{table_name}"
-                WHERE JSON_VALUE(METADATA, '$.tenancy') = :2
-                ORDER BY distance ASC
-                FETCH FIRST {limit} ROWS ONLY
-            """, [vec, tenancy])
-        else:
-            cur.execute(f"""
-                SELECT ID, TEXT, METADATA,
-                       VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance
-                FROM "{table_name}"
-                ORDER BY distance ASC
-                FETCH FIRST {limit} ROWS ONLY
-            """, [vec])
+            conditions.append(f"JSON_VALUE(METADATA, '$.tenancy') = :{param_idx}")
+            params.append(tenancy)
+            param_idx += 1
+        if text_filter:
+            conditions.append(f"TEXT LIKE :{param_idx}")
+            params.append(f"%{text_filter}%")
+            param_idx += 1
+        where = f"WHERE {' AND '.join(conditions)}" if conditions else ""
+        cur.execute(f"""
+            SELECT ID, TEXT, METADATA,
+                   VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance
+            FROM "{table_name}"
+            {where}
+            ORDER BY distance ASC
+            FETCH FIRST {limit} ROWS ONLY
+        """, params)
        results = []
        for row in cur:
            content = row[1]
@@ -3826,11 +3831,23 @@ def _chunk_findings_csv(csv_path: str, tenancy: str, extract_date: str, max_char
        return []
    skip_cols = {"extract_date", "deep_link", "domain_deeplink", "defined_tags",
                 "freeform_tags", "system_tags", "external_identifier"}
-    meta = json.dumps({"tenancy": tenancy, "extract_date": extract_date, "source": p.name})
+    # Extract CIS recommendation number from filename (e.g., cis_Identity_and_Access_Management_1-1.csv → 1.1)
+    rec_match = _re.search(r'_(\d+)-(\d+(?:\.\d+)?)\.csv$', p.name)
+    cis_rec = f"{rec_match.group(1)}.{rec_match.group(2)}" if rec_match else ""
+    # Extract section name from filename
+    sec_match = _re.search(r'^cis_(.+?)_\d+-', p.name)
+    cis_section = sec_match.group(1).replace("_", " ") if sec_match else ""
+
+    meta = json.dumps({"tenancy": tenancy, "extract_date": extract_date, "source": p.name, "cis_recommendation": cis_rec})

    for row in rows:
        # Build context header (always repeated in each chunk)
        header_parts = [f"Tenancy: {tenancy}", f"Extract Date: {extract_date}"]
+        if cis_rec:
+            header_parts.append(f"CIS Recommendation: {cis_rec}")
+        if cis_section:
+            header_parts.append(f"Section: {cis_section}")
+        header_parts.append(f"Status: Non-Compliant")
        body_parts = []
        # Identify key fields for the header
        name = row.get("name") or row.get("display_name") or row.get("username") or ""
@@ -4311,6 +4328,7 @@ class ConsultQuery(BaseModel):
    query: str
    table_name: str = ""
    top_k: int = 10
+    oci_config_id: str = ""

@app.post("/api/embeddings/consult")
 async def consult_embeddings(req: ConsultQuery, u=Depends(current_user)):
@@ -4320,8 +4338,25 @@ async def consult_embeddings(req: ConsultQuery, u=Depends(current_user)):
    adb_configs = _get_active_adb_configs(u["id"])
    if not adb_configs:
        raise HTTPException(400, "Nenhuma conexão ADB ativa configurada")
+    # Resolve tenancy for filtered search
+    rag_tenancy = None
+    if req.oci_config_id:
+        with db() as c:
+            oci_row = c.execute("SELECT tenancy_name FROM oci_configs WHERE id=?", (req.oci_config_id,)).fetchone()
+        if oci_row:
+            rag_tenancy = oci_row["tenancy_name"]
+            log.info(f"Consult: filtering by tenancy '{rag_tenancy}'")
+
+    # Detect CIS recommendation number in query for exact text filtering
+    import re as _re
+    cis_match = _re.search(r'(?:cis|recommendation)\s*(\d+\.\d+)', req.query, _re.IGNORECASE)
+    cis_text_filter = f"CIS Recommendation: {cis_match.group(1)}" if cis_match else None
+    if cis_text_filter:
+        log.info(f"Consult: detected CIS filter '{cis_text_filter}'")
+
    # Collect results from all active ADB configs + tables
    all_docs = []
+    rag_errors = []
    for adb_cfg in adb_configs:
        try:
            emb_genai = _resolve_embed_config(oci_config_id=adb_cfg.get("oci_config_id"))
@@ -4331,36 +4366,45 @@ async def consult_embeddings(req: ConsultQuery, u=Depends(current_user)):
        tables = _get_tables_for_config(adb_cfg["id"], active_only=True)
        if req.table_name:
            tables = [t for t in tables if t["table_name"] == req.table_name]
-        # Group tables by embedding dimension and generate one query embedding per dimension
-        embeddings_cache = {}  # dim -> query_embedding
+        embeddings_cache = {}
        for tbl in tables:
            tbl_name = tbl["table_name"]
            try:
                dim = _get_table_embedding_dim(adb_cfg, tbl_name)
                if not dim:
-                    log.warning(f"Consult: table {tbl_name} is empty or has no embeddings")
                    continue
                if dim not in embeddings_cache:
                    model = _DIM_TO_MODEL.get(dim, adb_cfg.get("embedding_model_id", ""))
                    if not model:
-                        log.warning(f"Consult: no model for dim={dim} on {tbl_name}")
                        continue
                    embeddings_cache[dim] = _embed_text(req.query, emb_genai, model)
-                    log.info(f"Consult: embedded query for dim={dim} using {model}")
                query_embedding = embeddings_cache[dim]
-                docs = _vector_search(adb_cfg, query_embedding, top_k=req.top_k, table_name=tbl_name)
+                # Apply tenancy filter (skip for global tables) + CIS text filter
+                search_tenancy = rag_tenancy if tbl_name.lower() not in _GLOBAL_TABLES else None
+                tbl_top_k = 10 if cis_text_filter else 3
+                docs = _vector_search(adb_cfg, query_embedding, top_k=tbl_top_k, table_name=tbl_name, tenancy=search_tenancy, text_filter=cis_text_filter if tbl_name.lower() not in _GLOBAL_TABLES else None)
                for d in docs:
                    d["source"] = tbl_name
                all_docs.extend(docs)
+                if docs:
+                    log.info(f"Consult: {len(docs)} docs from {tbl_name}")
            except Exception as e:
-                log.warning(f"Consult: failed on {tbl_name}: {type(e).__name__}: {str(e)[:200]}")
+                err = str(e)[:150]
+                log.warning(f"Consult: failed on {tbl_name}: {err}")
+                if "DPY-6001" in str(e) or "DPY-6005" in str(e) or "timeout" in str(e).lower():
+                    rag_errors.append(f"ADB offline ou timeout ({adb_cfg.get('config_name','?')})")
    if not all_docs:
+        if rag_errors:
+            return {"answer": "⚠️ " + "; ".join(set(rag_errors)) + ". A base de conhecimento não está disponível no momento.", "documents": [], "total": 0}
        return {"answer": "Nenhum resultado encontrado nas bases vetoriais.", "documents": [], "total": 0}
    # Sort by distance and take top results
    all_docs.sort(key=lambda d: d.get("distance", 999))
-    top_docs = all_docs[:req.top_k]
-    # Build context and call GenAI
-    rag_context = _build_rag_context(top_docs)
+    top_limit = 15 if cis_text_filter else 8
+    top_docs = all_docs[:top_limit]
+    # Build context with dates and sources
+    rag_context = _build_rag_context(top_docs, max_total_chars=16000 if cis_text_filter else 12000)
+    if rag_errors:
+        rag_context += "\n\n⚠️ Algumas bases não puderam ser consultadas: " + "; ".join(set(rag_errors))
    augmented = RAG_CONTEXT_TEMPLATE.format(context=rag_context, question=req.query)
    # Get GenAI config for answering — try saved config first, then auto-resolve from OCI
    gc = None
@@ -4379,11 +4423,11 @@ async def consult_embeddings(req: ConsultQuery, u=Depends(current_user)):
                "endpoint": resolved.get("endpoint", f"https://inference.generativeai.{resolved.get('genai_region','us-ashburn-1')}.oci.oraclecloud.com"),
                "compartment_id": resolved.get("compartment_id", ""),
                "genai_region": resolved.get("genai_region", "us-ashburn-1"),
-                "model_id": "openai.gpt-4.1",
+                "model_id": "openai.gpt-5.2",
                "model_ocid": "",
                "serving_type": "ON_DEMAND",
                "temperature": 0.3,
-                "max_tokens": 4000,
+                "max_tokens": 8000,
                "top_p": 0.9,
                "top_k": 1,
                "frequency_penalty": 0,