feat: RAG optimizations — single connection, smart table skip, follow-up context

- Single ADB connection: _vector_search_multi searches all tables with 1 connection (was 11)
- Smart table skip: _relevant_tables matches query keywords to tables, skips irrelevant ones
- Follow-up enrichment: short questions (≤8 words) enriched with previous user message for RAG context
- CIS filter applied to enriched query (follow-up "qual o comando cli?" inherits "cis 1.1" from previous)
- RAG system prompt: remediation must be COMPLETE with CLI commands in code blocks, never summarized
- Table keyword mapping for all 11 tables (IAM, networking, compute, etc.)
- Applied to both Chat Agent and Consultar Embeddings

backend/app.py

@@ -2516,20 +2516,28 @@ RAG_DEFAULT_SYSTEM_PROMPT = """Você é um assistente RAG especializado em Oracl
 - Use somente **1–3 linhas** de evidência por item para manter respostas compactas.
 - **Sempre informe a data de extração** dos dados apresentados.
 
-### Formato de resposta (compacto, porém completo)
+### Formato de resposta
 - **Tenancy:** <tenancy>
+- **Data de extração:** <date>
 - **Recomendação <número> — <título>**
   - **Seção/Capítulo:** …
   - **Nível/Tipo:** Manual/Automated (se disponível)
-  - **O que isso exige (CIS):** …
+  - **O que isso exige (CIS):** breve explicação
-  - **Como auditar (critério):** …
+  - **Recursos afetados:** listar recursos non-compliant com nome/OCID (se disponíveis no contexto)
-  - **Como corrigir (remediação):** (somente se solicitado ou relevante)
+  - **Como corrigir (remediação):** passos COMPLETOS e DETALHADOS
-    - Passos (Console/OCI CLI **apenas se estiverem nas evidências**)
+    - Passos via Console OCI (se estiverem nas evidências)
-    - Observações/alertas
+    - Passos via OCI CLI com comandos completos em code blocks (```bash)
-  - **Evidência (citação curta):** "…"
+    - Passos via API (se disponíveis)
-  - **Fontes consultadas:**
+    - Observações/alertas importantes
-    - CIS_REPORT: <metadata essenciais>
+  - **Como auditar:** como verificar se a correção foi aplicada
-    - CIS_RECOMMENDATIONS: <metadata essenciais> (se usado)"""
+  - **Fontes:** tabela e data de extração de cada informação usada
+
+### Regras de formatação
+- Comandos CLI e policies devem ser apresentados em **code blocks** (```bash ou ```hcl)
+- Listar TODOS os passos de remediação disponíveis no contexto — nunca resumir ou omitir passos
+- Se a remediação tem passos via Console E via CLI, apresentar AMBOS
+- Usar tabelas markdown para listar recursos afetados quando houver múltiplos
+- Ser completo na remediação mas conciso nas explicações"""
 
 CONSULT_SYSTEM_PROMPT = """Você é um consultor de dados vetorizados. Sua função é responder perguntas com base nos documentos recuperados do banco vetorial.
 
@@ -2963,6 +2971,94 @@ def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name:
     finally:
         conn.close()
 
+# ── Table relevance mapping for smart skip ──
+_TABLE_KEYWORDS = {
+    "identityandaccess": {"iam", "user", "users", "policy", "policies", "mfa", "api key", "group", "password", "credential", "authentication", "identity", "access"},
+    "networking": {"network", "vcn", "subnet", "security list", "nsg", "firewall", "route", "gateway", "nat", "internet", "port", "ingress", "egress", "cidr", "ip"},
+    "computeinstances": {"compute", "instance", "vm", "boot", "metadata", "secure boot", "encryption"},
+    "loggingandmonitoring": {"log", "logging", "monitor", "alarm", "event", "notification", "audit", "cloud guard", "flow"},
+    "objectstorage": {"bucket", "object storage", "storage", "visibility", "public"},
+    "storageblockvolume": {"block volume", "volume", "disk", "cmk", "encryption"},
+    "filestorageservice": {"file storage", "file system", "mount", "nfs"},
+    "assetmanagement": {"compartment", "tag", "asset", "resource"},
+    "summaryreportcsvvector": {"summary", "score", "compliance", "report", "overview", "total"},
+    "cisrecom": {"remediation", "fix", "correct", "resolve", "recommendation", "how to", "como corrigir", "remediação"},
+    "engineerknowledgebase": {"documentation", "guide", "tutorial", "blog", "best practice", "knowledge"},
+}
+
+def _relevant_tables(query: str, tables: list[str]) -> list[str]:
+    """Filter tables to only those relevant to the query. Always includes cisrecom and engineerknowledgebase."""
+    q = query.lower()
+    # If query mentions a CIS number, include all tables (the text filter handles precision)
+    import re as _re
+    if _re.search(r'cis\s*\d+\.\d+', q):
+        return tables
+    relevant = []
+    for tbl in tables:
+        tbl_lower = tbl.lower()
+        # Always include global knowledge tables
+        if tbl_lower in _GLOBAL_TABLES or tbl_lower == "summaryreportcsvvector":
+            relevant.append(tbl)
+            continue
+        keywords = _TABLE_KEYWORDS.get(tbl_lower, set())
+        if any(kw in q for kw in keywords):
+            relevant.append(tbl)
+    # If no specific table matched, search all (generic question)
+    return relevant if len(relevant) > 2 else tables
+
+
+def _vector_search_multi(cfg: dict, query_embedding: list, tables: list[str], top_k_per_table: int = 3,
+                         tenancy: str = None, text_filter: str = None) -> list:
+    """Search multiple tables using a SINGLE ADB connection. Returns all documents with source."""
+    import array
+    conn = _get_adb_connection(cfg)
+    all_results = []
+    try:
+        conn.call_timeout = 30000
+        vec = array.array('f', query_embedding)
+        for table_name in tables:
+            try:
+                cur = conn.cursor()
+                limit = int(top_k_per_table)
+                use_tenancy = tenancy and table_name.lower() not in _GLOBAL_TABLES
+                conditions = []
+                params = [vec]
+                param_idx = 2
+                if use_tenancy:
+                    conditions.append(f"JSON_VALUE(METADATA, '$.tenancy') = :{param_idx}")
+                    params.append(tenancy)
+                    param_idx += 1
+                tbl_filter = text_filter if (text_filter and table_name.lower() not in _GLOBAL_TABLES) else None
+                if tbl_filter:
+                    conditions.append(f"TEXT LIKE :{param_idx}")
+                    params.append(f"%{tbl_filter}%")
+                    param_idx += 1
+                where = f"WHERE {' AND '.join(conditions)}" if conditions else ""
+                cur.execute(f"""
+                    SELECT ID, TEXT, METADATA,
+                           VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance
+                    FROM "{table_name}"
+                    {where}
+                    ORDER BY distance ASC
+                    FETCH FIRST {limit} ROWS ONLY
+                """, params)
+                for row in cur:
+                    content = row[1]
+                    if hasattr(content, 'read'):
+                        content = content.read()
+                    all_results.append({
+                        "id": row[0], "content": content or "",
+                        "metadata": row[2], "distance": float(row[3]),
+                        "source": table_name,
+                    })
+                cur.close()
+            except Exception as e:
+                log.warning(f"Multi-search failed for {table_name}: {str(e)[:120]}")
+    finally:
+        conn.close()
+    return all_results
+
+
 def _enrich_doc_content(doc: dict) -> str:
     """Extract the best text content from a document, checking metadata.text as fallback."""
     content = doc.get("content", "")
@@ -4366,33 +4462,39 @@ async def consult_embeddings(req: ConsultQuery, u=Depends(current_user)):
         tables = _get_tables_for_config(adb_cfg["id"], active_only=True)
         if req.table_name:
             tables = [t for t in tables if t["table_name"] == req.table_name]
-        embeddings_cache = {}
+        all_table_names = [t["table_name"] for t in tables if t["table_name"]]
-        for tbl in tables:
+        # Smart skip
-            tbl_name = tbl["table_name"]
+        relevant = _relevant_tables(req.query, all_table_names) if not req.table_name else all_table_names
+        skipped = set(all_table_names) - set(relevant)
+        if skipped:
+            log.info(f"Consult: skipped {', '.join(skipped)}")
+        # Auto-detect model
+        emb_model = adb_cfg.get("embedding_model_id", "cohere.embed-v4.0")
+        for tbl_name in relevant:
             try:
                 dim = _get_table_embedding_dim(adb_cfg, tbl_name)
-                if not dim:
+                if dim and dim in _DIM_TO_MODEL:
-                    continue
+                    emb_model = _DIM_TO_MODEL[dim]
-                if dim not in embeddings_cache:
+                    break
-                    model = _DIM_TO_MODEL.get(dim, adb_cfg.get("embedding_model_id", ""))
+            except Exception:
-                    if not model:
+                pass
-                        continue
+        try:
-                    embeddings_cache[dim] = _embed_text(req.query, emb_genai, model)
+            query_embedding = _embed_text(req.query, emb_genai, emb_model)
-                query_embedding = embeddings_cache[dim]
+            tbl_top_k = 10 if cis_text_filter else 3
-                # Apply tenancy filter (skip for global tables) + CIS text filter
+            docs = _vector_search_multi(adb_cfg, query_embedding, relevant,
-                search_tenancy = rag_tenancy if tbl_name.lower() not in _GLOBAL_TABLES else None
+                                       top_k_per_table=tbl_top_k, tenancy=rag_tenancy,
-                tbl_top_k = 10 if cis_text_filter else 3
+                                       text_filter=cis_text_filter)
-                docs = _vector_search(adb_cfg, query_embedding, top_k=tbl_top_k, table_name=tbl_name, tenancy=search_tenancy, text_filter=cis_text_filter if tbl_name.lower() not in _GLOBAL_TABLES else None)
+            all_docs.extend(docs)
+            if docs:
+                sources = {}
                 for d in docs:
-                    d["source"] = tbl_name
+                    sources[d["source"]] = sources.get(d["source"], 0) + 1
-                all_docs.extend(docs)
+                log.info(f"Consult: {len(docs)} docs — {', '.join(f'{k}:{v}' for k,v in sources.items())}")
-                if docs:
+        except Exception as e:
-                    log.info(f"Consult: {len(docs)} docs from {tbl_name}")
+            err = str(e)[:150]
-            except Exception as e:
+            log.warning(f"Consult: search failed: {err}")
-                err = str(e)[:150]
+            if "DPY-6001" in str(e) or "DPY-6005" in str(e) or "timeout" in str(e).lower():
-                log.warning(f"Consult: failed on {tbl_name}: {err}")
+                rag_errors.append(f"ADB offline ou timeout ({adb_cfg.get('config_name','?')})")
-                if "DPY-6001" in str(e) or "DPY-6005" in str(e) or "timeout" in str(e).lower():
-                    rag_errors.append(f"ADB offline ou timeout ({adb_cfg.get('config_name','?')})")
     if not all_docs:
         if rag_errors:
             return {"answer": "⚠️ " + "; ".join(set(rag_errors)) + ". A base de conhecimento não está disponível no momento.", "documents": [], "total": 0}
@@ -5864,6 +5966,28 @@ async def _chat_background(mid: str, sid: str, msg: ChatMsg, user: dict, genai_c
                 log.info(f"RAG: filtering by tenancy '{rag_tenancy}'")
 
         rag_context = ""
+        # For short follow-up questions, enrich with previous context for better RAG search
+        import re as _re_chat
+        rag_query = msg.message
+        if len(msg.message.split()) <= 8 and history:
+            # Short question — get previous user messages (excluding current which is already in history)
+            prev_user_msgs = [h["content"] for h in history if h["role"] == "user" and h["content"] != msg.message]
+            if prev_user_msgs:
+                rag_query = prev_user_msgs[-1] + " " + msg.message
+                log.info(f"RAG: enriched short query → '{rag_query[:100]}'")
+            elif len(history) >= 2:
+                # Fallback: use last assistant response for context
+                last_assistant = [h["content"][:200] for h in history if h["role"] == "assistant"]
+                if last_assistant:
+                    rag_query = last_assistant[-1] + " " + msg.message
+                    log.info(f"RAG: enriched from assistant context")
+
+        # Detect CIS recommendation number for exact text filtering
+        cis_chat_match = _re_chat.search(r'(?:cis|recommendation)\s*(\d+\.\d+)', rag_query, _re_chat.IGNORECASE)
+        cis_chat_filter = f"CIS Recommendation: {cis_chat_match.group(1)}" if cis_chat_match else None
+        if cis_chat_filter:
+            log.info(f"RAG: detected CIS filter '{cis_chat_filter}'")
+
         adb_cfgs = _get_active_adb_configs(user["id"]) if agent_type != "terraform" else []
         rag_errors = []
         if adb_cfgs:
@@ -5882,44 +6006,43 @@ async def _chat_background(mid: str, sid: str, msg: ChatMsg, user: dict, genai_c
                     tables = _get_tables_for_config(adb_cfg["id"], active_only=True)
                     if not tables:
                         tables = [{"table_name": adb_cfg.get("table_name", "")}]
-                    # Cache embeddings by dimension to avoid re-computing
+                    all_table_names = [t["table_name"] for t in tables if t["table_name"]]
-                    embedding_cache: dict[str, list] = {}
+                    # Smart skip: only search relevant tables
-                    for tbl in tables:
+                    relevant = _relevant_tables(rag_query, all_table_names)
-                        tbl_name = tbl["table_name"]
+                    skipped = set(all_table_names) - set(relevant)
-                        if not tbl_name:
+                    if skipped:
-                            continue
+                        log.info(f"RAG: skipped {', '.join(skipped)} (not relevant)")
+                    # Auto-detect embedding model (use first table's dim as representative)
+                    emb_model = default_model
+                    for tbl_name in relevant:
                         try:
-                            # Auto-detect model by table dimension
+                            dim = _get_table_embedding_dim(adb_cfg, tbl_name)
-                            emb_model = default_model
+                            if dim and dim in _DIM_TO_MODEL:
-                            try:
+                                emb_model = _DIM_TO_MODEL[dim]
-                                actual_dim = _get_table_embedding_dim(adb_cfg, tbl_name)
+                                break
-                                if actual_dim and actual_dim in _DIM_TO_MODEL:
+                        except Exception:
-                                    emb_model = _DIM_TO_MODEL[actual_dim]
+                            pass
-                            except Exception:
+                    query_embedding = _embed_text(rag_query, emb_genai, emb_model)
-                                pass
+                    # Single connection, multi-table search
-                            # Reuse cached embedding if same model
+                    tbl_top_k = 10 if cis_chat_filter else 3
-                            if emb_model not in embedding_cache:
+                    docs = _vector_search_multi(adb_cfg, query_embedding, relevant,
-                                embedding_cache[emb_model] = _embed_text(msg.message, emb_genai, emb_model)
+                                               top_k_per_table=tbl_top_k, tenancy=rag_tenancy,
-                            query_embedding = embedding_cache[emb_model]
+                                               text_filter=cis_chat_filter)
-                            documents = _vector_search(adb_cfg, query_embedding, top_k=3, table_name=tbl_name, tenancy=rag_tenancy)
+                    if docs:
-                            if documents:
+                        all_documents.extend(docs)
-                                for doc in documents:
+                        sources = {}
-                                    doc["source"] = tbl_name
+                        for d in docs:
-                                all_documents.extend(documents)
+                            sources[d["source"]] = sources.get(d["source"], 0) + 1
-                                log.info(f"RAG: {len(documents)} docs from {tbl_name} (dim={emb_model})")
+                        log.info(f"RAG: {len(docs)} docs — {', '.join(f'{k}:{v}' for k,v in sources.items())}")
-                        except Exception as te:
-                            err_short = str(te)[:100]
-                            log.warning(f"RAG search failed for {tbl_name}: {err_short}")
-                            if "DPY-6001" in str(te) or "DPY-6005" in str(te) or "timeout" in str(te).lower():
-                                rag_errors.append(f"ADB offline ou timeout ({adb_cfg.get('config_name','?')})")
                 except Exception as e:
-                    err_short = str(e)[:100]
+                    err_short = str(e)[:150]
                     log.warning(f"RAG retrieval failed for {adb_cfg.get('config_name','?')}: {err_short}")
                     if "DPY-6001" in str(e) or "DPY-6005" in str(e) or "timeout" in str(e).lower() or "connect" in str(e).lower():
                         rag_errors.append(f"Não foi possível conectar ao ADB ({adb_cfg.get('config_name','?')})")
             if all_documents:
                 all_documents.sort(key=lambda d: d["distance"])
-                rag_context = _build_rag_context(all_documents[:8])
+                top_limit = 15 if cis_chat_filter else 8
+                rag_context = _build_rag_context(all_documents[:top_limit], max_total_chars=16000 if cis_chat_filter else 12000)
             # Append connection errors to context so LLM can inform the user
             if rag_errors and not rag_context:
                 rag_context = "⚠️ AVISO: " + "; ".join(set(rag_errors)) + ". A base de conhecimento não está disponível no momento. Responda com base no seu conhecimento geral, informando que os dados do ADB não puderam ser consultados."