From c6b7cd75a9036fab4f22aabf99696d11016e2791 Mon Sep 17 00:00:00 2001 From: nogueiraguh Date: Tue, 24 Mar 2026 22:12:15 -0300 Subject: [PATCH] feat: CIS number in embeddings, tenancy dropdown in consult, text filter for exact CIS search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Chunk findings CSV: includes CIS Recommendation number, Section, Status in document header - Consult embeddings: tenancy dropdown (OCI config selector) for filtered search - CIS number detection: regex extracts "cis X.Y" from query → TEXT LIKE filter for exact match - Dynamic top_k: 10 per table when CIS filter active (vs 3 default), 15 global results - Vector search text_filter: combined vector similarity + TEXT LIKE for precise results - Purged 121174 legacy docs without tenancy metadata from all CIS tables - Re-embedded 4364 docs across 7 tables with full CIS metadata - GPT-5.2 for consult (was GPT-4.1), max_tokens 8000 --- backend/app.py | 106 +++++++++++++----- .../src/api/endpoints/embeddings.ts | 3 +- .../src/pages/config/EmbConsultPage.tsx | 19 +++- 3 files changed, 94 insertions(+), 34 deletions(-) diff --git a/backend/app.py b/backend/app.py index 58f2911..bab4687 100644 --- a/backend/app.py +++ b/backend/app.py @@ -2915,9 +2915,10 @@ def _get_table_embedding_dim(cfg: dict, table_name: str) -> int: _GLOBAL_TABLES = {"cisrecom", "engineerknowledgebase"} # Tables without tenancy filter (generic knowledge) -def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name: str = None, tenancy: str = None) -> list: +def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name: str = None, tenancy: str = None, text_filter: str = None) -> list: """Search ADB vector store using cosine similarity. Returns top-K documents. - If tenancy is provided and table is not global, filters by tenancy.""" + If tenancy is provided and table is not global, filters by tenancy. + If text_filter is provided, also filters TEXT content with LIKE.""" import array table_name = table_name or cfg.get("table_name", "") conn = _get_adb_connection(cfg) @@ -2927,23 +2928,27 @@ def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name: vec = array.array('f', query_embedding) limit = int(top_k) use_tenancy_filter = tenancy and table_name.lower() not in _GLOBAL_TABLES + # Build WHERE clauses + conditions = [] + params = [vec] + param_idx = 2 if use_tenancy_filter: - cur.execute(f""" - SELECT ID, TEXT, METADATA, - VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance - FROM "{table_name}" - WHERE JSON_VALUE(METADATA, '$.tenancy') = :2 - ORDER BY distance ASC - FETCH FIRST {limit} ROWS ONLY - """, [vec, tenancy]) - else: - cur.execute(f""" - SELECT ID, TEXT, METADATA, - VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance - FROM "{table_name}" - ORDER BY distance ASC - FETCH FIRST {limit} ROWS ONLY - """, [vec]) + conditions.append(f"JSON_VALUE(METADATA, '$.tenancy') = :{param_idx}") + params.append(tenancy) + param_idx += 1 + if text_filter: + conditions.append(f"TEXT LIKE :{param_idx}") + params.append(f"%{text_filter}%") + param_idx += 1 + where = f"WHERE {' AND '.join(conditions)}" if conditions else "" + cur.execute(f""" + SELECT ID, TEXT, METADATA, + VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance + FROM "{table_name}" + {where} + ORDER BY distance ASC + FETCH FIRST {limit} ROWS ONLY + """, params) results = [] for row in cur: content = row[1] @@ -3826,11 +3831,23 @@ def _chunk_findings_csv(csv_path: str, tenancy: str, extract_date: str, max_char return [] skip_cols = {"extract_date", "deep_link", "domain_deeplink", "defined_tags", "freeform_tags", "system_tags", "external_identifier"} - meta = json.dumps({"tenancy": tenancy, "extract_date": extract_date, "source": p.name}) + # Extract CIS recommendation number from filename (e.g., cis_Identity_and_Access_Management_1-1.csv → 1.1) + rec_match = _re.search(r'_(\d+)-(\d+(?:\.\d+)?)\.csv$', p.name) + cis_rec = f"{rec_match.group(1)}.{rec_match.group(2)}" if rec_match else "" + # Extract section name from filename + sec_match = _re.search(r'^cis_(.+?)_\d+-', p.name) + cis_section = sec_match.group(1).replace("_", " ") if sec_match else "" + + meta = json.dumps({"tenancy": tenancy, "extract_date": extract_date, "source": p.name, "cis_recommendation": cis_rec}) for row in rows: # Build context header (always repeated in each chunk) header_parts = [f"Tenancy: {tenancy}", f"Extract Date: {extract_date}"] + if cis_rec: + header_parts.append(f"CIS Recommendation: {cis_rec}") + if cis_section: + header_parts.append(f"Section: {cis_section}") + header_parts.append(f"Status: Non-Compliant") body_parts = [] # Identify key fields for the header name = row.get("name") or row.get("display_name") or row.get("username") or "" @@ -4311,6 +4328,7 @@ class ConsultQuery(BaseModel): query: str table_name: str = "" top_k: int = 10 + oci_config_id: str = "" @app.post("/api/embeddings/consult") async def consult_embeddings(req: ConsultQuery, u=Depends(current_user)): @@ -4320,8 +4338,25 @@ async def consult_embeddings(req: ConsultQuery, u=Depends(current_user)): adb_configs = _get_active_adb_configs(u["id"]) if not adb_configs: raise HTTPException(400, "Nenhuma conexão ADB ativa configurada") + # Resolve tenancy for filtered search + rag_tenancy = None + if req.oci_config_id: + with db() as c: + oci_row = c.execute("SELECT tenancy_name FROM oci_configs WHERE id=?", (req.oci_config_id,)).fetchone() + if oci_row: + rag_tenancy = oci_row["tenancy_name"] + log.info(f"Consult: filtering by tenancy '{rag_tenancy}'") + + # Detect CIS recommendation number in query for exact text filtering + import re as _re + cis_match = _re.search(r'(?:cis|recommendation)\s*(\d+\.\d+)', req.query, _re.IGNORECASE) + cis_text_filter = f"CIS Recommendation: {cis_match.group(1)}" if cis_match else None + if cis_text_filter: + log.info(f"Consult: detected CIS filter '{cis_text_filter}'") + # Collect results from all active ADB configs + tables all_docs = [] + rag_errors = [] for adb_cfg in adb_configs: try: emb_genai = _resolve_embed_config(oci_config_id=adb_cfg.get("oci_config_id")) @@ -4331,36 +4366,45 @@ async def consult_embeddings(req: ConsultQuery, u=Depends(current_user)): tables = _get_tables_for_config(adb_cfg["id"], active_only=True) if req.table_name: tables = [t for t in tables if t["table_name"] == req.table_name] - # Group tables by embedding dimension and generate one query embedding per dimension - embeddings_cache = {} # dim -> query_embedding + embeddings_cache = {} for tbl in tables: tbl_name = tbl["table_name"] try: dim = _get_table_embedding_dim(adb_cfg, tbl_name) if not dim: - log.warning(f"Consult: table {tbl_name} is empty or has no embeddings") continue if dim not in embeddings_cache: model = _DIM_TO_MODEL.get(dim, adb_cfg.get("embedding_model_id", "")) if not model: - log.warning(f"Consult: no model for dim={dim} on {tbl_name}") continue embeddings_cache[dim] = _embed_text(req.query, emb_genai, model) - log.info(f"Consult: embedded query for dim={dim} using {model}") query_embedding = embeddings_cache[dim] - docs = _vector_search(adb_cfg, query_embedding, top_k=req.top_k, table_name=tbl_name) + # Apply tenancy filter (skip for global tables) + CIS text filter + search_tenancy = rag_tenancy if tbl_name.lower() not in _GLOBAL_TABLES else None + tbl_top_k = 10 if cis_text_filter else 3 + docs = _vector_search(adb_cfg, query_embedding, top_k=tbl_top_k, table_name=tbl_name, tenancy=search_tenancy, text_filter=cis_text_filter if tbl_name.lower() not in _GLOBAL_TABLES else None) for d in docs: d["source"] = tbl_name all_docs.extend(docs) + if docs: + log.info(f"Consult: {len(docs)} docs from {tbl_name}") except Exception as e: - log.warning(f"Consult: failed on {tbl_name}: {type(e).__name__}: {str(e)[:200]}") + err = str(e)[:150] + log.warning(f"Consult: failed on {tbl_name}: {err}") + if "DPY-6001" in str(e) or "DPY-6005" in str(e) or "timeout" in str(e).lower(): + rag_errors.append(f"ADB offline ou timeout ({adb_cfg.get('config_name','?')})") if not all_docs: + if rag_errors: + return {"answer": "⚠️ " + "; ".join(set(rag_errors)) + ". A base de conhecimento não está disponível no momento.", "documents": [], "total": 0} return {"answer": "Nenhum resultado encontrado nas bases vetoriais.", "documents": [], "total": 0} # Sort by distance and take top results all_docs.sort(key=lambda d: d.get("distance", 999)) - top_docs = all_docs[:req.top_k] - # Build context and call GenAI - rag_context = _build_rag_context(top_docs) + top_limit = 15 if cis_text_filter else 8 + top_docs = all_docs[:top_limit] + # Build context with dates and sources + rag_context = _build_rag_context(top_docs, max_total_chars=16000 if cis_text_filter else 12000) + if rag_errors: + rag_context += "\n\n⚠️ Algumas bases não puderam ser consultadas: " + "; ".join(set(rag_errors)) augmented = RAG_CONTEXT_TEMPLATE.format(context=rag_context, question=req.query) # Get GenAI config for answering — try saved config first, then auto-resolve from OCI gc = None @@ -4379,11 +4423,11 @@ async def consult_embeddings(req: ConsultQuery, u=Depends(current_user)): "endpoint": resolved.get("endpoint", f"https://inference.generativeai.{resolved.get('genai_region','us-ashburn-1')}.oci.oraclecloud.com"), "compartment_id": resolved.get("compartment_id", ""), "genai_region": resolved.get("genai_region", "us-ashburn-1"), - "model_id": "openai.gpt-4.1", + "model_id": "openai.gpt-5.2", "model_ocid": "", "serving_type": "ON_DEMAND", "temperature": 0.3, - "max_tokens": 4000, + "max_tokens": 8000, "top_p": 0.9, "top_k": 1, "frequency_penalty": 0, diff --git a/frontend-react/src/api/endpoints/embeddings.ts b/frontend-react/src/api/endpoints/embeddings.ts index 1f51d75..93ac0ee 100644 --- a/frontend-react/src/api/endpoints/embeddings.ts +++ b/frontend-react/src/api/endpoints/embeddings.ts @@ -88,10 +88,11 @@ export const embeddingsApi = { }) as unknown as Promise, /** Consult embeddings with a question */ - consult: (query: string, tableName?: string, topK = 10) => + consult: (query: string, tableName?: string, topK = 10, ociConfigId?: string) => client.post('/embeddings/consult', { query, table_name: tableName || '', top_k: topK, + oci_config_id: ociConfigId || '', }) as unknown as Promise, }; diff --git a/frontend-react/src/pages/config/EmbConsultPage.tsx b/frontend-react/src/pages/config/EmbConsultPage.tsx index 2fa832f..c728977 100644 --- a/frontend-react/src/pages/config/EmbConsultPage.tsx +++ b/frontend-react/src/pages/config/EmbConsultPage.tsx @@ -39,10 +39,11 @@ function renderMarkdown(text: string): string { /* ── Main ── */ export default function EmbConsultPage() { const { t } = useI18n(); - const { adbCfg } = useAppStore(); + const { adbCfg, ociCfg } = useAppStore(); // Config selectors const [selTable, setSelTable] = useState(''); + const [selOci, setSelOci] = useState(ociCfg.length > 0 ? ociCfg[0].id : ''); const [topK, setTopK] = useState(10); // Chat @@ -101,7 +102,7 @@ export default function EmbConsultPage() { setLoading(true); try { - const d = await embeddingsApi.consult(q, selTable || undefined, topK); + const d = await embeddingsApi.consult(q, selTable || undefined, topK, selOci || undefined); let answer = d.answer || t('ec.noAnswer'); const assistantMsg: ChatMessage = { @@ -162,6 +163,20 @@ export default function EmbConsultPage() { {/* Config bar */}
+
+ + +