From 749f4e479b82b9a3ed2b068ca488f1be486f4403 Mon Sep 17 00:00:00 2001 From: nogueiraguh Date: Wed, 11 Mar 2026 03:06:53 -0300 Subject: [PATCH] feat: consult embeddings UI, vector search FLOAT32 fix, and base64 compartment decode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add Consult Embeddings sub-menu with chat-like Q&A interface - Fix VECTOR_DISTANCE FLOAT32/FLOAT64 mismatch (array 'd' → 'f') - Decode base64 compartment_id in _resolve_embed_config - Sidebar sub-item navigation for Embeddings hierarchy - Fallback to raw document display when no GenAI config available --- README.md | 4 ++- backend/app.py | 83 +++++++++++++++++++++++++++++++++++++++++++-- frontend/index.html | 57 ++++++++++++++++++++++++++++--- 3 files changed, 136 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 30177c8..8a5508d 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,7 @@ The platform combines security compliance scanning, AI-powered chat with **RAG ( - **Purge & re-embed**: option to purge old embeddings by tenancy before re-embedding updated reports - **Preview chunks before embedding**: review generated sections before creating embeddings - **OCI GenAI Embeddings**: uses Cohere Embed models (v4.0, multilingual) via OCI GenAI `embed_text` API +- **Consult Embeddings**: dedicated sub-menu under Embeddings with chat-like interface — query the vector store in natural language, AI searches all active tables via cosine similarity and returns contextual answers with source citations - Browse and inspect embeddings from the ADB vector store - 3 embedding models supported: Cohere Embed v4.0, OpenAI Text Embedding 3 Large/Small @@ -536,6 +537,7 @@ oci-cis-agent/ | POST | `/api/embeddings/report/{rid}` | Generate embeddings from CIS report (chunked by section, accepts `table_name`, `report_date`) | | POST | `/api/embeddings/upload` | Upload file (.txt/.pdf/.csv/.json/.md) and generate embeddings | | POST | `/api/embeddings/upload-url` | Import URL (web page or PDF), extract text, and generate embeddings | +| POST | `/api/embeddings/consult` | Query embeddings via vector search + GenAI (natural language Q&A) | | POST | `/api/embeddings/{vid}/purge` | Purge old embeddings by table + tenancy before re-embedding | | GET | `/api/embeddings/{vid}/list` | List embeddings in ADB (paginated, accepts `table_name` query param) | | DELETE | `/api/embeddings/{vid}/{doc_id}` | Delete individual embedding (accepts `table_name` query param) | @@ -667,7 +669,7 @@ All models include OCID mapping for `us-ashburn-1`. For other regions, use the " | Version | Date | Changes | |---------|------|---------| -| **v2.3** | 2026-03 | **Knowledge Base & RAG Enhancements**: Knowledge Base (Base de Conhecimento) with file upload + URL import to `engineerknowledgebase` vector table, CIS Recommendations upload to `cisrecom` table, URL content extraction (HTML tag/script stripping, remote PDF parsing), auto-resolve embedding config from OCI credentials (no separate GenAI config required), `report_date` metadata tracking for embedding freshness, purge & re-embed flow for updated reports. **ADB Vector Improvements**: case-insensitive table matching with quoted identifiers for Oracle ADB, table validation endpoint against `user_tables`, tables section moved inside edit connection card, case-preserving table registration (removed forced uppercase). **Embeddings tab cleanup**: removed delete buttons, simplified to read-only view, updated descriptions. **Dependencies**: added PyPDF2 for PDF text extraction | +| **v2.3** | 2026-03 | **Consult Embeddings**: dedicated sub-menu with chat-like interface for natural language Q&A against vector store — queries all active tables via cosine similarity, returns contextual answers with source citations (works with or without GenAI config). **Knowledge Base & RAG Enhancements**: Knowledge Base (Base de Conhecimento) with file upload + URL import to `engineerknowledgebase` vector table, CIS Recommendations upload to `cisrecom` table, URL content extraction (HTML tag/script stripping, remote PDF parsing), auto-resolve embedding config from OCI credentials (no separate GenAI config required), `report_date` metadata tracking for embedding freshness, purge & re-embed flow for updated reports. **ADB Vector Improvements**: case-insensitive table matching with quoted identifiers for Oracle ADB, table validation endpoint against `user_tables`, tables section moved inside edit connection card, case-preserving table registration (removed forced uppercase). **Vector search fix**: FLOAT32 compatibility for VECTOR_DISTANCE queries, base64-decoded compartment_id for auto-resolved OCI configs. **Embeddings tab cleanup**: removed delete buttons, simplified to read-only view, updated descriptions. **Dependencies**: added PyPDF2 for PDF text extraction | | **v2.2** | 2026-03 | **UI Modernization (Phases 1B-5)**: Oracle Dark Premium theme, 45+ Lucide SVG icons replacing all emojis, KPI dashboard with compliance gauge + Chart.js donut/bar charts, report summary API endpoint, Phase 5 animations (staggered fade-in, smooth theme transitions, hover effects) — animations scoped to tab switch only (no flickering on re-render). **Terraform Intelligence**: official resource docs injection from registry.terraform.io (Example Usage + Argument Reference, on-demand fetch + SQLite cache), smart tfvars auto-generation from declared variables, multi-region provider management with `var.region` support, RPC anti-cycle rule, system prompt auto-sync to DB, 60-min polling timeout for long-running applies. **ADB fix**: wallet_password always passed for thin mode mTLS (fixes DPY-6005). CIS_EMBEDDINGS default removed from ADB config | | **v2.1** | 2026-03 | Terraform Agent (AI-powered IaC generation with plan/apply/destroy lifecycle, workspace management, Terraform CLI in container, inline file editor, split-panel terminal, multi-file generation via `// filename:` markers, **smart file correction** with merge-based updates, **14-point validation checklist**, 100K max_tokens), **3-layer auto-split pipeline** (frontend + backend monolithic HCL splitting into categorized files + backend deduplication), **HCL syntax auto-fix** (single-line blocks expanded to multi-line), **SQLite-based resource type validation** (~937 types with `difflib` suggestions for invalid types), **Terraform Resource Reference** (auto-generated OCI provider catalog from `terraform providers schema -json`, generated at startup to survive volume mounts, stored in SQLite, refreshable via menu button), compact system prompt for Terraform agent, ChatGPT-style chat history for both Chat Agent and Terraform Agent (rename/delete sessions), OCI Resource Actions (start/stop instances and Autonomous DBs from **OCI Account Explorer**), OCI Explorer expanded to 40+ resource types across 8 categories with tree-view navigation and resizable panels, compartment filtering for MCP CIS scans, chat sidebar with model parameters, chat audit logs, collection error tracking in MCP server, tool timeout increased to 30min with auto-retry, memory compaction tuned (6K threshold, 20 recent messages), model catalog trimmed from 69 to 15 curated models, improved GenAI response extraction with fallback for empty responses, fixed layout overflow (no page scroll), provider alias detection with brace-matching parser, multi-region Terraform support | | **v2.0** | 2026-03 | Async background chat processing (no more 504 timeouts), frontend polling with timestamps, 8 uvicorn workers + 16-thread chat executor for ~12 simultaneous chats, parallelized MCP data collection (5-thread base + 8-thread regional), 2-hour MCP session cache, 5-min tool timeout, full dead code cleanup across backend/frontend/MCP | diff --git a/backend/app.py b/backend/app.py index f72c239..758d1bf 100644 --- a/backend/app.py +++ b/backend/app.py @@ -2419,7 +2419,7 @@ def _resolve_embed_config(oci_config_id: str = None, genai_cfg: dict = None) -> "oci_config_id": oc["id"], "genai_region": oc["region"], "endpoint": f"https://inference.generativeai.{oc['region']}.oci.oraclecloud.com", - "compartment_id": (dict(oc).get("compartment_id") or oc["tenancy_ocid"]), + "compartment_id": _safe_dec(dict(oc).get("compartment_id") or oc["tenancy_ocid"]), } # Last resort: any genai config with db() as c: @@ -2432,7 +2432,7 @@ def _resolve_embed_config(oci_config_id: str = None, genai_cfg: dict = None) -> "oci_config_id": oc["id"], "genai_region": oc["region"], "endpoint": f"https://inference.generativeai.{oc['region']}.oci.oraclecloud.com", - "compartment_id": (dict(oc).get("compartment_id") or oc["tenancy_ocid"]), + "compartment_id": _safe_dec(dict(oc).get("compartment_id") or oc["tenancy_ocid"]), } raise HTTPException(400, "Nenhuma credencial OCI configurada para gerar embeddings.") @@ -2466,7 +2466,7 @@ def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name: conn = _get_adb_connection(cfg) try: cur = conn.cursor() - vec = array.array('d', query_embedding) + vec = array.array('f', query_embedding) if tenancy: # Filter by tenancy in METADATA JSON field using LIKE for broad compatibility # Matches both structured JSON {"tenancy":"X",...} and legacy "tenancy: X, ..." @@ -3319,6 +3319,83 @@ async def embed_upload_url( _audit(u["id"], u["username"], "embed_url", url, f"{len(documents)} chunks") return {"ok": True, "message": f"Embedding de {len(documents)} chunks iniciado", "chunks": len(documents), "url": url} +class ConsultQuery(BaseModel): + query: str + table_name: str = "" + top_k: int = 10 + +@app.post("/api/embeddings/consult") +async def consult_embeddings(req: ConsultQuery, u=Depends(current_user)): + """Query embeddings via vector search + GenAI to get a formatted answer.""" + if not req.query.strip(): + raise HTTPException(400, "Query não pode ser vazia") + adb_configs = _get_active_adb_configs(u["id"]) + if not adb_configs: + raise HTTPException(400, "Nenhuma conexão ADB ativa configurada") + # Collect results from all active ADB configs + tables + all_docs = [] + for adb_cfg in adb_configs: + log.info(f"Consult: processing ADB {adb_cfg['id']} ({adb_cfg.get('config_name','')})") + try: + emb_genai = _resolve_embed_config(oci_config_id=adb_cfg.get("oci_config_id")) + log.info(f"Consult: resolved embed config, compartment={emb_genai.get('compartment_id','?')[:30]}, region={emb_genai.get('genai_region','?')}") + except Exception as e: + log.warning(f"Consult: resolve config failed for {adb_cfg['id']}: {e}") + continue + emb_model = adb_cfg.get("embedding_model_id", "") + if not emb_model: + log.warning(f"Consult: no embedding_model_id for {adb_cfg['id']}") + continue + try: + query_embedding = _embed_text(req.query, emb_genai, emb_model) + log.info(f"Consult: embedded query, dims={len(query_embedding)}") + except Exception as e: + log.warning(f"Consult: embed failed for {adb_cfg['id']}: {e}") + continue + tables = _get_tables_for_config(adb_cfg["id"], active_only=True) + log.info(f"Consult: {len(tables)} active tables, filter='{req.table_name}'") + if req.table_name: + tables = [t for t in tables if t["table_name"] == req.table_name] + for tbl in tables: + try: + docs = _vector_search(adb_cfg, query_embedding, top_k=req.top_k, table_name=tbl["table_name"]) + for d in docs: + d["source"] = f"{tbl['table_name']}" + all_docs.extend(docs) + except Exception as e: + log.warning(f"Consult: search failed on {tbl['table_name']}: {type(e).__name__}: {e}") + if not all_docs: + return {"answer": "Nenhum resultado encontrado nas bases vetoriais.", "documents": [], "total": 0} + # Sort by distance and take top results + all_docs.sort(key=lambda d: d.get("distance", 999)) + top_docs = all_docs[:req.top_k] + # Build context and call GenAI + rag_context = _build_rag_context(top_docs) + augmented = RAG_CONTEXT_TEMPLATE.format(context=rag_context, question=req.query) + # Get GenAI config for answering + with db() as c: + gc_row = c.execute("SELECT * FROM genai_configs WHERE is_default=1 ORDER BY created_at DESC").fetchone() + if not gc_row: + gc_row = c.execute("SELECT * FROM genai_configs ORDER BY created_at DESC").fetchone() + if not gc_row: + # No GenAI config — return raw documents formatted as answer + parts = [] + for i, d in enumerate(top_docs, 1): + content = d.get("content", "") + if len(content) > 800: content = content[:800] + "..." + parts.append(f"**Documento {i}** — `{d.get('source', '?')}` (distância: {d.get('distance', 0):.4f})\n\n{content}") + raw_answer = "**Resultados da busca vetorial** (sem GenAI configurado para sumarizar):\n\n---\n\n" + "\n\n---\n\n".join(parts) + doc_list = [{"content": d.get("content", "")[:500], "source": d.get("source", ""), "distance": round(d.get("distance", 0), 4), "metadata": d.get("metadata", "")} for d in top_docs] + return {"answer": raw_answer, "documents": doc_list, "total": len(all_docs)} + gc = dict(gc_row) + try: + answer, _, _ = _call_genai(gc, [{"role": "user", "content": augmented}], system_prompt=RAG_DEFAULT_SYSTEM_PROMPT) + except Exception as e: + log.error(f"Consult GenAI error: {e}") + answer = f"Erro ao consultar GenAI: {str(e)[:300]}" + doc_list = [{"content": d.get("content", "")[:500], "source": d.get("source", ""), "distance": round(d.get("distance", 0), 4), "metadata": d.get("metadata", "")} for d in top_docs] + return {"answer": answer, "documents": doc_list, "total": len(all_docs)} + @app.get("/api/embeddings/{vid}/list") async def list_embeddings(vid: str, table_name: str = Query(""), limit: int = Query(50), offset: int = Query(0), u=Depends(current_user)): with db() as c: diff --git a/frontend/index.html b/frontend/index.html index 0a04f9c..1c36bc9 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -613,7 +613,7 @@ function rApp(){return`
${rSb()}
${rTb()}
@@ -621,17 +621,17 @@ function rSb(){
Principal
${tabs.map(t=>`
${t[1]}${t[2]}
`).join('')}
Configuração
-${ctabs.map(t=>`
${t[1]}${t[2]}
`).join('')} +${ctabs.map(t=>`
${t[1]}${t[2]}
`).join('')} ${S.user?.role==='admin'?`
Administração
${atabs.map(t=>`
${t[1]}${t[2]}
`).join('')}`:''}
${i}
${S.user?.first_name?S.user.first_name+' '+S.user.last_name:S.user?.username}
${S.user?.role}
${document.documentElement.classList.contains('dark')?IC.sun:IC.moon}
⏻
`} -function rTb(){const t={'chat':IC.chat+' AI Agent Chat','terraform':TF_IC+' Terraform Agent','explorer':IC.search+' OCI Account Explorer','report':IC.chart+' Compliance Reports','downloads':IC.folder+' Downloads','oci-config':IC.cloud+' Credenciais OCI','genai':IC.brain+' OCI Generative AI','mcp':IC.plug+' MCP Servers','adb':IC.db+' Autonomous DB Vector','embeddings':IC.dna+' Embeddings','users':IC.users+' Gerenciar Usuários','mfa':IC.lock+' Autenticação MFA','audit':IC.log+' Audit Log'}; +function rTb(){const t={'chat':IC.chat+' AI Agent Chat','terraform':TF_IC+' Terraform Agent','explorer':IC.search+' OCI Account Explorer','report':IC.chart+' Compliance Reports','downloads':IC.folder+' Downloads','oci-config':IC.cloud+' Credenciais OCI','genai':IC.brain+' OCI Generative AI','mcp':IC.plug+' MCP Servers','adb':IC.db+' Autonomous DB Vector','embeddings':IC.dna+' Embeddings','emb-consult':IC.search+' Consultar Embeddings','users':IC.users+' Gerenciar Usuários','mfa':IC.lock+' Autenticação MFA','audit':IC.log+' Audit Log'}; return`
${t[S.tab]||''}
v${V}
`} -function rPg(){switch(S.tab){case'chat':return rChat();case'terraform':return rTerraform();case'explorer':return rExplorer();case'report':return rReport();case'downloads':return rDl();case'oci-config':return rOci();case'genai':return rGenAI();case'mcp':return rMCP();case'adb':return rADB();case'embeddings':return rEmbeddings();case'users':return rUsers();case'mfa':return rMfa();case'audit':return rAudit();default:return''}} +function rPg(){switch(S.tab){case'chat':return rChat();case'terraform':return rTerraform();case'explorer':return rExplorer();case'report':return rReport();case'downloads':return rDl();case'oci-config':return rOci();case'genai':return rGenAI();case'mcp':return rMCP();case'adb':return rADB();case'embeddings':return rEmbeddings();case'emb-consult':return rEmbConsultPage();case'users':return rUsers();case'mfa':return rMfa();case'audit':return rAudit();default:return''}} /* ── Chat ── */ function rChat(){ @@ -2592,6 +2592,8 @@ async function removeTable(vid,tid){if(!confirm('Excluir esta tabela do registro /* ── Embeddings ── */ function tblOpts(vid){const cfg=S.adbCfg.find(c=>c.id===vid);if(!cfg||!(cfg.tables||[]).length)return''; return cfg.tables.filter(t=>t.is_active).map(t=>'').join('')} +function tblOptsAll(vid){const cfg=S.adbCfg.find(c=>c.id===vid);if(!cfg||!(cfg.tables||[]).length)return''; + return''+cfg.tables.filter(t=>t.is_active).map(t=>'').join('')} function rEmbeddings(){ const adbOpts=S.adbCfg.map(c=>'').join(''); const rptOpts=S.reports.filter(r=>r.status==='completed').map(r=>'').join(''); @@ -2686,6 +2688,53 @@ async function delEmb(vid,docId){if(!confirm('Excluir este embedding?'))return; const tbl=document.getElementById('eltbl')?.value||''; try{await $api('/embeddings/'+vid+'/'+docId+(tbl?'?table_name='+encodeURIComponent(tbl):''),{method:'DELETE'});loadEmbs()}catch(e){alert('Erro: '+e.message)}} +/* ── Consult Embeddings ── */ +if(!S.consultHistory)S.consultHistory=[]; +function rEmbConsultPage(){ + if(!S.adbCfg.length)return`
${IC.search}

Nenhuma conexão ADB configurada.

Configure em ADB Vector.

`; + const firstVid=S.adbCfg.find(c=>c.is_active)?.id||S.adbCfg[0]?.id||''; + return rEmbConsult(firstVid)} +function rEmbConsult(firstVid){ + const msgs=S.consultHistory.map(m=>`
${m.r==='bot'?fm(m.c):m.c}
`).join(''); + return`
+
${IC.search} Consultar Embeddings
+
Faça perguntas sobre os dados armazenados nas bases vetoriais. O modelo de IA consulta os embeddings e retorna uma resposta contextualizada.
+
+
+
+
+
+${msgs||`
${IC.search}

Faça uma consulta para pesquisar nos embeddings.

Ex: "findings para CIS 1.1", "recomendações de networking", "como corrigir MFA"

`} +
+
+ + +
`} + +async function consultEmb(){ + const q=document.getElementById('ecQuery').value.trim();if(!q)return; + const tbl=document.getElementById('ecTbl')?.value||''; + const topK=parseInt(document.getElementById('ecTopK')?.value)||10; + S.consultHistory.push({r:'user',c:q}); + document.getElementById('ecQuery').value=''; + const msgEl=document.getElementById('ecMsgs'); + msgEl.innerHTML=S.consultHistory.map(m=>`
${m.r==='bot'?fm(m.c):m.c}
`).join('')+ + '
Consultando bases vetoriais...
'; + msgEl.scrollTop=msgEl.scrollHeight; + const btn=document.getElementById('ecBtn');btn.disabled=true; + try{ + const d=await $api('/embeddings/consult',{method:'POST',body:{query:q,table_name:tbl,top_k:topK}}); + let resp=d.answer||'Sem resposta do modelo.'; + if(d.documents?.length){resp+='\n\n---\n**Fontes consultadas ('+d.documents.length+' documentos, '+d.total+' total):**\n'; + d.documents.forEach((doc,i)=>{resp+='\n'+(i+1)+'. **'+doc.source+'** (distância: '+doc.distance+')'}); + } + S.consultHistory.push({r:'bot',c:resp}); + }catch(e){S.consultHistory.push({r:'bot',c:'Erro: '+e.message})} + btn.disabled=false; + msgEl.innerHTML=S.consultHistory.map(m=>`
${m.r==='bot'?fm(m.c):m.c}
`).join(''); + msgEl.scrollTop=msgEl.scrollHeight; +} + /* ── Users ── */ function rUsers(){return`
${IC.users} Gerenciamento de Usuários
NomeUsuárioEmailRoleMFAStatusÚltimo LoginAções