feat: add dedicated Embeddings tab with report chunking and file upload

Add a new Embeddings menu in the sidebar for managing vector embeddings
separately from ADB config. Supports embedding CIS reports (chunked by
section) and uploading .txt files (chunked by paragraphs). Includes
listing and deleting individual embeddings from the ADB vector store.

- Add _chunk_report_by_section() for CIS report sectional chunking
- Add _chunk_text_file() for paragraph-based text chunking
- Add POST /api/embeddings/report/{rid} endpoint
- Add POST /api/embeddings/upload endpoint (multipart)
- Add GET /api/embeddings/{vid}/list with pagination
- Add DELETE /api/embeddings/{vid}/{doc_id}
- Add Embeddings tab in frontend with 3 cards (list, report, upload)
- Remove ingestion section from ADB Vector tab

backend/app.py

@@ -953,6 +953,140 @@ def _ingest_documents_task(cfg: dict, genai_cfg: dict, documents: list, user_id:
     finally:
         conn.close()
 
+# ── Embeddings ────────────────────────────────────────────────────────────────
+def _chunk_report_by_section(report_data: dict) -> list:
+    """Chunk a CIS report into documents grouped by section."""
+    if isinstance(report_data, str):
+        report_data = json.loads(report_data)
+    findings = report_data.get("findings", {})
+    tenancy = report_data.get("tenancy", "unknown")
+    generated_at = report_data.get("generated_at", "")
+    sections = {}
+    for cid, check in findings.items():
+        sec = check.get("section", "Other")
+        sections.setdefault(sec, [])
+        sections[sec].append(check)
+    documents = []
+    for section_name, checks in sections.items():
+        passed = sum(1 for c in checks if c.get("status") == "PASS")
+        failed = sum(1 for c in checks if c.get("status") == "FAIL")
+        review = sum(1 for c in checks if c.get("status") == "REVIEW")
+        lines = [f"Section: {section_name}", f"Total checks: {len(checks)}, Passed: {passed}, Failed: {failed}, Review: {review}", ""]
+        for c in checks:
+            status = c.get("status", "REVIEW")
+            lines.append(f"- [{c.get('id', '')}] {c.get('title', '')} — Status: {status}")
+            if c.get("findings"):
+                for f in c["findings"]:
+                    lines.append(f"  Finding: {f}")
+        documents.append({
+            "content": "\n".join(lines),
+            "source": f"CIS Report - {tenancy} - {generated_at}",
+            "metadata": f"section: {section_name}, total: {len(checks)}, passed: {passed}, failed: {failed}, review: {review}"
+        })
+    return documents
+
+def _chunk_text_file(text: str, filename: str, chunk_size: int = 1000) -> list:
+    """Split text into chunks by paragraphs or fixed size."""
+    paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()]
+    documents = []
+    current_chunk = ""
+    chunk_num = 1
+    for para in paragraphs:
+        if len(current_chunk) + len(para) + 2 > chunk_size and current_chunk:
+            documents.append({"content": current_chunk, "source": filename, "metadata": f"chunk: {chunk_num}"})
+            chunk_num += 1
+            current_chunk = para
+        else:
+            current_chunk = current_chunk + "\n\n" + para if current_chunk else para
+    if current_chunk:
+        documents.append({"content": current_chunk, "source": filename, "metadata": f"chunk: {chunk_num}"})
+    return documents
+
+def _get_adb_and_genai(vid: str):
+    """Load ADB config and its linked GenAI config. Returns (adb_cfg, genai_cfg) or raises."""
+    with db() as c:
+        cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
+    if not cfg: raise HTTPException(404, "ADB config not found")
+    cfg = dict(cfg)
+    if not cfg.get("genai_config_id"):
+        raise HTTPException(400, "GenAI config not linked to this ADB connection")
+    with db() as c:
+        gc = c.execute("SELECT * FROM genai_configs WHERE id=?", (cfg["genai_config_id"],)).fetchone()
+    if not gc: raise HTTPException(400, "Linked GenAI config not found")
+    return cfg, dict(gc)
+
+@app.post("/api/embeddings/report/{rid}")
+async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(require("admin","user"))):
+    vid = req.get("adb_config_id")
+    if not vid: raise HTTPException(400, "adb_config_id is required")
+    with db() as c:
+        r = c.execute("SELECT report_data,tenancy_name FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone()
+    if not r: raise HTTPException(404, "Report not found or not completed")
+    report_data = r["report_data"]
+    if isinstance(report_data, str):
+        try: report_data = json.loads(report_data)
+        except: raise HTTPException(400, "Invalid report data")
+    documents = _chunk_report_by_section(report_data)
+    if not documents: raise HTTPException(400, "No sections found in report")
+    cfg, gc = _get_adb_and_genai(vid)
+    bg.add_task(_ingest_documents_task, cfg, gc, documents, u["id"], u["username"])
+    _audit(u["id"], u["username"], "embed_report", rid, f"{len(documents)} sections")
+    return {"ok": True, "message": f"Embedding de {len(documents)} seções iniciado", "sections": len(documents)}
+
+@app.post("/api/embeddings/upload")
+async def embed_upload(adb_config_id: str = Form(...), file: UploadFile = File(...), bg: BackgroundTasks = None, u=Depends(require("admin","user"))):
+    if not file.filename.lower().endswith(".txt"):
+        raise HTTPException(400, "Only .txt files are supported")
+    content = (await file.read()).decode("utf-8", errors="replace")
+    if not content.strip(): raise HTTPException(400, "File is empty")
+    documents = _chunk_text_file(content, file.filename)
+    if not documents: raise HTTPException(400, "No content chunks found")
+    cfg, gc = _get_adb_and_genai(adb_config_id)
+    bg.add_task(_ingest_documents_task, cfg, gc, documents, u["id"], u["username"])
+    _audit(u["id"], u["username"], "embed_upload", file.filename, f"{len(documents)} chunks")
+    return {"ok": True, "message": f"Embedding de {len(documents)} chunks iniciado", "chunks": len(documents), "filename": file.filename}
+
+@app.get("/api/embeddings/{vid}/list")
+async def list_embeddings(vid: str, limit: int = Query(50), offset: int = Query(0), u=Depends(current_user)):
+    with db() as c:
+        cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
+    if not cfg: raise HTTPException(404)
+    try:
+        conn = _get_adb_connection(dict(cfg))
+        cur = conn.cursor()
+        table_name = cfg["table_name"] or "CIS_EMBEDDINGS"
+        cur.execute(f"SELECT COUNT(*) FROM {table_name}")
+        total = cur.fetchone()[0]
+        cur.execute(f"""
+            SELECT ID, SOURCE, METADATA, CREATED_AT FROM {table_name}
+            ORDER BY CREATED_AT DESC
+            OFFSET :1 ROWS FETCH NEXT :2 ROWS ONLY
+        """, [offset, limit])
+        rows = []
+        for row in cur:
+            rows.append({"id": row[0], "source": row[1], "metadata": row[2],
+                         "created_at": str(row[3]) if row[3] else None})
+        cur.close(); conn.close()
+        return {"total": total, "offset": offset, "limit": limit, "documents": rows}
+    except Exception as e:
+        raise HTTPException(500, f"Erro ao listar embeddings: {str(e)[:500]}")
+
+@app.delete("/api/embeddings/{vid}/{doc_id}")
+async def delete_embedding(vid: str, doc_id: str, u=Depends(require("admin","user"))):
+    with db() as c:
+        cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
+    if not cfg: raise HTTPException(404)
+    try:
+        conn = _get_adb_connection(dict(cfg))
+        cur = conn.cursor()
+        table_name = cfg["table_name"] or "CIS_EMBEDDINGS"
+        cur.execute(f"DELETE FROM {table_name} WHERE ID = :1", [doc_id])
+        conn.commit()
+        cur.close(); conn.close()
+        return {"ok": True}
+    except Exception as e:
+        raise HTTPException(500, f"Erro ao deletar: {str(e)[:500]}")
+
 # ── Reports ───────────────────────────────────────────────────────────────────
 @app.post("/api/reports/run")
 async def run_report(req: RunReportReq, bg: BackgroundTasks, u=Depends(require("admin","user"))):

frontend/index.html

@@ -233,7 +233,7 @@ function rApp(){return`<div class="app">${rSb()}<div class="mc">${rTb()}<div cla
 
 function rSb(){
   const tabs=[['chat','💬','Chat Agent'],['explorer','🔍','OCI Explorer'],['report','📊','Reports'],['downloads','📁','Downloads']];
-  const ctabs=[['oci-config','☁️','Credenciais OCI'],['genai','🧠','GenAI Config'],['mcp','🔌','MCP Servers'],['adb','🗄️','ADB Vector']];
+  const ctabs=[['oci-config','☁️','Credenciais OCI'],['genai','🧠','GenAI Config'],['mcp','🔌','MCP Servers'],['adb','🗄️','ADB Vector'],['embeddings','🧬','Embeddings']];
   const atabs=[['users','👥','Usuários'],['mfa','🔐','MFA'],['audit','📋','Audit Log']];
   const i=(S.user?.first_name||S.user?.username||'?')[0].toUpperCase();
   return`<div class="sb">
@@ -247,10 +247,10 @@ ${atabs.map(t=>`<div class="ni ${S.tab===t[0]?'on':''}" onclick="switchTab('${t[
 <div class="sb-f"><div class="ui"><div class="ua">${i}</div><div><div class="un">${S.user?.first_name?S.user.first_name+' '+S.user.last_name:S.user?.username}</div><div class="ur">${S.user?.role}</div></div>
 <div class="lo-btn" onclick="doLogout()" title="Sair">⏻</div></div></div></div>`}
 
-function rTb(){const t={'chat':'💬 AI Agent Chat','explorer':'🔍 OCI Account Explorer','report':'📊 Compliance Reports','downloads':'📁 Downloads','oci-config':'☁️ Credenciais OCI','genai':'🧠 OCI Generative AI','mcp':'🔌 MCP Servers','adb':'🗄️ Autonomous DB Vector','users':'👥 Gerenciar Usuários','mfa':'🔐 Autenticação MFA','audit':'📋 Audit Log'};
+function rTb(){const t={'chat':'💬 AI Agent Chat','explorer':'🔍 OCI Account Explorer','report':'📊 Compliance Reports','downloads':'📁 Downloads','oci-config':'☁️ Credenciais OCI','genai':'🧠 OCI Generative AI','mcp':'🔌 MCP Servers','adb':'🗄️ Autonomous DB Vector','embeddings':'🧬 Embeddings','users':'👥 Gerenciar Usuários','mfa':'🔐 Autenticação MFA','audit':'📋 Audit Log'};
   return`<div class="tb"><div class="tb-t">${t[S.tab]||''}</div><div class="tb-a"><span class="tag">v${V}</span></div></div>`}
 
-function rPg(){switch(S.tab){case'chat':return rChat();case'explorer':return rExplorer();case'report':return rReport();case'downloads':return rDl();case'oci-config':return rOci();case'genai':return rGenAI();case'mcp':return rMCP();case'adb':return rADB();case'users':return rUsers();case'mfa':return rMfa();case'audit':return rAudit();default:return''}}
+function rPg(){switch(S.tab){case'chat':return rChat();case'explorer':return rExplorer();case'report':return rReport();case'downloads':return rDl();case'oci-config':return rOci();case'genai':return rGenAI();case'mcp':return rMCP();case'adb':return rADB();case'embeddings':return rEmbeddings();case'users':return rUsers();case'mfa':return rMfa();case'audit':return rAudit();default:return''}}
 
 /* ── Chat ── */
 function rChat(){
@@ -466,15 +466,7 @@ ${!S.adbCfg.length?'<div class="emp" style="padding:.85rem"><p>Nenhuma conexão
 <div class="cd"><div class="ct">📤 Upload Wallet</div>
 <div class="g2"><div class="ig"><label>Conexão ADB</label><select id="awsel">${S.adbCfg.map(c=>`<option value="${c.id}">${c.config_name}</option>`).join('')}</select></div>
 <div class="ig"><label>Wallet ZIP</label><input type="file" id="awf" accept=".zip"></div></div>
-<button class="btn bs" onclick="uWallet()">Upload Wallet</button></div>
+<button class="btn bs" onclick="uWallet()">Upload Wallet</button></div>`}
-<div class="cd"><div class="ct">📄 Ingerir Documentos</div>
-<div class="cdesc">Gere embeddings e insira documentos na tabela ADB para uso com RAG.</div>
-<div class="g2"><div class="ig"><label>Conexão ADB</label>
-<select id="aisel">${S.adbCfg.filter(c=>c.genai_config_id).map(c=>'<option value="'+c.id+'">'+c.config_name+'</option>').join('')}</select></div>
-<div class="ig"><label>Documentos (JSON)</label>
-<div class="ht">Array de objetos: [{"content":"texto","source":"fonte","metadata":"info"}]</div>
-<textarea id="adocs" rows="6" style="font-family:var(--fm);font-size:.72rem;width:100%;padding:.5rem;border:1px solid var(--bd);border-radius:var(--r)" placeholder='[{"content":"...","source":"doc.pdf","metadata":"page 1"}]'></textarea></div></div>
-<button class="btn bp" onclick="ingestDocs()">Ingerir Documentos</button></div>`}
 async function sAdb(){try{await $api('/adb/config',{method:'POST',body:{config_name:document.getElementById('an').value,dsn:document.getElementById('adsn').value,
   username:document.getElementById('auser').value,password:document.getElementById('apw').value,
   wallet_password:document.getElementById('awpw').value||null,table_name:document.getElementById('atbl').value,
@@ -487,11 +479,49 @@ async function uWallet(){const fd=new FormData();const f=document.getElementById
   fd.append('wallet',f);const id=document.getElementById('awsel').value;if(!id)return alert('Selecione conexão');
   try{const d=await $api('/adb/'+id+'/upload-wallet',{method:'POST',body:fd,headers:{}});alert('Wallet enviada! Arquivos: '+d.files.join(', '))}catch(e){alert(e.message)}}
 async function ensureTable(id){try{const d=await $api('/adb/'+id+'/ensure-table',{method:'POST'});alert('Tabela criada/verificada: '+d.table)}catch(e){alert('Erro: '+e.message)}}
-async function ingestDocs(){const id=document.getElementById('aisel').value;
+
-  if(!id)return alert('Selecione conexão ADB com GenAI configurado');
+/* ── Embeddings ── */
-  let docs;try{docs=JSON.parse(document.getElementById('adocs').value)}catch(e){return alert('JSON inválido')}
+function rEmbeddings(){
-  if(!Array.isArray(docs)||!docs.length)return alert('Forneça array de documentos');
+  const adbOpts=S.adbCfg.filter(c=>c.genai_config_id).map(c=>'<option value="'+c.id+'">'+c.config_name+'</option>').join('');
-  try{const d=await $api('/adb/'+id+'/ingest',{method:'POST',body:{adb_config_id:id,documents:docs}});alert(d.message)}catch(e){alert('Erro: '+e.message)}}
+  const rptOpts=S.reports.filter(r=>r.status==='completed').map(r=>'<option value="'+r.id+'">'+r.tenancy_name+' ('+r.created_at?.substring(0,10)+')</option>').join('');
+  if(!S.adbCfg.filter(c=>c.genai_config_id).length)return`<div class="cd"><div class="emp"><div class="eic">🧬</div><p>Nenhuma conexão ADB com GenAI configurada.</p><p style="font-size:.74rem;margin-top:.35rem">Configure em <strong>ADB Vector</strong> vinculando uma Config GenAI.</p></div></div>`;
+  return`<div class="cd"><div class="ct">📋 Embeddings Existentes</div>
+<div class="cdesc">Consulte os documentos armazenados na tabela de embeddings do ADB.</div>
+<div class="g2"><div class="ig"><label>Conexão ADB</label><select id="elsel">${adbOpts}</select></div>
+<div class="ig" style="align-self:end"><button class="btn bs" onclick="loadEmbs()">Carregar</button></div></div>
+<div id="emb-list"></div></div>
+<div class="cd"><div class="ct">📊 Embed Relatório CIS</div>
+<div class="cdesc">Gere embeddings a partir de relatórios CIS. Cada seção (IAM, Networking, etc.) gera um embedding separado.</div><div id="erm"></div>
+<div class="g2"><div class="ig"><label>Conexão ADB</label><select id="ersel">${adbOpts}</select></div>
+<div class="ig"><label>Relatório</label><select id="errpt">${rptOpts||'<option value="">Nenhum relatório disponível</option>'}</select></div></div>
+<button class="btn bp" onclick="embedReport()">Gerar Embeddings do Relatório</button></div>
+<div class="cd"><div class="ct">📄 Upload de Arquivo</div>
+<div class="cdesc">Faça upload de um arquivo .txt para gerar embeddings automaticamente (chunking por parágrafos).</div><div id="efm"></div>
+<div class="g2"><div class="ig"><label>Conexão ADB</label><select id="efsel">${adbOpts}</select></div>
+<div class="ig"><label>Arquivo (.txt)</label><input type="file" id="eff" accept=".txt"></div></div>
+<button class="btn bp" onclick="embedFile()">Upload e Gerar Embeddings</button></div>`}
+
+async function loadEmbs(){const vid=document.getElementById('elsel').value;if(!vid)return;
+  try{const d=await $api('/embeddings/'+vid+'/list');
+    if(!d.documents.length){document.getElementById('emb-list').innerHTML='<div class="emp" style="padding:.65rem"><p>Nenhum embedding encontrado.</p></div>';return}
+    document.getElementById('emb-list').innerHTML='<table style="margin-top:.65rem"><thead><tr><th>ID</th><th>Source</th><th>Metadata</th><th>Data</th><th>Ações</th></tr></thead><tbody>'+
+      d.documents.map(e=>'<tr><td style="font-family:var(--fm);font-size:.62rem">'+e.id.substring(0,8)+'...</td><td>'+
+        (e.source||'—')+'</td><td style="font-size:.72rem;color:var(--t3)">'+(e.metadata||'—')+'</td><td style="font-size:.72rem;color:var(--t4)">'+
+        (e.created_at||'—')+'</td><td><button class="btn bd bsm" onclick="delEmb(\''+vid+'\',\''+e.id+'\')">Excluir</button></td></tr>').join('')+
+      '</tbody></table><div style="padding:.5rem;font-size:.72rem;color:var(--t4)">Total: '+d.total+' documentos</div>'}
+  catch(e){document.getElementById('emb-list').innerHTML='<div style="color:var(--rd);padding:.5rem">Erro: '+e.message+'</div>'}}
+
+async function embedReport(){const vid=document.getElementById('ersel').value;const rid=document.getElementById('errpt').value;
+  if(!vid||!rid)return alert('Selecione conexão ADB e relatório');
+  try{const d=await $api('/embeddings/report/'+rid,{method:'POST',body:{adb_config_id:vid}});sm('erm','✅ '+d.message,'s')}catch(e){sm('erm',e.message,'e')}}
+
+async function embedFile(){const vid=document.getElementById('efsel').value;const f=document.getElementById('eff').files[0];
+  if(!vid||!f)return alert('Selecione conexão ADB e arquivo');
+  const fd=new FormData();fd.append('file',f);fd.append('adb_config_id',vid);
+  try{const d=await $api('/embeddings/upload',{method:'POST',body:fd,headers:{}});sm('efm','✅ '+d.message,'s')}catch(e){sm('efm',e.message,'e')}}
+
+async function delEmb(vid,docId){if(!confirm('Excluir este embedding?'))return;
+  try{await $api('/embeddings/'+vid+'/'+docId,{method:'DELETE'});loadEmbs()}catch(e){alert('Erro: '+e.message)}}
 
 /* ── Users ── */
 function rUsers(){return`<div class="cd"><div class="ct">👥 Gerenciamento de Usuários</div><div id="um"></div>