feat: CIS number in embeddings, tenancy dropdown in consult, text filter for exact CIS search
- Chunk findings CSV: includes CIS Recommendation number, Section, Status in document header - Consult embeddings: tenancy dropdown (OCI config selector) for filtered search - CIS number detection: regex extracts "cis X.Y" from query → TEXT LIKE filter for exact match - Dynamic top_k: 10 per table when CIS filter active (vs 3 default), 15 global results - Vector search text_filter: combined vector similarity + TEXT LIKE for precise results - Purged 121174 legacy docs without tenancy metadata from all CIS tables - Re-embedded 4364 docs across 7 tables with full CIS metadata - GPT-5.2 for consult (was GPT-4.1), max_tokens 8000
This commit is contained in:
nogueiraguh
@@ -2915,9 +2915,10 @@ def _get_table_embedding_dim(cfg: dict, table_name: str) -> int:
|
||||
|
||||
_GLOBAL_TABLES = {"cisrecom", "engineerknowledgebase"} # Tables without tenancy filter (generic knowledge)
|
||||
|
||||
def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name: str = None, tenancy: str = None) -> list:
|
||||
def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name: str = None, tenancy: str = None, text_filter: str = None) -> list:
|
||||
"""Search ADB vector store using cosine similarity. Returns top-K documents.
|
||||
If tenancy is provided and table is not global, filters by tenancy."""
|
||||
If tenancy is provided and table is not global, filters by tenancy.
|
||||
If text_filter is provided, also filters TEXT content with LIKE."""
|
||||
import array
|
||||
table_name = table_name or cfg.get("table_name", "")
|
||||
conn = _get_adb_connection(cfg)
|
||||
@@ -2927,23 +2928,27 @@ def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name:
|
||||
vec = array.array('f', query_embedding)
|
||||
limit = int(top_k)
|
||||
use_tenancy_filter = tenancy and table_name.lower() not in _GLOBAL_TABLES
|
||||
# Build WHERE clauses
|
||||
conditions = []
|
||||
params = [vec]
|
||||
param_idx = 2
|
||||
if use_tenancy_filter:
|
||||
conditions.append(f"JSON_VALUE(METADATA, '$.tenancy') = :{param_idx}")
|
||||
params.append(tenancy)
|
||||
param_idx += 1
|
||||
if text_filter:
|
||||
conditions.append(f"TEXT LIKE :{param_idx}")
|
||||
params.append(f"%{text_filter}%")
|
||||
param_idx += 1
|
||||
where = f"WHERE {' AND '.join(conditions)}" if conditions else ""
|
||||
cur.execute(f"""
|
||||
SELECT ID, TEXT, METADATA,
|
||||
VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance
|
||||
FROM "{table_name}"
|
||||
WHERE JSON_VALUE(METADATA, '$.tenancy') = :2
|
||||
{where}
|
||||
ORDER BY distance ASC
|
||||
FETCH FIRST {limit} ROWS ONLY
|
||||
""", [vec, tenancy])
|
||||
else:
|
||||
cur.execute(f"""
|
||||
SELECT ID, TEXT, METADATA,
|
||||
VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance
|
||||
FROM "{table_name}"
|
||||
ORDER BY distance ASC
|
||||
FETCH FIRST {limit} ROWS ONLY
|
||||
""", [vec])
|
||||
""", params)
|
||||
results = []
|
||||
for row in cur:
|
||||
content = row[1]
|
||||
@@ -3826,11 +3831,23 @@ def _chunk_findings_csv(csv_path: str, tenancy: str, extract_date: str, max_char
|
||||
return []
|
||||
skip_cols = {"extract_date", "deep_link", "domain_deeplink", "defined_tags",
|
||||
"freeform_tags", "system_tags", "external_identifier"}
|
||||
meta = json.dumps({"tenancy": tenancy, "extract_date": extract_date, "source": p.name})
|
||||
# Extract CIS recommendation number from filename (e.g., cis_Identity_and_Access_Management_1-1.csv → 1.1)
|
||||
rec_match = _re.search(r'_(\d+)-(\d+(?:\.\d+)?)\.csv$', p.name)
|
||||
cis_rec = f"{rec_match.group(1)}.{rec_match.group(2)}" if rec_match else ""
|
||||
# Extract section name from filename
|
||||
sec_match = _re.search(r'^cis_(.+?)_\d+-', p.name)
|
||||
cis_section = sec_match.group(1).replace("_", " ") if sec_match else ""
|
||||
|
||||
meta = json.dumps({"tenancy": tenancy, "extract_date": extract_date, "source": p.name, "cis_recommendation": cis_rec})
|
||||
|
||||
for row in rows:
|
||||
# Build context header (always repeated in each chunk)
|
||||
header_parts = [f"Tenancy: {tenancy}", f"Extract Date: {extract_date}"]
|
||||
if cis_rec:
|
||||
header_parts.append(f"CIS Recommendation: {cis_rec}")
|
||||
if cis_section:
|
||||
header_parts.append(f"Section: {cis_section}")
|
||||
header_parts.append(f"Status: Non-Compliant")
|
||||
body_parts = []
|
||||
# Identify key fields for the header
|
||||
name = row.get("name") or row.get("display_name") or row.get("username") or ""
|
||||
@@ -4311,6 +4328,7 @@ class ConsultQuery(BaseModel):
|
||||
query: str
|
||||
table_name: str = ""
|
||||
top_k: int = 10
|
||||
oci_config_id: str = ""
|
||||
|
||||
@app.post("/api/embeddings/consult")
|
||||
async def consult_embeddings(req: ConsultQuery, u=Depends(current_user)):
|
||||
@@ -4320,8 +4338,25 @@ async def consult_embeddings(req: ConsultQuery, u=Depends(current_user)):
|
||||
adb_configs = _get_active_adb_configs(u["id"])
|
||||
if not adb_configs:
|
||||
raise HTTPException(400, "Nenhuma conexão ADB ativa configurada")
|
||||
# Resolve tenancy for filtered search
|
||||
rag_tenancy = None
|
||||
if req.oci_config_id:
|
||||
with db() as c:
|
||||
oci_row = c.execute("SELECT tenancy_name FROM oci_configs WHERE id=?", (req.oci_config_id,)).fetchone()
|
||||
if oci_row:
|
||||
rag_tenancy = oci_row["tenancy_name"]
|
||||
log.info(f"Consult: filtering by tenancy '{rag_tenancy}'")
|
||||
|
||||
# Detect CIS recommendation number in query for exact text filtering
|
||||
import re as _re
|
||||
cis_match = _re.search(r'(?:cis|recommendation)\s*(\d+\.\d+)', req.query, _re.IGNORECASE)
|
||||
cis_text_filter = f"CIS Recommendation: {cis_match.group(1)}" if cis_match else None
|
||||
if cis_text_filter:
|
||||
log.info(f"Consult: detected CIS filter '{cis_text_filter}'")
|
||||
|
||||
# Collect results from all active ADB configs + tables
|
||||
all_docs = []
|
||||
rag_errors = []
|
||||
for adb_cfg in adb_configs:
|
||||
try:
|
||||
emb_genai = _resolve_embed_config(oci_config_id=adb_cfg.get("oci_config_id"))
|
||||
@@ -4331,36 +4366,45 @@ async def consult_embeddings(req: ConsultQuery, u=Depends(current_user)):
|
||||
tables = _get_tables_for_config(adb_cfg["id"], active_only=True)
|
||||
if req.table_name:
|
||||
tables = [t for t in tables if t["table_name"] == req.table_name]
|
||||
# Group tables by embedding dimension and generate one query embedding per dimension
|
||||
embeddings_cache = {} # dim -> query_embedding
|
||||
embeddings_cache = {}
|
||||
for tbl in tables:
|
||||
tbl_name = tbl["table_name"]
|
||||
try:
|
||||
dim = _get_table_embedding_dim(adb_cfg, tbl_name)
|
||||
if not dim:
|
||||
log.warning(f"Consult: table {tbl_name} is empty or has no embeddings")
|
||||
continue
|
||||
if dim not in embeddings_cache:
|
||||
model = _DIM_TO_MODEL.get(dim, adb_cfg.get("embedding_model_id", ""))
|
||||
if not model:
|
||||
log.warning(f"Consult: no model for dim={dim} on {tbl_name}")
|
||||
continue
|
||||
embeddings_cache[dim] = _embed_text(req.query, emb_genai, model)
|
||||
log.info(f"Consult: embedded query for dim={dim} using {model}")
|
||||
query_embedding = embeddings_cache[dim]
|
||||
docs = _vector_search(adb_cfg, query_embedding, top_k=req.top_k, table_name=tbl_name)
|
||||
# Apply tenancy filter (skip for global tables) + CIS text filter
|
||||
search_tenancy = rag_tenancy if tbl_name.lower() not in _GLOBAL_TABLES else None
|
||||
tbl_top_k = 10 if cis_text_filter else 3
|
||||
docs = _vector_search(adb_cfg, query_embedding, top_k=tbl_top_k, table_name=tbl_name, tenancy=search_tenancy, text_filter=cis_text_filter if tbl_name.lower() not in _GLOBAL_TABLES else None)
|
||||
for d in docs:
|
||||
d["source"] = tbl_name
|
||||
all_docs.extend(docs)
|
||||
if docs:
|
||||
log.info(f"Consult: {len(docs)} docs from {tbl_name}")
|
||||
except Exception as e:
|
||||
log.warning(f"Consult: failed on {tbl_name}: {type(e).__name__}: {str(e)[:200]}")
|
||||
err = str(e)[:150]
|
||||
log.warning(f"Consult: failed on {tbl_name}: {err}")
|
||||
if "DPY-6001" in str(e) or "DPY-6005" in str(e) or "timeout" in str(e).lower():
|
||||
rag_errors.append(f"ADB offline ou timeout ({adb_cfg.get('config_name','?')})")
|
||||
if not all_docs:
|
||||
if rag_errors:
|
||||
return {"answer": "⚠️ " + "; ".join(set(rag_errors)) + ". A base de conhecimento não está disponível no momento.", "documents": [], "total": 0}
|
||||
return {"answer": "Nenhum resultado encontrado nas bases vetoriais.", "documents": [], "total": 0}
|
||||
# Sort by distance and take top results
|
||||
all_docs.sort(key=lambda d: d.get("distance", 999))
|
||||
top_docs = all_docs[:req.top_k]
|
||||
# Build context and call GenAI
|
||||
rag_context = _build_rag_context(top_docs)
|
||||
top_limit = 15 if cis_text_filter else 8
|
||||
top_docs = all_docs[:top_limit]
|
||||
# Build context with dates and sources
|
||||
rag_context = _build_rag_context(top_docs, max_total_chars=16000 if cis_text_filter else 12000)
|
||||
if rag_errors:
|
||||
rag_context += "\n\n⚠️ Algumas bases não puderam ser consultadas: " + "; ".join(set(rag_errors))
|
||||
augmented = RAG_CONTEXT_TEMPLATE.format(context=rag_context, question=req.query)
|
||||
# Get GenAI config for answering — try saved config first, then auto-resolve from OCI
|
||||
gc = None
|
||||
@@ -4379,11 +4423,11 @@ async def consult_embeddings(req: ConsultQuery, u=Depends(current_user)):
|
||||
"endpoint": resolved.get("endpoint", f"https://inference.generativeai.{resolved.get('genai_region','us-ashburn-1')}.oci.oraclecloud.com"),
|
||||
"compartment_id": resolved.get("compartment_id", ""),
|
||||
"genai_region": resolved.get("genai_region", "us-ashburn-1"),
|
||||
"model_id": "openai.gpt-4.1",
|
||||
"model_id": "openai.gpt-5.2",
|
||||
"model_ocid": "",
|
||||
"serving_type": "ON_DEMAND",
|
||||
"temperature": 0.3,
|
||||
"max_tokens": 4000,
|
||||
"max_tokens": 8000,
|
||||
"top_p": 0.9,
|
||||
"top_k": 1,
|
||||
"frequency_penalty": 0,
|
||||
|
||||
@@ -88,10 +88,11 @@ export const embeddingsApi = {
|
||||
}) as unknown as Promise<PurgeResult>,
|
||||
|
||||
/** Consult embeddings with a question */
|
||||
consult: (query: string, tableName?: string, topK = 10) =>
|
||||
consult: (query: string, tableName?: string, topK = 10, ociConfigId?: string) =>
|
||||
client.post('/embeddings/consult', {
|
||||
query,
|
||||
table_name: tableName || '',
|
||||
top_k: topK,
|
||||
oci_config_id: ociConfigId || '',
|
||||
}) as unknown as Promise<ConsultResult>,
|
||||
};
|
||||
|
||||
@@ -39,10 +39,11 @@ function renderMarkdown(text: string): string {
|
||||
/* ── Main ── */
|
||||
export default function EmbConsultPage() {
|
||||
const { t } = useI18n();
|
||||
const { adbCfg } = useAppStore();
|
||||
const { adbCfg, ociCfg } = useAppStore();
|
||||
|
||||
// Config selectors
|
||||
const [selTable, setSelTable] = useState('');
|
||||
const [selOci, setSelOci] = useState(ociCfg.length > 0 ? ociCfg[0].id : '');
|
||||
const [topK, setTopK] = useState(10);
|
||||
|
||||
// Chat
|
||||
@@ -101,7 +102,7 @@ export default function EmbConsultPage() {
|
||||
setLoading(true);
|
||||
|
||||
try {
|
||||
const d = await embeddingsApi.consult(q, selTable || undefined, topK);
|
||||
const d = await embeddingsApi.consult(q, selTable || undefined, topK, selOci || undefined);
|
||||
let answer = d.answer || t('ec.noAnswer');
|
||||
|
||||
const assistantMsg: ChatMessage = {
|
||||
@@ -162,6 +163,20 @@ export default function EmbConsultPage() {
|
||||
|
||||
{/* Config bar */}
|
||||
<div className="flex gap-3 flex-wrap flex-shrink-0">
|
||||
<div className="min-w-[160px]">
|
||||
<label className="block text-[.68rem] font-semibold mb-1" style={{ color: 'var(--t4)' }}>Tenancy</label>
|
||||
<select
|
||||
value={selOci}
|
||||
onChange={(e) => setSelOci(e.target.value)}
|
||||
className="w-full px-3 py-1.5 rounded-lg text-[.76rem] outline-none"
|
||||
style={{ background: 'var(--bg1)', border: '1px solid var(--bd)', color: 'var(--t1)' }}
|
||||
>
|
||||
<option value="">Todas</option>
|
||||
{ociCfg.map((c) => (
|
||||
<option key={c.id} value={c.id}>{c.tenancy_name}</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
<div className="flex-1 min-w-[180px]">
|
||||
<label className="block text-[.68rem] font-semibold mb-1" style={{ color: 'var(--t4)' }}>{t('ec.tableOptional')}</label>
|
||||
<select
|
||||
|
||||
Reference in New Issue
Block a user