feat: knowledge base with URL import, ADB case-sensitivity fix, and RAG enhancements
- Add Knowledge Base (Base de Conhecimento) with file upload + URL import - URL content extraction: HTML stripping, remote PDF parsing - Auto-resolve embedding config from OCI credentials - Case-insensitive ADB table matching with quoted identifiers - Table validation endpoint, report_date metadata, purge & re-embed flow - Embeddings tab cleanup, PyPDF2 dependency, README updated to v2.3
This commit is contained in:
nogueiraguh
594
backend/app.py
594
backend/app.py
@@ -466,6 +466,16 @@ async def current_user(cred: HTTPAuthorizationCredentials = Depends(security)):
|
||||
if not u or not s: raise HTTPException(401, "Sessão inválida")
|
||||
return dict(u)
|
||||
|
||||
def _user_from_token(token: str):
|
||||
"""Resolve user from a raw JWT token string (for query-param auth on downloads)."""
|
||||
try: p = pyjwt.decode(token, APP_SECRET, algorithms=[JWT_ALG])
|
||||
except Exception: raise HTTPException(401, "Token inválido")
|
||||
with db() as c:
|
||||
u = c.execute("SELECT * FROM users WHERE id=? AND is_active=1", (p["sub"],)).fetchone()
|
||||
s = c.execute("SELECT * FROM sessions WHERE id=? AND is_active=1", (p["sid"],)).fetchone()
|
||||
if not u or not s: raise HTTPException(401, "Sessão inválida")
|
||||
return dict(u)
|
||||
|
||||
def require(*roles):
|
||||
async def dep(u=Depends(current_user)):
|
||||
if u["role"] not in roles: raise HTTPException(403, f"Requer role: {', '.join(roles)}")
|
||||
@@ -929,7 +939,7 @@ async def explore_security_lists(cid: str, compartment_id: str = Query(None), re
|
||||
_, default_comp = _explore_comp(cid)
|
||||
comp = compartment_id or default_comp
|
||||
sls = vn.list_security_lists(comp).data
|
||||
def _fmt_rule(r, direction):
|
||||
def _fmt_rule(r, direction, idx):
|
||||
proto_map = {"6":"TCP","17":"UDP","1":"ICMP","all":"ALL"}
|
||||
proto = proto_map.get(r.protocol, r.protocol)
|
||||
src_dst = r.source if direction == "ingress" else r.destination
|
||||
@@ -941,17 +951,98 @@ async def explore_security_lists(cid: str, compartment_id: str = Query(None), re
|
||||
dp = r.udp_options.destination_port_range
|
||||
if dp: ports = f"{dp.min}-{dp.max}" if dp.min != dp.max else str(dp.min)
|
||||
return {"direction":direction,"protocol":proto,"source_dest":src_dst or "","ports":ports,
|
||||
"stateless":r.is_stateless,"description":getattr(r,'description','') or ""}
|
||||
"stateless":r.is_stateless,"description":getattr(r,'description','') or "","rule_index":idx}
|
||||
result = []
|
||||
for s in sls:
|
||||
ingress = [_fmt_rule(r,"ingress") for r in (s.ingress_security_rules or [])]
|
||||
egress = [_fmt_rule(r,"egress") for r in (s.egress_security_rules or [])]
|
||||
ingress = [_fmt_rule(r,"ingress",i) for i,r in enumerate(s.ingress_security_rules or [])]
|
||||
egress = [_fmt_rule(r,"egress",i) for i,r in enumerate(s.egress_security_rules or [])]
|
||||
result.append({"id":s.id,"display_name":s.display_name,"vcn_id":s.vcn_id,"lifecycle_state":s.lifecycle_state,
|
||||
"ingress_count":len(ingress),"egress_count":len(egress),"rules":ingress+egress})
|
||||
return result
|
||||
except Exception as e:
|
||||
return {"error": str(e)[:500]}
|
||||
|
||||
@app.post("/api/oci/explore/{cid}/security_lists/{sl_id}/rules")
|
||||
async def add_security_list_rule(cid: str, sl_id: str, req: dict, u=Depends(require("admin", "user"))):
|
||||
"""Add a rule to a Security List (appends to existing rules)."""
|
||||
try:
|
||||
import oci
|
||||
config = _get_oci_config(cid)
|
||||
region = req.get("region")
|
||||
if region: config["region"] = region
|
||||
vn = oci.core.VirtualNetworkClient(config)
|
||||
sl = vn.get_security_list(sl_id).data
|
||||
direction = req.get("direction", "ingress").lower()
|
||||
protocol = str(req.get("protocol", "6"))
|
||||
source_dest = req.get("source_dest", "0.0.0.0/0")
|
||||
port_min = req.get("port_min")
|
||||
port_max = req.get("port_max")
|
||||
description = req.get("description", "").strip() or None
|
||||
is_stateless = req.get("is_stateless", False)
|
||||
tcp_opts = None
|
||||
udp_opts = None
|
||||
if protocol == "6" and port_min is not None:
|
||||
tcp_opts = oci.core.models.TcpOptions(
|
||||
destination_port_range=oci.core.models.PortRange(min=int(port_min), max=int(port_max or port_min)))
|
||||
elif protocol == "17" and port_min is not None:
|
||||
udp_opts = oci.core.models.UdpOptions(
|
||||
destination_port_range=oci.core.models.PortRange(min=int(port_min), max=int(port_max or port_min)))
|
||||
ingress_rules = list(sl.ingress_security_rules or [])
|
||||
egress_rules = list(sl.egress_security_rules or [])
|
||||
if direction == "ingress":
|
||||
ingress_rules.append(oci.core.models.IngressSecurityRule(
|
||||
protocol=protocol, source=source_dest, source_type="CIDR_BLOCK",
|
||||
is_stateless=is_stateless, description=description,
|
||||
tcp_options=tcp_opts, udp_options=udp_opts))
|
||||
else:
|
||||
egress_rules.append(oci.core.models.EgressSecurityRule(
|
||||
protocol=protocol, destination=source_dest, destination_type="CIDR_BLOCK",
|
||||
is_stateless=is_stateless, description=description,
|
||||
tcp_options=tcp_opts, udp_options=udp_opts))
|
||||
update = oci.core.models.UpdateSecurityListDetails(
|
||||
ingress_security_rules=ingress_rules, egress_security_rules=egress_rules)
|
||||
vn.update_security_list(sl_id, update)
|
||||
_audit(u["id"], u["username"], "seclist_add_rule", sl_id,
|
||||
f"{direction} {protocol} {source_dest} port={port_min or 'ALL'}")
|
||||
return {"ok": True, "message": "Regra adicionada com sucesso"}
|
||||
except Exception as e:
|
||||
raise HTTPException(400, str(e)[:500])
|
||||
|
||||
@app.delete("/api/oci/explore/{cid}/security_lists/{sl_id}/rules")
|
||||
async def remove_security_list_rule(cid: str, sl_id: str, req: dict, u=Depends(require("admin", "user"))):
|
||||
"""Remove a rule from a Security List by index."""
|
||||
try:
|
||||
import oci
|
||||
config = _get_oci_config(cid)
|
||||
region = req.get("region")
|
||||
if region: config["region"] = region
|
||||
vn = oci.core.VirtualNetworkClient(config)
|
||||
sl = vn.get_security_list(sl_id).data
|
||||
direction = req.get("direction", "ingress").lower()
|
||||
rule_index = req.get("rule_index")
|
||||
if rule_index is None: raise HTTPException(400, "rule_index obrigatório")
|
||||
ingress_rules = list(sl.ingress_security_rules or [])
|
||||
egress_rules = list(sl.egress_security_rules or [])
|
||||
if direction == "ingress":
|
||||
if 0 <= rule_index < len(ingress_rules):
|
||||
removed = ingress_rules.pop(rule_index)
|
||||
desc = f"ingress idx={rule_index} src={removed.source}"
|
||||
else:
|
||||
raise HTTPException(400, "Índice inválido")
|
||||
else:
|
||||
if 0 <= rule_index < len(egress_rules):
|
||||
removed = egress_rules.pop(rule_index)
|
||||
desc = f"egress idx={rule_index} dst={removed.destination}"
|
||||
else:
|
||||
raise HTTPException(400, "Índice inválido")
|
||||
update = oci.core.models.UpdateSecurityListDetails(
|
||||
ingress_security_rules=ingress_rules, egress_security_rules=egress_rules)
|
||||
vn.update_security_list(sl_id, update)
|
||||
_audit(u["id"], u["username"], "seclist_remove_rule", sl_id, desc)
|
||||
return {"ok": True, "message": "Regra removida com sucesso"}
|
||||
except Exception as e:
|
||||
raise HTTPException(400, str(e)[:500])
|
||||
|
||||
@app.get("/api/oci/explore/{cid}/block_volumes")
|
||||
async def explore_block_volumes(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)):
|
||||
try:
|
||||
@@ -1029,11 +1120,97 @@ async def explore_nsgs(cid: str, compartment_id: str = Query(None), region: str
|
||||
_, default_comp = _explore_comp(cid)
|
||||
comp = compartment_id or default_comp
|
||||
nsgs = vn.list_network_security_groups(compartment_id=comp).data
|
||||
return [{"id":n.id,"display_name":n.display_name,"vcn_id":n.vcn_id,"lifecycle_state":n.lifecycle_state,
|
||||
"time_created":str(n.time_created)} for n in nsgs]
|
||||
result = []
|
||||
proto_map = {"1": "ICMP", "6": "TCP", "17": "UDP", "all": "ALL"}
|
||||
for n in nsgs:
|
||||
rules_data = oci.pagination.list_call_get_all_results(
|
||||
vn.list_network_security_group_security_rules, n.id).data
|
||||
rules = []
|
||||
for r in rules_data:
|
||||
ports = ""
|
||||
if r.tcp_options and r.tcp_options.destination_port_range:
|
||||
pr = r.tcp_options.destination_port_range
|
||||
ports = str(pr.min) if pr.min == pr.max else f"{pr.min}-{pr.max}"
|
||||
elif r.udp_options and r.udp_options.destination_port_range:
|
||||
pr = r.udp_options.destination_port_range
|
||||
ports = str(pr.min) if pr.min == pr.max else f"{pr.min}-{pr.max}"
|
||||
src_dst = ""
|
||||
src_dst_type = ""
|
||||
if r.direction == "INGRESS":
|
||||
src_dst = r.source or ""
|
||||
src_dst_type = r.source_type or ""
|
||||
else:
|
||||
src_dst = r.destination or ""
|
||||
src_dst_type = r.destination_type or ""
|
||||
rules.append({
|
||||
"id": r.id, "direction": r.direction,
|
||||
"protocol": proto_map.get(r.protocol, r.protocol),
|
||||
"protocol_num": r.protocol,
|
||||
"source_dest": src_dst, "source_dest_type": src_dst_type,
|
||||
"ports": ports, "is_stateless": r.is_stateless or False,
|
||||
"description": r.description or ""
|
||||
})
|
||||
result.append({"id": n.id, "display_name": n.display_name, "vcn_id": n.vcn_id,
|
||||
"lifecycle_state": n.lifecycle_state, "time_created": str(n.time_created),
|
||||
"rule_count": len(rules), "rules": rules})
|
||||
return result
|
||||
except Exception as e:
|
||||
return {"error": str(e)[:500]}
|
||||
|
||||
@app.post("/api/oci/explore/{cid}/nsgs/{nsg_id}/rules")
|
||||
async def add_nsg_rule(cid: str, nsg_id: str, req: dict, u=Depends(require("admin", "user"))):
|
||||
"""Add a security rule to an NSG."""
|
||||
try:
|
||||
import oci
|
||||
config = _get_oci_config(cid)
|
||||
region = req.get("region")
|
||||
if region: config["region"] = region
|
||||
vn = oci.core.VirtualNetworkClient(config)
|
||||
direction = req.get("direction", "INGRESS").upper()
|
||||
protocol = str(req.get("protocol", "6"))
|
||||
source_dest = req.get("source_dest", "0.0.0.0/0")
|
||||
source_dest_type = req.get("source_dest_type", "CIDR_BLOCK")
|
||||
port_min = req.get("port_min")
|
||||
port_max = req.get("port_max")
|
||||
description = req.get("description", "")
|
||||
is_stateless = req.get("is_stateless", False)
|
||||
rule = {"direction": direction, "protocol": protocol, "is_stateless": is_stateless, "description": description}
|
||||
if direction == "INGRESS":
|
||||
rule["source"] = source_dest
|
||||
rule["source_type"] = source_dest_type
|
||||
else:
|
||||
rule["destination"] = source_dest
|
||||
rule["destination_type"] = source_dest_type
|
||||
if protocol == "6" and port_min is not None:
|
||||
rule["tcp_options"] = oci.core.models.TcpOptions(
|
||||
destination_port_range=oci.core.models.PortRange(min=int(port_min), max=int(port_max or port_min)))
|
||||
elif protocol == "17" and port_min is not None:
|
||||
rule["udp_options"] = oci.core.models.UdpOptions(
|
||||
destination_port_range=oci.core.models.PortRange(min=int(port_min), max=int(port_max or port_min)))
|
||||
details = oci.core.models.AddNetworkSecurityGroupSecurityRulesDetails(
|
||||
security_rules=[oci.core.models.AddSecurityRuleDetails(**rule)])
|
||||
resp = vn.add_network_security_group_security_rules(nsg_id, details)
|
||||
_audit(u["id"], u["username"], "nsg_add_rule", nsg_id, f"{direction} {protocol} {source_dest} {port_min or 'ALL'}")
|
||||
added = [{"id": r.id} for r in (resp.data.security_rules or [])]
|
||||
return {"ok": True, "message": "Regra adicionada com sucesso", "rules_added": added}
|
||||
except Exception as e:
|
||||
raise HTTPException(400, str(e)[:500])
|
||||
|
||||
@app.delete("/api/oci/explore/{cid}/nsgs/{nsg_id}/rules/{rule_id}")
|
||||
async def remove_nsg_rule(cid: str, nsg_id: str, rule_id: str, region: str = Query(None), u=Depends(require("admin", "user"))):
|
||||
"""Remove a security rule from an NSG."""
|
||||
try:
|
||||
import oci
|
||||
config = _get_oci_config(cid)
|
||||
if region: config["region"] = region
|
||||
vn = oci.core.VirtualNetworkClient(config)
|
||||
details = oci.core.models.RemoveNetworkSecurityGroupSecurityRulesDetails(security_rule_ids=[rule_id])
|
||||
vn.remove_network_security_group_security_rules(nsg_id, details)
|
||||
_audit(u["id"], u["username"], "nsg_remove_rule", nsg_id, f"rule_id={rule_id}")
|
||||
return {"ok": True, "message": "Regra removida com sucesso"}
|
||||
except Exception as e:
|
||||
raise HTTPException(400, str(e)[:500])
|
||||
|
||||
@app.get("/api/oci/explore/{cid}/route_tables")
|
||||
async def explore_route_tables(cid: str, compartment_id: str = Query(None), region: str = Query(None), u=Depends(current_user)):
|
||||
try:
|
||||
@@ -2225,44 +2402,92 @@ def _get_adb_connection(cfg: dict):
|
||||
params["wallet_password"] = _dec(cfg["wallet_password_enc"]) if cfg.get("wallet_password_enc") else ""
|
||||
return oracledb.connect(**params)
|
||||
|
||||
def _resolve_embed_config(oci_config_id: str = None, genai_cfg: dict = None) -> dict:
|
||||
"""Resolve embedding config from genai_cfg or oci_config. Returns dict with oci_config_id, endpoint, genai_region, compartment_id."""
|
||||
if genai_cfg:
|
||||
return genai_cfg
|
||||
if oci_config_id:
|
||||
with db() as c:
|
||||
# Try linked genai config first
|
||||
gc = c.execute("SELECT * FROM genai_configs WHERE oci_config_id=? ORDER BY is_default DESC, created_at DESC",
|
||||
(oci_config_id,)).fetchone()
|
||||
if gc: return dict(gc)
|
||||
# Fallback: build from oci_config directly
|
||||
oc = c.execute("SELECT * FROM oci_configs WHERE id=?", (oci_config_id,)).fetchone()
|
||||
if oc:
|
||||
return {
|
||||
"oci_config_id": oc["id"],
|
||||
"genai_region": oc["region"],
|
||||
"endpoint": f"https://inference.generativeai.{oc['region']}.oci.oraclecloud.com",
|
||||
"compartment_id": oc.get("compartment_id") or oc["tenancy_ocid"],
|
||||
}
|
||||
# Last resort: any genai config
|
||||
with db() as c:
|
||||
gc = c.execute("SELECT * FROM genai_configs ORDER BY is_default DESC, created_at DESC").fetchone()
|
||||
if gc: return dict(gc)
|
||||
# Or any oci config
|
||||
oc = c.execute("SELECT * FROM oci_configs ORDER BY created_at DESC").fetchone()
|
||||
if oc:
|
||||
return {
|
||||
"oci_config_id": oc["id"],
|
||||
"genai_region": oc["region"],
|
||||
"endpoint": f"https://inference.generativeai.{oc['region']}.oci.oraclecloud.com",
|
||||
"compartment_id": oc.get("compartment_id") or oc["tenancy_ocid"],
|
||||
}
|
||||
raise HTTPException(400, "Nenhuma credencial OCI configurada para gerar embeddings.")
|
||||
|
||||
def _embed_text(text: str, genai_cfg: dict, embedding_model_id: str) -> list:
|
||||
"""Generate embedding using OCI GenAI embed endpoint."""
|
||||
import oci
|
||||
config_path = str(OCI_DIR / genai_cfg["oci_config_id"] / "config")
|
||||
config = oci.config.from_file(config_path, "DEFAULT")
|
||||
endpoint = genai_cfg["endpoint"]
|
||||
endpoint = genai_cfg.get("endpoint") or f"https://inference.generativeai.{genai_cfg.get('genai_region','us-ashburn-1')}.oci.oraclecloud.com"
|
||||
client = oci.generative_ai_inference.GenerativeAiInferenceClient(
|
||||
config=config, service_endpoint=endpoint,
|
||||
retry_strategy=oci.retry.NoneRetryStrategy(), timeout=(10, 120)
|
||||
)
|
||||
embed_detail = oci.generative_ai_inference.models.EmbedTextDetails()
|
||||
embed_detail.inputs = [text]
|
||||
# Resolve OCID for embedding model by region
|
||||
emb_info = EMBEDDING_MODELS.get(embedding_model_id, {})
|
||||
region = genai_cfg.get("genai_region", "")
|
||||
emb_ref = emb_info.get("ocids", {}).get(region) or embedding_model_id
|
||||
embed_detail.serving_mode = oci.generative_ai_inference.models.OnDemandServingMode(model_id=emb_ref)
|
||||
embed_detail.compartment_id = genai_cfg["compartment_id"]
|
||||
embed_detail.compartment_id = genai_cfg.get("compartment_id", "")
|
||||
embed_detail.truncate = "NONE"
|
||||
embed_detail.input_type = "SEARCH_QUERY"
|
||||
response = client.embed_text(embed_detail)
|
||||
return response.data.embeddings[0]
|
||||
|
||||
def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name: str = None) -> list:
|
||||
"""Search ADB vector store using cosine similarity. Returns top-K documents."""
|
||||
def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name: str = None, tenancy: str = None) -> list:
|
||||
"""Search ADB vector store using cosine similarity. Returns top-K documents.
|
||||
If tenancy is provided, filters METADATA JSON to match that tenancy only."""
|
||||
import array
|
||||
table_name = table_name or cfg.get("table_name", "")
|
||||
conn = _get_adb_connection(cfg)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
vec = array.array('d', query_embedding)
|
||||
cur.execute(f"""
|
||||
SELECT ID, CONTENT, METADATA, SOURCE,
|
||||
VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance
|
||||
FROM {table_name}
|
||||
ORDER BY distance ASC
|
||||
FETCH FIRST :2 ROWS ONLY
|
||||
""", [vec, top_k])
|
||||
if tenancy:
|
||||
# Filter by tenancy in METADATA JSON field using LIKE for broad compatibility
|
||||
# Matches both structured JSON {"tenancy":"X",...} and legacy "tenancy: X, ..."
|
||||
tenancy_filter = f'%"tenancy":"{tenancy}"%'
|
||||
tenancy_filter_legacy = f'%tenancy: {tenancy}%'
|
||||
cur.execute(f"""
|
||||
SELECT ID, TEXT, METADATA,
|
||||
VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance
|
||||
FROM "{table_name}"
|
||||
WHERE (METADATA LIKE :3 OR METADATA LIKE :4)
|
||||
ORDER BY distance ASC
|
||||
FETCH FIRST :2 ROWS ONLY
|
||||
""", [vec, top_k, tenancy_filter, tenancy_filter_legacy])
|
||||
else:
|
||||
cur.execute(f"""
|
||||
SELECT ID, TEXT, METADATA,
|
||||
VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance
|
||||
FROM "{table_name}"
|
||||
ORDER BY distance ASC
|
||||
FETCH FIRST :2 ROWS ONLY
|
||||
""", [vec, top_k])
|
||||
results = []
|
||||
for row in cur:
|
||||
content = row[1]
|
||||
@@ -2270,7 +2495,7 @@ def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name:
|
||||
content = content.read()
|
||||
results.append({
|
||||
"id": row[0], "content": content or "",
|
||||
"metadata": row[2], "source": row[3], "distance": float(row[4])
|
||||
"metadata": row[2], "distance": float(row[3])
|
||||
})
|
||||
cur.close()
|
||||
return results
|
||||
@@ -2291,15 +2516,15 @@ def _build_rag_context(documents: list) -> str:
|
||||
return "\n\n---\n\n".join(parts)
|
||||
|
||||
def _get_active_adb_configs(user_id: str) -> list[dict]:
|
||||
"""Get all active ADB vector configs with a linked GenAI config."""
|
||||
"""Get all active ADB vector configs. GenAI config is resolved automatically if not linked."""
|
||||
with db() as c:
|
||||
rows = c.execute(
|
||||
"SELECT * FROM adb_vector_configs WHERE user_id=? AND is_active=1 AND genai_config_id IS NOT NULL ORDER BY created_at DESC",
|
||||
"SELECT * FROM adb_vector_configs WHERE user_id=? AND is_active=1 ORDER BY created_at DESC",
|
||||
(user_id,)
|
||||
).fetchall()
|
||||
if not rows:
|
||||
rows = c.execute(
|
||||
"SELECT * FROM adb_vector_configs WHERE is_active=1 AND genai_config_id IS NOT NULL ORDER BY created_at DESC"
|
||||
"SELECT * FROM adb_vector_configs WHERE is_active=1 ORDER BY created_at DESC"
|
||||
).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
@@ -2623,6 +2848,50 @@ async def test_adb(vid: str, u=Depends(require("admin","user"))):
|
||||
_config_log("adb", vid, cname, "error", "test", msg, u["id"], u["username"])
|
||||
return {"status":"error","message":msg}
|
||||
|
||||
@app.post("/api/adb/{vid}/tables/check")
|
||||
async def check_adb_tables(vid: str, u=Depends(require("admin","user"))):
|
||||
"""Check which registered tables exist in ADB and list all user tables."""
|
||||
with db() as c:
|
||||
cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
||||
if not cfg: raise HTTPException(404)
|
||||
registered = _get_tables_for_config(vid, active_only=False)
|
||||
reg_names_upper = {t["table_name"].upper() for t in registered}
|
||||
try:
|
||||
conn = _get_adb_connection(dict(cfg))
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT USER FROM DUAL")
|
||||
current_user_name = cur.fetchone()[0]
|
||||
cur.execute("SELECT table_name FROM user_tables ORDER BY table_name")
|
||||
adb_tables = [r[0] for r in cur.fetchall()]
|
||||
adb_lookup = {t.upper(): t for t in adb_tables}
|
||||
results = []
|
||||
for t in registered:
|
||||
tname_reg = t["table_name"]
|
||||
tname_upper = tname_reg.upper()
|
||||
real_name = adb_lookup.get(tname_upper)
|
||||
if real_name:
|
||||
try:
|
||||
cur.execute(f'SELECT COUNT(*) FROM "{real_name}"')
|
||||
cnt = cur.fetchone()[0]
|
||||
case_note = f" (ADB: {real_name})" if real_name != tname_reg else ""
|
||||
results.append({"table": tname_reg, "adb_name": real_name, "status": "ok", "rows": cnt, "message": f"{cnt} registros{case_note}"})
|
||||
except Exception as e:
|
||||
results.append({"table": tname_reg, "status": "error", "message": str(e)[:200]})
|
||||
else:
|
||||
results.append({"table": tname_reg, "status": "not_found", "message": f"Não existe no schema {current_user_name}"})
|
||||
unregistered = [t for t in adb_tables if t.upper() not in reg_names_upper]
|
||||
cur.close()
|
||||
conn.close()
|
||||
return {
|
||||
"status": "success",
|
||||
"schema_user": current_user_name,
|
||||
"adb_tables_total": len(adb_tables),
|
||||
"registered": results,
|
||||
"unregistered_in_adb": unregistered[:50]
|
||||
}
|
||||
except Exception as e:
|
||||
return {"status": "error", "message": f"Erro de conexão: {str(e)[:500]}"}
|
||||
|
||||
@app.delete("/api/adb/configs/{vid}")
|
||||
async def del_adb(vid: str, u=Depends(require("admin","user"))):
|
||||
with db() as c: c.execute("DELETE FROM adb_vector_configs WHERE id=?", (vid,))
|
||||
@@ -2668,7 +2937,7 @@ async def update_adb(
|
||||
_config_log("adb", vid, config_name, "success", "save", f"Conexão atualizada: {config_name} ({dsn})", u["id"], u["username"])
|
||||
return {"id": vid, "config_name": config_name}
|
||||
|
||||
_TABLE_NAME_RE = re.compile(r'^[A-Z][A-Z0-9_]{0,127}$')
|
||||
_TABLE_NAME_RE = re.compile(r'^[A-Za-z][A-Za-z0-9_]{0,127}$')
|
||||
|
||||
@app.get("/api/adb/{vid}/tables")
|
||||
async def list_adb_tables(vid: str, u=Depends(current_user)):
|
||||
@@ -2679,7 +2948,7 @@ async def list_adb_tables(vid: str, u=Depends(current_user)):
|
||||
|
||||
@app.post("/api/adb/{vid}/tables")
|
||||
async def add_adb_table(vid: str, req: dict, u=Depends(require("admin","user"))):
|
||||
table_name = req.get("table_name", "").strip().upper()
|
||||
table_name = req.get("table_name", "").strip()
|
||||
description = req.get("description", "").strip()
|
||||
if not table_name: raise HTTPException(400, "table_name é obrigatório")
|
||||
if not _TABLE_NAME_RE.match(table_name): raise HTTPException(400, "Nome da tabela inválido. Use letras maiúsculas, números e underscores.")
|
||||
@@ -2712,7 +2981,7 @@ async def update_adb_table(vid: str, tid: str, req: dict, u=Depends(require("adm
|
||||
if not row: raise HTTPException(404, "Table entry not found")
|
||||
sets, vals = [], []
|
||||
if "table_name" in req:
|
||||
tn = req["table_name"].strip().upper()
|
||||
tn = req["table_name"].strip()
|
||||
if not tn: raise HTTPException(400, "table_name é obrigatório")
|
||||
if not _TABLE_NAME_RE.match(tn): raise HTTPException(400, "Nome da tabela inválido")
|
||||
sets.append("table_name=?"); vals.append(tn)
|
||||
@@ -2743,11 +3012,45 @@ async def ingest_documents(vid: str, req: IngestDocReq, bg: BackgroundTasks, u=D
|
||||
bg.add_task(_ingest_documents_task, cfg, dict(gc), req.documents, u["id"], u["username"], table_name=req.table_name)
|
||||
return {"ok": True, "message": f"Ingestão iniciada para {len(req.documents)} documentos", "config_id": vid}
|
||||
|
||||
def _ingest_documents_task(cfg: dict, genai_cfg: dict, documents: list, user_id: str, username: str, table_name: str = None):
|
||||
"""Background task: embed and insert documents into ADB via OCI GenAI."""
|
||||
def _build_metadata_json(tenancy: str = "", compartments: str = "", section: str = "",
|
||||
report_date: str = "", extra: dict = None) -> str:
|
||||
"""Build a structured JSON metadata string for vector embeddings."""
|
||||
meta = {}
|
||||
if tenancy:
|
||||
meta["tenancy"] = tenancy
|
||||
if compartments:
|
||||
meta["compartments"] = compartments
|
||||
if section:
|
||||
meta["section"] = section
|
||||
if report_date:
|
||||
meta["report_date"] = report_date
|
||||
if extra:
|
||||
meta.update(extra)
|
||||
return json.dumps(meta, ensure_ascii=False) if meta else ""
|
||||
|
||||
|
||||
def _auto_register_table(adb_config_id: str, table_name: str, description: str = ""):
|
||||
"""Auto-register a table in adb_vector_tables if not already present."""
|
||||
if not table_name:
|
||||
return
|
||||
with db() as c:
|
||||
exists = c.execute("SELECT 1 FROM adb_vector_tables WHERE adb_config_id=? AND table_name=? COLLATE NOCASE",
|
||||
(adb_config_id, table_name)).fetchone()
|
||||
if not exists:
|
||||
c.execute("INSERT INTO adb_vector_tables (id, adb_config_id, table_name, description) VALUES (?,?,?,?)",
|
||||
(str(uuid.uuid4()), adb_config_id, table_name, description))
|
||||
log.info(f"Auto-registered table '{table_name}' for ADB config {adb_config_id}")
|
||||
|
||||
def _ingest_documents_task(cfg: dict, genai_cfg: dict, documents: list, user_id: str, username: str,
|
||||
table_name: str = None, tenancy: str = None, compartments: str = None,
|
||||
report_date: str = None):
|
||||
"""Background task: embed and insert documents into ADB via OCI GenAI.
|
||||
Tenancy and compartments are stored in METADATA as structured JSON for filtering."""
|
||||
import array
|
||||
emb_model = cfg.get("embedding_model_id", "cohere.embed-v4.0")
|
||||
table_name = table_name or cfg.get("table_name", "")
|
||||
# Auto-register table so it appears in multi-table RAG search
|
||||
_auto_register_table(cfg["id"], table_name)
|
||||
conn = _get_adb_connection(cfg)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
@@ -2758,18 +3061,31 @@ def _ingest_documents_task(cfg: dict, genai_cfg: dict, documents: list, user_id:
|
||||
if not content: continue
|
||||
embedding = _embed_text(content, genai_cfg, emb_model)
|
||||
vec = array.array('d', embedding)
|
||||
# Build structured metadata with tenancy isolation
|
||||
doc_tenancy = tenancy or doc.get("tenancy", "")
|
||||
doc_compartments = compartments or doc.get("compartments", "")
|
||||
metadata = _build_metadata_json(
|
||||
tenancy=doc_tenancy,
|
||||
compartments=doc_compartments,
|
||||
section=doc.get("section", ""),
|
||||
report_date=report_date or "",
|
||||
extra={"legacy_metadata": doc.get("metadata", "")} if doc.get("metadata") else None
|
||||
)
|
||||
cur.execute(f"""
|
||||
INSERT INTO {table_name} (ID, CONTENT, EMBEDDING, METADATA, SOURCE)
|
||||
VALUES (:1, :2, :3, :4, :5)
|
||||
""", [str(uuid.uuid4()), content, vec, doc.get("metadata", ""), doc.get("source", "manual_upload")])
|
||||
INSERT INTO "{table_name}" (ID, TEXT, EMBEDDING, METADATA)
|
||||
VALUES (:1, :2, :3, :4)
|
||||
""", [str(uuid.uuid4()), content, vec, metadata])
|
||||
inserted += 1
|
||||
except Exception as e:
|
||||
log.error(f"Failed to ingest document: {e}")
|
||||
conn.commit()
|
||||
cur.close()
|
||||
log.info(f"Ingested {inserted}/{len(documents)} documents into {table_name}")
|
||||
log.info(f"Ingested {inserted}/{len(documents)} documents into {table_name}" +
|
||||
(f" (tenancy={tenancy})" if tenancy else ""))
|
||||
_audit(user_id, username, "ingest_documents", cfg["id"], f"{inserted} documents")
|
||||
_config_log("adb", cfg["id"], cfg.get("config_name"), "success", "ingest", f"{inserted}/{len(documents)} documentos ingeridos em {table_name}", user_id, username)
|
||||
_config_log("adb", cfg["id"], cfg.get("config_name"), "success", "ingest",
|
||||
f"{inserted}/{len(documents)} documentos ingeridos em {table_name}" +
|
||||
(f" (tenancy: {tenancy})" if tenancy else ""), user_id, username)
|
||||
except Exception as e:
|
||||
log.error(f"Ingestion task failed: {e}")
|
||||
_config_log("adb", cfg["id"], cfg.get("config_name"), "error", "ingest", str(e)[:500], user_id, username)
|
||||
@@ -2781,6 +3097,8 @@ def _chunk_report_by_section(report_data: dict) -> list:
|
||||
"""Chunk a CIS report into documents grouped by section."""
|
||||
if isinstance(report_data, str):
|
||||
report_data = json.loads(report_data)
|
||||
if isinstance(report_data, list):
|
||||
report_data = {"findings": {str(i): item for i, item in enumerate(report_data)}, "tenancy": "unknown"}
|
||||
findings = report_data.get("findings", {})
|
||||
tenancy = report_data.get("tenancy", "unknown")
|
||||
generated_at = report_data.get("generated_at", "")
|
||||
@@ -2813,6 +3131,9 @@ def _chunk_report_by_section(report_data: dict) -> list:
|
||||
documents.append({
|
||||
"content": "\n".join(lines),
|
||||
"source": f"CIS Report - {tenancy} - {generated_at}",
|
||||
"section": section_name,
|
||||
"tenancy": tenancy,
|
||||
"compartments": ", ".join(compartments[:50]),
|
||||
"metadata": f"tenancy: {tenancy}, section: {section_name}, total: {len(checks)}, passed: {passed}, failed: {failed}, review: {review}"
|
||||
})
|
||||
return documents
|
||||
@@ -2834,18 +3155,20 @@ def _chunk_text_file(text: str, filename: str, chunk_size: int = 1000) -> list:
|
||||
documents.append({"content": current_chunk, "source": filename, "metadata": f"chunk: {chunk_num}"})
|
||||
return documents
|
||||
|
||||
def _get_adb_and_genai(vid: str):
|
||||
"""Load ADB config and its linked GenAI config. Returns (adb_cfg, genai_cfg) or raises."""
|
||||
def _get_adb_and_genai(vid: str, oci_config_id: str = None):
|
||||
"""Load ADB config and resolve embed config.
|
||||
Priority: ADB.genai_config_id → genai by oci_config_id → oci_config directly → any available."""
|
||||
with db() as c:
|
||||
cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
||||
if not cfg: raise HTTPException(404, "ADB config not found")
|
||||
cfg = dict(cfg)
|
||||
if not cfg.get("genai_config_id"):
|
||||
raise HTTPException(400, "GenAI config not linked to this ADB connection")
|
||||
with db() as c:
|
||||
gc = c.execute("SELECT * FROM genai_configs WHERE id=?", (cfg["genai_config_id"],)).fetchone()
|
||||
if not gc: raise HTTPException(400, "Linked GenAI config not found")
|
||||
return cfg, dict(gc)
|
||||
genai_cfg = None
|
||||
if cfg.get("genai_config_id"):
|
||||
with db() as c:
|
||||
row = c.execute("SELECT * FROM genai_configs WHERE id=?", (cfg["genai_config_id"],)).fetchone()
|
||||
if row: genai_cfg = dict(row)
|
||||
gc = _resolve_embed_config(oci_config_id=oci_config_id, genai_cfg=genai_cfg)
|
||||
return cfg, gc
|
||||
|
||||
@app.get("/api/embeddings/preview/{rid}")
|
||||
async def preview_report_chunks(rid: str, u=Depends(current_user)):
|
||||
@@ -2861,9 +3184,10 @@ async def preview_report_chunks(rid: str, u=Depends(current_user)):
|
||||
except Exception:
|
||||
raise HTTPException(400, "Invalid report data")
|
||||
documents = _chunk_report_by_section(report_data)
|
||||
return {"tenancy": report_data.get("tenancy", "unknown"),
|
||||
"regions": report_data.get("regions", []),
|
||||
"compartments": report_data.get("compartments", []),
|
||||
rd = report_data if isinstance(report_data, dict) else {}
|
||||
return {"tenancy": rd.get("tenancy", "unknown"),
|
||||
"regions": rd.get("regions", []),
|
||||
"compartments": rd.get("compartments", []),
|
||||
"total_chunks": len(documents),
|
||||
"chunks": documents}
|
||||
|
||||
@@ -2872,7 +3196,7 @@ async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(requi
|
||||
vid = req.get("adb_config_id")
|
||||
if not vid: raise HTTPException(400, "adb_config_id is required")
|
||||
with db() as c:
|
||||
r = c.execute("SELECT json_path,tenancy_name FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone()
|
||||
r = c.execute("SELECT json_path,tenancy_name,config_id FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone()
|
||||
if not r: raise HTTPException(404, "Report not found or not completed")
|
||||
json_path = r["json_path"]
|
||||
if not json_path or not Path(json_path).exists():
|
||||
@@ -2883,17 +3207,63 @@ async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(requi
|
||||
raise HTTPException(400, "Invalid report data")
|
||||
documents = _chunk_report_by_section(report_data)
|
||||
if not documents: raise HTTPException(400, "No sections found in report")
|
||||
cfg, gc = _get_adb_and_genai(vid)
|
||||
# Optional section filter — embed only a specific section
|
||||
section_filter = req.get("section")
|
||||
if section_filter:
|
||||
documents = [d for d in documents if d.get("section") == section_filter]
|
||||
if not documents: raise HTTPException(400, f"Section '{section_filter}' not found in report")
|
||||
cfg, gc = _get_adb_and_genai(vid, oci_config_id=r.get("config_id"))
|
||||
target_table = req.get("table_name") or None
|
||||
bg.add_task(_ingest_documents_task, cfg, gc, documents, u["id"], u["username"], table_name=target_table)
|
||||
_audit(u["id"], u["username"], "embed_report", rid, f"{len(documents)} sections")
|
||||
return {"ok": True, "message": f"Embedding de {len(documents)} seções iniciado", "sections": len(documents)}
|
||||
# Extract tenancy and compartments for isolation
|
||||
tenancy = report_data.get("tenancy", r["tenancy_name"] or "unknown")
|
||||
report_date = report_data.get("generated_at", "")
|
||||
compartments_list = report_data.get("compartments", [])
|
||||
compartments_str = json.dumps(compartments_list) if compartments_list else ""
|
||||
bg.add_task(_ingest_documents_task, cfg, gc, documents, u["id"], u["username"],
|
||||
table_name=target_table, tenancy=tenancy, compartments=compartments_str,
|
||||
report_date=report_date)
|
||||
label = f"section={section_filter}" if section_filter else f"{len(documents)} sections"
|
||||
_audit(u["id"], u["username"], "embed_report", rid, f"{label}, tenancy={tenancy}")
|
||||
return {"ok": True, "message": f"Embedding de {len(documents)} seção(ões) iniciado (tenancy: {tenancy})", "sections": len(documents), "tenancy": tenancy}
|
||||
|
||||
def _extract_pdf_text(file_bytes: bytes) -> str:
|
||||
"""Extract text from a PDF file using PyPDF2 or pdfplumber."""
|
||||
try:
|
||||
import PyPDF2
|
||||
import io
|
||||
reader = PyPDF2.PdfReader(io.BytesIO(file_bytes))
|
||||
pages = []
|
||||
for page in reader.pages:
|
||||
text = page.extract_text()
|
||||
if text:
|
||||
pages.append(text.strip())
|
||||
return "\n\n".join(pages)
|
||||
except ImportError:
|
||||
pass
|
||||
try:
|
||||
import pdfplumber
|
||||
import io
|
||||
pages = []
|
||||
with pdfplumber.open(io.BytesIO(file_bytes)) as pdf:
|
||||
for page in pdf.pages:
|
||||
text = page.extract_text()
|
||||
if text:
|
||||
pages.append(text.strip())
|
||||
return "\n\n".join(pages)
|
||||
except ImportError:
|
||||
raise HTTPException(400, "PDF support requires PyPDF2 or pdfplumber. Install: pip install PyPDF2")
|
||||
|
||||
@app.post("/api/embeddings/upload")
|
||||
async def embed_upload(adb_config_id: str = Form(...), table_name: str = Form(""), file: UploadFile = File(...), bg: BackgroundTasks = None, u=Depends(require("admin","user"))):
|
||||
if not file.filename.lower().endswith(".txt"):
|
||||
raise HTTPException(400, "Only .txt files are supported")
|
||||
content = (await file.read()).decode("utf-8", errors="replace")
|
||||
fname = file.filename.lower()
|
||||
allowed = ('.txt', '.pdf', '.csv', '.json', '.md')
|
||||
if not any(fname.endswith(ext) for ext in allowed):
|
||||
raise HTTPException(400, f"Formatos aceitos: {', '.join(allowed)}")
|
||||
raw = await file.read()
|
||||
if fname.endswith('.pdf'):
|
||||
content = _extract_pdf_text(raw)
|
||||
else:
|
||||
content = raw.decode("utf-8", errors="replace")
|
||||
if not content.strip(): raise HTTPException(400, "File is empty")
|
||||
documents = _chunk_text_file(content, file.filename)
|
||||
if not documents: raise HTTPException(400, "No content chunks found")
|
||||
@@ -2903,6 +3273,52 @@ async def embed_upload(adb_config_id: str = Form(...), table_name: str = Form(""
|
||||
_audit(u["id"], u["username"], "embed_upload", file.filename, f"{len(documents)} chunks")
|
||||
return {"ok": True, "message": f"Embedding de {len(documents)} chunks iniciado", "chunks": len(documents), "filename": file.filename}
|
||||
|
||||
def _extract_text_from_html(html: str) -> str:
|
||||
"""Extract readable text from HTML, stripping tags and scripts."""
|
||||
import re as _re
|
||||
text = _re.sub(r'<script[^>]*>[\s\S]*?</script>', ' ', html, flags=_re.IGNORECASE)
|
||||
text = _re.sub(r'<style[^>]*>[\s\S]*?</style>', ' ', text, flags=_re.IGNORECASE)
|
||||
text = _re.sub(r'<[^>]+>', ' ', text)
|
||||
text = _re.sub(r'&[a-zA-Z]+;', ' ', text)
|
||||
text = _re.sub(r'&#\d+;', ' ', text)
|
||||
text = _re.sub(r'\s+', ' ', text).strip()
|
||||
return text
|
||||
|
||||
@app.post("/api/embeddings/upload-url")
|
||||
async def embed_upload_url(
|
||||
adb_config_id: str = Form(...),
|
||||
table_name: str = Form(""),
|
||||
url: str = Form(...),
|
||||
bg: BackgroundTasks = None,
|
||||
u=Depends(require("admin", "user"))
|
||||
):
|
||||
import requests as req
|
||||
url = url.strip()
|
||||
if not url.startswith(("http://", "https://")):
|
||||
raise HTTPException(400, "URL inválida — deve começar com http:// ou https://")
|
||||
try:
|
||||
resp = req.get(url, timeout=30, headers={"User-Agent": "Mozilla/5.0 OCI-CIS-Agent/1.0"})
|
||||
resp.raise_for_status()
|
||||
except Exception as e:
|
||||
raise HTTPException(400, f"Erro ao acessar URL: {str(e)[:300]}")
|
||||
ct = resp.headers.get("content-type", "")
|
||||
if "pdf" in ct:
|
||||
content = _extract_pdf_text(resp.content)
|
||||
elif "html" in ct or "text" in ct:
|
||||
content = _extract_text_from_html(resp.text)
|
||||
else:
|
||||
content = resp.text
|
||||
if not content or not content.strip():
|
||||
raise HTTPException(400, "Nenhum conteúdo extraído da URL")
|
||||
documents = _chunk_text_file(content, url)
|
||||
if not documents:
|
||||
raise HTTPException(400, "Nenhum chunk gerado do conteúdo")
|
||||
cfg, gc = _get_adb_and_genai(adb_config_id)
|
||||
target_table = table_name.strip() or None
|
||||
bg.add_task(_ingest_documents_task, cfg, gc, documents, u["id"], u["username"], table_name=target_table)
|
||||
_audit(u["id"], u["username"], "embed_url", url, f"{len(documents)} chunks")
|
||||
return {"ok": True, "message": f"Embedding de {len(documents)} chunks iniciado", "chunks": len(documents), "url": url}
|
||||
|
||||
@app.get("/api/embeddings/{vid}/list")
|
||||
async def list_embeddings(vid: str, table_name: str = Query(""), limit: int = Query(50), offset: int = Query(0), u=Depends(current_user)):
|
||||
with db() as c:
|
||||
@@ -2913,17 +3329,18 @@ async def list_embeddings(vid: str, table_name: str = Query(""), limit: int = Qu
|
||||
cur = conn.cursor()
|
||||
table_name = table_name.strip() or cfg["table_name"]
|
||||
if not table_name: raise HTTPException(400, "Nenhuma tabela selecionada")
|
||||
cur.execute(f"SELECT COUNT(*) FROM {table_name}")
|
||||
cur.execute(f'SELECT COUNT(*) FROM "{table_name}"')
|
||||
total = cur.fetchone()[0]
|
||||
cur.execute(f"""
|
||||
SELECT ID, SOURCE, METADATA, CREATED_AT FROM {table_name}
|
||||
ORDER BY CREATED_AT DESC
|
||||
SELECT ID, METADATA FROM "{table_name}"
|
||||
OFFSET :1 ROWS FETCH NEXT :2 ROWS ONLY
|
||||
""", [offset, limit])
|
||||
rows = []
|
||||
for row in cur:
|
||||
rows.append({"id": row[0], "source": row[1], "metadata": row[2],
|
||||
"created_at": str(row[3]) if row[3] else None})
|
||||
rid = row[0].hex() if isinstance(row[0], bytes) else str(row[0])
|
||||
meta = row[1]
|
||||
if hasattr(meta, 'read'): meta = meta.read()
|
||||
rows.append({"id": rid, "metadata": meta})
|
||||
cur.close(); conn.close()
|
||||
return {"total": total, "offset": offset, "limit": limit, "documents": rows}
|
||||
except Exception as e:
|
||||
@@ -2939,13 +3356,42 @@ async def delete_embedding(vid: str, doc_id: str, table_name: str = Query(""), u
|
||||
cur = conn.cursor()
|
||||
table_name = table_name.strip() or cfg["table_name"]
|
||||
if not table_name: raise HTTPException(400, "Nenhuma tabela selecionada")
|
||||
cur.execute(f"DELETE FROM {table_name} WHERE ID = :1", [doc_id])
|
||||
cur.execute(f'DELETE FROM "{table_name}" WHERE ID = :1', [doc_id])
|
||||
conn.commit()
|
||||
cur.close(); conn.close()
|
||||
return {"ok": True}
|
||||
except Exception as e:
|
||||
raise HTTPException(500, f"Erro ao deletar: {str(e)[:500]}")
|
||||
|
||||
@app.post("/api/embeddings/{vid}/purge")
|
||||
async def purge_embeddings(vid: str, req: dict, u=Depends(require("admin","user"))):
|
||||
"""Delete old embeddings from a table, optionally filtered by tenancy."""
|
||||
table_name = req.get("table_name", "").strip()
|
||||
tenancy = req.get("tenancy", "").strip()
|
||||
if not table_name: raise HTTPException(400, "table_name é obrigatório")
|
||||
with db() as c:
|
||||
cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
|
||||
if not cfg: raise HTTPException(404)
|
||||
try:
|
||||
conn = _get_adb_connection(dict(cfg))
|
||||
cur = conn.cursor()
|
||||
if tenancy:
|
||||
cur.execute(f'SELECT COUNT(*) FROM "{table_name}" WHERE METADATA LIKE :1',
|
||||
[f'%"tenancy":"{tenancy}"%'])
|
||||
count = cur.fetchone()[0]
|
||||
cur.execute(f'DELETE FROM "{table_name}" WHERE METADATA LIKE :1',
|
||||
[f'%"tenancy":"{tenancy}"%'])
|
||||
else:
|
||||
cur.execute(f'SELECT COUNT(*) FROM "{table_name}"')
|
||||
count = cur.fetchone()[0]
|
||||
cur.execute(f'DELETE FROM "{table_name}"')
|
||||
conn.commit()
|
||||
cur.close(); conn.close()
|
||||
_audit(u["id"], u["username"], "purge_embeddings", vid, f"table={table_name}, tenancy={tenancy or 'ALL'}, deleted={count}")
|
||||
return {"ok": True, "deleted": count, "table": table_name, "tenancy": tenancy or "ALL"}
|
||||
except Exception as e:
|
||||
raise HTTPException(500, f"Erro ao limpar: {str(e)[:500]}")
|
||||
|
||||
# ── Reports ───────────────────────────────────────────────────────────────────
|
||||
|
||||
def _classify_report_file(fname: str) -> str:
|
||||
@@ -3158,7 +3604,9 @@ async def list_report_files(rid: str, u=Depends(current_user)):
|
||||
return [dict(f) for f in files]
|
||||
|
||||
@app.get("/api/reports/{rid}/files/{fid}/download")
|
||||
async def download_report_file(rid: str, fid: str, u=Depends(current_user)):
|
||||
async def download_report_file(rid: str, fid: str, token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))):
|
||||
u = _user_from_token(token) if token else (await current_user(cred) if cred else None)
|
||||
if not u: raise HTTPException(401, "Not authenticated")
|
||||
with db() as c:
|
||||
r = c.execute("SELECT user_id FROM reports WHERE id=?", (rid,)).fetchone()
|
||||
if not r: raise HTTPException(404)
|
||||
@@ -3176,7 +3624,9 @@ async def report_html(rid):
|
||||
return FileResponse(r["html_path"], media_type="text/html")
|
||||
|
||||
@app.get("/api/reports/{rid}/download")
|
||||
async def report_dl(rid, fmt: str = Query("json"), u=Depends(current_user)):
|
||||
async def report_dl(rid, fmt: str = Query("json"), token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))):
|
||||
u = _user_from_token(token) if token else (await current_user(cred) if cred else None)
|
||||
if not u: raise HTTPException(401, "Not authenticated")
|
||||
with db() as c: r=c.execute("SELECT * FROM reports WHERE id=?",(rid,)).fetchone()
|
||||
if not r: raise HTTPException(404)
|
||||
p = r["json_path"] if fmt=="json" else r["html_path"]
|
||||
@@ -3265,23 +3715,37 @@ async def _chat_background(mid: str, sid: str, msg: ChatMsg, user: dict, genai_c
|
||||
history = [{"role":r["role"],"content":r["content"]} for r in prev]
|
||||
|
||||
# ── RAG: augment with vector context from ALL active ADB configs ──
|
||||
# Resolve active tenancy for filtered vector search
|
||||
rag_tenancy = None
|
||||
active_oci_for_rag = genai_cfg.get("oci_config_id") or (msg.oci_config_id if hasattr(msg, 'oci_config_id') and msg.oci_config_id else None)
|
||||
if active_oci_for_rag:
|
||||
with db() as c:
|
||||
oci_for_rag = c.execute("SELECT tenancy_name FROM oci_configs WHERE id=?", (active_oci_for_rag,)).fetchone()
|
||||
if oci_for_rag:
|
||||
rag_tenancy = oci_for_rag["tenancy_name"]
|
||||
log.info(f"RAG: filtering by tenancy '{rag_tenancy}'")
|
||||
|
||||
rag_context = ""
|
||||
adb_cfgs = _get_active_adb_configs(user["id"])
|
||||
if adb_cfgs:
|
||||
all_documents = []
|
||||
for adb_cfg in adb_cfgs:
|
||||
try:
|
||||
with db() as c:
|
||||
emb_genai = c.execute("SELECT * FROM genai_configs WHERE id=?", (adb_cfg["genai_config_id"],)).fetchone()
|
||||
genai_linked = None
|
||||
if adb_cfg.get("genai_config_id"):
|
||||
with db() as c:
|
||||
row = c.execute("SELECT * FROM genai_configs WHERE id=?", (adb_cfg["genai_config_id"],)).fetchone()
|
||||
if row: genai_linked = dict(row)
|
||||
emb_genai = _resolve_embed_config(oci_config_id=active_oci_for_rag, genai_cfg=genai_linked)
|
||||
if emb_genai:
|
||||
emb_model = adb_cfg.get("embedding_model_id", "cohere.embed-v4.0")
|
||||
query_embedding = _embed_text(msg.message, dict(emb_genai), emb_model)
|
||||
query_embedding = _embed_text(msg.message, emb_genai, emb_model)
|
||||
tables = _get_tables_for_config(adb_cfg["id"], active_only=True)
|
||||
if not tables:
|
||||
tables = [{"table_name": adb_cfg.get("table_name", "")}]
|
||||
for tbl in tables:
|
||||
try:
|
||||
documents = _vector_search(adb_cfg, query_embedding, top_k=5, table_name=tbl["table_name"])
|
||||
documents = _vector_search(adb_cfg, query_embedding, top_k=5, table_name=tbl["table_name"], tenancy=rag_tenancy)
|
||||
if documents:
|
||||
for doc in documents:
|
||||
doc["source"] = f"{doc.get('source', 'unknown')} [{tbl['table_name']}]"
|
||||
@@ -3993,7 +4457,9 @@ async def tf_workspace_status(wid: str, u=Depends(current_user)):
|
||||
|
||||
|
||||
@app.get("/api/terraform/workspaces/{wid}/download")
|
||||
async def tf_download(wid: str, u=Depends(current_user)):
|
||||
async def tf_download(wid: str, token: str = Query(None), cred: HTTPAuthorizationCredentials = Depends(HTTPBearer(auto_error=False))):
|
||||
u = _user_from_token(token) if token else (await current_user(cred) if cred else None)
|
||||
if not u: raise HTTPException(401, "Not authenticated")
|
||||
with db() as c:
|
||||
r = c.execute("SELECT tf_code, name FROM terraform_workspaces WHERE id=? AND user_id=?", (wid, u["id"])).fetchone()
|
||||
if not r or not r["tf_code"]: raise HTTPException(404, "No code found")
|
||||
|
||||
Reference in New Issue
Block a user