feat: detailed embed progress — current section, global bar, queue display
- Backend: tracks current_table, current_inserted, current_total, queue per embed task - Frontend: dual progress bars (section + global), queue of upcoming sections - Section bar shows table name + individual progress - Global bar shows inserted/skipped/total with green/red segments - Queue shows remaining sections with doc counts
This commit is contained in:
@@ -3885,36 +3885,44 @@ async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(requi
|
||||
"message": f"Embedding {total_docs} documentos em {len(tables_used)} tabelas..."
|
||||
})
|
||||
|
||||
# Build queue info: [(table, doc_count), ...]
|
||||
table_queue = [(tbl, len(docs)) for tbl, docs in table_docs.items()]
|
||||
|
||||
def _bg_embed_all():
|
||||
"""Background: embed documents into their respective tables sequentially."""
|
||||
import array
|
||||
default_model = cfg.get("embedding_model_id", "cohere.embed-v4.0")
|
||||
inserted_total = 0
|
||||
skipped_total = 0
|
||||
processed_total = 0
|
||||
errors = []
|
||||
for tbl, docs in table_docs.items():
|
||||
for tbl_idx, (tbl, docs) in enumerate(table_docs.items()):
|
||||
remaining = table_queue[tbl_idx + 1:] if tbl_idx + 1 < len(table_queue) else []
|
||||
_auto_register_table(cfg["id"], tbl)
|
||||
# Purge old data for this tenancy/date before inserting
|
||||
purged = _purge_table_by_tenancy(cfg, tbl, tenancy, extract_date)
|
||||
if purged:
|
||||
_update_embed_status(task_id, {"message": f"Purged {purged} old docs from {tbl}..."})
|
||||
# Auto-detect embedding model based on table dimension
|
||||
_update_embed_status(task_id, {"message": f"Purged {purged} old docs from {tbl}...",
|
||||
"current_table": tbl, "current_inserted": 0, "current_total": len(docs),
|
||||
"queue": [{"table": t, "docs": n} for t, n in remaining]})
|
||||
emb_model = default_model
|
||||
try:
|
||||
actual_dim = _get_table_embedding_dim(cfg, tbl)
|
||||
if actual_dim and actual_dim in _DIM_TO_MODEL:
|
||||
emb_model = _DIM_TO_MODEL[actual_dim]
|
||||
log.info(f"Table {tbl}: dim={actual_dim}, using model {emb_model}")
|
||||
except Exception as e:
|
||||
log.warning(f"Could not detect dim for {tbl}: {e}")
|
||||
except Exception:
|
||||
pass
|
||||
current_inserted = 0
|
||||
current_skipped = 0
|
||||
try:
|
||||
conn = _get_adb_connection(cfg)
|
||||
cur = conn.cursor()
|
||||
for doc in docs:
|
||||
processed = 0
|
||||
try:
|
||||
content = doc.get("content", "")
|
||||
if not content:
|
||||
processed += 1
|
||||
skipped_total += 1
|
||||
current_skipped += 1
|
||||
processed_total += 1
|
||||
continue
|
||||
embedding = _embed_text(content, gc, emb_model)
|
||||
vec = array.array('f', [float(x) for x in embedding])
|
||||
@@ -3926,17 +3934,29 @@ async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(requi
|
||||
cur.execute(f'INSERT INTO "{tbl}" (ID, TEXT, EMBEDDING, METADATA) VALUES (HEXTORAW(:1), :2, :3, :4)',
|
||||
[uuid.uuid4().hex.upper(), content, vec, metadata])
|
||||
inserted_total += 1
|
||||
current_inserted += 1
|
||||
except Exception as e:
|
||||
log.warning(f"Embed skip in {tbl}: {str(e)[:120]}")
|
||||
processed += 1
|
||||
err_str = str(e)
|
||||
if "message" in err_str:
|
||||
import re as _re
|
||||
m = _re.search(r'"message"\s*:\s*"(.*?)"', err_str)
|
||||
log.warning(f"Embed skip in {tbl}: {m.group(1)[:200] if m else err_str[:200]}")
|
||||
else:
|
||||
log.warning(f"Embed skip in {tbl}: {err_str[:200]}")
|
||||
skipped_total += 1
|
||||
current_skipped += 1
|
||||
processed_total += 1
|
||||
_update_embed_status(task_id, {
|
||||
"inserted": inserted_total,
|
||||
"message": f"Embedding {inserted_total}/{processed} OK — {tbl}"
|
||||
"inserted": inserted_total, "skipped": skipped_total, "processed": processed_total,
|
||||
"current_table": tbl, "current_inserted": current_inserted, "current_skipped": current_skipped,
|
||||
"current_total": len(docs),
|
||||
"queue": [{"table": t, "docs": n} for t, n in remaining],
|
||||
"message": f"{tbl}: {current_inserted}/{len(docs)} — global: {inserted_total}/{total_docs}"
|
||||
})
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
log.info(f"Embedded {len(docs)} docs into {tbl} (tenancy={tenancy})")
|
||||
log.info(f"Embedded {current_inserted}/{len(docs)} docs into {tbl} (tenancy={tenancy})")
|
||||
except Exception as e:
|
||||
log.error(f"Failed to connect/embed to {tbl}: {e}")
|
||||
errors.append(f"{tbl}: {str(e)[:100]}")
|
||||
|
||||
Reference in New Issue
Block a user