feat: detailed embed progress — current section, global bar, queue display

- Backend: tracks current_table, current_inserted, current_total, queue per embed task
- Frontend: dual progress bars (section + global), queue of upcoming sections
- Section bar shows table name + individual progress
- Global bar shows inserted/skipped/total with green/red segments
- Queue shows remaining sections with doc counts
This commit is contained in:
nogueiraguh
2026-03-21 12:48:31 -03:00
parent dcc202ff9f
commit 8db3e42d31
2 changed files with 88 additions and 35 deletions

View File

@@ -3885,36 +3885,44 @@ async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(requi
"message": f"Embedding {total_docs} documentos em {len(tables_used)} tabelas..."
})
# Build queue info: [(table, doc_count), ...]
table_queue = [(tbl, len(docs)) for tbl, docs in table_docs.items()]
def _bg_embed_all():
"""Background: embed documents into their respective tables sequentially."""
import array
default_model = cfg.get("embedding_model_id", "cohere.embed-v4.0")
inserted_total = 0
skipped_total = 0
processed_total = 0
errors = []
for tbl, docs in table_docs.items():
for tbl_idx, (tbl, docs) in enumerate(table_docs.items()):
remaining = table_queue[tbl_idx + 1:] if tbl_idx + 1 < len(table_queue) else []
_auto_register_table(cfg["id"], tbl)
# Purge old data for this tenancy/date before inserting
purged = _purge_table_by_tenancy(cfg, tbl, tenancy, extract_date)
if purged:
_update_embed_status(task_id, {"message": f"Purged {purged} old docs from {tbl}..."})
# Auto-detect embedding model based on table dimension
_update_embed_status(task_id, {"message": f"Purged {purged} old docs from {tbl}...",
"current_table": tbl, "current_inserted": 0, "current_total": len(docs),
"queue": [{"table": t, "docs": n} for t, n in remaining]})
emb_model = default_model
try:
actual_dim = _get_table_embedding_dim(cfg, tbl)
if actual_dim and actual_dim in _DIM_TO_MODEL:
emb_model = _DIM_TO_MODEL[actual_dim]
log.info(f"Table {tbl}: dim={actual_dim}, using model {emb_model}")
except Exception as e:
log.warning(f"Could not detect dim for {tbl}: {e}")
except Exception:
pass
current_inserted = 0
current_skipped = 0
try:
conn = _get_adb_connection(cfg)
cur = conn.cursor()
for doc in docs:
processed = 0
try:
content = doc.get("content", "")
if not content:
processed += 1
skipped_total += 1
current_skipped += 1
processed_total += 1
continue
embedding = _embed_text(content, gc, emb_model)
vec = array.array('f', [float(x) for x in embedding])
@@ -3926,17 +3934,29 @@ async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(requi
cur.execute(f'INSERT INTO "{tbl}" (ID, TEXT, EMBEDDING, METADATA) VALUES (HEXTORAW(:1), :2, :3, :4)',
[uuid.uuid4().hex.upper(), content, vec, metadata])
inserted_total += 1
current_inserted += 1
except Exception as e:
log.warning(f"Embed skip in {tbl}: {str(e)[:120]}")
processed += 1
err_str = str(e)
if "message" in err_str:
import re as _re
m = _re.search(r'"message"\s*:\s*"(.*?)"', err_str)
log.warning(f"Embed skip in {tbl}: {m.group(1)[:200] if m else err_str[:200]}")
else:
log.warning(f"Embed skip in {tbl}: {err_str[:200]}")
skipped_total += 1
current_skipped += 1
processed_total += 1
_update_embed_status(task_id, {
"inserted": inserted_total,
"message": f"Embedding {inserted_total}/{processed} OK — {tbl}"
"inserted": inserted_total, "skipped": skipped_total, "processed": processed_total,
"current_table": tbl, "current_inserted": current_inserted, "current_skipped": current_skipped,
"current_total": len(docs),
"queue": [{"table": t, "docs": n} for t, n in remaining],
"message": f"{tbl}: {current_inserted}/{len(docs)} — global: {inserted_total}/{total_docs}"
})
conn.commit()
cur.close()
conn.close()
log.info(f"Embedded {len(docs)} docs into {tbl} (tenancy={tenancy})")
log.info(f"Embedded {current_inserted}/{len(docs)} docs into {tbl} (tenancy={tenancy})")
except Exception as e:
log.error(f"Failed to connect/embed to {tbl}: {e}")
errors.append(f"{tbl}: {str(e)[:100]}")