feat: detailed embed progress — current section, global bar, queue display
- Backend: tracks current_table, current_inserted, current_total, queue per embed task - Frontend: dual progress bars (section + global), queue of upcoming sections - Section bar shows table name + individual progress - Global bar shows inserted/skipped/total with green/red segments - Queue shows remaining sections with doc counts
This commit is contained in:
nogueiraguh
@@ -3885,36 +3885,44 @@ async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(requi
|
||||
"message": f"Embedding {total_docs} documentos em {len(tables_used)} tabelas..."
|
||||
})
|
||||
|
||||
# Build queue info: [(table, doc_count), ...]
|
||||
table_queue = [(tbl, len(docs)) for tbl, docs in table_docs.items()]
|
||||
|
||||
def _bg_embed_all():
|
||||
"""Background: embed documents into their respective tables sequentially."""
|
||||
import array
|
||||
default_model = cfg.get("embedding_model_id", "cohere.embed-v4.0")
|
||||
inserted_total = 0
|
||||
skipped_total = 0
|
||||
processed_total = 0
|
||||
errors = []
|
||||
for tbl, docs in table_docs.items():
|
||||
for tbl_idx, (tbl, docs) in enumerate(table_docs.items()):
|
||||
remaining = table_queue[tbl_idx + 1:] if tbl_idx + 1 < len(table_queue) else []
|
||||
_auto_register_table(cfg["id"], tbl)
|
||||
# Purge old data for this tenancy/date before inserting
|
||||
purged = _purge_table_by_tenancy(cfg, tbl, tenancy, extract_date)
|
||||
if purged:
|
||||
_update_embed_status(task_id, {"message": f"Purged {purged} old docs from {tbl}..."})
|
||||
# Auto-detect embedding model based on table dimension
|
||||
_update_embed_status(task_id, {"message": f"Purged {purged} old docs from {tbl}...",
|
||||
"current_table": tbl, "current_inserted": 0, "current_total": len(docs),
|
||||
"queue": [{"table": t, "docs": n} for t, n in remaining]})
|
||||
emb_model = default_model
|
||||
try:
|
||||
actual_dim = _get_table_embedding_dim(cfg, tbl)
|
||||
if actual_dim and actual_dim in _DIM_TO_MODEL:
|
||||
emb_model = _DIM_TO_MODEL[actual_dim]
|
||||
log.info(f"Table {tbl}: dim={actual_dim}, using model {emb_model}")
|
||||
except Exception as e:
|
||||
log.warning(f"Could not detect dim for {tbl}: {e}")
|
||||
except Exception:
|
||||
pass
|
||||
current_inserted = 0
|
||||
current_skipped = 0
|
||||
try:
|
||||
conn = _get_adb_connection(cfg)
|
||||
cur = conn.cursor()
|
||||
for doc in docs:
|
||||
processed = 0
|
||||
try:
|
||||
content = doc.get("content", "")
|
||||
if not content:
|
||||
processed += 1
|
||||
skipped_total += 1
|
||||
current_skipped += 1
|
||||
processed_total += 1
|
||||
continue
|
||||
embedding = _embed_text(content, gc, emb_model)
|
||||
vec = array.array('f', [float(x) for x in embedding])
|
||||
@@ -3926,17 +3934,29 @@ async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(requi
|
||||
cur.execute(f'INSERT INTO "{tbl}" (ID, TEXT, EMBEDDING, METADATA) VALUES (HEXTORAW(:1), :2, :3, :4)',
|
||||
[uuid.uuid4().hex.upper(), content, vec, metadata])
|
||||
inserted_total += 1
|
||||
current_inserted += 1
|
||||
except Exception as e:
|
||||
log.warning(f"Embed skip in {tbl}: {str(e)[:120]}")
|
||||
processed += 1
|
||||
err_str = str(e)
|
||||
if "message" in err_str:
|
||||
import re as _re
|
||||
m = _re.search(r'"message"\s*:\s*"(.*?)"', err_str)
|
||||
log.warning(f"Embed skip in {tbl}: {m.group(1)[:200] if m else err_str[:200]}")
|
||||
else:
|
||||
log.warning(f"Embed skip in {tbl}: {err_str[:200]}")
|
||||
skipped_total += 1
|
||||
current_skipped += 1
|
||||
processed_total += 1
|
||||
_update_embed_status(task_id, {
|
||||
"inserted": inserted_total,
|
||||
"message": f"Embedding {inserted_total}/{processed} OK — {tbl}"
|
||||
"inserted": inserted_total, "skipped": skipped_total, "processed": processed_total,
|
||||
"current_table": tbl, "current_inserted": current_inserted, "current_skipped": current_skipped,
|
||||
"current_total": len(docs),
|
||||
"queue": [{"table": t, "docs": n} for t, n in remaining],
|
||||
"message": f"{tbl}: {current_inserted}/{len(docs)} — global: {inserted_total}/{total_docs}"
|
||||
})
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
log.info(f"Embedded {len(docs)} docs into {tbl} (tenancy={tenancy})")
|
||||
log.info(f"Embedded {current_inserted}/{len(docs)} docs into {tbl} (tenancy={tenancy})")
|
||||
except Exception as e:
|
||||
log.error(f"Failed to connect/embed to {tbl}: {e}")
|
||||
errors.append(f"{tbl}: {str(e)[:100]}")
|
||||
|
||||
@@ -585,7 +585,11 @@ export default function ReportsPage() {
|
||||
const setEmbedTaskId = store.setRptEmbedTaskId;
|
||||
const [embedLoading, setEmbedLoading] = useState('');
|
||||
const [embedMsg, setEmbedMsg] = useState<{ type: 's' | 'e'; text: string } | null>(null);
|
||||
const [embedProgress, setEmbedProgress] = useState<{ inserted: number; skipped: number; processed: number; total: number } | null>(null);
|
||||
const [embedProgress, setEmbedProgress] = useState<{
|
||||
inserted: number; skipped: number; processed: number; total: number;
|
||||
currentTable?: string; currentInserted?: number; currentTotal?: number; currentSkipped?: number;
|
||||
queue?: { table: string; docs: number }[];
|
||||
} | null>(null);
|
||||
|
||||
/* ── HTML iframe (from store) ── */
|
||||
const showIframe = store.rptShowIframe;
|
||||
@@ -797,7 +801,14 @@ export default function ReportsPage() {
|
||||
const processed = (data.processed as number) || inserted;
|
||||
|
||||
if (total > 0) {
|
||||
setEmbedProgress({ inserted, skipped, processed, total });
|
||||
setEmbedProgress({
|
||||
inserted, skipped, processed, total,
|
||||
currentTable: (data.current_table as string) || undefined,
|
||||
currentInserted: (data.current_inserted as number) || 0,
|
||||
currentTotal: (data.current_total as number) || 0,
|
||||
currentSkipped: (data.current_skipped as number) || 0,
|
||||
queue: (data.queue as { table: string; docs: number }[]) || [],
|
||||
});
|
||||
}
|
||||
|
||||
if (s.status === 'done') {
|
||||
@@ -1491,29 +1502,51 @@ export default function ReportsPage() {
|
||||
|
||||
{/* Embed progress + message */}
|
||||
{embedProgress && embedProgress.total > 0 && (() => {
|
||||
const pct = Math.round((embedProgress.processed / embedProgress.total) * 100);
|
||||
const globalPct = Math.round((embedProgress.processed / embedProgress.total) * 100);
|
||||
const okPct = Math.round((embedProgress.inserted / embedProgress.total) * 100);
|
||||
const failPct = Math.round((embedProgress.skipped / embedProgress.total) * 100);
|
||||
const ct = embedProgress.currentTable;
|
||||
const ci = embedProgress.currentInserted || 0;
|
||||
const cTotal = embedProgress.currentTotal || 0;
|
||||
const cPct = cTotal > 0 ? Math.round((ci / cTotal) * 100) : 0;
|
||||
const queue = embedProgress.queue || [];
|
||||
return (
|
||||
<div className="mx-5 mt-3">
|
||||
<div className="flex items-center justify-between mb-1">
|
||||
<span className="text-[.65rem] font-medium" style={{ color: 'var(--t2)' }}>
|
||||
{embedProgress.inserted} OK
|
||||
{embedProgress.skipped > 0 && <span style={{ color: 'var(--rd)' }}> / {embedProgress.skipped} falhas</span>}
|
||||
{' '}({embedProgress.processed}/{embedProgress.total})
|
||||
</span>
|
||||
<span className="text-[.65rem] font-bold" style={{ color: embedProgress.skipped > 0 ? 'var(--yl)' : 'var(--gn)' }}>
|
||||
{pct}%
|
||||
</span>
|
||||
</div>
|
||||
<div className="w-full h-2 rounded-full overflow-hidden flex" style={{ background: 'var(--bg2)' }}>
|
||||
{okPct > 0 && (
|
||||
<div className="h-full transition-all duration-500" style={{ width: `${okPct}%`, background: 'var(--gn)' }} />
|
||||
)}
|
||||
{failPct > 0 && (
|
||||
<div className="h-full transition-all duration-500" style={{ width: `${failPct}%`, background: 'var(--rd)' }} />
|
||||
)}
|
||||
<div className="mx-5 mt-3 flex flex-col gap-2">
|
||||
{/* Current section */}
|
||||
{ct && (
|
||||
<div>
|
||||
<div className="flex items-center justify-between mb-1">
|
||||
<span className="text-[.65rem] font-semibold" style={{ color: 'var(--t1)' }}>
|
||||
{ct} <span style={{ color: 'var(--t4)', fontWeight: 400 }}>({ci}/{cTotal})</span>
|
||||
</span>
|
||||
<span className="text-[.65rem] font-bold" style={{ color: 'var(--ac)' }}>{cPct}%</span>
|
||||
</div>
|
||||
<div className="w-full h-1.5 rounded-full overflow-hidden" style={{ background: 'var(--bg2)' }}>
|
||||
<div className="h-full rounded-full transition-all duration-500" style={{ width: `${cPct}%`, background: 'var(--ac)' }} />
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{/* Global progress */}
|
||||
<div>
|
||||
<div className="flex items-center justify-between mb-1">
|
||||
<span className="text-[.6rem] font-medium" style={{ color: 'var(--t3)' }}>
|
||||
Global: {embedProgress.inserted} OK
|
||||
{embedProgress.skipped > 0 && <span style={{ color: 'var(--rd)' }}> / {embedProgress.skipped} falhas</span>}
|
||||
{' '}({embedProgress.processed}/{embedProgress.total})
|
||||
</span>
|
||||
<span className="text-[.6rem] font-bold" style={{ color: embedProgress.skipped > 0 ? 'var(--yl)' : 'var(--gn)' }}>{globalPct}%</span>
|
||||
</div>
|
||||
<div className="w-full h-1.5 rounded-full overflow-hidden flex" style={{ background: 'var(--bg2)' }}>
|
||||
{okPct > 0 && <div className="h-full transition-all duration-500" style={{ width: `${okPct}%`, background: 'var(--gn)' }} />}
|
||||
{failPct > 0 && <div className="h-full transition-all duration-500" style={{ width: `${failPct}%`, background: 'var(--rd)' }} />}
|
||||
</div>
|
||||
</div>
|
||||
{/* Queue */}
|
||||
{queue.length > 0 && (
|
||||
<div className="text-[.58rem]" style={{ color: 'var(--t4)' }}>
|
||||
Fila: {queue.map((q) => `${q.table} (${q.docs})`).join(' → ')}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})()}
|
||||
|
||||
Reference in New Issue
Block a user