feat: detailed embed progress — current section, global bar, queue display
- Backend: tracks current_table, current_inserted, current_total, queue per embed task - Frontend: dual progress bars (section + global), queue of upcoming sections - Section bar shows table name + individual progress - Global bar shows inserted/skipped/total with green/red segments - Queue shows remaining sections with doc counts
This commit is contained in:
@@ -3885,36 +3885,44 @@ async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(requi
|
|||||||
"message": f"Embedding {total_docs} documentos em {len(tables_used)} tabelas..."
|
"message": f"Embedding {total_docs} documentos em {len(tables_used)} tabelas..."
|
||||||
})
|
})
|
||||||
|
|
||||||
|
# Build queue info: [(table, doc_count), ...]
|
||||||
|
table_queue = [(tbl, len(docs)) for tbl, docs in table_docs.items()]
|
||||||
|
|
||||||
def _bg_embed_all():
|
def _bg_embed_all():
|
||||||
"""Background: embed documents into their respective tables sequentially."""
|
"""Background: embed documents into their respective tables sequentially."""
|
||||||
import array
|
import array
|
||||||
default_model = cfg.get("embedding_model_id", "cohere.embed-v4.0")
|
default_model = cfg.get("embedding_model_id", "cohere.embed-v4.0")
|
||||||
inserted_total = 0
|
inserted_total = 0
|
||||||
|
skipped_total = 0
|
||||||
|
processed_total = 0
|
||||||
errors = []
|
errors = []
|
||||||
for tbl, docs in table_docs.items():
|
for tbl_idx, (tbl, docs) in enumerate(table_docs.items()):
|
||||||
|
remaining = table_queue[tbl_idx + 1:] if tbl_idx + 1 < len(table_queue) else []
|
||||||
_auto_register_table(cfg["id"], tbl)
|
_auto_register_table(cfg["id"], tbl)
|
||||||
# Purge old data for this tenancy/date before inserting
|
|
||||||
purged = _purge_table_by_tenancy(cfg, tbl, tenancy, extract_date)
|
purged = _purge_table_by_tenancy(cfg, tbl, tenancy, extract_date)
|
||||||
if purged:
|
if purged:
|
||||||
_update_embed_status(task_id, {"message": f"Purged {purged} old docs from {tbl}..."})
|
_update_embed_status(task_id, {"message": f"Purged {purged} old docs from {tbl}...",
|
||||||
# Auto-detect embedding model based on table dimension
|
"current_table": tbl, "current_inserted": 0, "current_total": len(docs),
|
||||||
|
"queue": [{"table": t, "docs": n} for t, n in remaining]})
|
||||||
emb_model = default_model
|
emb_model = default_model
|
||||||
try:
|
try:
|
||||||
actual_dim = _get_table_embedding_dim(cfg, tbl)
|
actual_dim = _get_table_embedding_dim(cfg, tbl)
|
||||||
if actual_dim and actual_dim in _DIM_TO_MODEL:
|
if actual_dim and actual_dim in _DIM_TO_MODEL:
|
||||||
emb_model = _DIM_TO_MODEL[actual_dim]
|
emb_model = _DIM_TO_MODEL[actual_dim]
|
||||||
log.info(f"Table {tbl}: dim={actual_dim}, using model {emb_model}")
|
except Exception:
|
||||||
except Exception as e:
|
pass
|
||||||
log.warning(f"Could not detect dim for {tbl}: {e}")
|
current_inserted = 0
|
||||||
|
current_skipped = 0
|
||||||
try:
|
try:
|
||||||
conn = _get_adb_connection(cfg)
|
conn = _get_adb_connection(cfg)
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
for doc in docs:
|
for doc in docs:
|
||||||
processed = 0
|
|
||||||
try:
|
try:
|
||||||
content = doc.get("content", "")
|
content = doc.get("content", "")
|
||||||
if not content:
|
if not content:
|
||||||
processed += 1
|
skipped_total += 1
|
||||||
|
current_skipped += 1
|
||||||
|
processed_total += 1
|
||||||
continue
|
continue
|
||||||
embedding = _embed_text(content, gc, emb_model)
|
embedding = _embed_text(content, gc, emb_model)
|
||||||
vec = array.array('f', [float(x) for x in embedding])
|
vec = array.array('f', [float(x) for x in embedding])
|
||||||
@@ -3926,17 +3934,29 @@ async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(requi
|
|||||||
cur.execute(f'INSERT INTO "{tbl}" (ID, TEXT, EMBEDDING, METADATA) VALUES (HEXTORAW(:1), :2, :3, :4)',
|
cur.execute(f'INSERT INTO "{tbl}" (ID, TEXT, EMBEDDING, METADATA) VALUES (HEXTORAW(:1), :2, :3, :4)',
|
||||||
[uuid.uuid4().hex.upper(), content, vec, metadata])
|
[uuid.uuid4().hex.upper(), content, vec, metadata])
|
||||||
inserted_total += 1
|
inserted_total += 1
|
||||||
|
current_inserted += 1
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.warning(f"Embed skip in {tbl}: {str(e)[:120]}")
|
err_str = str(e)
|
||||||
processed += 1
|
if "message" in err_str:
|
||||||
|
import re as _re
|
||||||
|
m = _re.search(r'"message"\s*:\s*"(.*?)"', err_str)
|
||||||
|
log.warning(f"Embed skip in {tbl}: {m.group(1)[:200] if m else err_str[:200]}")
|
||||||
|
else:
|
||||||
|
log.warning(f"Embed skip in {tbl}: {err_str[:200]}")
|
||||||
|
skipped_total += 1
|
||||||
|
current_skipped += 1
|
||||||
|
processed_total += 1
|
||||||
_update_embed_status(task_id, {
|
_update_embed_status(task_id, {
|
||||||
"inserted": inserted_total,
|
"inserted": inserted_total, "skipped": skipped_total, "processed": processed_total,
|
||||||
"message": f"Embedding {inserted_total}/{processed} OK — {tbl}"
|
"current_table": tbl, "current_inserted": current_inserted, "current_skipped": current_skipped,
|
||||||
|
"current_total": len(docs),
|
||||||
|
"queue": [{"table": t, "docs": n} for t, n in remaining],
|
||||||
|
"message": f"{tbl}: {current_inserted}/{len(docs)} — global: {inserted_total}/{total_docs}"
|
||||||
})
|
})
|
||||||
conn.commit()
|
conn.commit()
|
||||||
cur.close()
|
cur.close()
|
||||||
conn.close()
|
conn.close()
|
||||||
log.info(f"Embedded {len(docs)} docs into {tbl} (tenancy={tenancy})")
|
log.info(f"Embedded {current_inserted}/{len(docs)} docs into {tbl} (tenancy={tenancy})")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.error(f"Failed to connect/embed to {tbl}: {e}")
|
log.error(f"Failed to connect/embed to {tbl}: {e}")
|
||||||
errors.append(f"{tbl}: {str(e)[:100]}")
|
errors.append(f"{tbl}: {str(e)[:100]}")
|
||||||
|
|||||||
@@ -585,7 +585,11 @@ export default function ReportsPage() {
|
|||||||
const setEmbedTaskId = store.setRptEmbedTaskId;
|
const setEmbedTaskId = store.setRptEmbedTaskId;
|
||||||
const [embedLoading, setEmbedLoading] = useState('');
|
const [embedLoading, setEmbedLoading] = useState('');
|
||||||
const [embedMsg, setEmbedMsg] = useState<{ type: 's' | 'e'; text: string } | null>(null);
|
const [embedMsg, setEmbedMsg] = useState<{ type: 's' | 'e'; text: string } | null>(null);
|
||||||
const [embedProgress, setEmbedProgress] = useState<{ inserted: number; skipped: number; processed: number; total: number } | null>(null);
|
const [embedProgress, setEmbedProgress] = useState<{
|
||||||
|
inserted: number; skipped: number; processed: number; total: number;
|
||||||
|
currentTable?: string; currentInserted?: number; currentTotal?: number; currentSkipped?: number;
|
||||||
|
queue?: { table: string; docs: number }[];
|
||||||
|
} | null>(null);
|
||||||
|
|
||||||
/* ── HTML iframe (from store) ── */
|
/* ── HTML iframe (from store) ── */
|
||||||
const showIframe = store.rptShowIframe;
|
const showIframe = store.rptShowIframe;
|
||||||
@@ -797,7 +801,14 @@ export default function ReportsPage() {
|
|||||||
const processed = (data.processed as number) || inserted;
|
const processed = (data.processed as number) || inserted;
|
||||||
|
|
||||||
if (total > 0) {
|
if (total > 0) {
|
||||||
setEmbedProgress({ inserted, skipped, processed, total });
|
setEmbedProgress({
|
||||||
|
inserted, skipped, processed, total,
|
||||||
|
currentTable: (data.current_table as string) || undefined,
|
||||||
|
currentInserted: (data.current_inserted as number) || 0,
|
||||||
|
currentTotal: (data.current_total as number) || 0,
|
||||||
|
currentSkipped: (data.current_skipped as number) || 0,
|
||||||
|
queue: (data.queue as { table: string; docs: number }[]) || [],
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if (s.status === 'done') {
|
if (s.status === 'done') {
|
||||||
@@ -1491,30 +1502,52 @@ export default function ReportsPage() {
|
|||||||
|
|
||||||
{/* Embed progress + message */}
|
{/* Embed progress + message */}
|
||||||
{embedProgress && embedProgress.total > 0 && (() => {
|
{embedProgress && embedProgress.total > 0 && (() => {
|
||||||
const pct = Math.round((embedProgress.processed / embedProgress.total) * 100);
|
const globalPct = Math.round((embedProgress.processed / embedProgress.total) * 100);
|
||||||
const okPct = Math.round((embedProgress.inserted / embedProgress.total) * 100);
|
const okPct = Math.round((embedProgress.inserted / embedProgress.total) * 100);
|
||||||
const failPct = Math.round((embedProgress.skipped / embedProgress.total) * 100);
|
const failPct = Math.round((embedProgress.skipped / embedProgress.total) * 100);
|
||||||
|
const ct = embedProgress.currentTable;
|
||||||
|
const ci = embedProgress.currentInserted || 0;
|
||||||
|
const cTotal = embedProgress.currentTotal || 0;
|
||||||
|
const cPct = cTotal > 0 ? Math.round((ci / cTotal) * 100) : 0;
|
||||||
|
const queue = embedProgress.queue || [];
|
||||||
return (
|
return (
|
||||||
<div className="mx-5 mt-3">
|
<div className="mx-5 mt-3 flex flex-col gap-2">
|
||||||
|
{/* Current section */}
|
||||||
|
{ct && (
|
||||||
|
<div>
|
||||||
<div className="flex items-center justify-between mb-1">
|
<div className="flex items-center justify-between mb-1">
|
||||||
<span className="text-[.65rem] font-medium" style={{ color: 'var(--t2)' }}>
|
<span className="text-[.65rem] font-semibold" style={{ color: 'var(--t1)' }}>
|
||||||
{embedProgress.inserted} OK
|
{ct} <span style={{ color: 'var(--t4)', fontWeight: 400 }}>({ci}/{cTotal})</span>
|
||||||
|
</span>
|
||||||
|
<span className="text-[.65rem] font-bold" style={{ color: 'var(--ac)' }}>{cPct}%</span>
|
||||||
|
</div>
|
||||||
|
<div className="w-full h-1.5 rounded-full overflow-hidden" style={{ background: 'var(--bg2)' }}>
|
||||||
|
<div className="h-full rounded-full transition-all duration-500" style={{ width: `${cPct}%`, background: 'var(--ac)' }} />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{/* Global progress */}
|
||||||
|
<div>
|
||||||
|
<div className="flex items-center justify-between mb-1">
|
||||||
|
<span className="text-[.6rem] font-medium" style={{ color: 'var(--t3)' }}>
|
||||||
|
Global: {embedProgress.inserted} OK
|
||||||
{embedProgress.skipped > 0 && <span style={{ color: 'var(--rd)' }}> / {embedProgress.skipped} falhas</span>}
|
{embedProgress.skipped > 0 && <span style={{ color: 'var(--rd)' }}> / {embedProgress.skipped} falhas</span>}
|
||||||
{' '}({embedProgress.processed}/{embedProgress.total})
|
{' '}({embedProgress.processed}/{embedProgress.total})
|
||||||
</span>
|
</span>
|
||||||
<span className="text-[.65rem] font-bold" style={{ color: embedProgress.skipped > 0 ? 'var(--yl)' : 'var(--gn)' }}>
|
<span className="text-[.6rem] font-bold" style={{ color: embedProgress.skipped > 0 ? 'var(--yl)' : 'var(--gn)' }}>{globalPct}%</span>
|
||||||
{pct}%
|
|
||||||
</span>
|
|
||||||
</div>
|
</div>
|
||||||
<div className="w-full h-2 rounded-full overflow-hidden flex" style={{ background: 'var(--bg2)' }}>
|
<div className="w-full h-1.5 rounded-full overflow-hidden flex" style={{ background: 'var(--bg2)' }}>
|
||||||
{okPct > 0 && (
|
{okPct > 0 && <div className="h-full transition-all duration-500" style={{ width: `${okPct}%`, background: 'var(--gn)' }} />}
|
||||||
<div className="h-full transition-all duration-500" style={{ width: `${okPct}%`, background: 'var(--gn)' }} />
|
{failPct > 0 && <div className="h-full transition-all duration-500" style={{ width: `${failPct}%`, background: 'var(--rd)' }} />}
|
||||||
)}
|
|
||||||
{failPct > 0 && (
|
|
||||||
<div className="h-full transition-all duration-500" style={{ width: `${failPct}%`, background: 'var(--rd)' }} />
|
|
||||||
)}
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
{/* Queue */}
|
||||||
|
{queue.length > 0 && (
|
||||||
|
<div className="text-[.58rem]" style={{ color: 'var(--t4)' }}>
|
||||||
|
Fila: {queue.map((q) => `${q.table} (${q.docs})`).join(' → ')}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
);
|
);
|
||||||
})()}
|
})()}
|
||||||
{embedMsg && (
|
{embedMsg && (
|
||||||
|
|||||||
Reference in New Issue
Block a user