diff --git a/backend/app.py b/backend/app.py index 3093d85..8d31aea 100644 --- a/backend/app.py +++ b/backend/app.py @@ -3885,36 +3885,44 @@ async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(requi "message": f"Embedding {total_docs} documentos em {len(tables_used)} tabelas..." }) + # Build queue info: [(table, doc_count), ...] + table_queue = [(tbl, len(docs)) for tbl, docs in table_docs.items()] + def _bg_embed_all(): """Background: embed documents into their respective tables sequentially.""" import array default_model = cfg.get("embedding_model_id", "cohere.embed-v4.0") inserted_total = 0 + skipped_total = 0 + processed_total = 0 errors = [] - for tbl, docs in table_docs.items(): + for tbl_idx, (tbl, docs) in enumerate(table_docs.items()): + remaining = table_queue[tbl_idx + 1:] if tbl_idx + 1 < len(table_queue) else [] _auto_register_table(cfg["id"], tbl) - # Purge old data for this tenancy/date before inserting purged = _purge_table_by_tenancy(cfg, tbl, tenancy, extract_date) if purged: - _update_embed_status(task_id, {"message": f"Purged {purged} old docs from {tbl}..."}) - # Auto-detect embedding model based on table dimension + _update_embed_status(task_id, {"message": f"Purged {purged} old docs from {tbl}...", + "current_table": tbl, "current_inserted": 0, "current_total": len(docs), + "queue": [{"table": t, "docs": n} for t, n in remaining]}) emb_model = default_model try: actual_dim = _get_table_embedding_dim(cfg, tbl) if actual_dim and actual_dim in _DIM_TO_MODEL: emb_model = _DIM_TO_MODEL[actual_dim] - log.info(f"Table {tbl}: dim={actual_dim}, using model {emb_model}") - except Exception as e: - log.warning(f"Could not detect dim for {tbl}: {e}") + except Exception: + pass + current_inserted = 0 + current_skipped = 0 try: conn = _get_adb_connection(cfg) cur = conn.cursor() for doc in docs: - processed = 0 try: content = doc.get("content", "") if not content: - processed += 1 + skipped_total += 1 + current_skipped += 1 + processed_total += 1 continue embedding = _embed_text(content, gc, emb_model) vec = array.array('f', [float(x) for x in embedding]) @@ -3926,17 +3934,29 @@ async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(requi cur.execute(f'INSERT INTO "{tbl}" (ID, TEXT, EMBEDDING, METADATA) VALUES (HEXTORAW(:1), :2, :3, :4)', [uuid.uuid4().hex.upper(), content, vec, metadata]) inserted_total += 1 + current_inserted += 1 except Exception as e: - log.warning(f"Embed skip in {tbl}: {str(e)[:120]}") - processed += 1 + err_str = str(e) + if "message" in err_str: + import re as _re + m = _re.search(r'"message"\s*:\s*"(.*?)"', err_str) + log.warning(f"Embed skip in {tbl}: {m.group(1)[:200] if m else err_str[:200]}") + else: + log.warning(f"Embed skip in {tbl}: {err_str[:200]}") + skipped_total += 1 + current_skipped += 1 + processed_total += 1 _update_embed_status(task_id, { - "inserted": inserted_total, - "message": f"Embedding {inserted_total}/{processed} OK — {tbl}" + "inserted": inserted_total, "skipped": skipped_total, "processed": processed_total, + "current_table": tbl, "current_inserted": current_inserted, "current_skipped": current_skipped, + "current_total": len(docs), + "queue": [{"table": t, "docs": n} for t, n in remaining], + "message": f"{tbl}: {current_inserted}/{len(docs)} — global: {inserted_total}/{total_docs}" }) conn.commit() cur.close() conn.close() - log.info(f"Embedded {len(docs)} docs into {tbl} (tenancy={tenancy})") + log.info(f"Embedded {current_inserted}/{len(docs)} docs into {tbl} (tenancy={tenancy})") except Exception as e: log.error(f"Failed to connect/embed to {tbl}: {e}") errors.append(f"{tbl}: {str(e)[:100]}") diff --git a/frontend-react/src/pages/ReportsPage.tsx b/frontend-react/src/pages/ReportsPage.tsx index aa6b5c5..e47ee54 100644 --- a/frontend-react/src/pages/ReportsPage.tsx +++ b/frontend-react/src/pages/ReportsPage.tsx @@ -585,7 +585,11 @@ export default function ReportsPage() { const setEmbedTaskId = store.setRptEmbedTaskId; const [embedLoading, setEmbedLoading] = useState(''); const [embedMsg, setEmbedMsg] = useState<{ type: 's' | 'e'; text: string } | null>(null); - const [embedProgress, setEmbedProgress] = useState<{ inserted: number; skipped: number; processed: number; total: number } | null>(null); + const [embedProgress, setEmbedProgress] = useState<{ + inserted: number; skipped: number; processed: number; total: number; + currentTable?: string; currentInserted?: number; currentTotal?: number; currentSkipped?: number; + queue?: { table: string; docs: number }[]; + } | null>(null); /* ── HTML iframe (from store) ── */ const showIframe = store.rptShowIframe; @@ -797,7 +801,14 @@ export default function ReportsPage() { const processed = (data.processed as number) || inserted; if (total > 0) { - setEmbedProgress({ inserted, skipped, processed, total }); + setEmbedProgress({ + inserted, skipped, processed, total, + currentTable: (data.current_table as string) || undefined, + currentInserted: (data.current_inserted as number) || 0, + currentTotal: (data.current_total as number) || 0, + currentSkipped: (data.current_skipped as number) || 0, + queue: (data.queue as { table: string; docs: number }[]) || [], + }); } if (s.status === 'done') { @@ -1491,29 +1502,51 @@ export default function ReportsPage() { {/* Embed progress + message */} {embedProgress && embedProgress.total > 0 && (() => { - const pct = Math.round((embedProgress.processed / embedProgress.total) * 100); + const globalPct = Math.round((embedProgress.processed / embedProgress.total) * 100); const okPct = Math.round((embedProgress.inserted / embedProgress.total) * 100); const failPct = Math.round((embedProgress.skipped / embedProgress.total) * 100); + const ct = embedProgress.currentTable; + const ci = embedProgress.currentInserted || 0; + const cTotal = embedProgress.currentTotal || 0; + const cPct = cTotal > 0 ? Math.round((ci / cTotal) * 100) : 0; + const queue = embedProgress.queue || []; return ( -
-
- - {embedProgress.inserted} OK - {embedProgress.skipped > 0 && / {embedProgress.skipped} falhas} - {' '}({embedProgress.processed}/{embedProgress.total}) - - 0 ? 'var(--yl)' : 'var(--gn)' }}> - {pct}% - -
-
- {okPct > 0 && ( -
- )} - {failPct > 0 && ( -
- )} +
+ {/* Current section */} + {ct && ( +
+
+ + {ct} ({ci}/{cTotal}) + + {cPct}% +
+
+
+
+
+ )} + {/* Global progress */} +
+
+ + Global: {embedProgress.inserted} OK + {embedProgress.skipped > 0 && / {embedProgress.skipped} falhas} + {' '}({embedProgress.processed}/{embedProgress.total}) + + 0 ? 'var(--yl)' : 'var(--gn)' }}>{globalPct}% +
+
+ {okPct > 0 &&
} + {failPct > 0 &&
} +
+ {/* Queue */} + {queue.length > 0 && ( +
+ Fila: {queue.map((q) => `${q.table} (${q.docs})`).join(' → ')} +
+ )}
); })()}