GustavoNogueira/A-Team-Security-Infra-Agent-Engineer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: embedding reliability - chunking, truncation, progress bar, shared status

Browse Source 
- Text chunking: max 8000 chars per document, large CSV rows split with context header
- Text truncation: _embed_text truncates at 8000 chars before sending to model (prevents 400 errors)
- Purge before re-embed: deletes old docs by tenancy+date before inserting new ones
- Shared embedding status: file-based status (/data/.embed_status/) instead of in-memory dict (fixes multi-worker visibility)
- Progress bar: shows green (OK) + red (failures) segments, percentage, processed/total count
- Better error logging: extracts readable error message from OCI API errors
This commit is contained in:
nogueiraguh
2026-03-21 10:48:21 -03:00
parent 73c60212f7
commit ab2d0a3e9b
2 changed files with 197 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
frontend-react/src/pages/ReportsPage.tsx
View File

@@ -583,6 +583,7 @@ export default function ReportsPage() {
const setEmbedTable = store.setRptEmbedTable;
const [embedLoading, setEmbedLoading] = useState(''); // '' = idle, 'all' = full embed, section name = section embed
const [embedMsg, setEmbedMsg] = useState<{ type: 's' | 'e'; text: string } | null>(null);
const [embedProgress, setEmbedProgress] = useState<{ inserted: number; skipped: number; processed: number; total: number } | null>(null);
/* ── HTML iframe (from store) ── */
const showIframe = store.rptShowIframe;
@@ -783,23 +784,34 @@ export default function ReportsPage() {
const pollEmbeddingStatus = useCallback(async (taskId: string) => {
setEmbedMsg({ type: 's', text: 'Embedding iniciado...' });
setEmbedProgress(null);
const poll = setInterval(async () => {
try {
const s = await reportsApi.embeddingStatus(taskId);
if (s.total) {
setEmbedProgress({
inserted: s.inserted || 0,
skipped: (s as unknown as Record<string, number>).skipped || 0,
processed: (s as unknown as Record<string, number>).processed || s.inserted || 0,
total: s.total,
});
}
if (s.status === 'running') {
setEmbedMsg({ type: 's', text: s.message });
} else if (s.status === 'done') {
clearInterval(poll);
setEmbedMsg({ type: 's', text: s.message });
setEmbedProgress(null);
setEmbedLoading('');
} else if (s.status === 'error') {
clearInterval(poll);
setEmbedMsg({ type: 'e', text: s.message });
setEmbedProgress(null);
setEmbedLoading('');
}
} catch { /* keep polling */ }
}, 2000);
setTimeout(() => { clearInterval(poll); setEmbedLoading(''); }, 300000);
setTimeout(() => { clearInterval(poll); setEmbedLoading(''); setEmbedProgress(null); }, 300000);
}, []);
const handleEmbed = async (rid: string) => {
@@ -1459,7 +1471,34 @@ export default function ReportsPage() {
)}
</div>
{/* Embed message */}
{/* Embed progress + message */}
{embedProgress && embedProgress.total > 0 && (() => {
const pct = Math.round((embedProgress.processed / embedProgress.total) * 100);
const okPct = Math.round((embedProgress.inserted / embedProgress.total) * 100);
const failPct = Math.round((embedProgress.skipped / embedProgress.total) * 100);
return (
<div className="mx-5 mt-3">
<div className="flex items-center justify-between mb-1">
<span className="text-[.65rem] font-medium" style={{ color: 'var(--t2)' }}>
{embedProgress.inserted} OK
{embedProgress.skipped > 0 && <span style={{ color: 'var(--rd)' }}> / {embedProgress.skipped} falhas</span>}
{' '}({embedProgress.processed}/{embedProgress.total})
</span>
<span className="text-[.65rem] font-bold" style={{ color: embedProgress.skipped > 0 ? 'var(--yl)' : 'var(--gn)' }}>
{pct}%
</span>
</div>
<div className="w-full h-2 rounded-full overflow-hidden flex" style={{ background: 'var(--bg2)' }}>
{okPct > 0 && (
<div className="h-full transition-all duration-500" style={{ width: `${okPct}%`, background: 'var(--gn)' }} />
)}
{failPct > 0 && (
<div className="h-full transition-all duration-500" style={{ width: `${failPct}%`, background: 'var(--rd)' }} />
)}
</div>
</div>
);
})()}
{embedMsg && (
<div
className="mx-5 mt-3 flex items-center gap-2 px-3 py-2 rounded-lg text-xs"