From 1b3c02d10bbae6b91987cdcec41f2c317d60ebae Mon Sep 17 00:00:00 2001 From: nogueiraguh Date: Tue, 31 Mar 2026 10:44:13 -0300 Subject: [PATCH] =?UTF-8?q?feat:=20v3.0=20=E2=80=94=20CIS=20PDF=20chunker,?= =?UTF-8?q?=20DOCX=20professional,=20RAG=20complete,=20progress=20bar,=20c?= =?UTF-8?q?ode=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CIS PDF chunker: segments by recommendation (54/54 complete with desc+rationale+rem), 7000 chars target, 500 overlap - RAG: 60s timeout + retry, direct SQL fetch by recommendationNumber, concatenate all chunks - Auto-detect embedding dimension from DDL (empty tables), auto-detect model per dimension - DOCX: camo strip (Pillow), A4 centered cover, code blocks (Consolas), lists with hanging indent - Compliance report: progress bar (RAG X/35 percentage), Audit/Verification bold - Code cleanup: removed redundant imports (io, base64, time, Response, HTMLResponse), duplicate DB query - EmbeddingsPage: auto-select ADB config fix - ReportsPage: removed DOCX preview (docx-preview lib) - README updated to v3.0 with all new features --- README.md | 232 ++--- backend/app.py | 852 ++++++++++++++---- frontend-react/src/pages/ReportsPage.tsx | 41 +- .../src/pages/config/EmbeddingsPage.tsx | 11 + 4 files changed, 852 insertions(+), 284 deletions(-) diff --git a/README.md b/README.md index 9518107..c9ff818 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@

- Version + Version Python FastAPI OCI @@ -29,67 +29,57 @@ The platform combines security compliance scanning, AI-powered chat with **RAG ( ## Features -### πŸ€– AI Chat Agent with RAG + MCP Tool Use +### AI Chat Agent with RAG + MCP Tool Use - **OCI Generative AI** integration via official SDK (`oci.generative_ai_inference`) -- **RAG (Retrieval-Augmented Generation)**: automatically queries ADB vector store for relevant context before generating responses β€” single ADB connection per search, smart table skip (only queries relevant tables based on question topic), CIS recommendation text filter for exact matching, follow-up context enrichment for short questions +- **RAG (Retrieval-Augmented Generation)**: queries ADB vector store for relevant context before generating responses β€” single ADB connection per search, smart table skip, CIS recommendation text filter for exact matching, follow-up context enrichment - **Source hierarchy**: findings tables > cisrecom (official remediation) > engineerknowledgebase (complementary) β€” temporal awareness with extract dates, tenancy isolation via JSON_VALUE filter -- **MCP Tool Use (Function Calling)**: GenAI models can call tools from registered MCP servers during chat β€” supports both Cohere and Generic (OpenAI-style) function calling formats with automatic tool execution loop (max 5 iterations) -- **Chat Memory Compaction**: automatic summarization of older messages when conversation exceeds ~6000 tokens β€” keeps 20 recent messages (10 interactions) intact and generates an LLM-based summary of older context -- **Chat Sidebar**: collapsible right-side panel with model parameters (temperature, max_tokens, top_p, top_k, penalties) for quick adjustment during conversation -- **Multimodal Chat**: upload images (PNG/JPG/GIF/WebP), PDFs, and text files directly in the chat for AI analysis β€” supports up to 5 files per message via OCI GenAI `ImageContent` and `DocumentContent` -- **Async Background Processing**: chat requests return immediately, GenAI + MCP tools process in background via dedicated thread pool (10 threads), frontend polls for results β€” eliminates 504 timeouts on long-running scans -- **Thinking Indicator**: button disables and shows spinner + "Pensando..." while waiting for GenAI response, with message timestamps +- **MCP Tool Use (Function Calling)**: GenAI models call tools from registered MCP servers during chat β€” supports Cohere and Generic (OpenAI-style) function calling with automatic tool execution loop (max 5 iterations) +- **Chat Memory Compaction**: automatic summarization of older messages when conversation exceeds ~6000 tokens β€” keeps 20 recent messages intact +- **Chat Sidebar**: collapsible panel with model parameters (temperature, max_tokens, top_p, top_k, penalties) +- **Multimodal Chat**: upload images (PNG/JPG/GIF/WebP), PDFs, and text files for AI analysis β€” up to 5 files per message via OCI GenAI `ImageContent` and `DocumentContent` +- **Async Background Processing**: chat requests return immediately, GenAI + MCP tools process in background via dedicated thread pool (10 threads), frontend polls for results - 16 chat models + 3 embedding models across 5 providers: **Meta** (Llama 4), **Google** (Gemini 2.5), **OpenAI** (GPT-5.2/5.1/5 Mini/4.1/4o, o3/o4-mini), **xAI** (Grok 4/3) -- OCID-based model resolution: catalog maps model IDs to OCI resource IDs per region for reliable API calls +- OCID-based model resolution: catalog maps model IDs to OCI resource IDs per region - 16 OCI regions supported with auto-generated endpoints -- **Provider parameter matrix**: automatic parameter segmentation per model β€” reasoning models (o3/o4-mini) only get `max_completion_tokens` + `reasoning_effort`, OpenAI GPT-4.x gets freq/pres penalties, Google/Meta get `top_k`, xAI gets basic params. Unsupported parameters are explicitly nulled to prevent SDK serialization errors -- Full parameter control: temperature, max_tokens, top_p, top_k, frequency/presence penalty (when supported by model) +- **Provider parameter matrix**: automatic parameter segmentation per model β€” reasoning models (o3/o4-mini) only get `max_completion_tokens` + `reasoning_effort`, OpenAI GPT-4.x gets freq/pres penalties, Google/Meta get `top_k` +- Full parameter control: temperature, max_tokens, top_p, top_k, frequency/presence penalty (when supported) - Toggle MCP tools on/off per chat session - Conversation history with session management - On-Demand and Dedicated serving modes -### πŸ—οΈ Terraform Agent (IaC) +### Terraform Agent (IaC) - **AI-powered Terraform code generation** for OCI infrastructure provisioning - Dedicated chat interface with OCI Terraform provider context - **Workspace management**: create, plan, apply, destroy Terraform workspaces -- **Compartment-aware**: browse and select target compartment for provisioning, with existing OCI resources injected as context +- **Compartment-aware**: browse and select target compartment, with existing OCI resources injected as context - **Plan/Apply/Destroy lifecycle**: full Terraform workflow with real-time output tracking - **File management**: view generated `.tf` files, copy, download individually or as bundle -- **Confirmation modal**: type "DESTROY" to confirm resource destruction -- **Inline file editor**: click any generated `.tf` file to edit in-place with monospace editor, Tab support, and save β€” auto-syncs with workspace code -- **Split-panel terminal**: real-time Plan/Apply/Destroy output in a dark terminal panel alongside files/plan/resources (always visible) +- **Inline file editor**: click any `.tf` file to edit in-place with monospace editor, Tab support, and save +- **Split-panel terminal**: real-time Plan/Apply/Destroy output in a dark terminal panel alongside files/plan/resources - **Multi-file generation**: AI generates separate `.tf` files via `// filename:` markers (main.tf, variables.tf, outputs.tf, etc.) -- **Smart file correction**: when fixing errors, model generates only changed files β€” frontend merges with existing workspace, preserving untouched files. Existing files, plan output, and apply errors are injected as context automatically -- **14-point validation checklist**: system prompt enforces cross-references, CIDRs, security lists, route tables, DRG attachments, RPC peering, HCL block syntax, resource type validation, and variable declarations -- **Auto-split pipeline (3 layers)**: frontend + backend auto-split of monolithic HCL files into categorized files (networking.tf, compute.tf, etc.) + backend deduplication to prevent duplicate resource errors -- **HCL syntax fix**: automatic expansion of single-line blocks with multiple arguments into valid multi-line HCL format -- **Resource type validation**: SQLite-based validation of ~937 OCI resource types with `difflib.get_close_matches` suggestions for invalid types β€” warnings appended automatically to model responses +- **Smart file correction**: when fixing errors, model generates only changed files β€” frontend merges with existing workspace, preserving untouched files +- **14-point validation checklist**: system prompt enforces cross-references, CIDRs, security lists, route tables, DRG attachments, RPC peering, HCL syntax, resource type validation, and variable declarations +- **Auto-split pipeline (3 layers)**: frontend + backend auto-split of monolithic HCL files into categorized files + backend deduplication +- **HCL syntax fix**: automatic expansion of single-line blocks into valid multi-line HCL format +- **Resource type validation**: SQLite-based validation of ~937 OCI resource types with `difflib.get_close_matches` suggestions - **Chat history**: ChatGPT-style session history with rename/delete, shared between Chat Agent and Terraform Agent - **Resizable split view**: chat on top (50%), files/plan/resources + terminal on bottom (50%) with drag-to-resize - Terraform CLI installed in container (v1.14.7) -- **Terraform Resource Reference**: auto-generated OCI provider resource catalog (~937 resources) injected into system prompt β€” built from `terraform providers schema -json` at container startup, stored in SQLite, refreshable via menu button -- **Official Resource Docs Injection**: on-demand fetch of Example Usage + Argument Reference from the official OCI Terraform provider docs (registry.terraform.io) β€” cached in SQLite, injected into model context based on detected resource types -- **Smart variable resolution**: auto-generates `terraform.tfvars` by scanning declared variables and mapping them to OCI config values (`tenancy_ocid`, `user_ocid`, `fingerprint`, `compartment_id`, `ssh_public_key`) β€” region is never forced, respecting variable defaults -- **Multi-region provider management**: auto-detects provider aliases and region variables in generated code, generates `provider.tf` with credentials and correct region references (`var.region` instead of hardcoded values) -- **Region safety guard**: plan is blocked if the model does not declare `variable "region"` β€” prevents accidental provisioning in the wrong region by never falling back to hardcoded defaults -- **Rollback button**: when `terraform apply` fails, a Rollback button appears allowing manual `terraform destroy` to clean up partially created resources (requires typing "ROLLBACK" to confirm) -- **System prompt sync**: Terraform system prompt in code is automatically synced to DB on every restart β€” single source of truth +- **Terraform Resource Reference**: auto-generated OCI provider resource catalog (~937 resources) injected into system prompt β€” built from `terraform providers schema -json`, stored in SQLite, refreshable via UI +- **Official Resource Docs Injection**: on-demand fetch of Example Usage + Argument Reference from the official OCI Terraform provider docs β€” cached in SQLite +- **Smart variable resolution**: auto-generates `terraform.tfvars` by scanning declared variables and mapping them to OCI config values +- **Multi-region provider management**: auto-detects provider aliases and region variables, generates `provider.tf` with correct region references +- **Region safety guard**: plan is blocked if the model does not declare `variable "region"` β€” prevents accidental provisioning in the wrong region +- **Rollback button**: when `terraform apply` fails, a Rollback button appears for `terraform destroy` with confirmation +- **Prompt Generator**: dedicated sub-menu for AI-powered prompt generation β€” curated models, same knowledge pipeline as Terraform Agent, official docs injection, copy & send to Terraform Agent +- **Prompt Editor**: inline system prompt editor with auto-save (800ms debounce) - Compact system prompt optimized for OCI Terraform provider best practices (100K max_tokens) -- **Flat workspace enforcement**: system prompt prohibits `module` blocks (workspace has no subdirectories) and enforces single-file variable declarations (`variables.tf` only) -### πŸ“ Terraform Prompt Generator -- **AI-powered prompt generation** for OCI infrastructure β€” generates structured, optimized prompts to feed into the Terraform Agent -- Dedicated chat interface (sub-menu under Terraform) matching Terraform Agent layout -- **Curated model selection**: only 7 recommended models (GPT-4.1, o3, o4-mini, GPT-5.1, GPT-5.2, Gemini 2.5 Pro, Gemini 2.5 Flash), grouped by provider -- **Same knowledge pipeline as Terraform Agent**: compact resource reference + auto-detected resource types + official docs from GitHub (cached in SQLite) -- **Official Terraform docs injection**: detects resource types from user message (keywords + explicit `oci_*` mentions), fetches Example Usage + Argument Reference from GitHub, cached in `tf_resource_docs` table -- **Conversation history**: persistent sessions (agent_type `tf-prompt`) with sidebar history panel, rename/delete, session restore -- **Contextual follow-up**: conversation history sent to GenAI for iterative prompt refinement -- **Copy & Send**: action buttons on assistant messages to copy the prompt or send directly to the Terraform Agent input -- Robust system prompt with OCI Provider constraints, inference rules, architecture templates, and workspace restrictions -- Example prompts as clickable buttons for quick start +### OCI Services +- **Service Status tab**: auto-detect 6 security services (VSS, Data Safe, Cloud Guard, Bastion, Security Zones, Vault) per tenancy via OCI API with user overrides +- **OCI Health tab**: real-time Oracle service health from ocistatus.oraclecloud.com β€” 49 regions, search, geographic filters -### ⚑ OCI Resource Actions +### OCI Resource Actions - **Start/Stop Compute Instances** directly from OCI Account Explorer with one click - **Start/Stop Autonomous Databases** from Explorer with confirmation prompt - **Start/Stop DB Systems, MySQL, Container Instances** β€” 5 resource types with operational control @@ -99,8 +89,8 @@ The platform combines security compliance scanning, AI-powered chat with **RAG ( - Lifecycle state badges (RUNNING/STOPPED/AVAILABLE) with color indicators - Actions audited in the audit log -### πŸ” OCI Account Explorer -- **KPI stats bar**: real-time resource counts per category with tooltip breakdown, cached in SQLite for instant loading +### OCI Account Explorer +- **KPI stats bar**: real-time resource counts per category with tooltip breakdown, cached in SQLite - **Tree-view navigation** with resizable side panel (280px default) and drag handle - **36 resource types** across 9 categories: - **Compute**: Instances, Boot Volumes, Instance Pools @@ -118,128 +108,124 @@ The platform combines security compliance scanning, AI-powered chat with **RAG ( - **Dead state filtering**: automatically hides TERMINATED/DELETED/DELETING resources - **Background count refresh**: counts cached in SQLite, refreshed on compartment access without blocking UI - **Security list detail**: view ingress/egress rules directly in the explorer -- Select which OCI connection to explore - Real-time API calls via OCI Python SDK -### πŸ“Š CIS Compliance Reports (Oracle Official Engine) +### CIS Compliance Reports (Oracle Official Engine) - Powered by Oracle's official `cis_reports.py` (6660 lines, 48 CIS + 11 OBP checks) - **Granular execution parameters**: CIS Level (1/2), OCI Best Practices, Raw Data, OCID Redaction - - **Level 1**: Essential security controls that can be implemented with minimal impact on operations. Recommended as baseline for all organizations. - - **Level 2**: Advanced security controls that may restrict functionality or require more effort to implement. Recommended for high-security environments. - **Multiple output formats**: HTML summary, CSV per section/finding, JSON summary, optional XLSX -- **Report history**: full execution history with status, tenancy filter, and download actions (HTML/JSON) in the Reports tab -- **Delete reports**: remove completed reports and their associated files from the system -- **File browser**: files grouped by section (Summary, CIS Findings, OBP, Raw Data, etc.) within the Reports page, with per-section embedding -- **Embed individual files**: embed any report file (CSV, TXT, JSON, PDF) directly into the ADB vector store for RAG enrichment +- **Report history**: full execution history with status, tenancy filter, and download actions +- **Delete reports**: remove completed reports and their associated files +- **File browser**: files grouped by section within the Reports page, with per-section embedding +- **Embed individual files**: embed any report file (CSV, TXT, JSON, PDF) into the ADB vector store for RAG enrichment - Region filtering with multi-select - Real-time progress tracking with phase-based progress bar -- **CIS Engine auto-update**: check for new versions of `cis_reports.py` from Oracle's GitHub repository and update with one click (admin only). Custom patches are automatically reapplied after update +- **CIS Engine auto-update**: check for new versions from Oracle's GitHub and update with one click (admin only) -### πŸ“‹ LAD A-Team CIS Compliance Report +### LAD A-Team CIS Compliance Report - **Professional compliance report** following Oracle Cloud Security Assessment format (cover page, purpose statement, disclaimer, copyright) - **Table of Contents** with links to each section and recommendation - **Security Overview**: 7 Oracle security pillars (Customer Isolation, Data Encryption, Data Management, etc.) +- **OCI Services section**: summary table of 6 security services per tenancy + detailed descriptions + Cloud Guard recommendations table - **CIS Assessment Summary**: table with Domains / Total Controls / Failed / Passed per section - **CIS Benchmark Recommendation Summary**: full table with Item / Description / Status (OK/FAILED) - **Detailed findings**: per-recommendation cards with compliance percentage bar, result description, and remediation steps -- **RAG-powered remediation**: enriched remediation steps fetched from ADB vector store (`CISRECOM` table) using cosine similarity search β€” filters by exact recommendation number, extracts only remediation section from matched chunks -- **Affected resources CSV links**: non-compliant findings show a clickable link to download the CSV with affected resource details (authenticated via JWT token) -- **Download PDF**: `window.print()` with CSS `@page A4` formatting for professional PDF output -- **Caching**: generated HTML cached to disk, subsequent views are instant; "Regenerar" button clears cache and re-generates +- **RAG-powered remediation**: enriched remediation steps from ADB vector store (`CISRECOM` table) using cosine similarity β€” filters by exact recommendation number +- **Affected resources CSV links**: non-compliant findings show a clickable link to download the CSV with affected resource details (JWT auth) +- **Back page**: Connect with us + copyright +- **Download PDF**: `window.print()` with CSS `@page A4` formatting +- **Download DOCX**: standalone DOCX export with Pillow camouflage strip, green header tables, result boxes, code blocks, A4 format +- **compliance_data.json**: shared data source between HTML and DOCX (identical content) +- **Progress bar during generation**: shows RAG progress (X/35), percentage, and current recommendation +- **ADB connection pre-check**: fail fast with 503 if ADB is unreachable before starting generation +- **Caching**: generated HTML cached to disk; "Regenerar" button clears cache and re-generates - **Collapsible sections**: HTML report and compliance report start minimized, expandable on demand -### πŸ›‘οΈ Built-in CIS MCP Server (Granular Per-Section) +### Built-in CIS MCP Server (Granular Per-Section) - **Auto-registered** CIS Compliance Scanner MCP server β€” available out of the box - **12 granular tools** instead of monolithic full-tenancy scan: - - `cis_scan_iam` β€” IAM checks (CIS 1.1–1.17): users, policies, groups, MFA, API keys - - `cis_scan_networking` β€” Network checks (CIS 2.1–2.8): security lists, NSGs, VCNs - - `cis_scan_compute` β€” Compute checks (CIS 3.1–3.3): instance metadata, monitoring - - `cis_scan_logging_monitoring` β€” Logging/Monitoring checks (CIS 4.1–4.17): audit, alarms, events - - `cis_scan_storage` β€” Storage checks (CIS 5.1–5.3): buckets, block volumes, file systems - - `cis_scan_asset_management` β€” Asset checks (CIS 6.1–6.2): compartments, tagging + - `cis_scan_iam` β€” IAM checks (CIS 1.1-1.17): users, policies, groups, MFA, API keys + - `cis_scan_networking` β€” Network checks (CIS 2.1-2.8): security lists, NSGs, VCNs + - `cis_scan_compute` β€” Compute checks (CIS 3.1-3.3): instance metadata, monitoring + - `cis_scan_logging_monitoring` β€” Logging/Monitoring checks (CIS 4.1-4.17): audit, alarms, events + - `cis_scan_storage` β€” Storage checks (CIS 5.1-5.3): buckets, block volumes, file systems + - `cis_scan_asset_management` β€” Asset checks (CIS 6.1-6.2): compartments, tagging - `cis_list_configs` / `cis_list_checks` β€” list available OCI configs and CIS checks - `cis_get_check` / `cis_get_remediation` β€” detailed findings and remediation guidance - `cis_get_scan_status` / `cis_invalidate_cache` β€” session status and cache management -- **Per-section data collection**: each scan tool collects only the OCI data needed for that section, avoiding unnecessary API calls -- **Region-specific scanning**: all scan tools accept optional `regions` parameter to target specific OCI regions (e.g., `["us-ashburn-1"]`) instead of scanning the entire tenancy -- **Session caching**: collected data is cached per config+regions scope (2-hour TTL), so subsequent scans on different sections reuse shared prerequisites (compartments, identity domains) -- **Compartment filtering**: scan results can be filtered by specific compartment, showing only relevant findings -- **Collection error tracking**: errors during data collection are captured per section and reported to the caller -- **Parallelized data collection**: base collectors and regional collectors run in parallel thread pools (up to 8 workers), with 30-minute timeout per tool call and 1 automatic retry on failure -- Based on Oracle's official `cis_reports.py` (6660 lines, 48 CIS + 11 OBP checks) +- **Per-section data collection**: each scan tool collects only the OCI data needed for that section +- **Region-specific scanning**: all scan tools accept optional `regions` parameter to target specific OCI regions +- **Session caching**: collected data cached per config+regions scope (2-hour TTL) +- **Compartment filtering**: scan results can be filtered by specific compartment +- **Parallelized data collection**: base and regional collectors run in parallel (up to 8 workers), 30-minute timeout with 1 automatic retry -### πŸ”Œ MCP Server Registry + Tool Discovery +### MCP Server Registry + Tool Discovery - Register multiple MCP servers (stdio, SSE, Python module) - **Automatic tool discovery**: connect to MCP servers and discover available tools with names, descriptions, and input schemas - **Manual tool definition**: add/edit tools with JSON Schema parameter definitions -- **Chat Agent integration**: discovered tools are automatically available as GenAI function calls during chat +- **Chat Agent integration**: discovered tools are automatically available as GenAI function calls - Upload `.py` scripts directly to servers - **Link MCP servers to ADB Vector** databases as tools - Activate/deactivate servers -- Select which MCP server to use per report execution -### πŸ—„οΈ Autonomous Database Vector Storage +### Autonomous Database Vector Storage - Oracle Autonomous Database connection with **mTLS Wallet** authentication - `python-oracledb` Thin mode (no Oracle Client needed) - Wallet ZIP upload and automatic extraction - Connection testing with **case-insensitive table validation** against ADB `user_tables` -- **Multiple vector tables per ADB**: register, edit, toggle active/inactive β€” tables managed inside the edit connection card +- **Multiple vector tables per ADB**: register, edit, toggle active/inactive - **Multi-table RAG search**: queries all active tables across all ADB configs, merges results by cosine distance -- **Auto-resolve embedding config**: automatically uses existing OCI credentials for GenAI embeddings β€” no separate GenAI config required -- **Case-insensitive table matching**: supports both uppercase and lowercase table names in Oracle ADB with quoted identifiers +- **Auto-resolve embedding config**: automatically uses existing OCI credentials for GenAI embeddings +- **Case-insensitive table matching**: supports both uppercase and lowercase table names with quoted identifiers -### 🧬 Embeddings & Knowledge Base -- **Auto-embed CIS Reports**: one-click embedding of all report CSVs β€” each CSV automatically mapped to its ADB table (identityandaccess, networking, computeinstances, etc.) -- **Per-section embedding**: embed individual sections (Identity, Networking, Compute, etc.) with dedicated button per section -- **CIS Recommendation tagging**: each embedded document includes CIS recommendation number, section, and status in the content for precise search -- **Auto-detect embedding dimension**: detects table dimension (1536/3072) and selects correct model automatically -- **Purge before re-embed**: automatically deletes old data for same tenancy/date before inserting new embeddings +### Embeddings & Knowledge Base +- **Auto-embed CIS Reports**: one-click embedding of all report CSVs β€” each CSV automatically mapped to its ADB table +- **Per-section embedding**: embed individual sections with dedicated button per section +- **CIS Recommendation tagging**: each embedded document includes CIS recommendation number, section, and status +- **Auto-detect embedding dimension**: detects table dimension (1536/3072) from DDL (even on empty tables) and selects correct model automatically (1536 = small, 3072 = large) +- **Purge before re-embed**: automatically deletes old data for same tenancy/date before inserting - **Real-time progress**: progress bar with current section, global progress, queue of upcoming sections - **Background persistence**: embedding continues when navigating away, progress resumes on return - **Text chunking**: large CSV rows split into 8000-char chunks with context header (Tenancy, Resource, CIS Recommendation) repeated -- **CIS Recommendations**: upload CIS PDF to populate the `cisrecom` vector table +- **CIS PDF Chunker**: segments CIS PDF by recommendation number (X.X Ensure...), target 7000 chars with 500-char overlap, filters TOC/appendix/page headers β€” 54/54 recommendations with complete Description + Rationale + Remediation +- **RAG timeout 60s + retry**: increased from 30s with no retry for improved reliability +- **Direct SQL fetch**: retrieves chunks by metadata `recommendationNumber` for overlap resolution - **Knowledge Base**: upload documents (`.txt`, `.pdf`, `.csv`, `.json`, `.md`) or **import URLs** to populate `engineerknowledgebase` - **Consult Embeddings**: chat-like interface with tenancy selector β€” queries vector store with CIS number detection for exact matching, smart table skip, source citations - **OCI GenAI Embeddings**: uses OpenAI Text Embedding 3 Large/Small via OCI GenAI - 11 ADB vector tables supported (9 CIS report + 2 knowledge base) -### πŸ“œ Configuration & Chat Logs +### Configuration & Chat Logs - **Persistent activity log** per configuration tab (OCI, GenAI, ADB, MCP) -- **Chat audit logs**: all chat interactions (user messages, AI responses, tool calls, errors) logged with session/message ID, severity, and source +- **Chat audit logs**: all chat interactions logged with session/message ID, severity, and source - Logs all test, save, upload, report, and ingest operations with severity (success/error/info) - **Inline log panel** at the bottom of each config tab with severity filter - Auto-cleanup of logs older than 30 days - Admin can view all logs; users see only their own -- API endpoints for listing and clearing logs -### πŸ”„ Export / Import Configuration +### Export / Import Configuration - **Full config export** as JSON: OCI credentials (with private keys), GenAI configs, MCP servers, system prompts, app settings, ADB vector configs - **Import** from exported JSON with merge-by-ID (skips existing, prevents duplicates) - Private keys exported as base64 and restored to disk on import -- Enables full environment migration between machines (Mac ↔ Windows ↔ Linux) +- Enables full environment migration between machines -### 🎨 Theme System & UI +### Theme System & UI - **Light/Dark mode toggle** in the sidebar footer - **Oracle Dark Premium** design: deep dark backgrounds (#0D0F12), dark cards (#1A1D24), Oracle Red accent (#C74634) - **Oracle-branded light mode**: warm light backgrounds (original theme) - Theme preference persisted in localStorage across sessions - All CSS driven by CSS custom properties β€” zero hardcoded colors -- **SVG icon system**: 45+ Lucide-based inline SVG icons replace all emojis for cross-platform consistency (Windows/Mac/Linux) -- **KPI Dashboard**: compliance score gauge (SVG semicircular arc with gradient), pass/fail/total KPI cards, donut chart (severity distribution), horizontal bar chart (findings by section) β€” powered by Recharts -- **Animated UI**: staggered fade-in for cards/KPIs/tables, smooth theme transitions, hover effects with transforms, shimmer loading states, pulse alerts +- **SVG icon system**: 45+ Lucide-based inline SVG icons for cross-platform consistency +- **KPI Dashboard**: compliance score gauge (SVG semicircular arc with gradient), pass/fail/total KPI cards, donut chart, horizontal bar chart β€” powered by Recharts +- **Animated UI**: staggered fade-in, smooth theme transitions, hover effects, shimmer loading states, pulse alerts -### ✏️ Terraform Prompt Editor -- **Inline prompt editor** accessible from the Terraform menu (3-dot menu β†’ Editar Prompt) -- Edit the active Terraform system prompt name and content -- **Auto-save with debounce** (800ms): changes saved to database automatically as you type -- Visual "Salvando..." β†’ "Salvo βœ“" indicator - -### πŸ” Security +### Security - **JWT authentication** with configurable expiry - **TOTP MFA** (Google Authenticator / Authy compatible) - **RBAC** with 3 roles: Admin, User, Viewer - Audit logging for all operations - Encrypted credential storage +- **Per-user timezone** setting (not global) --- @@ -266,6 +252,7 @@ The platform combines security compliance scanning, AI-powered chat with **RAG ( | | - GenAI --> LLM (16 models) | | | | - oracledb --> ADB Vector | | | | - Chromium --> PDF gen | | +| | - Pillow --> DOCX gen | | | | - Terraform CLI 1.14.7 | | | | - SQLite (agent.db) | | | | | | @@ -325,7 +312,7 @@ Default credentials: - **Username:** `admin` - **Password:** `admin123` -> ⚠️ Change the default password immediately after first login. +> Change the default password immediately after first login. --- @@ -460,7 +447,7 @@ CREATE TABLE engineerknowledgebase (ID RAW(16), TEXT CLOB, METADATA JSON, EMBEDD | Table Name | Purpose | How to populate | |-----------|---------|-----------------| -| `cisrecom` | CIS Benchmark official recommendations β€” remediation steps, rationale, audit procedures per control | Upload CIS PDF in Embeddings tab | +| `cisrecom` | CIS Benchmark official recommendations β€” remediation steps, rationale, audit procedures per control | Upload CIS PDF in Embeddings tab (auto-chunked by recommendation) | | `engineerknowledgebase` | General knowledge base β€” blogs, documentation, PDFs, URLs for complementary context | Upload files or import URLs in Embeddings tab | > **How embedding works**: When you click **"Embed Report"** on a completed CIS report, the system automatically: @@ -506,19 +493,19 @@ Allow group to read buckets in compartment ``` oci-cis-agent/ β”œβ”€β”€ backend/ -β”‚ β”œβ”€β”€ app.py # FastAPI application (~7700 lines) +β”‚ β”œβ”€β”€ app.py # FastAPI application (~9800 lines) β”‚ β”œβ”€β”€ cis_reports.py # Oracle CIS Benchmark checker (6660 lines, report engine) β”‚ β”œβ”€β”€ mcp_cis_server.py # MCP server with 12 granular CIS tools (~700 lines) β”‚ β”œβ”€β”€ gen_tf_reference.py # OCI Terraform provider resource catalog generator β”‚ β”œβ”€β”€ Dockerfile # Python 3.12 + OCI CLI + Terraform 1.14.7 + Chromium β”‚ └── requirements.txt # Dependencies β”œβ”€β”€ frontend-react/ -β”‚ β”œβ”€β”€ src/ # React 19 SPA (TypeScript, 38 source files, ~15500 lines) -β”‚ β”‚ β”œβ”€β”€ pages/ # 15 page components (Chat, Terraform, Explorer, Reports, Config, Admin) +β”‚ β”œβ”€β”€ src/ # React 19 SPA (TypeScript, 39 source files, ~15800 lines) +β”‚ β”‚ β”œβ”€β”€ pages/ # 16 page components (Chat, Terraform, Explorer, OCI Services, Reports, Config, Admin) β”‚ β”‚ β”œβ”€β”€ api/ # API client + endpoint modules β”‚ β”‚ β”œβ”€β”€ stores/ # Zustand stores (app state persistence across navigation) β”‚ β”‚ β”œβ”€β”€ hooks/ # Custom hooks (theme, polling) -β”‚ β”‚ └── i18n/ # Internationalization (pt/en, 700+ keys) +β”‚ β”‚ └── i18n/ # Internationalization (pt/en, 703 keys) β”‚ └── dist/ # Built SPA served at / β”œβ”€β”€ nginx/ β”‚ └── default.conf # Reverse proxy (React SPA + API /api/) @@ -598,6 +585,14 @@ oci-cis-agent/ | GET | `/api/oci/explore/{id}/counts` | Get cached resource counts (instant from SQLite) | | POST | `/api/oci/explore/{id}/counts/refresh` | Refresh all resource counts in background | +### OCI Services + +| Method | Endpoint | Description | +|--------|----------|-------------| +| GET | `/api/oci-services/{config_id}` | Get service status per tenancy | +| PUT | `/api/oci-services/{config_id}` | Update service overrides | +| GET | `/api/oci-health` | Oracle Cloud status proxy (49 regions) | + ### OCI Resource Actions | Method | Endpoint | Description | @@ -704,7 +699,16 @@ oci-cis-agent/ | GET | `/api/reports/{rid}/files/{fid}/download` | Download individual report file | | GET | `/api/reports/{rid}/compliance-report` | Serve cached LAD A-Team compliance report HTML | | POST | `/api/reports/{rid}/compliance-report/generate` | Generate compliance report (background, with RAG) | -| GET | `/api/reports/{rid}/compliance-report/status` | Check if compliance report is ready | +| GET | `/api/reports/{rid}/compliance-report/status` | Check compliance report status (returns progress: current, total, step, rec) | +| GET | `/api/reports/{rid}/compliance-report/docx` | Download compliance report as DOCX | + +### User Settings + +| Method | Endpoint | Description | +|--------|----------|-------------| +| GET | `/api/users/me/timezone` | Get user timezone | +| PUT | `/api/users/me/timezone` | Set user timezone | +| GET | `/api/settings/timezone/options` | List available timezone options | ### CIS Engine @@ -785,6 +789,7 @@ All models include OCID mapping for `us-ashburn-1`. For other regions, use the " | OCI SDK | `oci` 2.133.0, `oci-cli` | | ADB | `python-oracledb` 2.4.1 (Thin mode) | | GenAI | `oci.generative_ai_inference` | +| Image Processing | Pillow (DOCX report generation) | | Container | Docker Compose, Nginx reverse proxy | | MCP | Model Context Protocol SDK (stdio/SSE) with tool discovery + execution | | Terraform | Terraform CLI 1.14.7, OCI Provider | @@ -796,9 +801,10 @@ All models include OCID mapping for `us-ashburn-1`. For other regions, use the " | Version | Date | Changes | |---------|------|---------| +| **v3.0** | 2026-03 | **OCI Services page**: new menu item with Service Status tab (auto-detect 6 security services β€” VSS, Data Safe, Cloud Guard, Bastion, Security Zones, Vault β€” per tenancy via OCI API + user overrides) and OCI Health tab (real-time Oracle service health from ocistatus.oraclecloud.com, 49 regions, search, geo filters). **Compliance Report v3.0**: OCI Services section with summary table + detailed descriptions + Cloud Guard recommendations, back page (Connect with us + copyright), DOCX download (Pillow camouflage strip, green header tables, result boxes, code blocks, A4 format), compliance_data.json shared between HTML and DOCX, progress bar during RAG generation (X/35, percentage), ADB connection pre-check (fail fast 503). **CIS PDF Chunker**: segments CIS PDF by recommendation number (X.X Ensure...), target 7000 chars with 500-char overlap, filters TOC/appendix/page headers β€” 54/54 recommendations with complete Description + Rationale + Remediation. **Embedding improvements**: auto-detect dimension from DDL (even on empty tables), auto-detect model per dimension (1536=small, 3072=large), RAG timeout 60s + retry (was 30s), direct SQL fetch by metadata recommendationNumber for overlap chunks. **Per-user timezone** (not global). **New API endpoints**: OCI Health proxy, OCI Services status/overrides, user timezone CRUD, DOCX download, compliance progress details, timezone options. | | **v2.8** | 2026-03 | **Legacy frontend removed**: React SPA now served at root `/` (no more `/app/` prefix), legacy vanilla JS SPA removed. **Compliance Report ZIP download**: Chromium headless PDF generation (identical to browser print) + CSV findings bundled in ZIP. **Compliance generation state persisted server-side**: `.compliance_generating` marker file ensures generation status survives page navigation β€” spinner auto-resumes on return. **RAG remediation enriched**: extracts Description, Rationale, Audit, and Remediation sections from CIS vector chunks, combines up to 3 matched chunks. **Chat markdown styling**: full markdown rendering with syntax highlighting (Catppuccin Mocha theme via rehype-highlight), styled headings, lists, tables, blockquotes, inline/block code. **Default model auto-select**: Chat Agent auto-selects first GenAI config on page load. **Explorer silent polling**: no more flickering during start/stop resource actions. | | **v2.7** | 2026-03 | **LAD A-Team CIS Compliance Report**: professional Oracle-format compliance report with cover page, TOC, Security Overview (7 pillars), CIS Assessment Summary, detailed findings with compliance % bars, RAG-powered remediation from ADB vector store (`CISRECOM` table with strict recommendation number filtering), affected resources CSV download links per non-compliant finding (JWT auth via query param). **React SPA**: 15-page React 19 frontend (TypeScript, Vite, Zustand, 38 source files). **i18n**: full internationalization with 625 keys (pt/en), language switcher persisted in localStorage. **Report management**: delete reports endpoint, embed individual report files (CSV/TXT/JSON/PDF) into vector store. **Terraform ZIP download**: replaced individual file downloads with client-side ZIP generation (pure JS, no external lib). **Explorer UX**: silent polling during start/stop actions eliminates flickering (resources update in-place without clearing the list). **Report sections collapsed by default**: HTML report and compliance report iframes start minimized. | -| **v2.6** | 2026-03 | **Rename**: app renamed to "AI Agent β€” Infrastructure & Security Engineer". **Session Token Auth**: OCI session token authentication support (type selector API Key/Session Token, conditional form fields, collapsible credential form, type tags in credentials table, `SecurityTokenSigner` backend). **Terraform Workspaces submenu**: sidebar sub-tab listing all workspaces grouped by date with accordion UI, filtered by existing chat sessions (INNER JOIN), actions (plan/apply/destroy/delete/open). **Retry on failure**: "Reenviar" button on failed messages across Chat Agent, Terraform Agent, and Prompt Generator. **Terraform fixes**: `f2` undefined variable in dedup logic, `oci_core_services` false positive validation (data sources excluded), `S.tfCode` reconstruction from `S.tfFiles` for Re-plan, auto-fix button visible without pre-selected model, DRG attachment rules in system prompt. **Resilience**: stuck workspace recovery on container restart (planning/applying/destroying β†’ reset + lock file cleanup). | +| **v2.6** | 2026-03 | **Rename**: app renamed to "AI Agent β€” Infrastructure & Security Engineer". **Session Token Auth**: OCI session token authentication support (type selector API Key/Session Token, conditional form fields, collapsible credential form, type tags in credentials table, `SecurityTokenSigner` backend). **Terraform Workspaces submenu**: sidebar sub-tab listing all workspaces grouped by date with accordion UI, filtered by existing chat sessions (INNER JOIN), actions (plan/apply/destroy/delete/open). **Retry on failure**: "Reenviar" button on failed messages across Chat Agent, Terraform Agent, and Prompt Generator. **Terraform fixes**: `f2` undefined variable in dedup logic, `oci_core_services` false positive validation (data sources excluded), `S.tfCode` reconstruction from `S.tfFiles` for Re-plan, auto-fix button visible without pre-selected model, DRG attachment rules in system prompt. **Resilience**: stuck workspace recovery on container restart (planning/applying/destroying reset + lock file cleanup). | | **v2.5** | 2026-03 | **Explorer Expansion**: KPI stats bar with per-category resource counts (cached in SQLite `explorer_counts_cache`, background refresh on compartment access), enhanced views for VCNs/Subnets/Load Balancers/Buckets with expandable cards, start/stop for DB Systems + MySQL + Container Instances (5 resource types total), dead state filtering (`TERMINATED`/`DELETED`/`DELETING` hidden across all 36 endpoints), wider compartment tree (280px), IAM excluded from compartment totals. **Prompt Generator Intelligence**: same knowledge pipeline as Terraform Agent β€” auto-detects resource types from user message, fetches official docs (Example Usage + Arguments) from GitHub cached in SQLite, uses compact categorized resource reference. **Network Firewall fix**: `NetworkFirewallPolicySummaryCollection` not iterable β€” added `.items` accessor for collection objects. | | **v2.4** | 2026-03 | **Terraform Safety & UX**: region safety guard (plan blocked if model omits `variable "region"` β€” prevents wrong-region provisioning), rollback button on apply failure (manual `terraform destroy` with "ROLLBACK" confirmation), SSH public key field in OCI configs (auto-injected into `terraform.tfvars` for compute instances), reasoning_effort uppercase fix for OCI SDK, split-panel layout (terminal right 40%, files/plan/resources below chat 60%), plan button persists after destroy. **Terraform tfvars**: expanded `oci_var_map` with `ssh_public_key`/`ssh_authorized_keys` auto-mapping from OCI config | | **v2.3** | 2026-03 | **Consult Embeddings**: dedicated sub-menu with chat-like interface for natural language Q&A against vector store β€” queries all active tables via cosine similarity, returns contextual answers with source citations (works with or without GenAI config). **Knowledge Base & RAG Enhancements**: Knowledge Base (Base de Conhecimento) with file upload + URL import to `engineerknowledgebase` vector table, CIS Recommendations upload to `cisrecom` table, URL content extraction (HTML tag/script stripping, remote PDF parsing), auto-resolve embedding config from OCI credentials (no separate GenAI config required), `report_date` metadata tracking for embedding freshness, purge & re-embed flow for updated reports. **ADB Vector Improvements**: case-insensitive table matching with quoted identifiers for Oracle ADB, table validation endpoint against `user_tables`, tables section moved inside edit connection card, case-preserving table registration (removed forced uppercase). **Vector search fix**: FLOAT32 compatibility for VECTOR_DISTANCE queries, base64-decoded compartment_id for auto-resolved OCI configs. **Embeddings tab cleanup**: removed delete buttons, simplified to read-only view, updated descriptions. **Dependencies**: added PyPDF2 for PDF text extraction | @@ -813,7 +819,7 @@ All models include OCID mapping for `us-ashburn-1`. For other regions, use the " | **v1.4** | 2026-03 | In-place config editing, 69 chat + 11 embedding models with OCID resolution (OpenAI GPT-5.3/5.2/5.1/5/4.1/4o/Codex/Image/Audio, xAI, Google, Meta, ProtectAI), OpenAI Text Embedding 3, custom OCID support, wallet auto-parse with DSN extraction | | **v1.3** | 2026-03 | Persistent config logs per tab, GenAI auto-fill from OCI credentials, inline UX feedback with loading spinners, MCP type-switch fix, encrypted key passphrase support, Docker stdin hang fix | | **v1.2** | 2026-03 | RAG pipeline (OCI GenAI embeddings + ADB vector search), dedicated Embeddings tab, CIS report chunking, file upload embedding | -| **v1.1** | 2025-02 | OCI SDK GenAI (exact pattern), OCI Account Explorer, MCP↔ADB linking, full chat parameters | +| **v1.1** | 2025-02 | OCI SDK GenAI (exact pattern), OCI Account Explorer, MCP-ADB linking, full chat parameters | | **v1.0** | 2025-02 | Initial release: OCI theme, GenAI integration, MCP servers, ADB vector, JWT+MFA+RBAC | --- @@ -872,5 +878,5 @@ MIT ---

- Built with ❀️ for Oracle Cloud Infrastructure security compliance -

\ No newline at end of file + Built for Oracle Cloud Infrastructure security compliance +

diff --git a/backend/app.py b/backend/app.py index d14295e..53d039d 100644 --- a/backend/app.py +++ b/backend/app.py @@ -17,7 +17,7 @@ from fastapi import ( Query, Body, BackgroundTasks ) from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import JSONResponse, FileResponse, StreamingResponse, HTMLResponse +from fastapi.responses import JSONResponse, FileResponse, StreamingResponse, HTMLResponse, Response from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials from pydantic import BaseModel import jwt as pyjwt @@ -2914,14 +2914,29 @@ def _embed_text(text: str, genai_cfg: dict, embedding_model_id: str, max_chars: _DIM_TO_MODEL = {1536: "openai.text-embedding-3-small", 3072: "openai.text-embedding-3-large"} def _get_table_embedding_dim(cfg: dict, table_name: str) -> int: - """Detect the embedding dimension of a table by sampling one row.""" + """Detect the embedding dimension of a table by sampling one row, or from column definition if empty.""" conn = _get_adb_connection(cfg) try: cur = conn.cursor() + # Try from data first cur.execute(f'SELECT VECTOR_DIMENSION_COUNT(EMBEDDING) FROM "{table_name}" FETCH FIRST 1 ROWS ONLY') row = cur.fetchone() - cur.close() - return int(row[0]) if row and row[0] else 0 + if row and row[0]: + cur.close() + return int(row[0]) + # Table is empty β€” check column definition from DDL + try: + cur.execute(f"""SELECT DBMS_METADATA.GET_DDL('TABLE', :1) FROM DUAL""", [table_name]) + ddl_row = cur.fetchone() + cur.close() + if ddl_row and ddl_row[0]: + import re + m = re.search(r'VECTOR\((\d+)', str(ddl_row[0])) + if m: + return int(m.group(1)) + except Exception: + pass + return 0 except Exception as e: log.warning(f"Failed to get embedding dimension for table '{table_name}': {e}") return 0 @@ -3677,6 +3692,16 @@ def _ingest_documents_task(cfg: dict, genai_cfg: dict, documents: list, user_id: import array emb_model = cfg.get("embedding_model_id", "cohere.embed-v4.0") table_name = table_name or cfg.get("table_name", "") + # Auto-detect embedding dimension from existing table data and use matching model + try: + actual_dim = _get_table_embedding_dim(cfg, table_name) + if actual_dim and actual_dim in _DIM_TO_MODEL: + detected_model = _DIM_TO_MODEL[actual_dim] + if detected_model != emb_model: + log.info(f"Ingest: table '{table_name}' has {actual_dim} dims, switching model from {emb_model} to {detected_model}") + emb_model = detected_model + except Exception as e: + log.warning(f"Ingest: failed to detect dimension for '{table_name}': {e}") total = len(documents) # Track status if task_id: @@ -3776,17 +3801,198 @@ def _chunk_report_by_section(report_data: dict) -> list: }) return documents -def _chunk_text_file(text: str, filename: str, chunk_size: int = 1000) -> list: - """Split text into chunks by paragraphs or fixed size.""" +def _chunk_cis_pdf(text: str, filename: str, target_chars: int = 7000, overlap_chars: int = 500) -> list: + """Chunk a CIS Foundations Benchmark PDF by recommendation number. + Each recommendation (1.1, 1.2, etc.) becomes one or more chunks with overlap. + Port of the JavaScript embedding pipeline.""" + import re as _re + + def normalize(t): + t = t.replace('\r', '\n') + t = _re.sub(r'[ \t]+\n', '\n', t) + t = _re.sub(r'\n{3,}', '\n\n', t) + return t.strip() + + def strip_page_headers(t): + # Remove "Page XX" both standalone and at start of lines + t = _re.sub(r'^\s*Page\s+\d+\s*$', '', t, flags=_re.MULTILINE | _re.IGNORECASE) + t = _re.sub(r'^Page\s+\d+\s+', '', t, flags=_re.MULTILINE | _re.IGNORECASE) + return t + + def remove_toc(t): + # Remove everything from "Table of Contents" up to the actual recommendations section + # The real content starts with "Recommendations\n1 Identity" or "Profile Applicability" + toc_start = _re.search(r'\bTable of Contents\b', t, _re.IGNORECASE) + if not toc_start: + return t + # Find where actual recommendation content begins + content_start = _re.search(r'\bRecommendations\s*\n\s*1\s+Identity', t[toc_start.start():], _re.IGNORECASE) + if not content_start: + content_start = _re.search(r'\bProfile Applicability\b', t[toc_start.start():], _re.IGNORECASE) + if not content_start: + content_start = _re.search(r'\bOverview\b', t[toc_start.start():], _re.IGNORECASE) + if not content_start: + return t + end_pos = toc_start.start() + content_start.start() + if end_pos <= toc_start.start(): + return t + return normalize(t[:toc_start.start()] + '\n\n' + t[end_pos:]) + + def is_chapter_header(line): + l = line.strip() + return bool(_re.match(r'^\d+\s+[A-Za-z].+', l)) and not _re.match(r'^\d+\.\d+', l) + + def is_rec_header_start(line): + l = line.strip() + # Must be "1.1 Word..." but NOT a TOC line (with dots/page numbers) + if not _re.match(r'^\d+\.\d+(\.\d+)?\s+[A-Z]', l): + return False + # Skip TOC lines: contain "...." or end with a page number + if '....' in l or _re.search(r'\.\s*\d+\s*$', l): + return False + return True + + def header_looks_complete(h): + # Complete if has (Manual)/(Automated) or ends with a closing paren + if _re.search(r'\(\s*(Manual|Automated)\s*\)', h, _re.IGNORECASE): + return True + # Also stop if next line starts a known section like "Profile Applicability" + return False + + def chunk_text(t): + if not t: + return [] + paragraphs = [p.strip() for p in t.split('\n\n') if p.strip()] + chunks = [] + buf = "" + def push(): + nonlocal buf + b = buf.strip() + if b: + chunks.append(b) + buf = "" + for p in paragraphs: + if len(p) > target_chars: + push() + i = 0 + while i < len(p): + chunks.append(p[i:i + target_chars].strip()) + i += max(1, target_chars - overlap_chars) + continue + candidate = f"{buf}\n\n{p}" if buf else p + if len(candidate) <= target_chars: + buf = candidate + else: + push() + if chunks and overlap_chars > 0: + prev = chunks[-1] + overlap = prev[max(0, len(prev) - overlap_chars):] + buf = f"{overlap}\n\n{p}".strip() + else: + buf = p + push() + return chunks + + def remove_appendix(t): + """Remove appendix sections (Assessment Status, Change History, etc.) that pollute embeddings.""" + for marker in [r'\bAppendix\b', r'\bAssessment Status\b', r'\bChange History\b', + r'\bCIS Controls v\d', r'\bDate\s+Version\s+Changes']: + m = _re.search(marker, t, _re.IGNORECASE) + if m and m.start() > len(t) * 0.7: # only cut if in last 30% of doc + t = t[:m.start()].rstrip() + break + return t + + # Pipeline + text = normalize(text) + text = strip_page_headers(text) + text = remove_toc(text) + text = remove_appendix(text) + lines = text.split('\n') + + # Segment by recommendation + segments = [] + current = None + current_chapter = "" + + i = 0 + while i < len(lines): + line = lines[i] + if is_chapter_header(line): + current_chapter = line.strip() + if is_rec_header_start(line): + if current: + segments.append(current) + header = line.strip() + j = i + 1 + while j < len(lines) and not header_looks_complete(header): + next_line = lines[j].strip() + if is_rec_header_start(next_line): + break + # Stop consuming if we hit a known section start + if next_line.startswith('Profile Applicability') or next_line.startswith('Description:'): + break + if next_line: + header = _re.sub(r'\s+', ' ', f"{header} {next_line}").strip() + j += 1 + i = j - 1 + current = {"header": header, "chapter": current_chapter, "body_lines": []} + i += 1 + continue + if current: + current["body_lines"].append(line) + i += 1 + + if current: + segments.append(current) + + # Generate chunks + documents = [] + for seg in segments: + body = normalize('\n'.join(seg["body_lines"])) + if not body: + continue + rec_match = _re.match(r'^(\d+(\.\d+)+)', seg["header"]) + rec_number = rec_match.group(1) if rec_match else "unknown" + canonical = normalize('\n'.join(filter(None, [ + f"Recommendation: {seg['header']}", + f"Chapter: {seg['chapter']}" if seg['chapter'] else "", + "", + body, + ]))) + chunks = chunk_text(canonical) + for idx, chunk in enumerate(chunks): + documents.append({ + "content": chunk, + "source": filename, + "metadata": json.dumps({ + "filename": filename, + "recommendationNumber": rec_number, + "chapter": seg["chapter"], + "source": "CIS-OCI-PDF", + "chunkIndex": idx + 1, + "chunkCount": len(chunks), + }), + }) + + log.info(f"CIS PDF chunking: {len(segments)} recommendations β†’ {len(documents)} chunks from {filename}") + return documents + + +def _chunk_text_file(text: str, filename: str, chunk_size: int = 1000, overlap: int = 200) -> list: + """Split text into chunks by paragraphs with overlap to avoid losing context at boundaries.""" paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()] documents = [] current_chunk = "" + prev_tail = "" # last N chars of previous chunk for overlap chunk_num = 1 for para in paragraphs: if len(current_chunk) + len(para) + 2 > chunk_size and current_chunk: documents.append({"content": current_chunk, "source": filename, "metadata": f"chunk: {chunk_num}"}) chunk_num += 1 - current_chunk = para + # Keep overlap from end of current chunk + prev_tail = current_chunk[-overlap:] if len(current_chunk) > overlap else current_chunk + current_chunk = prev_tail + "\n\n" + para else: current_chunk = current_chunk + "\n\n" + para if current_chunk else para if current_chunk: @@ -4334,9 +4540,9 @@ async def embed_report_file(rid: str, fid: str, req: dict, bg: BackgroundTasks, def _extract_pdf_text(file_bytes: bytes) -> str: """Extract text from a PDF file using PyPDF2 or pdfplumber.""" + import io try: import PyPDF2 - import io reader = PyPDF2.PdfReader(io.BytesIO(file_bytes)) pages = [] for page in reader.pages: @@ -4348,7 +4554,6 @@ def _extract_pdf_text(file_bytes: bytes) -> str: pass try: import pdfplumber - import io pages = [] with pdfplumber.open(io.BytesIO(file_bytes)) as pdf: for page in pdf.pages: @@ -4373,10 +4578,16 @@ async def embed_upload(adb_config_id: str = Form(...), table_name: str = Form("" else: content = raw.decode("utf-8", errors="replace") if not content.strip(): raise HTTPException(400, "File is empty") - documents = _chunk_text_file(content, file.filename) + target_table = table_name.strip() or None + # Use CIS-specific chunking for cisrecom table (segments by recommendation number with overlap) + if target_table and 'cisrecom' in target_table.lower(): + documents = _chunk_cis_pdf(content, file.filename) + if not documents: + documents = _chunk_text_file(content, file.filename) # fallback + else: + documents = _chunk_text_file(content, file.filename) if not documents: raise HTTPException(400, "No content chunks found") cfg, gc = _get_adb_and_genai(adb_config_id) - target_table = table_name.strip() or None bg.add_task(_ingest_documents_task, cfg, gc, documents, u["id"], u["username"], table_name=target_table) _audit(u["id"], u["username"], "embed_upload", file.filename, f"{len(documents)} chunks") return {"ok": True, "message": f"Embedding de {len(documents)} chunks iniciado", "chunks": len(documents), "filename": file.filename} @@ -5024,8 +5235,8 @@ def _format_remediation_html(text: str) -> str: code_buf.append(stripped) else: flush_code() - # "From Console:", "From CLI:", "From Command Line:" β†’ bold header - header_match = _re.match(r'^(From\s+(?:Console|CLI|Command Line|the Console|the CLI|OCI Console|OCI CLI)[:\s]*)(.*)', stripped, _re.IGNORECASE) + # "From Console:", "From CLI:", "From Command Line:", "Audit / Verification:" β†’ bold header + header_match = _re.match(r'^(From\s+(?:Console|CLI|Command Line|the Console|the CLI|OCI Console|OCI CLI)[:\s]*|Audit\s*/?\s*Verification[:\s]*)(.*)', stripped, _re.IGNORECASE) if header_match: rest = esc(header_match.group(2)) if header_match.group(2) else '' html_parts.append(f'

{esc(header_match.group(1).strip())} {rest}

') @@ -5094,22 +5305,53 @@ def _search_cis_remediation(title: str, rec_num: str, user_id: str) -> dict: if all_docs: import re as _re all_docs.sort(key=lambda d: d.get("distance", 999)) - # STRICT FILTER: only keep chunks that mention THIS exact recommendation number + # FILTER: keep chunks that mention THIS recommendation number in text OR metadata exact_pattern = _re.compile( r'(?:Recommendation[:\s]*' + _re.escape(rec_num) + r'|^' + _re.escape(rec_num) + r')\s', _re.MULTILINE | _re.IGNORECASE ) - matched = [d["content"] for d in all_docs - if d.get("content") and len(d["content"]) > 500 - and exact_pattern.search(d["content"])] + matched = [] + for d in all_docs: + content = d.get("content", "") + if not content or len(content) < 200: + continue + # Match by text content + if exact_pattern.search(content): + matched.append(content) + continue + # Match by metadata recommendationNumber (for overlap chunks) + meta = d.get("metadata", "") + if isinstance(meta, str) and f'"recommendationNumber": "{rec_num}"' in meta: + matched.append(content) + continue + if isinstance(meta, str) and f'"recommendationNumber":"{rec_num}"' in meta: + matched.append(content) + # Also fetch ALL chunks for this rec from ADB directly (catches overlap chunks) + try: + conn2 = _get_adb_connection(adb_cfg) + cur2 = conn2.cursor() + for tbl_name in search_tables: + cur2.execute(f'''SELECT "TEXT" FROM "{tbl_name}" + WHERE JSON_VALUE("METADATA", '$.recommendationNumber') = :1 + ORDER BY JSON_VALUE("METADATA", '$.chunkIndex')''', [rec_num]) + for row in cur2: + txt = row[0].read() if hasattr(row[0], 'read') else row[0] + if txt and txt not in matched: + matched.append(txt) + cur2.close() + conn2.close() + except Exception as e: + log.warning(f"Direct chunk fetch for {rec_num}: {e}") + if matched: - best = matched[0] - desc = _extract_description_section(best) - rationale = _extract_rationale_section(best) - rem = _extract_remediation_section(best) - audit = _extract_audit_section(best) - # Try additional chunks for fuller content - for extra in matched[1:3]: + # Concatenate all chunks for the best possible extraction + full_text = '\n\n'.join(matched) + desc = _extract_description_section(full_text) + rationale = _extract_rationale_section(full_text) + rem = _extract_remediation_section(full_text) + audit = _extract_audit_section(full_text) + # Fallback: try individual chunks if full concatenation missed something + for extra in matched[1:4]: if not rem: rem = _extract_remediation_section(extra) elif len(rem) < 200: @@ -5120,8 +5362,25 @@ def _search_cis_remediation(title: str, rec_num: str, user_id: str) -> dict: audit = _extract_audit_section(extra) if not desc: desc = _extract_description_section(extra) + elif desc and not desc.rstrip().endswith(('.', ':', '!', '?', ';')): + # Description looks truncated β€” append continuation from next chunk + extra_desc = _extract_description_section(extra) + if extra_desc: + desc = desc.rstrip() + "\n" + extra_desc if not rationale: rationale = _extract_rationale_section(extra) + + # Also check non-matched docs for continuation (same CIS but different chunk) + all_contents = [d["content"] for d in all_docs if d.get("content") and len(d["content"]) > 200] + for extra_content in all_contents[:6]: + if extra_content in matched: + continue + if not rem: + rem = _extract_remediation_section(extra_content) + if desc and not desc.rstrip().endswith(('.', ':', '!', '?', ';')): + extra_desc = _extract_description_section(extra_content) + if extra_desc and extra_desc not in desc: + desc = desc.rstrip() + "\n" + extra_desc # Combine remediation + audit for complete guidance full_rem = rem or "" if audit and len(audit) > 50: @@ -5245,7 +5504,6 @@ def _check_oci_services(config_id: str) -> list[dict]: # DB stores base64 β€” decode if needed compartment_raw = (cfg["compartment_id"] or "") if cfg else "" if compartment_raw and not compartment_raw.startswith("ocid"): - import base64 try: compartment_raw = base64.b64decode(compartment_raw).decode("utf-8") except Exception: @@ -5464,6 +5722,35 @@ def _build_compliance_html(filtered_rows: list, tenancy_name: str, extract_date: def esc(v): return (v or "").replace("&","&").replace("<","<").replace(">",">").replace('"',""") def slug(s): return _re.sub(r"[^\w-]","", _re.sub(r"\s+","-",(s or "").strip().lower())) + def _join_broken_lines(text): + """Join single-newline broken lines into flowing paragraphs. + Preserves: double-newline paragraph breaks, numbered lists, bullet lists, headers.""" + if not text: + return text + lines = text.split('\n') + result = [] + for line in lines: + line = line.strip() + if not line: + result.append('\n') + continue + # Never join if this line starts a list item or header + starts_new = ( + _re.match(r'^\d+[\.\)]\s', line) or # "1. ", "2) " + _re.match(r'^[a-z][\.\)]\s', line) or # "a. ", "b) " + line.startswith('- ') or line.startswith('* ') or # bullets + line.startswith('From ') or # "From Console:", "From CLI:" + line.startswith('Audit') or # "Audit / Verification:" + line.startswith('Note:') or line.startswith('See ') + ) + if starts_new: + result.append(line) + elif result and result[-1] != '\n' and not result[-1].endswith(('.', ':', '!', '?', ';')): + result[-1] = result[-1] + ' ' + line + else: + result.append(line) + return '\n'.join(result).strip() + total_all = len(filtered_rows) total_passed = sum(1 for r in filtered_rows if r["__compliant"]) total_failed = total_all - total_passed @@ -5529,22 +5816,52 @@ def _build_compliance_html(filtered_rows: list, tenancy_name: str, extract_date: detail = f" ({findings_num} of {esc(total_items)} items)" result_html = f'
Result: Tenancy {esc(tenancy_name)}: Non-Compliant{detail}
' - # Description β€” split into paragraphs on double newlines, justify each + def _text_to_html_paras(text): + """Convert text to HTML paragraphs, rendering numbered lists as
    and bullets as