diff --git a/README.md b/README.md index 3807846..00204b5 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@

- Version + Version Python FastAPI OCI @@ -35,7 +35,8 @@ The platform combines security compliance scanning, AI-powered chat with **RAG ( - **MCP Tool Use (Function Calling)**: GenAI models can call tools from registered MCP servers during chat — supports both Cohere and Generic (OpenAI-style) function calling formats with automatic tool execution loop (max 5 iterations) - **Chat Memory Compaction**: automatic summarization of older messages when conversation exceeds ~8000 tokens — keeps 6 recent messages intact and generates an LLM-based summary of older context, similar to Claude Code's context compression - **Multimodal Chat**: upload images (PNG/JPG/GIF/WebP), PDFs, and text files directly in the chat for AI analysis — supports up to 5 files per message via OCI GenAI `ImageContent` and `DocumentContent` -- **Thinking Indicator**: button disables and shows spinner + "Pensando..." while waiting for GenAI response +- **Async Background Processing**: chat requests return immediately, GenAI + MCP tools process in background via dedicated thread pool (16 threads), frontend polls for results — eliminates 504 timeouts on long-running scans +- **Thinking Indicator**: button disables and shows spinner + "Pensando..." while waiting for GenAI response, with message timestamps - 69 chat models + 11 embedding models across 6 providers: **Cohere**, **Meta**, **Google**, **OpenAI** (GPT-5.3/5.2/5.1/5/4.1/4o, Codex, Image, Audio, o1/o3/o4-mini, GPT-oss), **xAI** (Grok 4.1/4/3), **ProtectAI** - OCID-based model resolution: catalog maps model IDs to OCI resource IDs per region for reliable API calls - 16 OCI regions supported with auto-generated endpoints @@ -76,7 +77,8 @@ The platform combines security compliance scanning, AI-powered chat with **RAG ( - `cis_get_scan_status` / `cis_invalidate_cache` — session status and cache management - **Per-section data collection**: each scan tool collects only the OCI data needed for that section, avoiding unnecessary API calls - **Region-specific scanning**: all scan tools accept optional `regions` parameter to target specific OCI regions (e.g., `["us-ashburn-1"]`) instead of scanning the entire tenancy -- **Session caching**: collected data is cached per config+regions scope, so subsequent scans on different sections reuse shared prerequisites (compartments, identity domains) +- **Session caching**: collected data is cached per config+regions scope (2-hour TTL), so subsequent scans on different sections reuse shared prerequisites (compartments, identity domains) +- **Parallelized data collection**: base collectors and regional collectors run in parallel thread pools (up to 8 workers), with 5-minute timeout per tool call - Based on Oracle's official `cis_reports.py` (6660 lines, 48 CIS + 11 OBP checks) ### 🔌 MCP Server Registry + Tool Discovery @@ -579,6 +581,7 @@ All models include OCID mapping for `us-ashburn-1`. For other regions, use the " | Version | Date | Changes | |---------|------|---------| +| **v2.0** | 2026-03 | Async background chat processing (no more 504 timeouts), frontend polling with timestamps, 8 uvicorn workers + 16-thread chat executor for ~12 simultaneous chats, parallelized MCP data collection (5-thread base + 8-thread regional), 2-hour MCP session cache, 5-min tool timeout, full dead code cleanup across backend/frontend/MCP | | **v1.9** | 2026-03 | Multimodal chat (image/PDF/text file upload with OCI GenAI ImageContent/DocumentContent), region-specific MCP scanning (`regions` param on all scan tools), orphaned report auto-detection on progress poll, nginx timeout increased to 15min, improved API error handling for non-JSON responses | | **v1.8** | 2026-03 | CIS Engine auto-update from Oracle GitHub with automatic patch reapplication, version check UI card (admin), new `/api/cis-engine/*` endpoints, report file listing and individual download endpoints, reorganized Reports tab (execution history + status) and Downloads tab (file browser only with expandable cards per report), CIS Level description tooltip, persistent log expand during report generation | | **v1.7** | 2026-03 | Oracle official CIS report engine (replaces lightweight checker), granular report parameters (Level, OBP, Raw Data, Redact), per-report file storage with category browser, tenancy filter in Downloads, individual file download | diff --git a/backend/Dockerfile b/backend/Dockerfile index 7f9a009..5ddbc4f 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -22,4 +22,4 @@ RUN mkdir -p /data EXPOSE 8000 -CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "2"] +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "8"] diff --git a/backend/app.py b/backend/app.py index 93b4e97..c5b04b2 100644 --- a/backend/app.py +++ b/backend/app.py @@ -5,11 +5,12 @@ OCI Account Explorer, MCP Server registry with VectorDB tool integration, Autonomous DB vector storage, CIS reports, chat agent, audit log. """ import os, json, uuid, hashlib, hmac, time, base64, struct, secrets, subprocess -import shutil, asyncio, sqlite3, logging, socket, re +import shutil, asyncio, sqlite3, logging, re, concurrent.futures from datetime import datetime, timedelta from pathlib import Path from typing import Optional, List, Dict, Any from contextlib import contextmanager +from functools import partial from fastapi import ( FastAPI, HTTPException, Depends, Request, UploadFile, File, Form, @@ -37,6 +38,7 @@ for d in [DATA, OCI_DIR, REPORTS, MCP_DIR, WALLET_DIR]: d.mkdir(parents=True, exist_ok=True) _running_reports: dict[str, asyncio.subprocess.Process] = {} # rid → subprocess +_chat_executor = concurrent.futures.ThreadPoolExecutor(max_workers=16, thread_name_prefix="chat") # ── Chat Memory Compaction Settings ── COMPACT_TOKEN_THRESHOLD = 8000 # estimated tokens before triggering compaction @@ -199,9 +201,10 @@ OCI_REGIONS = { # ── Database ────────────────────────────────────────────────────────────────── @contextmanager def db(): - conn = sqlite3.connect(str(DB_PATH)) + conn = sqlite3.connect(str(DB_PATH), timeout=30, check_same_thread=False) conn.row_factory = sqlite3.Row conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA busy_timeout=30000") conn.execute("PRAGMA foreign_keys=ON") try: yield conn @@ -379,6 +382,11 @@ def init_db(): c.execute(f"ALTER TABLE reports ADD COLUMN {col}") except sqlite3.OperationalError: pass + for col in ["status TEXT DEFAULT 'done'"]: + try: + c.execute(f"ALTER TABLE chat_messages ADD COLUMN {col}") + except sqlite3.OperationalError: + pass # Migrate legacy table_name from adb_vector_configs into adb_vector_tables try: for cfg_row in c.execute("SELECT id, table_name FROM adb_vector_configs WHERE table_name IS NOT NULL AND table_name != ''").fetchall(): @@ -496,10 +504,6 @@ class GenAIConfigReq(BaseModel): temperature: float = 1; max_tokens: int = 6000; top_p: float = 0.95 top_k: int = 1; frequency_penalty: float = 0; presence_penalty: float = 0 is_default: bool = False -class ADBVectorReq(BaseModel): - config_name: str; dsn: str; username: str; password: str - wallet_password: Optional[str] = None; table_name: str = "CIS_EMBEDDINGS"; use_mtls: bool = True - genai_config_id: Optional[str] = None; embedding_model_id: str = "cohere.embed-v4.0" class IngestDocReq(BaseModel): adb_config_id: str; documents: List[Dict[str, Any]]; table_name: Optional[str] = None class MCPServerReq(BaseModel): @@ -1035,7 +1039,7 @@ def _call_genai(gc: dict, message: str, history: list = None, tools: list = None config=config, service_endpoint=endpoint, retry_strategy=oci.retry.NoneRetryStrategy(), - timeout=(10, 240) + timeout=(10, 600) ) # System prompt @@ -1733,8 +1737,7 @@ async def update_adb( _config_log("adb", vid, config_name, "success", "save", f"Conexão atualizada: {config_name} ({dsn})", u["id"], u["username"]) return {"id": vid, "config_name": config_name} -import re as _re -_TABLE_NAME_RE = _re.compile(r'^[A-Z][A-Z0-9_]{0,127}$') +_TABLE_NAME_RE = re.compile(r'^[A-Z][A-Z0-9_]{0,127}$') @app.get("/api/adb/{vid}/tables") async def list_adb_tables(vid: str, u=Depends(current_user)): @@ -1917,12 +1920,15 @@ def _get_adb_and_genai(vid: str): async def preview_report_chunks(rid: str, u=Depends(current_user)): """Preview the chunks that will be generated from a CIS report before embedding.""" with db() as c: - r = c.execute("SELECT report_data,tenancy_name FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone() + r = c.execute("SELECT json_path,tenancy_name FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone() if not r: raise HTTPException(404, "Report not found or not completed") - report_data = r["report_data"] - if isinstance(report_data, str): - try: report_data = json.loads(report_data) - except: raise HTTPException(400, "Invalid report data") + json_path = r["json_path"] + if not json_path or not Path(json_path).exists(): + raise HTTPException(400, "Report JSON file not found") + try: + report_data = json.loads(Path(json_path).read_text()) + except Exception: + raise HTTPException(400, "Invalid report data") documents = _chunk_report_by_section(report_data) return {"tenancy": report_data.get("tenancy", "unknown"), "regions": report_data.get("regions", []), @@ -1935,12 +1941,15 @@ async def embed_report(rid: str, req: dict, bg: BackgroundTasks, u=Depends(requi vid = req.get("adb_config_id") if not vid: raise HTTPException(400, "adb_config_id is required") with db() as c: - r = c.execute("SELECT report_data,tenancy_name FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone() + r = c.execute("SELECT json_path,tenancy_name FROM reports WHERE id=? AND status='completed'", (rid,)).fetchone() if not r: raise HTTPException(404, "Report not found or not completed") - report_data = r["report_data"] - if isinstance(report_data, str): - try: report_data = json.loads(report_data) - except: raise HTTPException(400, "Invalid report data") + json_path = r["json_path"] + if not json_path or not Path(json_path).exists(): + raise HTTPException(400, "Report JSON file not found") + try: + report_data = json.loads(Path(json_path).read_text()) + except Exception: + raise HTTPException(400, "Invalid report data") documents = _chunk_report_by_section(report_data) if not documents: raise HTTPException(400, "No sections found in report") cfg, gc = _get_adb_and_genai(vid) @@ -2150,10 +2159,7 @@ async def get_report(rid, u=Depends(current_user)): with db() as c: r=c.execute("SELECT * FROM reports WHERE id=?",(rid,)).fetchone() if not r: raise HTTPException(404) if u["role"]!="admin" and r["user_id"]!=u["id"]: raise HTTPException(403) - d=dict(r) - d.pop("report_data", None) - d.pop("mcp_server_id", None) - return d + return dict(r) @app.get("/api/reports/{rid}/files") async def list_report_files(rid: str, u=Depends(current_user)): @@ -2196,12 +2202,13 @@ async def report_dl(rid, fmt: str = Query("json"), u=Depends(current_user)): # ── Chat Agent ──────────────────────────────────────────────────────────────── # (endpoints defined below _chat_core, after _agent_respond) -async def _chat_core_impl(msg: ChatMsg, u, attachments: list = None): - """Internal chat implementation — called by both /api/chat and /api/chat/upload.""" +def _chat_start(msg: ChatMsg, u, attachments: list = None): + """Start a chat: save user msg, resolve config, return (sid, mid, genai_cfg) or immediate response. + If genai_cfg is None, returns immediate fallback response in mid field as dict.""" sid = msg.session_id or str(uuid.uuid4()) with db() as c: - c.execute("INSERT INTO chat_messages (id,session_id,user_id,role,content,model_id) VALUES (?,?,?,?,?,?)", - (str(uuid.uuid4()), sid, u["id"], "user", msg.message, None)) + c.execute("INSERT INTO chat_messages (id,session_id,user_id,role,content,model_id,status) VALUES (?,?,?,?,?,?,?)", + (str(uuid.uuid4()), sid, u["id"], "user", msg.message, None, "done")) genai_cfg = None if msg.genai_config_id: @@ -2209,7 +2216,6 @@ async def _chat_core_impl(msg: ChatMsg, u, attachments: list = None): row = c.execute("SELECT * FROM genai_configs WHERE id=?", (msg.genai_config_id,)).fetchone() if row: genai_cfg = dict(row) - # Override params from inline chat settings if provided if msg.temperature is not None: genai_cfg["temperature"] = msg.temperature if msg.max_tokens is not None: genai_cfg["max_tokens"] = msg.max_tokens if msg.top_p is not None: genai_cfg["top_p"] = msg.top_p @@ -2217,7 +2223,6 @@ async def _chat_core_impl(msg: ChatMsg, u, attachments: list = None): if msg.frequency_penalty is not None: genai_cfg["frequency_penalty"] = msg.frequency_penalty if msg.presence_penalty is not None: genai_cfg["presence_penalty"] = msg.presence_penalty elif msg.model_id and msg.oci_config_id: - # Direct model mode: build synthetic config dict with db() as c: oci_row = c.execute("SELECT * FROM oci_configs WHERE id=?", (msg.oci_config_id,)).fetchone() if not oci_row: @@ -2243,148 +2248,158 @@ async def _chat_core_impl(msg: ChatMsg, u, attachments: list = None): "presence_penalty": msg.presence_penalty if msg.presence_penalty is not None else 0.0, } - if genai_cfg: - try: - history = [] - with db() as c: - prev = c.execute("SELECT role,content FROM chat_messages WHERE session_id=? AND role IN ('user','assistant') ORDER BY created_at ASC", (sid,)).fetchall() - history = [{"role":r["role"],"content":r["content"]} for r in prev] - - # ── RAG: augment with vector context from ALL active ADB configs ── - rag_context = "" - adb_cfgs = _get_active_adb_configs(u["id"]) - if adb_cfgs: - all_documents = [] - for adb_cfg in adb_cfgs: - try: - with db() as c: - emb_genai = c.execute("SELECT * FROM genai_configs WHERE id=?", (adb_cfg["genai_config_id"],)).fetchone() - if emb_genai: - emb_model = adb_cfg.get("embedding_model_id", "cohere.embed-v4.0") - query_embedding = _embed_text(msg.message, dict(emb_genai), emb_model) - tables = _get_tables_for_config(adb_cfg["id"], active_only=True) - if not tables: - tables = [{"table_name": adb_cfg.get("table_name", "CIS_EMBEDDINGS")}] - for tbl in tables: - try: - documents = _vector_search(adb_cfg, query_embedding, top_k=5, table_name=tbl["table_name"]) - if documents: - for doc in documents: - doc["source"] = f"{doc.get('source', 'unknown')} [{tbl['table_name']}]" - all_documents.extend(documents) - log.info(f"RAG: Retrieved {len(documents)} docs from {tbl['table_name']}") - except Exception as te: - log.warning(f"RAG search failed for table {tbl['table_name']}: {te}") - except Exception as e: - log.warning(f"RAG retrieval failed for {adb_cfg.get('config_name','?')} (non-fatal): {e}") - if all_documents: - all_documents.sort(key=lambda d: d["distance"]) - rag_context = _build_rag_context(all_documents[:10]) - - cfg_dict = dict(genai_cfg) - # Global system prompt from system_prompts table - with db() as c: - sp_row = c.execute("SELECT content FROM system_prompts WHERE agent='chat' AND is_active=1 LIMIT 1").fetchone() - global_prompt = sp_row["content"] if sp_row and sp_row["content"] else "" - # If RAG context found, wrap user message with context - if rag_context: - augmented_message = RAG_CONTEXT_TEMPLATE.format(context=rag_context, question=msg.message) - cfg_dict["system_prompt"] = global_prompt or RAG_DEFAULT_SYSTEM_PROMPT - else: - augmented_message = msg.message - if global_prompt: - cfg_dict["system_prompt"] = global_prompt - # Collect MCP tools if enabled - mcp_tools = [] - tool_defs = None - if msg.use_tools: - mcp_tools = _get_active_mcp_tools(u["id"]) - if mcp_tools: - tool_defs = [t["tool"] for t in mcp_tools] - log.info(f"Chat with {len(tool_defs)} MCP tools available") - - # ── Memory compaction: summarize old messages if history is too long ── - if history and _should_compact(history): - log.info(f"Compaction triggered: {len(history)} msgs, ~{_estimate_history_tokens(history)} est tokens") - history = _compact_history(sid, u["id"], cfg_dict, history) - log.info(f"Post-compaction: {len(history)} msgs, ~{_estimate_history_tokens(history)} est tokens") - - hist = history[:-1] if len(history) > 1 else None - resp_text, tool_calls, tool_calls_raw = _call_genai(cfg_dict, augmented_message, hist, tools=tool_defs, attachments=attachments) - - # Tool use loop (max 5 iterations) - # Accumulate extra_messages so the model sees the full tool use conversation - all_tool_results = [] - accumulated_msgs = [] # raw OCI SDK message objects for Generic format - iterations = 0 - api_format = GENAI_MODELS.get(genai_cfg.get("model_id", ""), {}).get("api_format", "GENERIC") - while tool_calls and iterations < 5: - iterations += 1 - log.info(f"Tool use iteration {iterations}: {len(tool_calls)} tool call(s)") - iteration_results = [] - for tc in tool_calls: - mcp_match = next((m for m in mcp_tools if m["tool"]["name"] == tc["name"]), None) - if mcp_match: - try: - result = await _execute_mcp_tool(mcp_match["server"], tc["name"], tc["arguments"]) - log.info(f"Tool {tc['name']} executed successfully ({len(result)} chars)") - except Exception as te: - result = f"Erro ao executar tool {tc['name']}: {str(te)[:300]}" - log.warning(f"Tool {tc['name']} failed: {te}") - else: - result = f"Tool {tc['name']} não encontrada nos MCP servers ativos" - iteration_results.append({"tool_call_id": tc["id"], "name": tc["name"], "content": result}) - all_tool_results.extend(iteration_results) - - # Build tool results in the appropriate format and call again - if api_format == "COHERE": - import oci - cohere_results = [] - for tr in iteration_results: - tc_obj = oci.generative_ai_inference.models.CohereToolCall() - tc_obj.name = tr["name"] - tc_obj.parameters = {} - tr_obj = oci.generative_ai_inference.models.CohereToolResult() - tr_obj.call = tc_obj - tr_obj.outputs = [{"result": tr["content"]}] - cohere_results.append(tr_obj) - resp_text, tool_calls, tool_calls_raw = _call_genai( - cfg_dict, augmented_message, hist, tools=tool_defs, - tool_results_cohere=cohere_results) - else: - import oci - # Accumulate: add assistant message with tool_calls + tool result messages - assistant_msg = oci.generative_ai_inference.models.AssistantMessage() - assistant_msg.tool_calls = tool_calls_raw - accumulated_msgs.append(assistant_msg) - for tr in iteration_results: - tool_msg = oci.generative_ai_inference.models.ToolMessage() - tool_msg.tool_call_id = tr["tool_call_id"] - tool_content = oci.generative_ai_inference.models.TextContent() - tool_content.text = tr["content"] - tool_msg.content = [tool_content] - accumulated_msgs.append(tool_msg) - - resp_text, tool_calls, tool_calls_raw = _call_genai( - cfg_dict, augmented_message, hist, tools=tool_defs, - extra_messages=accumulated_msgs) - - resp = resp_text - except Exception as e: - resp = f"❌ Erro GenAI: {str(e)[:400]}" - all_tool_results = [] - else: + if not genai_cfg: + # No GenAI config — return immediate fallback resp = _agent_respond(msg.message, u) - all_tool_results = [] + with db() as c: + c.execute("INSERT INTO chat_messages (id,session_id,user_id,role,content,model_id,status) VALUES (?,?,?,?,?,?,?)", + (str(uuid.uuid4()), sid, u["id"], "assistant", resp, None, "done")) + return sid, {"session_id": sid, "response": resp, "model_id": None, "status": "done"}, None - mid = genai_cfg["model_id"] if genai_cfg else None + # Create placeholder assistant message for background processing + mid = str(uuid.uuid4()) with db() as c: - c.execute("INSERT INTO chat_messages (id,session_id,user_id,role,content,model_id) VALUES (?,?,?,?,?,?)", - (str(uuid.uuid4()), sid, u["id"], "assistant", resp, mid)) - result = {"session_id": sid, "response": resp, "model_id": mid} - if all_tool_results: - result["tools_used"] = [{"name": tr["name"], "result_preview": tr["content"][:200]} for tr in all_tool_results] - return result + c.execute("INSERT INTO chat_messages (id,session_id,user_id,role,content,model_id,status) VALUES (?,?,?,?,?,?,?)", + (mid, sid, u["id"], "assistant", "", genai_cfg.get("model_id"), "processing")) + return sid, mid, genai_cfg + + +async def _chat_background(mid: str, sid: str, msg: ChatMsg, user: dict, genai_cfg: dict, attachments: list = None): + """Background worker — processes GenAI chat, updates DB when done.""" + log.info(f"Chat background started: mid={mid}, sid={sid}") + try: + history = [] + with db() as c: + prev = c.execute("SELECT role,content FROM chat_messages WHERE session_id=? AND role IN ('user','assistant') AND status='done' ORDER BY created_at ASC", (sid,)).fetchall() + history = [{"role":r["role"],"content":r["content"]} for r in prev] + + # ── RAG: augment with vector context from ALL active ADB configs ── + rag_context = "" + adb_cfgs = _get_active_adb_configs(user["id"]) + if adb_cfgs: + all_documents = [] + for adb_cfg in adb_cfgs: + try: + with db() as c: + emb_genai = c.execute("SELECT * FROM genai_configs WHERE id=?", (adb_cfg["genai_config_id"],)).fetchone() + if emb_genai: + emb_model = adb_cfg.get("embedding_model_id", "cohere.embed-v4.0") + query_embedding = _embed_text(msg.message, dict(emb_genai), emb_model) + tables = _get_tables_for_config(adb_cfg["id"], active_only=True) + if not tables: + tables = [{"table_name": adb_cfg.get("table_name", "CIS_EMBEDDINGS")}] + for tbl in tables: + try: + documents = _vector_search(adb_cfg, query_embedding, top_k=5, table_name=tbl["table_name"]) + if documents: + for doc in documents: + doc["source"] = f"{doc.get('source', 'unknown')} [{tbl['table_name']}]" + all_documents.extend(documents) + log.info(f"RAG: Retrieved {len(documents)} docs from {tbl['table_name']}") + except Exception as te: + log.warning(f"RAG search failed for table {tbl['table_name']}: {te}") + except Exception as e: + log.warning(f"RAG retrieval failed for {adb_cfg.get('config_name','?')} (non-fatal): {e}") + if all_documents: + all_documents.sort(key=lambda d: d["distance"]) + rag_context = _build_rag_context(all_documents[:10]) + + cfg_dict = dict(genai_cfg) + with db() as c: + sp_row = c.execute("SELECT content FROM system_prompts WHERE agent='chat' AND is_active=1 LIMIT 1").fetchone() + global_prompt = sp_row["content"] if sp_row and sp_row["content"] else "" + if rag_context: + augmented_message = RAG_CONTEXT_TEMPLATE.format(context=rag_context, question=msg.message) + cfg_dict["system_prompt"] = global_prompt or RAG_DEFAULT_SYSTEM_PROMPT + else: + augmented_message = msg.message + if global_prompt: + cfg_dict["system_prompt"] = global_prompt + + mcp_tools = [] + tool_defs = None + if msg.use_tools: + mcp_tools = _get_active_mcp_tools(user["id"]) + if mcp_tools: + tool_defs = [t["tool"] for t in mcp_tools] + log.info(f"Chat with {len(tool_defs)} MCP tools available") + + if history and _should_compact(history): + log.info(f"Compaction triggered: {len(history)} msgs, ~{_estimate_history_tokens(history)} est tokens") + history = _compact_history(sid, user["id"], cfg_dict, history) + log.info(f"Post-compaction: {len(history)} msgs, ~{_estimate_history_tokens(history)} est tokens") + + hist = history[:-1] if len(history) > 1 else None + loop = asyncio.get_event_loop() + resp_text, tool_calls, tool_calls_raw = await loop.run_in_executor( + _chat_executor, partial(_call_genai, cfg_dict, augmented_message, hist, + tool_defs, None, None, attachments)) + + all_tool_results = [] + accumulated_msgs = [] + iterations = 0 + api_format = GENAI_MODELS.get(genai_cfg.get("model_id", ""), {}).get("api_format", "GENERIC") + while tool_calls and iterations < 5: + iterations += 1 + log.info(f"Tool use iteration {iterations}: {len(tool_calls)} tool call(s)") + iteration_results = [] + for tc in tool_calls: + mcp_match = next((m for m in mcp_tools if m["tool"]["name"] == tc["name"]), None) + if mcp_match: + try: + result = await _execute_mcp_tool(mcp_match["server"], tc["name"], tc["arguments"]) + log.info(f"Tool {tc['name']} executed successfully ({len(result)} chars)") + except Exception as te: + result = f"Erro ao executar tool {tc['name']}: {str(te)[:300]}" + log.warning(f"Tool {tc['name']} failed: {te}") + else: + result = f"Tool {tc['name']} não encontrada nos MCP servers ativos" + iteration_results.append({"tool_call_id": tc["id"], "name": tc["name"], "content": result}) + all_tool_results.extend(iteration_results) + + if api_format == "COHERE": + import oci + cohere_results = [] + for tr in iteration_results: + tc_obj = oci.generative_ai_inference.models.CohereToolCall() + tc_obj.name = tr["name"] + tc_obj.parameters = {} + tr_obj = oci.generative_ai_inference.models.CohereToolResult() + tr_obj.call = tc_obj + tr_obj.outputs = [{"result": tr["content"]}] + cohere_results.append(tr_obj) + resp_text, tool_calls, tool_calls_raw = await loop.run_in_executor( + _chat_executor, partial(_call_genai, cfg_dict, augmented_message, hist, + tool_defs, cohere_results)) + else: + import oci + assistant_msg = oci.generative_ai_inference.models.AssistantMessage() + assistant_msg.tool_calls = tool_calls_raw + accumulated_msgs.append(assistant_msg) + for tr in iteration_results: + tool_msg = oci.generative_ai_inference.models.ToolMessage() + tool_msg.tool_call_id = tr["tool_call_id"] + tool_content = oci.generative_ai_inference.models.TextContent() + tool_content.text = tr["content"] + tool_msg.content = [tool_content] + accumulated_msgs.append(tool_msg) + resp_text, tool_calls, tool_calls_raw = await loop.run_in_executor( + _chat_executor, partial(_call_genai, cfg_dict, augmented_message, hist, + tool_defs, None, accumulated_msgs)) + + resp = resp_text + if all_tool_results: + tools_info = '\n\n🔧 **Tools utilizadas:** ' + ', '.join(tr["name"] for tr in all_tool_results) + resp += tools_info + + with db() as c: + c.execute("UPDATE chat_messages SET content=?, status='done' WHERE id=?", (resp, mid)) + log.info(f"Chat {mid} completed successfully") + except Exception as e: + log.error(f"Chat {mid} failed: {e}") + with db() as c: + c.execute("UPDATE chat_messages SET content=?, status='failed' WHERE id=?", + (f"❌ Erro GenAI: {str(e)[:400]}", mid)) MAX_UPLOAD_FILES = 5 MAX_UPLOAD_SIZE = 10 * 1024 * 1024 # 10MB @@ -2394,6 +2409,7 @@ DOC_MIMES = {"application/pdf"} @app.post("/api/chat/upload") async def chat_with_files( + bg: BackgroundTasks, message: str = Form(""), session_id: Optional[str] = Form(None), genai_config_id: Optional[str] = Form(None), @@ -2414,7 +2430,6 @@ async def chat_with_files( if len(files) > MAX_UPLOAD_FILES: raise HTTPException(400, f"Máximo {MAX_UPLOAD_FILES} arquivos por mensagem") - # Process uploaded files into attachments attachments = [] file_names = [] for f in files: @@ -2429,7 +2444,6 @@ async def chat_with_files( elif mime in DOC_MIMES: attachments.append({"type": "document", "mime": mime, "data_uri": data_uri}) else: - # Text-based files: read content and append to message try: text = data.decode("utf-8", errors="replace") except Exception: @@ -2437,7 +2451,6 @@ async def chat_with_files( message = f"{message}\n\n--- Conteúdo de {f.filename} ---\n{text[:50000]}" file_names.append(f.filename) - # Build a synthetic ChatMsg and delegate to the chat logic msg = ChatMsg( message=message or "Analise os arquivos anexados.", session_id=session_id, @@ -2455,19 +2468,35 @@ async def chat_with_files( presence_penalty=presence_penalty, ) - # Store attachments in request state for _call_genai - result = await _chat_core_impl(msg, u, attachments=attachments if attachments else None) + sid, mid_or_result, genai_cfg = _chat_start(msg, u, attachments=attachments if attachments else None) + if genai_cfg is None: + return mid_or_result # immediate fallback response + if file_names: - # Save file reference in user message with db() as c: - c.execute("UPDATE chat_messages SET content = content || ? WHERE session_id=? AND role='user' ORDER BY created_at DESC LIMIT 1", - (f"\n[📎 {', '.join(file_names)}]", result["session_id"])) - return result + c.execute("UPDATE chat_messages SET content = content || ? WHERE session_id=? AND role='user' AND status='done' ORDER BY created_at DESC LIMIT 1", + (f"\n[📎 {', '.join(file_names)}]", sid)) + + bg.add_task(_chat_background, mid_or_result, sid, msg, dict(u), genai_cfg, attachments if attachments else None) + return {"message_id": mid_or_result, "session_id": sid, "status": "processing"} @app.post("/api/chat") -async def chat(msg: ChatMsg, u=Depends(current_user)): - return await _chat_core_impl(msg, u) +async def chat(msg: ChatMsg, bg: BackgroundTasks, u=Depends(current_user)): + sid, mid_or_result, genai_cfg = _chat_start(msg, u) + if genai_cfg is None: + return mid_or_result # immediate fallback response + bg.add_task(_chat_background, mid_or_result, sid, msg, dict(u), genai_cfg) + return {"message_id": mid_or_result, "session_id": sid, "status": "processing"} + + +@app.get("/api/chat/{mid}/status") +async def chat_message_status(mid: str, u=Depends(current_user)): + with db() as c: + r = c.execute("SELECT id, session_id, role, content, model_id, status FROM chat_messages WHERE id=?", (mid,)).fetchone() + if not r: + raise HTTPException(404, "Message not found") + return dict(r) def _agent_respond(msg, user): @@ -2499,27 +2528,6 @@ async def clear_chat(sid, u=Depends(current_user)): with db() as c: c.execute("DELETE FROM chat_messages WHERE session_id=? AND user_id=?", (sid, u["id"])) return {"ok": True} -# ── Downloads ───────────────────────────────────────────────────────────────── -@app.get("/api/downloads") -async def list_dl(u=Depends(current_user)): - with db() as c: - q="SELECT id,tenancy_name,status,created_at,completed_at FROM reports WHERE status='completed'" - rows=c.execute(q+" ORDER BY created_at DESC").fetchall() if u["role"]=="admin" \ - else c.execute(q+" AND user_id=? ORDER BY created_at DESC",(u["id"],)).fetchall() - res=[] - for r in rows: - d=dict(r); d["files"]=[] - rd=REPORTS/d["id"] - if rd.exists(): d["files"]=[{"name":f.name,"size":f.stat().st_size} for f in rd.iterdir() if f.is_file()] - res.append(d) - return res - -@app.get("/api/downloads/{rid}/{fname}") -async def dl_file(rid, fname, u=Depends(current_user)): - p=REPORTS/rid/fname - if not p.exists(): raise HTTPException(404) - return FileResponse(p, filename=fname) - # ── Audit ───────────────────────────────────────────────────────────────────── @app.get("/api/audit-log") async def audit_log(limit:int=Query(100,le=500), u=Depends(require("admin"))): @@ -2544,19 +2552,6 @@ async def get_config_logs( with db() as c: rows = c.execute(query, params).fetchall() return [dict(r) for r in rows] -@app.delete("/api/config-logs") -async def clear_config_logs( - config_type: str = Query(None), config_id: str = Query(None), - u=Depends(require("admin")) -): - query = "DELETE FROM config_logs WHERE 1=1" - params = [] - if config_type: query += " AND config_type=?"; params.append(config_type) - if config_id: query += " AND config_id=?"; params.append(config_id) - with db() as c: c.execute(query, params) - return {"ok": True} - -# ── Health ──────────────────────────────────────────────────────────────────── # ── App Settings ────────────────────────────────────────────────────────────── @app.get("/api/settings/{key}") async def get_setting(key: str, u=Depends(current_user)): diff --git a/backend/mcp_cis_server.py b/backend/mcp_cis_server.py index 347d36a..66432de 100644 --- a/backend/mcp_cis_server.py +++ b/backend/mcp_cis_server.py @@ -11,13 +11,10 @@ import concurrent.futures import datetime import json import os -import sys import time import traceback from configparser import ConfigParser from functools import partial -from threading import Thread - from mcp.server import Server from mcp.server.stdio import stdio_server from mcp.types import Tool, TextContent @@ -27,7 +24,8 @@ server = Server("cis-compliance") # session_key -> {checker, sections_collected: set, sections_analyzed: set, ...} _sessions: dict = {} -SESSION_TTL = 1800 +SESSION_TTL = 7200 # 2 hours — reduce cold starts +TOOL_TIMEOUT = 300 # 5 min max per tool call def _session_key(config_id: str, regions: list[str] | None = None) -> str: @@ -42,15 +40,6 @@ OCI_CONFIGS_DIR = os.environ.get("OCI_CONFIGS_DIR", "/data/oci_configs") # Per-section data collectors: which private methods to call # ────────────────────────────────────────────────────────────────────── -# Home-region functions (run once, needed by most sections) -HOME_BASE = [ - "_CIS_Report__identity_read_compartments", - "_CIS_Report__cloud_guard_read_cloud_guard_configuration", - "_CIS_Report__identity_read_domains", - "_CIS_Report__identity_read_groups", - "_CIS_Report__identity_read_availability_domains", -] - # Section -> (home_region_collectors, regional_collectors, analysis_methods) SECTION_DEPS = { "iam": { @@ -217,12 +206,20 @@ def _ensure_base(session): if session["base_collected"]: return checker = session["checker"] - # Sequential: compartments + cloud_guard first, then domains, then groups - getattr(checker, "_CIS_Report__identity_read_compartments")() - getattr(checker, "_CIS_Report__cloud_guard_read_cloud_guard_configuration")() - getattr(checker, "_CIS_Report__identity_read_domains")() - getattr(checker, "_CIS_Report__identity_read_groups")() - getattr(checker, "_CIS_Report__identity_read_availability_domains")() + base_methods = [ + "_CIS_Report__identity_read_compartments", + "_CIS_Report__cloud_guard_read_cloud_guard_configuration", + "_CIS_Report__identity_read_domains", + "_CIS_Report__identity_read_groups", + "_CIS_Report__identity_read_availability_domains", + ] + def _run_base(fn_name): + try: + getattr(checker, fn_name)() + except Exception as e: + print(f"Warning: {fn_name} failed: {e}") + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as ex: + list(ex.map(_run_base, base_methods)) session["base_collected"] = True @@ -236,23 +233,21 @@ def _collect_section(session, section: str): if not deps: return - # Home region collectors (sequential for safety) - for method_name in deps["home"]: + def _run_method(fn_name): try: - getattr(checker, method_name)() + getattr(checker, fn_name)() except Exception as e: - print(f"Warning: {method_name} failed: {e}") + print(f"Warning: {fn_name} failed: {e}") + + # Home region collectors (parallel) + if deps["home"]: + with concurrent.futures.ThreadPoolExecutor(max_workers=len(deps["home"])) as ex: + list(ex.map(_run_method, deps["home"])) # Regional collectors (parallel with thread pool) if deps["regional"]: - def _run(fn_name): - try: - getattr(checker, fn_name)() - except Exception as e: - print(f"Warning: {fn_name} failed: {e}") - - with concurrent.futures.ThreadPoolExecutor(max_workers=4) as ex: - list(ex.map(_run, deps["regional"])) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as ex: + list(ex.map(_run_method, deps["regional"])) session["sections_collected"].add(section) @@ -627,8 +622,13 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: return [TextContent(type="text", text=json.dumps({"error": f"Tool '{name}' not found"}))] try: loop = asyncio.get_event_loop() - result = await loop.run_in_executor(None, partial(handler, arguments)) + result = await asyncio.wait_for( + loop.run_in_executor(None, partial(handler, arguments)), + timeout=TOOL_TIMEOUT + ) return [TextContent(type="text", text=json.dumps(result, default=str, ensure_ascii=False))] + except asyncio.TimeoutError: + return [TextContent(type="text", text=json.dumps({"error": f"Tool '{name}' timed out after {TOOL_TIMEOUT}s"}))] except Exception as e: return [TextContent(type="text", text=json.dumps({"error": str(e), "traceback": traceback.format_exc()}))] diff --git a/docker-compose.yml b/docker-compose.yml index 4c06497..bf60e80 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,8 +12,6 @@ services: - OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING=True volumes: - agent-data:/data - ports: - - "8000:8000" # Optional: direct API access for debugging networks: - agent-net healthcheck: diff --git a/frontend/index.html b/frontend/index.html index c632915..3a2915d 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -14,6 +14,9 @@ --gn:#16a34a;--gnl:rgba(22,163,74,.08);--rd:#dc2626;--rdl:rgba(220,38,38,.06); --bl:#2563eb;--bll:rgba(37,99,235,.06);--yl:#ca8a04;--yll:rgba(202,138,4,.06); --pp:#7c3aed;--ppl:rgba(124,58,237,.06); + --ok:#16a34a;--w:#ca8a04;--err:#dc2626; + --tx2:#78716c;--p:#1c1917;--pl:rgba(199,70,52,.06); + --b2:#ddd9d2;--b3:#c5c0b8; --r:12px;--rl:16px;--rr:20px; --sh1:0 1px 2px rgba(28,25,23,.04),0 1px 3px rgba(28,25,23,.06); --sh2:0 4px 6px -1px rgba(28,25,23,.05),0 2px 4px -2px rgba(28,25,23,.04); @@ -143,6 +146,7 @@ tbody tr:last-child td{border-bottom:none} .cm-user .cb strong{color:#ffe0db} .cb{max-width:72%;padding:.7rem .95rem;border-radius:var(--rl) var(--rl) var(--rl) 6px;background:var(--bg1);font-size:.82rem; line-height:1.6;border:1px solid var(--bd);white-space:pre-wrap;box-shadow:var(--sh1)} +.cm-ts{font-size:.65rem;color:#999;margin-top:3px;opacity:.7} .cb code{font-family:var(--fm);font-size:.73rem;background:var(--bg3);padding:.12rem .3rem;border-radius:5px} .cb strong{color:var(--ac)} .ch-i{display:flex;gap:.45rem;padding:.7rem .85rem;background:var(--bg1);border:1px solid var(--bd);border-top:none; @@ -212,7 +216,7 @@ const V='1.1'; const LOGO_W=``; const LOGO_R=``; -const S={user:null,token:null,tab:'chat',msgs:[],sid:null,reports:[],ociCfg:[],genaiCfg:[],adbCfg:[],mcpSvr:[],users:[],auditLogs:[],models:{},regions:[],embModels:{},selGenai:'',expData:null,editing:null, +const S={user:null,token:null,tab:'chat',msgs:[],sid:null,reports:[],ociCfg:[],genaiCfg:[],adbCfg:[],mcpSvr:[],users:[],models:{},regions:[],embModels:{},expData:null,editing:null, chatModel:'',chatOci:'',chatRegion:'',chatCompartment:'', chatParams:{temperature:1,max_tokens:6000,top_p:0.95,top_k:1,frequency_penalty:0,presence_penalty:0},chatParamsOpen:false,chatUseTools:true, chatPrompts:[],editingPrompt:null,trackingReportId:null,ociRegions:{},rptSelRegions:[],rptRegionsOpen:false,rptRegionFilter:'', @@ -240,7 +244,7 @@ async function loadData(){try{ if(S.user.role==='admin'){try{S.users=await $api('/users')}catch(e){}} try{S.cisVer=await $api('/cis-engine/version')}catch(e){} }catch(e){console.error(e)}} -function R(){document.getElementById('app').innerHTML=S.user?rApp():rLogin();bind();if(S.editing?.type==='mcp')setTimeout(mcpTypeFields,0)} +function R(){document.getElementById('app').innerHTML=S.user?rApp():rLogin();if(S.editing?.type==='mcp')setTimeout(mcpTypeFields,0)} function switchTab(t){S.tab=t;S.expData=null;R();if(t==='audit'&&S.user.role==='admin')loadAudit();if(t==='downloads')refreshDl(); const tm={'oci-config':'oci','genai':'genai','adb':'adb','mcp':'mcp'};if(tm[t])setTimeout(()=>refreshCLogs(tm[t]),100)} @@ -284,7 +288,7 @@ function rPg(){switch(S.tab){case'chat':return rChat();case'explorer':return rEx /* ── Chat ── */ function rChat(){ const ms=S.msgs.length===0?'

🤖

Inicie uma conversa com o agente.

Selecione um modelo para começar.

' - :S.msgs.map(m=>`
${fm(m.c)}
`).join(''); + :S.msgs.map(m=>`
${fm(m.c)}
${m.t?`
${m.t}
`:''}
`).join(''); // Build custom dropdown items // Skip non-chat models: Codex (completions only), Image, Audio, Guard, ProtectAI, GPT-oss, Pro (Responses API only) const skip=new Set(['openai.gpt-image-1','openai.gpt-image-1.5','openai.gpt-audio','meta.llama-guard-4-12b','protectai.deberta-v3-base-prompt-injection-v2', @@ -354,6 +358,7 @@ function addChatFiles(input){for(const f of input.files){if(S.chatFiles.length>= if(entry.type==='image'){const r=new FileReader();r.onload=e=>{entry.preview=e.target.result;R()};r.readAsDataURL(f)} S.chatFiles.push(entry)}input.value='';R()} function rmChatFile(i){S.chatFiles.splice(i,1);R()} +function tstamp(){const d=new Date();return d.toLocaleTimeString('pt-BR',{hour:'2-digit',minute:'2-digit'})} async function sChat(){const el=document.getElementById('chi');const m=el.value.trim(); if(!m&&!S.chatFiles.length)return; if(!S.chatModel){S.msgs.push({r:'assistant',c:'⚠️ Selecione um modelo antes de enviar uma mensagem.'});R();return} @@ -362,8 +367,7 @@ async function sChat(){const el=document.getElementById('chi');const m=el.value. const fileNames=S.chatFiles.map(f=>f.name); let userDisplay=m||''; if(hasFiles)userDisplay+=(userDisplay?'\n':'')+'📎 '+fileNames.join(', '); - S.msgs.push({r:'user',c:userDisplay});R();scCh(); - // Disable input and show thinking indicator + S.msgs.push({r:'user',c:userDisplay,t:tstamp()});R();scCh(); const btns=document.querySelectorAll('.ch-i .btn'); el.disabled=true;btns.forEach(b=>{b.disabled=true}); const sendBtn=btns[btns.length-1];if(sendBtn){sendBtn.dataset.origText=sendBtn.textContent;sendBtn.innerHTML='Pensando...'} @@ -396,12 +400,23 @@ async function sChat(){const el=document.getElementById('chi');const m=el.value. body.top_k=S.chatParams.top_k;body.frequency_penalty=S.chatParams.frequency_penalty;body.presence_penalty=S.chatParams.presence_penalty} d=await $api('/chat',{method:'POST',body})} S.sid=d.session_id; - S.msgs=S.msgs.filter(x=>!x.thinking); - let resp=d.response; - if(d.tools_used&&d.tools_used.length){resp+='\n\n🔧 **Tools utilizadas:** '+d.tools_used.map(t=>t.name).join(', ')} - S.msgs.push({r:'assistant',c:resp});R();scCh()} - catch(e){S.msgs=S.msgs.filter(x=>!x.thinking);S.msgs.push({r:'assistant',c:'❌ Erro: '+e.message});R()} + if(d.status==='processing'&&d.message_id){ + await pollChatResult(d.message_id) + }else{ + S.msgs=S.msgs.filter(x=>!x.thinking); + let resp=d.response; + if(d.tools_used&&d.tools_used.length){resp+='\n\n🔧 **Tools utilizadas:** '+d.tools_used.map(t=>t.name).join(', ')} + S.msgs.push({r:'assistant',c:resp,t:tstamp()});R();scCh()}} + catch(e){S.msgs=S.msgs.filter(x=>!x.thinking);S.msgs.push({r:'assistant',c:'❌ Erro: '+e.message,t:tstamp()});R()} finally{el.disabled=false;el.focus();btns.forEach(b=>{b.disabled=false});if(sendBtn)sendBtn.textContent=sendBtn.dataset.origText||'Enviar →'}} +async function pollChatResult(mid){ + for(let i=0;i<1200;i++){ + await new Promise(r=>setTimeout(r,i<10?1000:3000)); + try{const r=await $api('/chat/'+mid+'/status'); + if(r.status==='done'){S.msgs=S.msgs.filter(x=>!x.thinking);S.msgs.push({r:'assistant',c:r.content,t:tstamp()});R();scCh();return} + if(r.status==='failed'){S.msgs=S.msgs.filter(x=>!x.thinking);S.msgs.push({r:'assistant',c:'❌ '+r.content,t:tstamp()});R();scCh();return} + }catch(e){}} + S.msgs=S.msgs.filter(x=>!x.thinking);S.msgs.push({r:'assistant',c:'⏰ Timeout: a resposta está demorando muito. Tente novamente.',t:tstamp()});R()} function scCh(){setTimeout(()=>{const e=document.getElementById('chm');if(e)e.scrollTop=e.scrollHeight},50)} async function clrChat(){if(S.sid)try{await $api('/chat/'+S.sid,{method:'DELETE'})}catch(e){}S.msgs=[];S.sid=null;R()} @@ -513,7 +528,6 @@ ${status}`} async function cisLoadVersion(){try{S.cisVer=await $api('/cis-engine/version');R()}catch(e){console.error('cisVer',e)}} async function cisCheckUpdate(){S.cisCheckResult=null;R();try{S.cisCheckResult=await $api('/cis-engine/check-update');R()}catch(e){alert('Erro ao verificar: '+e.message)}} async function cisDoUpdate(){if(!confirm('Atualizar CIS Engine para a versão mais recente do GitHub?'))return;S.cisUpdating=true;R();try{const d=await $api('/cis-engine/update',{method:'POST'});alert(`Atualizado: ${d.old_version} → ${d.new_version}\nPatches aplicados: ${d.patches_applied.join(', ')||'nenhum'}`);S.cisCheckResult=null;S.cisUpdating=false;await cisLoadVersion()}catch(e){S.cisUpdating=false;R();alert('Erro: '+e.message)}} -function ldRpt(id){const f=document.getElementById('rfr');if(f)f.src=API+'/reports/'+id+'/html'} function rptPickRegion(r){S.rptSelRegions.push(r);S.rptRegionFilter='';R()} function rptRemoveRegion(r){S.rptSelRegions=S.rptSelRegions.filter(x=>x!==r);R()} function rptPickOci(id){S.rptOciVal=id;S.rptOciOpen=false;S.rptOciFilter='';R()} @@ -1128,7 +1142,6 @@ async function hLogin(){const u=document.getElementById('lu').value,p=document.g try{const d=await doLogin(u,p);if(d&&d.mfa_required){document.getElementById('lf').style.display='none';document.getElementById('mfa-s').style.display='block'}} catch(e){document.getElementById('le').innerHTML=`
${e.message}
`}} async function hMfa(){try{await doLogin(_lu,_lp,document.getElementById('mfa-c').value)}catch(e){document.getElementById('le').innerHTML=`
${e.message}
`}} -function bind(){} (async()=>{await checkAuth();R()})(); diff --git a/nginx/default.conf b/nginx/default.conf index 7885c93..3120def 100644 --- a/nginx/default.conf +++ b/nginx/default.conf @@ -9,8 +9,6 @@ server { index index.html; try_files $uri $uri/ /index.html; add_header Cache-Control "no-cache, no-store, must-revalidate"; - add_header Pragma "no-cache"; - add_header Expires 0; } location /api/ {