refactor: decompose monolith — app.py 10,600→143 lines, 30+ modules

Fase 0 complete: extract all business logic, auth, database, and 176
endpoints from monolithic app.py into dedicated modules.

Structure:
- app.py: FastAPI hub (CORS, routers, startup/shutdown)
- models.py: 13 Pydantic request models
- utils.py: shared utilities (embed status, upload validation, process registries)
- config.py: constants, env vars, model catalogs
- auth/: crypto, jwt_auth, oidc, rate_limit
- database/: db(), init_db(), schema DDL
- services/: genai, compliance, chat, terraform, embeddings, cis_reports, mcp
- routes/: 13 APIRouter modules (auth, users, oci_config, oci_explorer,
  genai, mcp, adb, embeddings, reports, chat, terraform, settings, cis_engine)

Also: README updated with OCIR auth instructions for manual docker run.

86 tests passing.

backend/services/chat_background.py

@@ -0,0 +1,469 @@
+"""Chat Agent background processing — GenAI call, RAG, MCP tool use."""
+import os, json, uuid, time, re, asyncio, concurrent.futures
+from datetime import datetime, timedelta
+from typing import Optional, List, Dict, Any
+from fastapi import HTTPException
+
+from config import (
+    DATA, OCI_DIR, REPORTS, log, _chat_executor,
+    COMPACT_TOKEN_THRESHOLD, COMPACT_KEEP_RECENT,
+)
+from database import db
+from auth.crypto import _make_token, _safe_dec
+from auth.jwt_auth import _audit, _chat_log, _verify_config_access
+from services.genai import (
+    _call_genai, _embed_text, _build_rag_context,
+    _get_active_adb_configs, _get_tables_for_config,
+    _resolve_embed_config, _DIM_TO_MODEL, _get_table_embedding_dim,
+    _vector_search_multi, _relevant_tables,
+    _estimate_tokens, _should_compact, _compact_history,
+    RAG_CONTEXT_TEMPLATE, RAG_DEFAULT_SYSTEM_PROMPT,
+)
+
+def _chat_start(msg: "ChatMsg", u, attachments: list = None, agent_type: str = "chat"):
+    """Start a chat: save user msg, resolve config, return (sid, mid, genai_cfg) or immediate response.
+    If genai_cfg is None, returns immediate fallback response in mid field as dict."""
+    is_new = not msg.session_id
+    sid = msg.session_id or str(uuid.uuid4())
+    with db() as c:
+        c.execute("INSERT INTO chat_messages (id,session_id,user_id,role,content,model_id,status) VALUES (?,?,?,?,?,?,?)",
+                  (str(uuid.uuid4()), sid, u["id"], "user", msg.message, None, "done"))
+        if is_new:
+            title = (msg.message or "Nova conversa")[:80].strip()
+            c.execute("INSERT OR IGNORE INTO chat_sessions (id,user_id,agent_type,title) VALUES (?,?,?,?)",
+                      (sid, u["id"], agent_type, title))
+        else:
+            c.execute("UPDATE chat_sessions SET updated_at=datetime('now') WHERE id=?", (sid,))
+
+    genai_cfg = None
+    if msg.genai_config_id:
+        _verify_config_access("genai", msg.genai_config_id, u)
+        with db() as c:
+            row = c.execute("SELECT * FROM genai_configs WHERE id=?", (msg.genai_config_id,)).fetchone()
+        if row:
+            genai_cfg = dict(row)
+            if msg.temperature is not None: genai_cfg["temperature"] = msg.temperature
+            if msg.max_tokens is not None: genai_cfg["max_tokens"] = msg.max_tokens
+            if msg.top_p is not None: genai_cfg["top_p"] = msg.top_p
+            if msg.top_k is not None: genai_cfg["top_k"] = msg.top_k
+            if msg.frequency_penalty is not None: genai_cfg["frequency_penalty"] = msg.frequency_penalty
+            if msg.presence_penalty is not None: genai_cfg["presence_penalty"] = msg.presence_penalty
+            if msg.reasoning_effort is not None: genai_cfg["reasoning_effort"] = msg.reasoning_effort
+    elif msg.model_id and msg.oci_config_id:
+        _verify_config_access("oci", msg.oci_config_id, u)
+        with db() as c:
+            oci_row = c.execute("SELECT * FROM oci_configs WHERE id=?", (msg.oci_config_id,)).fetchone()
+        if not oci_row:
+            raise HTTPException(400, "OCI config not found")
+        region = msg.genai_region or oci_row["region"]
+        compartment = _safe_dec(oci_row["compartment_id"]) if oci_row["compartment_id"] else ""
+        if not compartment:
+            raise HTTPException(400, "compartment_id required")
+        genai_cfg = {
+            "oci_config_id": msg.oci_config_id,
+            "model_id": msg.model_id,
+            "model_ocid": None,
+            "compartment_id": compartment,
+            "genai_region": region,
+            "endpoint": f"https://inference.generativeai.{region}.oci.oraclecloud.com",
+            "serving_type": "ON_DEMAND",
+            "dedicated_endpoint_id": None,
+            "temperature": msg.temperature if msg.temperature is not None else 1.0,
+            "max_tokens": msg.max_tokens if msg.max_tokens is not None else 6000,
+            "top_p": msg.top_p if msg.top_p is not None else 0.95,
+            "top_k": msg.top_k if msg.top_k is not None else 1,
+            "frequency_penalty": msg.frequency_penalty if msg.frequency_penalty is not None else 0.0,
+            "presence_penalty": msg.presence_penalty if msg.presence_penalty is not None else 0.0,
+            "reasoning_effort": msg.reasoning_effort,
+        }
+
+    if not genai_cfg:
+        # No GenAI config — return immediate fallback
+        resp = _agent_respond(msg.message, u)
+        with db() as c:
+            c.execute("INSERT INTO chat_messages (id,session_id,user_id,role,content,model_id,status) VALUES (?,?,?,?,?,?,?)",
+                      (str(uuid.uuid4()), sid, u["id"], "assistant", resp, None, "done"))
+        return sid, {"session_id": sid, "response": resp, "model_id": None, "status": "done"}, None
+
+    # Create placeholder assistant message for background processing
+    mid = str(uuid.uuid4())
+    with db() as c:
+        c.execute("INSERT INTO chat_messages (id,session_id,user_id,role,content,model_id,status) VALUES (?,?,?,?,?,?,?)",
+                  (mid, sid, u["id"], "assistant", "", genai_cfg.get("model_id"), "processing"))
+    return sid, mid, genai_cfg
+
+
+async def _chat_background(mid: str, sid: str, msg: "ChatMsg", user: dict, genai_cfg: dict, attachments: list = None, agent_type: str = "chat"):
+    """Background worker — processes GenAI chat, updates DB when done."""
+    log.info(f"Chat background started: mid={mid}, sid={sid}")
+    try:
+        history = []
+        with db() as c:
+            prev = c.execute("SELECT role,content FROM chat_messages WHERE session_id=? AND role IN ('user','assistant') AND status='done' ORDER BY created_at ASC", (sid,)).fetchall()
+            history = [{"role":r["role"],"content":r["content"]} for r in prev]
+
+        # ── RAG: augment with vector context from ALL active ADB configs ──
+        # Resolve active tenancy for filtered vector search
+        rag_tenancy = None
+        active_oci_for_rag = genai_cfg.get("oci_config_id") or (msg.oci_config_id if hasattr(msg, 'oci_config_id') and msg.oci_config_id else None)
+        if active_oci_for_rag:
+            with db() as c:
+                oci_for_rag = c.execute("SELECT tenancy_name FROM oci_configs WHERE id=?", (active_oci_for_rag,)).fetchone()
+            if oci_for_rag:
+                rag_tenancy = oci_for_rag["tenancy_name"]
+                log.info(f"RAG: filtering by tenancy '{rag_tenancy}'")
+
+        rag_context = ""
+        # For short follow-up questions, enrich with previous context for better RAG search
+        import re as _re_chat
+        rag_query = msg.message
+        if len(msg.message.split()) <= 8 and history:
+            # Short question — get previous user messages (excluding current which is already in history)
+            prev_user_msgs = [h["content"] for h in history if h["role"] == "user" and h["content"] != msg.message]
+            if prev_user_msgs:
+                rag_query = prev_user_msgs[-1] + " " + msg.message
+                log.info(f"RAG: enriched short query → '{rag_query[:100]}'")
+            elif len(history) >= 2:
+                # Fallback: use last assistant response for context
+                last_assistant = [h["content"][:200] for h in history if h["role"] == "assistant"]
+                if last_assistant:
+                    rag_query = last_assistant[-1] + " " + msg.message
+                    log.info(f"RAG: enriched from assistant context")
+
+        # Detect CIS recommendation number for exact text filtering
+        cis_chat_match = _re_chat.search(r'(?:cis|recommendation)\s*(\d+\.\d+)', rag_query, _re_chat.IGNORECASE)
+        cis_chat_filter = f"CIS Recommendation: {cis_chat_match.group(1)}" if cis_chat_match else None
+        if cis_chat_filter:
+            log.info(f"RAG: detected CIS filter '{cis_chat_filter}'")
+
+        adb_cfgs = _get_active_adb_configs(user["id"]) if agent_type != "terraform" else []
+        rag_errors = []
+        if adb_cfgs:
+            all_documents = []
+            for adb_cfg in adb_cfgs:
+                try:
+                    genai_linked = None
+                    if adb_cfg.get("genai_config_id"):
+                        with db() as c:
+                            row = c.execute("SELECT * FROM genai_configs WHERE id=?", (adb_cfg["genai_config_id"],)).fetchone()
+                            if row: genai_linked = dict(row)
+                    emb_genai = _resolve_embed_config(oci_config_id=active_oci_for_rag, genai_cfg=genai_linked, user_id=user["id"])
+                    if not emb_genai:
+                        continue
+                    default_model = adb_cfg.get("embedding_model_id", "cohere.embed-v4.0")
+                    tables = _get_tables_for_config(adb_cfg["id"], active_only=True)
+                    if not tables:
+                        tables = [{"table_name": adb_cfg.get("table_name", "")}]
+                    all_table_names = [t["table_name"] for t in tables if t["table_name"]]
+                    # Smart skip: only search relevant tables
+                    relevant = _relevant_tables(rag_query, all_table_names)
+                    skipped = set(all_table_names) - set(relevant)
+                    if skipped:
+                        log.info(f"RAG: skipped {', '.join(skipped)} (not relevant)")
+                    # Auto-detect embedding model (use first table's dim as representative)
+                    emb_model = default_model
+                    for tbl_name in relevant:
+                        try:
+                            dim = _get_table_embedding_dim(adb_cfg, tbl_name)
+                            if dim and dim in _DIM_TO_MODEL:
+                                emb_model = _DIM_TO_MODEL[dim]
+                                break
+                        except Exception:
+                            pass
+                    query_embedding = _embed_text(rag_query, emb_genai, emb_model)
+                    # Single connection, multi-table search
+                    tbl_top_k = 10 if cis_chat_filter else 3
+                    docs = _vector_search_multi(adb_cfg, query_embedding, relevant,
+                                               top_k_per_table=tbl_top_k, tenancy=rag_tenancy,
+                                               text_filter=cis_chat_filter, user_id=user["id"])
+                    if docs:
+                        all_documents.extend(docs)
+                        sources = {}
+                        for d in docs:
+                            sources[d["source"]] = sources.get(d["source"], 0) + 1
+                        log.info(f"RAG: {len(docs)} docs — {', '.join(f'{k}:{v}' for k,v in sources.items())}")
+                except Exception as e:
+                    err_short = str(e)[:150]
+                    log.warning(f"RAG retrieval failed for {adb_cfg.get('config_name','?')}: {err_short}")
+                    if "DPY-6001" in str(e) or "DPY-6005" in str(e) or "timeout" in str(e).lower() or "connect" in str(e).lower():
+                        rag_errors.append(f"Não foi possível conectar ao ADB ({adb_cfg.get('config_name','?')})")
+            if all_documents:
+                all_documents.sort(key=lambda d: d["distance"])
+                top_limit = 15 if cis_chat_filter else 8
+                rag_context = _build_rag_context(all_documents[:top_limit], max_total_chars=16000 if cis_chat_filter else 12000)
+            # Append connection errors to context so LLM can inform the user
+            if rag_errors and not rag_context:
+                rag_context = "⚠️ AVISO: " + "; ".join(set(rag_errors)) + ". A base de conhecimento não está disponível no momento. Responda com base no seu conhecimento geral, informando que os dados do ADB não puderam ser consultados."
+            elif rag_errors:
+                rag_context += "\n\n⚠️ AVISO: Algumas bases não puderam ser consultadas: " + "; ".join(set(rag_errors))
+
+        cfg_dict = dict(genai_cfg)
+        with db() as c:
+            sp_row = c.execute("SELECT content FROM system_prompts WHERE agent=? AND is_active=1 LIMIT 1", (agent_type,)).fetchone()
+        global_prompt = sp_row["content"] if sp_row and sp_row["content"] else ""
+        if rag_context:
+            augmented_message = RAG_CONTEXT_TEMPLATE.format(context=rag_context, question=msg.message)
+            cfg_dict["system_prompt"] = global_prompt or RAG_DEFAULT_SYSTEM_PROMPT
+        else:
+            augmented_message = msg.message
+            if global_prompt:
+                cfg_dict["system_prompt"] = global_prompt
+
+        # ── Inject all config context into system prompt so model auto-resolves IDs ──
+        ctx_parts = []
+        active_oci_id = cfg_dict.get("oci_config_id") or (msg.oci_config_id if msg.oci_config_id else None)
+        with db() as c:
+            oci_cfgs = c.execute("SELECT id,tenancy_name,region,compartment_id FROM oci_configs WHERE user_id=?", (user["id"],)).fetchall()
+            genai_cfgs = c.execute("SELECT id,name,model_id,genai_region,compartment_id,oci_config_id FROM genai_configs WHERE user_id=?", (user["id"],)).fetchall()
+            adb_cfgs = c.execute("SELECT id,config_name,dsn,table_name,is_active,genai_config_id,embedding_model_id FROM adb_vector_configs WHERE user_id=? AND is_active=1", (user["id"],)).fetchall()
+            mcp_srvs = c.execute("SELECT id,name,description,is_active FROM mcp_servers WHERE is_active=1 AND (user_id=? OR EXISTS (SELECT 1 FROM users WHERE id=? AND role='admin'))", (user["id"], user["id"])).fetchall()
+        # ── Active OCI config (selected by user in this session) ──
+        if active_oci_id:
+            for oc in oci_cfgs:
+                if oc["id"] == active_oci_id:
+                    comp = _safe_dec(oc["compartment_id"]) if oc["compartment_id"] else "N/A"
+                    ctx_parts.append(
+                        f"⚡ CONFIG OCI ATIVA (usar como config_id em TODAS as tools que precisarem): "
+                        f"config_id=\"{oc['id']}\" tenancy=\"{oc['tenancy_name']}\" region=\"{oc['region']}\" compartment_id=\"{comp}\""
+                    )
+                    break
+        if oci_cfgs:
+            ctx_parts.append("\nTodas as configurações OCI disponíveis (use 'id' como config_id):")
+            for oc in oci_cfgs:
+                comp = _safe_dec(oc["compartment_id"]) if oc["compartment_id"] else "N/A"
+                active_tag = " ← ATIVA" if oc["id"] == active_oci_id else ""
+                ctx_parts.append(f"  - id=\"{oc['id']}\" tenancy=\"{oc['tenancy_name']}\" region=\"{oc['region']}\" compartment_id=\"{comp}\"{active_tag}")
+        if genai_cfgs:
+            ctx_parts.append("\nConfigurações GenAI disponíveis (use 'id' como genai_config_id):")
+            for gc in genai_cfgs:
+                comp = _safe_dec(gc["compartment_id"]) if gc["compartment_id"] else "N/A"
+                ctx_parts.append(f"  - id=\"{gc['id']}\" name=\"{gc['name']}\" model=\"{gc['model_id']}\" region=\"{gc['genai_region']}\" compartment_id=\"{comp}\"")
+        if adb_cfgs:
+            ctx_parts.append("\nConfigurações ADB Vector Store disponíveis (use 'id' como adb_config_id):")
+            for ac in adb_cfgs:
+                emb = ac["embedding_model_id"] if ac["embedding_model_id"] else "N/A"
+                ctx_parts.append(f"  - id=\"{ac['id']}\" name=\"{ac['config_name']}\" table=\"{ac['table_name']}\" embedding_model=\"{emb}\"")
+        if mcp_srvs:
+            ctx_parts.append("\nMCP Servers ativos (tools disponíveis para uso):")
+            for ms in mcp_srvs:
+                desc = ms["description"] or ""
+                ctx_parts.append(f"  - id=\"{ms['id']}\" name=\"{ms['name']}\" desc=\"{desc}\"")
+        if ctx_parts:
+            ctx_parts.append("\nIMPORTANTE: Use automaticamente a config OCI ATIVA como config_id em todas as tools. NUNCA peça config_id, tenancy ou IDs ao usuário — já estão definidos acima.")
+            config_hint = "\n".join(ctx_parts)
+            base_prompt = cfg_dict.get("system_prompt", "")
+            cfg_dict["system_prompt"] = f"{base_prompt}\n\n{config_hint}" if base_prompt else config_hint
+
+        # ── Terraform agent: boost max_tokens (capped at 65K to avoid context overflow) + force reasoning_effort=high ──
+        if agent_type == "terraform":
+            tf_model_info = GENAI_MODELS.get(cfg_dict.get("model_id", ""), {})
+            tf_model_max = tf_model_info.get("max_tokens", 32768)
+            cfg_dict["max_tokens"] = min(tf_model_max, 65000)
+            if tf_model_info.get("reasoning") and not cfg_dict.get("reasoning_effort"):
+                cfg_dict["reasoning_effort"] = "HIGH"
+
+        # ── Inject existing OCI resources for terraform agent ──
+        if agent_type == "terraform" and active_oci_id:
+            try:
+                # Determine compartment: from msg or from OCI config
+                tf_compartment = getattr(msg, 'compartment_id', None) or None
+                if not tf_compartment:
+                    for oc in oci_cfgs:
+                        if oc["id"] == active_oci_id:
+                            tf_compartment = _safe_dec(oc["compartment_id"]) if oc["compartment_id"] else None
+                            break
+                if tf_compartment:
+                    tf_region = None
+                    for oc in oci_cfgs:
+                        if oc["id"] == active_oci_id:
+                            tf_region = oc["region"]
+                            break
+                    loop = asyncio.get_event_loop()
+                    resources = await loop.run_in_executor(
+                        _chat_executor, partial(_fetch_compartment_resources, active_oci_id, tf_compartment, tf_region))
+                    resource_ctx = _build_resource_context(resources)
+                    cfg_dict["system_prompt"] = cfg_dict.get("system_prompt", "") + "\n\n" + resource_ctx
+                    log.info(f"Terraform: injected resource context for compartment {tf_compartment[:20]}...")
+            except Exception as e:
+                log.warning(f"Failed to inject terraform resource context: {e}")
+
+        # ── Inject OCI Terraform resource reference for correct resource names ──
+        if agent_type == "terraform":
+            tf_ref = _load_tf_resource_reference()
+            if tf_ref:
+                # Extract only the categorized sections (not the full 900+ resource list)
+                # to keep prompt size manageable
+                sections = []
+                for line in tf_ref.split('\n'):
+                    if line.startswith('## All Resource Types'):
+                        break
+                    sections.append(line)
+                ref_compact = '\n'.join(sections).strip()
+                if ref_compact:
+                    cfg_dict["system_prompt"] = cfg_dict.get("system_prompt", "") + \
+                        "\n\n### Referência de Recursos OCI Terraform (gerado do provider schema)\n" + \
+                        "Use EXATAMENTE estes nomes de resource types. Se o recurso não estiver nesta lista, ele NÃO EXISTE no provider.\n\n" + \
+                        ref_compact
+                    log.info(f"Terraform: injected resource reference ({len(ref_compact)} chars)")
+
+        # ── Inject official Terraform resource docs (Example Usage + Arguments) ──
+        if agent_type == "terraform":
+            try:
+                # Detect resource types from user message + recent conversation history
+                detect_text = msg.message if hasattr(msg, 'message') else ""
+                if history:
+                    # Include last 3 messages for context
+                    for h in history[-3:]:
+                        detect_text += "\n" + (h.get("content", "") or "")
+                resource_types = _detect_tf_resource_types(detect_text)
+                if resource_types:
+                    loop = asyncio.get_event_loop()
+                    docs_ctx = await loop.run_in_executor(
+                        _chat_executor, partial(_get_tf_docs_for_resources, resource_types))
+                    if docs_ctx:
+                        cfg_dict["system_prompt"] = cfg_dict.get("system_prompt", "") + "\n\n" + docs_ctx
+                        log.info(f"Terraform: injected {len(resource_types)} resource docs ({len(docs_ctx)} chars)")
+            except Exception as e:
+                log.warning(f"Failed to inject terraform resource docs: {e}")
+
+        # Log total prompt sizes for debugging
+        if agent_type == "terraform":
+            sp_len = len(cfg_dict.get("system_prompt", ""))
+            msg_len = len(msg.message) if hasattr(msg, 'message') else 0
+            log.info(f"Terraform prompt sizes: system_prompt={sp_len} chars (~{sp_len//4} tokens), user_msg={msg_len} chars (~{msg_len//4} tokens), max_tokens={cfg_dict.get('max_tokens')}")
+
+        mcp_tools = []
+        tool_defs = None
+        if msg.use_tools:
+            mcp_tools = _get_active_mcp_tools(user["id"])
+            if mcp_tools:
+                tool_defs = [t["tool"] for t in mcp_tools]
+                log.info(f"Chat with {len(tool_defs)} MCP tools available")
+
+        if history and _should_compact(history):
+            log.info(f"Compaction triggered: {len(history)} msgs, ~{_estimate_history_tokens(history)} est tokens")
+            history = _compact_history(sid, user["id"], cfg_dict, history)
+            log.info(f"Post-compaction: {len(history)} msgs, ~{_estimate_history_tokens(history)} est tokens")
+
+        hist = history[:-1] if len(history) > 1 else None
+        loop = asyncio.get_event_loop()
+        try:
+            resp_text, tool_calls, tool_calls_raw = await asyncio.wait_for(
+                loop.run_in_executor(
+                    _chat_executor, partial(_call_genai, cfg_dict, augmented_message, hist,
+                                            tool_defs, None, None, attachments)),
+                timeout=300)
+        except asyncio.TimeoutError:
+            log.error(f"GenAI call timed out after 300s for session {sid}")
+            raise HTTPException(504, "O modelo de IA demorou demais para responder. Tente novamente.")
+
+        all_tool_results = []
+        accumulated_msgs = []
+        iterations = 0
+        api_format = GENAI_MODELS.get(genai_cfg.get("model_id", ""), {}).get("api_format", "GENERIC")
+        while tool_calls and iterations < 5:
+            iterations += 1
+            log.info(f"Tool use iteration {iterations}: {len(tool_calls)} tool call(s)")
+            iteration_results = []
+            for tc in tool_calls:
+                mcp_match = next((m for m in mcp_tools if m["tool"]["name"] == tc["name"]), None)
+                if mcp_match:
+                    try:
+                        result = await _execute_mcp_tool(mcp_match["server"], tc["name"], tc["arguments"])
+                        log.info(f"Tool {tc['name']} executed successfully ({len(result)} chars)")
+                        _chat_log(sid, mid, user["id"], "info", "tool", "tool_success", f"{tc['name']} ({len(result)} chars)")
+                    except Exception as te:
+                        result = f"Erro ao executar tool {tc['name']}: {str(te)[:300]}"
+                        log.warning(f"Tool {tc['name']} failed: {te}")
+                        _chat_log(sid, mid, user["id"], "error", "tool", "tool_error", f"{tc['name']}: {te}")
+                else:
+                    result = f"Tool {tc['name']} não encontrada nos MCP servers ativos"
+                    _chat_log(sid, mid, user["id"], "error", "tool", "tool_not_found", tc["name"])
+                iteration_results.append({"tool_call_id": tc["id"], "name": tc["name"], "content": result})
+            all_tool_results.extend(iteration_results)
+
+            if api_format == "COHERE":
+                import oci
+                cohere_results = []
+                for tr in iteration_results:
+                    tc_obj = oci.generative_ai_inference.models.CohereToolCall()
+                    tc_obj.name = tr["name"]
+                    tc_obj.parameters = {}
+                    tr_obj = oci.generative_ai_inference.models.CohereToolResult()
+                    tr_obj.call = tc_obj
+                    tr_obj.outputs = [{"result": tr["content"]}]
+                    cohere_results.append(tr_obj)
+                try:
+                    resp_text, tool_calls, tool_calls_raw = await asyncio.wait_for(
+                        loop.run_in_executor(
+                            _chat_executor, partial(_call_genai, cfg_dict, augmented_message, hist,
+                                                    tool_defs, cohere_results)),
+                        timeout=300)
+                except asyncio.TimeoutError:
+                    log.error(f"GenAI tool-use call timed out after 300s (Cohere, iteration {iterations})")
+                    raise HTTPException(504, "O modelo de IA demorou demais para responder. Tente novamente.")
+            else:
+                import oci
+                assistant_msg = oci.generative_ai_inference.models.AssistantMessage()
+                assistant_msg.tool_calls = tool_calls_raw
+                accumulated_msgs.append(assistant_msg)
+                for tr in iteration_results:
+                    tool_msg = oci.generative_ai_inference.models.ToolMessage()
+                    tool_msg.tool_call_id = tr["tool_call_id"]
+                    tool_content = oci.generative_ai_inference.models.TextContent()
+                    tool_content.text = tr["content"]
+                    tool_msg.content = [tool_content]
+                    accumulated_msgs.append(tool_msg)
+                try:
+                    resp_text, tool_calls, tool_calls_raw = await asyncio.wait_for(
+                        loop.run_in_executor(
+                            _chat_executor, partial(_call_genai, cfg_dict, augmented_message, hist,
+                                                    tool_defs, None, accumulated_msgs)),
+                        timeout=300)
+                except asyncio.TimeoutError:
+                    log.error(f"GenAI tool-use call timed out after 300s (Generic, iteration {iterations})")
+                    raise HTTPException(504, "O modelo de IA demorou demais para responder. Tente novamente.")
+
+        resp = resp_text
+        if all_tool_results:
+            tools_info = '\n\n🔧 **Tools utilizadas:** ' + ', '.join(tr["name"] for tr in all_tool_results)
+            resp += tools_info
+
+        if not resp or not resp.strip():
+            model_id = genai_cfg.get("model_id", "unknown")
+            resp = f"⚠️ O modelo **{model_id}** retornou uma resposta vazia. Isso pode ocorrer com alguns modelos. Tente novamente ou selecione outro modelo (ex: Gemini, Llama)."
+            log.warning(f"Chat {mid}: empty response from model {model_id}, returning fallback message")
+
+        # Validate terraform resource types against DB
+        if agent_type == "terraform" and resp and '```' in resp:
+            try:
+                tf_errors = _validate_tf_resource_types(resp)
+                if tf_errors:
+                    warning_lines = ["\n\n⚠️ **Validação automática — Resource types inválidos detectados:**"]
+                    for err in tf_errors:
+                        line = f"- `{err['type']}` — NÃO EXISTE no provider oracle/oci."
+                        if err['suggestions']:
+                            line += f" Sugestões: {', '.join('`' + s + '`' for s in err['suggestions'])}"
+                        warning_lines.append(line)
+                    warning_lines.append("\n**Clique em 'Re-plan' após o modelo corrigir, ou peça correção no chat.**")
+                    resp += '\n'.join(warning_lines)
+                    log.warning(f"Chat {mid}: {len(tf_errors)} invalid TF resource types detected: {[e['type'] for e in tf_errors]}")
+            except Exception as e:
+                log.warning(f"TF resource type validation failed: {e}")
+
+        with db() as c:
+            c.execute("UPDATE chat_messages SET content=?, status='done' WHERE id=?", (resp, mid))
+        log.info(f"Chat {mid} completed successfully")
+    except Exception as e:
+        log.error(f"Chat {mid} failed: {e}")
+        _chat_log(sid, mid, user["id"], "error", "genai", "genai_failed", str(e))
+        with db() as c:
+            c.execute("UPDATE chat_messages SET content=?, status='failed' WHERE id=?",
+                      (f"❌ Erro GenAI: {str(e)[:400]}", mid))
+
+MAX_UPLOAD_FILES = 5
+MAX_UPLOAD_SIZE = 10 * 1024 * 1024  # 10MB
+IMAGE_MIMES = {"image/png", "image/jpeg", "image/gif", "image/webp", "image/bmp"}
+DOC_MIMES = {"application/pdf"}
+
+

backend/services/cis_reports.py

@@ -0,0 +1,80 @@
+"""CIS report execution — subprocess management."""
+import os, json, uuid, asyncio
+from pathlib import Path
+from datetime import datetime
+
+from config import OCI_DIR, REPORTS, log
+from database import db
+from auth.jwt_auth import _audit, _config_log
+from utils import running_reports, classify_report_file
+
+
+async def _exec_report(rid, cfg, regions, level=2, obp=False, raw=False, redact_output=False):
+    rdir = REPORTS / rid; rdir.mkdir(parents=True, exist_ok=True)
+    config_path = str(OCI_DIR / cfg["id"] / "config")
+    try:
+        cmd = ["python3", "-u", "/app/cis_reports.py",
+               "-c", config_path,
+               "--report-directory", str(rdir),
+               "--level", str(level),
+               "--print-to-screen", "True",
+               "--report-summary-json"]
+        if regions: cmd += ["--regions", ",".join(regions)]
+        if obp: cmd.append("--obp")
+        if raw: cmd.append("--raw")
+        if redact_output: cmd.append("--redact-output")
+        proc = await asyncio.create_subprocess_exec(*cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
+        running_reports[rid] = proc
+        with db() as c:
+            c.execute("UPDATE reports SET worker_pid=? WHERE id=?", (proc.pid, rid))
+        progress_lines = []
+        try:
+            while True:
+                line = await proc.stdout.readline()
+                if not line:
+                    break
+                text = line.decode(errors="replace").strip()
+                if text:
+                    progress_lines.append(text)
+                    with db() as c:
+                        c.execute("UPDATE reports SET progress=? WHERE id=?",
+                                  ("\n".join(progress_lines[-50:]), rid))
+            await proc.wait()
+        finally:
+            running_reports.pop(rid, None)
+        stderr_data = await proc.stderr.read()
+        # Check if cancelled
+        with db() as c:
+            cur_status = c.execute("SELECT status FROM reports WHERE id=?", (rid,)).fetchone()
+        if cur_status and cur_status["status"] == "cancelled":
+            return
+        if proc.returncode == 0:
+            # Scan output directory for all generated files
+            html_path = None; json_path = None
+            with db() as c:
+                for fpath in rdir.iterdir():
+                    if fpath.is_file():
+                        fname = fpath.name
+                        ftype = fpath.suffix.lstrip(".")
+                        category = classify_report_file(fname)
+                        fsize = fpath.stat().st_size
+                        c.execute("INSERT INTO report_files (id,report_id,file_name,file_path,file_type,file_category,file_size) VALUES (?,?,?,?,?,?,?)",
+                                  (str(uuid.uuid4()), rid, fname, str(fpath), ftype, category, fsize))
+                        if "summary_report" in fname and fname.endswith(".html"):
+                            html_path = str(fpath)
+                        elif "summary_report" in fname and fname.endswith(".json"):
+                            json_path = str(fpath)
+                c.execute("UPDATE reports SET status='completed',progress=?,html_path=?,json_path=?,completed_at=datetime('now') WHERE id=?",
+                          ("\n".join(progress_lines), html_path, json_path, rid))
+            _config_log("oci", cfg["id"], cfg["tenancy_name"], "success", "report", f"Report completed: {rid}")
+        else:
+            err = (stderr_data.decode(errors="replace") if stderr_data else "Unknown")[:2000]
+            with db() as c:
+                c.execute("UPDATE reports SET status='failed',progress=?,error_msg=?,completed_at=datetime('now') WHERE id=?",
+                          ("\n".join(progress_lines), err, rid))
+            _config_log("oci", cfg["id"], cfg["tenancy_name"], "error", "report", err)
+    except Exception as e:
+        running_reports.pop(rid, None)
+        with db() as c:
+            c.execute("UPDATE reports SET status='failed',error_msg=?,completed_at=datetime('now') WHERE id=?", (str(e)[:2000], rid))
+        _config_log("oci", cfg["id"], cfg["tenancy_name"], "error", "report", str(e)[:2000])

backend/services/embeddings.py

@@ -0,0 +1,550 @@
+"""Embeddings service — chunking, metadata, ADB vector ingest."""
+import os, json, uuid, time, re, hashlib
+from pathlib import Path
+from typing import Optional, List, Dict, Any
+
+from config import DATA, REPORTS, WALLET_DIR, log, _chat_executor, _EMBED_STATUS_DIR
+from database import db
+from auth.jwt_auth import _config_log, _verify_config_access
+from services.genai import (
+    _get_adb_connection, _resolve_embed_config, _embed_text,
+    _DIM_TO_MODEL, _get_table_embedding_dim, _get_active_adb_configs,
+    _get_tables_for_config,
+)
+
+def _build_metadata_json(tenancy: str = "", compartments: str = "", section: str = "",
+                         report_date: str = "", user_id: str = "", extra: dict = None) -> str:
+    """Build a structured JSON metadata string for vector embeddings."""
+    meta = {}
+    if tenancy:
+        meta["tenancy"] = tenancy
+    if compartments:
+        meta["compartments"] = compartments
+    if section:
+        meta["section"] = section
+    if report_date:
+        meta["report_date"] = report_date
+    if user_id:
+        meta["user_id"] = user_id
+    if extra:
+        meta.update(extra)
+    return json.dumps(meta, ensure_ascii=False) if meta else ""
+
+
+def _auto_register_table(adb_config_id: str, table_name: str, description: str = ""):
+    """Auto-register a table in adb_vector_tables if not already present."""
+    if not table_name:
+        return
+    with db() as c:
+        exists = c.execute("SELECT 1 FROM adb_vector_tables WHERE adb_config_id=? AND table_name=? COLLATE NOCASE",
+                           (adb_config_id, table_name)).fetchone()
+        if not exists:
+            c.execute("INSERT INTO adb_vector_tables (id, adb_config_id, table_name, description) VALUES (?,?,?,?)",
+                      (str(uuid.uuid4()), adb_config_id, table_name, description))
+            log.info(f"Auto-registered table '{table_name}' for ADB config {adb_config_id}")
+
+def _ingest_documents_task(cfg: dict, genai_cfg: dict, documents: list, user_id: str, username: str,
+                           table_name: str = None, tenancy: str = None, compartments: str = None,
+                           report_date: str = None, task_id: str = None):
+    """Background task: embed and insert documents into ADB via OCI GenAI.
+    Tenancy and compartments are stored in METADATA as structured JSON for filtering."""
+    import array
+    emb_model = cfg.get("embedding_model_id", "cohere.embed-v4.0")
+    table_name = table_name or cfg.get("table_name", "")
+    # Auto-detect embedding dimension from existing table data and use matching model
+    try:
+        actual_dim = _get_table_embedding_dim(cfg, table_name)
+        if actual_dim and actual_dim in _DIM_TO_MODEL:
+            detected_model = _DIM_TO_MODEL[actual_dim]
+            if detected_model != emb_model:
+                log.info(f"Ingest: table '{table_name}' has {actual_dim} dims, switching model from {emb_model} to {detected_model}")
+                emb_model = detected_model
+    except Exception as e:
+        log.warning(f"Ingest: failed to detect dimension for '{table_name}': {e}")
+    total = len(documents)
+    # Track status
+    if task_id:
+        _set_embed_status(task_id, {"status": "running", "table": table_name, "tenancy": tenancy or "",
+                                       "inserted": 0, "total": total, "user_id": user_id, "message": "Iniciando embedding..."})
+    # Auto-register table so it appears in multi-table RAG search
+    _auto_register_table(cfg["id"], table_name)
+    conn = _get_adb_connection(cfg)
+    try:
+        cur = conn.cursor()
+        inserted = 0
+        for i, doc in enumerate(documents):
+            try:
+                content = doc.get("content", "")
+                if not content: continue
+                embedding = _embed_text(content, genai_cfg, emb_model)
+                vec = array.array('f', [float(x) for x in embedding])
+                # Build structured metadata with tenancy isolation
+                doc_tenancy = tenancy or doc.get("tenancy", "")
+                doc_compartments = compartments or doc.get("compartments", "")
+                metadata = _build_metadata_json(
+                    tenancy=doc_tenancy,
+                    compartments=doc_compartments,
+                    section=doc.get("section", ""),
+                    report_date=report_date or "",
+                    user_id=user_id,
+                    extra={"legacy_metadata": doc.get("metadata", "")} if doc.get("metadata") else None
+                )
+                cur.execute(f"""
+                    INSERT INTO "{table_name}" (ID, TEXT, EMBEDDING, METADATA)
+                    VALUES (HEXTORAW(:1), :2, :3, :4)
+                """, [uuid.uuid4().hex.upper(), content, vec, metadata])
+                inserted += 1
+                if task_id:
+                    _update_embed_status(task_id, {"inserted": inserted, "message": f"Embedding {inserted}/{total}..."})
+            except Exception as e:
+                log.error(f"Failed to ingest document: {e}")
+        conn.commit()
+        cur.close()
+        msg = f"{inserted}/{total} documentos ingeridos em {table_name}" + (f" (tenancy: {tenancy})" if tenancy else "")
+        log.info(f"Ingested {inserted}/{total} documents into {table_name}" +
+                 (f" (tenancy={tenancy})" if tenancy else ""))
+        _audit(user_id, username, "ingest_documents", cfg["id"], f"{inserted} documents")
+        _config_log("adb", cfg["id"], cfg.get("config_name"), "success", "ingest", msg, user_id, username)
+        if task_id:
+            _update_embed_status(task_id, {"status": "done", "inserted": inserted, "message": msg})
+    except Exception as e:
+        log.error(f"Ingestion task failed: {e}")
+        _config_log("adb", cfg["id"], cfg.get("config_name"), "error", "ingest", str(e)[:500], user_id, username)
+        if task_id:
+            _update_embed_status(task_id, {"status": "error", "message": str(e)[:300]})
+    finally:
+        conn.close()
+
+# ── Embeddings ────────────────────────────────────────────────────────────────
+def _chunk_report_by_section(report_data: dict) -> list:
+    """Chunk a CIS report into documents grouped by section."""
+    if isinstance(report_data, str):
+        report_data = json.loads(report_data)
+    if isinstance(report_data, list):
+        report_data = {"findings": {str(i): item for i, item in enumerate(report_data)}, "tenancy": "unknown"}
+    findings = report_data.get("findings", {})
+    tenancy = report_data.get("tenancy", "unknown")
+    generated_at = report_data.get("generated_at", "")
+    regions = report_data.get("regions", [])
+    compartments = report_data.get("compartments", [])
+    # Build context header for each chunk
+    ctx_parts = [f"Tenancy: {tenancy}"]
+    if regions:
+        ctx_parts.append(f"Regions: {', '.join(regions)}")
+    if compartments:
+        ctx_parts.append(f"Compartments: {', '.join(compartments[:50])}")
+    ctx_header = "\n".join(ctx_parts)
+    sections = {}
+    for cid, check in findings.items():
+        sec = check.get("section", "Other")
+        sections.setdefault(sec, [])
+        sections[sec].append(check)
+    documents = []
+    for section_name, checks in sections.items():
+        passed = sum(1 for c in checks if c.get("status") == "PASS")
+        failed = sum(1 for c in checks if c.get("status") == "FAIL")
+        review = sum(1 for c in checks if c.get("status") == "REVIEW")
+        lines = [ctx_header, "", f"Section: {section_name}", f"Total checks: {len(checks)}, Passed: {passed}, Failed: {failed}, Review: {review}", ""]
+        for c in checks:
+            status = c.get("status", "REVIEW")
+            lines.append(f"- [{c.get('id', '')}] {c.get('title', '')} — Status: {status}")
+            if c.get("findings"):
+                for f in c["findings"]:
+                    lines.append(f"  Finding: {f}")
+        documents.append({
+            "content": "\n".join(lines),
+            "source": f"CIS Report - {tenancy} - {generated_at}",
+            "section": section_name,
+            "tenancy": tenancy,
+            "compartments": ", ".join(compartments[:50]),
+            "metadata": f"tenancy: {tenancy}, section: {section_name}, total: {len(checks)}, passed: {passed}, failed: {failed}, review: {review}"
+        })
+    return documents
+
+def _chunk_cis_pdf(text: str, filename: str, target_chars: int = 7000, overlap_chars: int = 500) -> list:
+    """Chunk a CIS Foundations Benchmark PDF by recommendation number.
+    Each recommendation (1.1, 1.2, etc.) becomes one or more chunks with overlap.
+    Port of the JavaScript embedding pipeline."""
+    import re as _re
+
+    def normalize(t):
+        t = t.replace('\r', '\n')
+        t = _re.sub(r'[ \t]+\n', '\n', t)
+        t = _re.sub(r'\n{3,}', '\n\n', t)
+        return t.strip()
+
+    def strip_page_headers(t):
+        # Remove "Page XX" both standalone and at start of lines
+        t = _re.sub(r'^\s*Page\s+\d+\s*$', '', t, flags=_re.MULTILINE | _re.IGNORECASE)
+        t = _re.sub(r'^Page\s+\d+\s+', '', t, flags=_re.MULTILINE | _re.IGNORECASE)
+        return t
+
+    def remove_toc(t):
+        # Remove everything from "Table of Contents" up to the actual recommendations section
+        # The real content starts with "Recommendations\n1 Identity" or "Profile Applicability"
+        toc_start = _re.search(r'\bTable of Contents\b', t, _re.IGNORECASE)
+        if not toc_start:
+            return t
+        # Find where actual recommendation content begins
+        content_start = _re.search(r'\bRecommendations\s*\n\s*1\s+Identity', t[toc_start.start():], _re.IGNORECASE)
+        if not content_start:
+            content_start = _re.search(r'\bProfile Applicability\b', t[toc_start.start():], _re.IGNORECASE)
+        if not content_start:
+            content_start = _re.search(r'\bOverview\b', t[toc_start.start():], _re.IGNORECASE)
+        if not content_start:
+            return t
+        end_pos = toc_start.start() + content_start.start()
+        if end_pos <= toc_start.start():
+            return t
+        return normalize(t[:toc_start.start()] + '\n\n' + t[end_pos:])
+
+    def is_chapter_header(line):
+        l = line.strip()
+        return bool(_re.match(r'^\d+\s+[A-Za-z].+', l)) and not _re.match(r'^\d+\.\d+', l)
+
+    def is_rec_header_start(line):
+        l = line.strip()
+        # Must be "1.1 Word..." but NOT a TOC line (with dots/page numbers)
+        if not _re.match(r'^\d+\.\d+(\.\d+)?\s+[A-Z]', l):
+            return False
+        # Skip TOC lines: contain "...." or end with a page number
+        if '....' in l or _re.search(r'\.\s*\d+\s*$', l):
+            return False
+        return True
+
+    def header_looks_complete(h):
+        # Complete if has (Manual)/(Automated) or ends with a closing paren
+        if _re.search(r'\(\s*(Manual|Automated)\s*\)', h, _re.IGNORECASE):
+            return True
+        # Also stop if next line starts a known section like "Profile Applicability"
+        return False
+
+    def chunk_text(t):
+        if not t:
+            return []
+        paragraphs = [p.strip() for p in t.split('\n\n') if p.strip()]
+        chunks = []
+        buf = ""
+        def push():
+            nonlocal buf
+            b = buf.strip()
+            if b:
+                chunks.append(b)
+            buf = ""
+        for p in paragraphs:
+            if len(p) > target_chars:
+                push()
+                i = 0
+                while i < len(p):
+                    chunks.append(p[i:i + target_chars].strip())
+                    i += max(1, target_chars - overlap_chars)
+                continue
+            candidate = f"{buf}\n\n{p}" if buf else p
+            if len(candidate) <= target_chars:
+                buf = candidate
+            else:
+                push()
+                if chunks and overlap_chars > 0:
+                    prev = chunks[-1]
+                    overlap = prev[max(0, len(prev) - overlap_chars):]
+                    buf = f"{overlap}\n\n{p}".strip()
+                else:
+                    buf = p
+        push()
+        return chunks
+
+    def remove_appendix(t):
+        """Remove appendix sections (Assessment Status, Change History, etc.) that pollute embeddings."""
+        for marker in [r'\bAppendix\b', r'\bAssessment Status\b', r'\bChange History\b',
+                       r'\bCIS Controls v\d', r'\bDate\s+Version\s+Changes']:
+            m = _re.search(marker, t, _re.IGNORECASE)
+            if m and m.start() > len(t) * 0.7:  # only cut if in last 30% of doc
+                t = t[:m.start()].rstrip()
+                break
+        return t
+
+    # Pipeline
+    text = normalize(text)
+    text = strip_page_headers(text)
+    text = remove_toc(text)
+    text = remove_appendix(text)
+    lines = text.split('\n')
+
+    # Segment by recommendation
+    segments = []
+    current = None
+    current_chapter = ""
+
+    i = 0
+    while i < len(lines):
+        line = lines[i]
+        if is_chapter_header(line):
+            current_chapter = line.strip()
+        if is_rec_header_start(line):
+            if current:
+                segments.append(current)
+            header = line.strip()
+            j = i + 1
+            while j < len(lines) and not header_looks_complete(header):
+                next_line = lines[j].strip()
+                if is_rec_header_start(next_line):
+                    break
+                # Stop consuming if we hit a known section start
+                if next_line.startswith('Profile Applicability') or next_line.startswith('Description:'):
+                    break
+                if next_line:
+                    header = _re.sub(r'\s+', ' ', f"{header} {next_line}").strip()
+                j += 1
+            i = j - 1
+            current = {"header": header, "chapter": current_chapter, "body_lines": []}
+            i += 1
+            continue
+        if current:
+            current["body_lines"].append(line)
+        i += 1
+
+    if current:
+        segments.append(current)
+
+    # Generate chunks
+    documents = []
+    for seg in segments:
+        body = normalize('\n'.join(seg["body_lines"]))
+        if not body:
+            continue
+        rec_match = _re.match(r'^(\d+(\.\d+)+)', seg["header"])
+        rec_number = rec_match.group(1) if rec_match else "unknown"
+        canonical = normalize('\n'.join(filter(None, [
+            f"Recommendation: {seg['header']}",
+            f"Chapter: {seg['chapter']}" if seg['chapter'] else "",
+            "",
+            body,
+        ])))
+        chunks = chunk_text(canonical)
+        for idx, chunk in enumerate(chunks):
+            documents.append({
+                "content": chunk,
+                "source": filename,
+                "metadata": json.dumps({
+                    "filename": filename,
+                    "recommendationNumber": rec_number,
+                    "chapter": seg["chapter"],
+                    "source": "CIS-OCI-PDF",
+                    "chunkIndex": idx + 1,
+                    "chunkCount": len(chunks),
+                }),
+            })
+
+    log.info(f"CIS PDF chunking: {len(segments)} recommendations → {len(documents)} chunks from {filename}")
+    return documents
+
+
+def _chunk_text_file(text: str, filename: str, chunk_size: int = 1000, overlap: int = 200) -> list:
+    """Split text into chunks by paragraphs with overlap to avoid losing context at boundaries."""
+    paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()]
+    documents = []
+    current_chunk = ""
+    prev_tail = ""  # last N chars of previous chunk for overlap
+    chunk_num = 1
+    for para in paragraphs:
+        if len(current_chunk) + len(para) + 2 > chunk_size and current_chunk:
+            documents.append({"content": current_chunk, "source": filename, "metadata": f"chunk: {chunk_num}"})
+            chunk_num += 1
+            # Keep overlap from end of current chunk
+            prev_tail = current_chunk[-overlap:] if len(current_chunk) > overlap else current_chunk
+            current_chunk = prev_tail + "\n\n" + para
+        else:
+            current_chunk = current_chunk + "\n\n" + para if current_chunk else para
+    if current_chunk:
+        documents.append({"content": current_chunk, "source": filename, "metadata": f"chunk: {chunk_num}"})
+    return documents
+
+def _get_adb_and_genai(vid: str, oci_config_id: str = None, user_id: str = None):
+    """Load ADB config and resolve embed config (scoped to user_id).
+    Priority: ADB.genai_config_id → genai by oci_config_id → oci_config directly → user's default."""
+    with db() as c:
+        cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?", (vid,)).fetchone()
+    if not cfg: raise HTTPException(404, "ADB config not found")
+    cfg = dict(cfg)
+    genai_cfg = None
+    if cfg.get("genai_config_id"):
+        with db() as c:
+            row = c.execute("SELECT * FROM genai_configs WHERE id=?", (cfg["genai_config_id"],)).fetchone()
+            if row: genai_cfg = dict(row)
+    gc = _resolve_embed_config(oci_config_id=oci_config_id, genai_cfg=genai_cfg, user_id=user_id or cfg.get("user_id"))
+    return cfg, gc
+
+
+
+def _chunk_summary_csv(csv_path: str, tenancy: str, extract_date: str) -> list:
+    """Chunk the cis_summary_report.csv into documents with tenancy/date metadata.
+    Each row becomes a document with structured content for vector search."""
+    import csv as csvmod
+    p = Path(csv_path)
+    if not p.exists():
+        return []
+    documents = []
+    with open(p, "r", encoding="utf-8") as f:
+        rows = list(csvmod.DictReader(f))
+    if not rows:
+        return []
+    # Group rows by section for richer context
+    sections: dict = {}
+    for row in rows:
+        sec = row.get("Section", "Unknown")
+        sections.setdefault(sec, []).append(row)
+    for sec_name, sec_rows in sections.items():
+        lines = []
+        for r in sec_rows:
+            rec = r.get("Recommendation #", "")
+            title = r.get("Title", "")
+            compliant = r.get("Compliant", "")
+            pct = r.get("Compliance Percentage Per Recommendation", "")
+            findings = r.get("Findings", "0")
+            total = r.get("Total", "0")
+            lines.append(f"Recommendation {rec}: {title} | Status: {compliant} | Compliance: {pct}% | Findings: {findings}/{total}")
+        content = (
+            f"Tenancy: {tenancy}\n"
+            f"Extract Date: {extract_date}\n"
+            f"Section: {sec_name}\n"
+            f"Total Recommendations: {len(sec_rows)}\n\n"
+            + "\n".join(lines)
+        )
+        documents.append({
+            "content": content,
+            "section": sec_name,
+            "tenancy": tenancy,
+            "metadata": json.dumps({"tenancy": tenancy, "extract_date": extract_date, "section": sec_name})
+        })
+    return documents
+
+
+# ── CIS Report CSV → ADB Table Mapping ──
+_CIS_TABLE_MAP = {
+    "Identity_and_Access_Management": "identityandaccess",
+    "Networking": "networking",
+    "Compute": "computeinstances",
+    "Logging_and_Monitoring": "loggingandmonitoring",
+    "Storage_Object_Storage": "objectstorage",
+    "Storage_Block_Volumes": "storageblockvolume",
+    "Storage_File_Storage_Service": "filestorageservice",
+    "Asset_Management": "assetmanagement",
+}
+
+def _purge_table_by_tenancy(cfg: dict, table_name: str, tenancy: str, extract_date: str = "") -> int:
+    """Delete existing embeddings from a table for a specific tenancy (and optionally extract_date).
+    Returns number of rows deleted."""
+    try:
+        conn = _get_adb_connection(cfg)
+        cur = conn.cursor()
+        if extract_date:
+            cur.execute(f"""DELETE FROM "{table_name}" WHERE
+                JSON_VALUE(METADATA, '$.tenancy') = :1 AND
+                JSON_VALUE(METADATA, '$.extract_date') = :2""", [tenancy, extract_date])
+        else:
+            cur.execute(f"""DELETE FROM "{table_name}" WHERE
+                JSON_VALUE(METADATA, '$.tenancy') = :1""", [tenancy])
+        deleted = cur.rowcount
+        conn.commit()
+        cur.close()
+        conn.close()
+        if deleted:
+            log.info(f"Purged {deleted} rows from {table_name} (tenancy={tenancy}, date={extract_date or 'all'})")
+        return deleted
+    except Exception as e:
+        log.warning(f"Purge failed for {table_name}: {e}")
+        return 0
+
+
+def _resolve_table_for_csv(filename: str) -> str | None:
+    """Map a CIS report CSV filename to its ADB vector table."""
+    if filename == "cis_summary_report.csv":
+        return "summaryreportcsvvector"
+    for pattern, table in _CIS_TABLE_MAP.items():
+        if pattern in filename:
+            return table
+    return None
+
+
+def _chunk_findings_csv(csv_path: str, tenancy: str, extract_date: str, max_chars: int = 8000) -> list:
+    """Chunk a CIS findings CSV into documents. Each row becomes one or more documents.
+    If a row exceeds max_chars (~6000 tokens), it's split into smaller chunks with
+    a context header (tenancy, resource name, ID) repeated in each part."""
+    import csv as csvmod, re as _re
+    p = Path(csv_path)
+    if not p.exists():
+        return []
+    documents = []
+    with open(p, "r", encoding="utf-8") as f:
+        rows = list(csvmod.DictReader(f))
+    if not rows:
+        return []
+    skip_cols = {"extract_date", "deep_link", "domain_deeplink", "defined_tags",
+                 "freeform_tags", "system_tags", "external_identifier"}
+    # Extract CIS recommendation number from filename (e.g., cis_Identity_and_Access_Management_1-1.csv → 1.1)
+    rec_match = _re.search(r'_(\d+)-(\d+(?:\.\d+)?)\.csv$', p.name)
+    cis_rec = f"{rec_match.group(1)}.{rec_match.group(2)}" if rec_match else ""
+    # Extract section name from filename
+    sec_match = _re.search(r'^cis_(.+?)_\d+-', p.name)
+    cis_section = sec_match.group(1).replace("_", " ") if sec_match else ""
+
+    meta = json.dumps({"tenancy": tenancy, "extract_date": extract_date, "source": p.name, "cis_recommendation": cis_rec})
+
+    for row in rows:
+        # Build context header (always repeated in each chunk)
+        header_parts = [f"Tenancy: {tenancy}", f"Extract Date: {extract_date}"]
+        if cis_rec:
+            header_parts.append(f"CIS Recommendation: {cis_rec}")
+        if cis_section:
+            header_parts.append(f"Section: {cis_section}")
+        header_parts.append(f"Status: Non-Compliant")
+        body_parts = []
+        # Identify key fields for the header
+        name = row.get("name") or row.get("display_name") or row.get("username") or ""
+        rid = row.get("id", "")
+        if name:
+            header_parts.append(f"Resource: {name}")
+        if rid:
+            header_parts.append(f"ID: {rid}")
+
+        for col, val in row.items():
+            if col.lower() in skip_cols or not val or not val.strip():
+                continue
+            if col.lower() in ("name", "display_name", "username", "id"):
+                continue  # already in header
+            # Clean HYPERLINK formulas
+            if val.startswith("=HYPERLINK"):
+                m = _re.search(r',\s*"([^"]+)"', val)
+                val = m.group(1) if m else val
+            body_parts.append(f"{col}: {val}")
+
+        header = "\n".join(header_parts)
+        body = "\n".join(body_parts)
+        full_content = header + "\n" + body
+
+        if len(full_content) <= max_chars:
+            if len(full_content) > 50:
+                documents.append({"content": full_content, "tenancy": tenancy, "metadata": meta})
+        else:
+            # Split body into chunks, each prefixed with context header
+            chunk_size = max_chars - len(header) - 50  # reserve space for header + part label
+            chunks = []
+            current = ""
+            for line in body_parts:
+                if len(current) + len(line) + 2 > chunk_size and current:
+                    chunks.append(current)
+                    current = line
+                else:
+                    current = current + "\n" + line if current else line
+            if current:
+                chunks.append(current)
+
+            for i, chunk in enumerate(chunks):
+                part_label = f"(part {i + 1}/{len(chunks)})" if len(chunks) > 1 else ""
+                content = f"{header}\n{part_label}\n{chunk}".strip()
+                if len(content) > 50:
+                    documents.append({"content": content, "tenancy": tenancy, "metadata": meta})
+
+    return documents
+
+

backend/services/mcp_client.py

@@ -0,0 +1,93 @@
+"""MCP client — tool discovery from MCP servers."""
+import json
+from config import MCP_DIR, log
+from database import db
+
+async def _discover_mcp_tools(mcp_srv: dict) -> list[dict]:
+    """Connect to MCP server and list available tools."""
+    from mcp import ClientSession
+    if mcp_srv["server_type"] in ("stdio", "module"):
+        from mcp.client.stdio import stdio_client, StdioServerParameters
+        cmd = mcp_srv.get("command") or "python3"
+        args_raw = mcp_srv.get("args")
+        args = json.loads(args_raw) if isinstance(args_raw, str) else (args_raw or [])
+        env_raw = mcp_srv.get("env_vars")
+        env = json.loads(env_raw) if isinstance(env_raw, str) else (env_raw or None)
+        params = StdioServerParameters(command=cmd, args=args, env=env)
+        async with stdio_client(params) as streams:
+            async with ClientSession(*streams) as session:
+                await session.initialize()
+                result = await session.list_tools()
+                return [{"name": t.name, "description": t.description or "",
+                         "input_schema": t.inputSchema if hasattr(t, 'inputSchema') else {}} for t in result.tools]
+    elif mcp_srv["server_type"] == "sse":
+        from mcp.client.sse import sse_client
+        async with sse_client(mcp_srv["url"]) as streams:
+            async with ClientSession(*streams) as session:
+                await session.initialize()
+                result = await session.list_tools()
+                return [{"name": t.name, "description": t.description or "",
+                         "input_schema": t.inputSchema if hasattr(t, 'inputSchema') else {}} for t in result.tools]
+    return []
+
+async def _execute_mcp_tool(mcp_srv: dict, tool_name: str, arguments: dict) -> str:
+    """Connect to MCP server and execute a specific tool."""
+    from mcp import ClientSession
+    if mcp_srv["server_type"] in ("stdio", "module"):
+        from mcp.client.stdio import stdio_client, StdioServerParameters
+        cmd = mcp_srv.get("command") or "python3"
+        args_raw = mcp_srv.get("args")
+        args = json.loads(args_raw) if isinstance(args_raw, str) else (args_raw or [])
+        env_raw = mcp_srv.get("env_vars")
+        env = json.loads(env_raw) if isinstance(env_raw, str) else (env_raw or None)
+        params = StdioServerParameters(command=cmd, args=args, env=env)
+        async with stdio_client(params) as streams:
+            async with ClientSession(*streams) as session:
+                await session.initialize()
+                result = await session.call_tool(tool_name, arguments)
+                parts = []
+                for c in result.content:
+                    if hasattr(c, 'text'):
+                        parts.append(c.text)
+                    elif hasattr(c, 'data'):
+                        parts.append(str(c.data))
+                return "\n".join(parts) if parts else "Tool executed successfully (no output)"
+    elif mcp_srv["server_type"] == "sse":
+        from mcp.client.sse import sse_client
+        async with sse_client(mcp_srv["url"]) as streams:
+            async with ClientSession(*streams) as session:
+                await session.initialize()
+                result = await session.call_tool(tool_name, arguments)
+                parts = []
+                for c in result.content:
+                    if hasattr(c, 'text'):
+                        parts.append(c.text)
+                    elif hasattr(c, 'data'):
+                        parts.append(str(c.data))
+                return "\n".join(parts) if parts else "Tool executed successfully (no output)"
+    raise ValueError(f"Unsupported MCP server type: {mcp_srv['server_type']}")
+
+def _get_active_mcp_tools(user_id: str) -> list[dict]:
+    """Get all tools from active MCP servers for a user, with server reference."""
+    with db() as c:
+        rows = c.execute(
+            "SELECT * FROM mcp_servers WHERE is_active=1 AND (user_id=? OR is_global=1)",
+            (user_id,)
+        ).fetchall()
+    result = []
+    for r in rows:
+        srv = dict(r)
+        tools_raw = srv.get("tools")
+        if not tools_raw:
+            continue
+        try:
+            tools = json.loads(tools_raw) if isinstance(tools_raw, str) else tools_raw
+        except Exception as e:
+            log.warning(f"Failed to parse tools JSON for MCP server '{srv.get('name', '?')}': {e}")
+            continue
+        for t in tools:
+            if isinstance(t, dict) and t.get("name"):
+                result.append({"server": srv, "tool": t})
+    return result
+
+# ── ADB Vector DB Config ─────────────────────────────────────────────────────

backend/services/oci_helpers.py

@@ -0,0 +1,31 @@
+"""OCI SDK client helpers — config loading, signer creation, client factory."""
+from config import OCI_DIR
+
+
+def _get_oci_config(oci_config_id: str) -> dict:
+    import oci
+    config_path = str(OCI_DIR / oci_config_id / "config")
+    config = oci.config.from_file(config_path, "DEFAULT")
+    return config
+
+
+def _get_oci_signer(oci_config_id: str):
+    """Return (config, signer) tuple. For session_token auth, returns SecurityTokenSigner; for api_key returns None."""
+    import oci
+    config = _get_oci_config(oci_config_id)
+    if config.get("security_token_file"):
+        from oci.auth.signers import SecurityTokenSigner
+        token_path = config["security_token_file"]
+        with open(token_path) as f:
+            token = f.read().strip()
+        signer = SecurityTokenSigner(token, config["key_file"])
+        return config, signer
+    return config, None
+
+
+def _oci_client(client_class, oci_config_id: str, **kwargs):
+    """Create an OCI client with correct auth (api_key or session_token)."""
+    config, signer = _get_oci_signer(oci_config_id)
+    if signer:
+        return client_class(config=config, signer=signer, **kwargs)
+    return client_class(config, **kwargs)

backend/services/terraform_exec.py

@@ -0,0 +1,421 @@
+"""Terraform execution — split monolith, write files, plan/apply/destroy."""
+import os, json, uuid, asyncio, subprocess, re
+from pathlib import Path
+from datetime import datetime
+
+from config import TERRAFORM_DIR, OCI_DIR, log
+from database import db
+from auth.crypto import _safe_dec
+from auth.jwt_auth import _audit, _verify_config_access
+
+def _split_tf_monolith(content: str) -> dict:
+    """Split a monolithic HCL string into multiple logical files by resource type.
+    Returns dict of {filename: content} or None if splitting not needed."""
+    import re as _re
+    content = _fix_single_line_blocks(content)
+    lines = content.split('\n')
+    top_blocks = []
+    preamble = []
+    accum = []
+    in_block = False
+    b_depth = 0
+    cur_header = ''
+
+    for line in lines:
+        stripped = _re.sub(r'"[^"]*"', '', line)
+        opens = stripped.count('{')
+        closes = stripped.count('}')
+        if not in_block:
+            hdr = _re.match(r'^\s*(variable|output|resource|data|provider|module)\s+"', line) or \
+                  _re.match(r'^\s*(locals|terraform)\s*\{', line)
+            if hdr:
+                in_block = True
+                cur_header = line
+                accum = preamble + [line]
+                preamble = []
+                b_depth = opens - closes
+                if b_depth <= 0:
+                    b_depth = 0
+                if b_depth == 0 and opens > 0:
+                    top_blocks.append((cur_header, '\n'.join(accum)))
+                    in_block = False
+                    accum = []
+            else:
+                preamble.append(line)
+        else:
+            accum.append(line)
+            b_depth += opens - closes
+            if b_depth <= 0:
+                b_depth = 0
+                top_blocks.append((cur_header, '\n'.join(accum)))
+                in_block = False
+                accum = []
+                cur_header = ''
+    if accum:
+        top_blocks.append((cur_header or '', '\n'.join(accum)))
+
+    if len(top_blocks) < 3:
+        return None
+
+    cats = {'variables': [], 'outputs': [], 'networking': [], 'compute': [], 'database': [],
+            'firewall': [], 'loadbalancer': [], 'storage': [], 'iam': [], 'drg': [],
+            'data': [], 'providers': [], 'other': []}
+    for header, body in top_blocks:
+        h = header.strip()
+        if h.startswith('variable '):
+            cats['variables'].append(body)
+        elif h.startswith('output '):
+            cats['outputs'].append(body)
+        elif h.startswith('provider '):
+            cats['providers'].append(body)
+        elif _re.search(r'resource\s+"oci_core_(drg|remote_peering|drg_route|drg_attachment)', h):
+            cats['drg'].append(body)
+        elif _re.search(r'resource\s+"oci_core_(vcn|subnet|internet_gateway|nat_gateway|service_gateway|route_table|security_list|network_security_group|dhcp_options|local_peering_gateway|virtual_circuit)', h):
+            cats['networking'].append(body)
+        elif _re.search(r'resource\s+"oci_core_instance', h):
+            cats['compute'].append(body)
+        elif _re.search(r'resource\s+"oci_(database|nosql|mysql|psql)', h) or _re.search(r'resource\s+"oci_core_autonomous', h):
+            cats['database'].append(body)
+        elif _re.search(r'resource\s+"oci_network_firewall', h):
+            cats['firewall'].append(body)
+        elif _re.search(r'resource\s+"oci_(load_balancer|network_load_balancer)', h):
+            cats['loadbalancer'].append(body)
+        elif _re.search(r'resource\s+"oci_(objectstorage|file_storage)', h):
+            cats['storage'].append(body)
+        elif _re.search(r'resource\s+"oci_identity', h):
+            cats['iam'].append(body)
+        elif h.startswith('data '):
+            cats['data'].append(body)
+        else:
+            cats['other'].append(body)
+
+    result = {}
+    mapping = [('variables.tf', 'variables'), ('providers.tf', 'providers'), ('networking.tf', 'networking'),
+               ('drg.tf', 'drg'), ('compute.tf', 'compute'), ('database.tf', 'database'),
+               ('firewall.tf', 'firewall'), ('loadbalancer.tf', 'loadbalancer'), ('storage.tf', 'storage'),
+               ('iam.tf', 'iam'), ('data.tf', 'data'), ('main.tf', 'other'), ('outputs.tf', 'outputs')]
+    for fname, key in mapping:
+        if cats[key]:
+            result[fname] = '\n\n'.join(cats[key])
+    return result if len(result) >= 3 else None
+
+
+def _write_tf_files(wdir: Path, tf_code: str):
+    """Parse tf_code for '// filename: xxx.tf' markers and write separate files.
+    If only 1-2 large files, auto-split into multiple files by resource type."""
+    import re as _re
+    parts = _re.split(r'^//\s*filename:\s*(\S+)\s*$', tf_code, flags=_re.MULTILINE)
+
+    files = {}
+    if len(parts) >= 3:
+        if parts[0].strip():
+            files['main.tf'] = parts[0].strip()
+        for i in range(1, len(parts), 2):
+            fname = parts[i].strip().replace("/", "_").replace("..", "_")
+            content = parts[i + 1].strip() if i + 1 < len(parts) else ""
+            if fname and content:
+                files[fname] = content
+    else:
+        files['main.tf'] = tf_code
+
+    # Auto-split: if 1-2 large files, split by resource type
+    if len(files) <= 2:
+        all_content = '\n\n'.join(files.values())
+        if len(all_content) > 2000:
+            split = _split_tf_monolith(all_content)
+            if split:
+                files = split
+                log.info(f"Backend auto-split monolithic TF into {len(files)} files: {list(files.keys())}")
+
+    # Deduplicate: if a resource appears in multiple files, keep only in the split file
+    # This handles edge cases where monolithic + split files both exist
+    if len(files) > 2:
+        resource_locations = {}  # "type.name" -> [(filename, line)]
+        dupes_in = set()
+        for fname, content in files.items():
+            for m in _re.finditer(r'^resource\s+"(\S+)"\s+"(\S+)"', content, _re.MULTILINE):
+                key = f'{m.group(1)}.{m.group(2)}'
+                resource_locations.setdefault(key, []).append(fname)
+        # Find files that contain ONLY duplicate resources (= monolithic leftovers)
+        for key, fnames in resource_locations.items():
+            if len(fnames) > 1:
+                # The largest file is likely the monolithic one
+                largest = max(fnames, key=lambda f: len(files.get(f, '')))
+                dupes_in.add(largest)
+        # Remove monolithic files that are fully duplicated
+        for fname in dupes_in:
+            if all(
+                any(f2 != fname for f2 in resource_locations.get(key, []))
+                for key, flist in resource_locations.items()
+                if fname in flist
+            ) and len(files) - 1 >= 2:
+                log.info(f"Removing duplicate monolithic file: {fname}")
+                del files[fname]
+
+    # Fix single-line blocks in all files
+    for fname in files:
+        files[fname] = _fix_single_line_blocks(files[fname])
+
+    # Write files to disk
+    for fname, content in files.items():
+        (wdir / fname).write_text(content)
+
+
+async def _terraform_exec(wid: str, action: str, user: dict):
+    """Background: run terraform init + plan/apply/destroy in workspace dir."""
+    log.info(f"Terraform {action} started: wid={wid}")
+    status_col = f"{action}_output"
+    final_ok = {"plan": "planned", "apply": "applied", "destroy": "destroyed"}[action]
+
+    try:
+        with db() as c:
+            ws = c.execute("SELECT * FROM terraform_workspaces WHERE id=?", (wid,)).fetchone()
+        if not ws:
+            return
+
+        wdir = TERRAFORM_DIR / wid
+        wdir.mkdir(parents=True, exist_ok=True)
+
+        # Clean old .tf files (keep .terraform/, state, lock)
+        for old_tf in wdir.glob("*.tf"):
+            old_tf.unlink()
+
+        # Write .tf files — parse // filename: comments to split into separate files
+        _write_tf_files(wdir, ws["tf_code"] or "")
+
+        # Auto-generate provider.tf from OCI config
+        # Detect provider aliases declared by the model in generated files
+        import re as _re2
+        oci_cfg = _get_oci_config(ws["oci_config_id"])
+        alias_blocks = []  # list of (alias_name, region_ref)
+
+        # Scan all .tf files for provider "oci" { alias = "..." ... region = ... }
+        provider_block_re = _re2.compile(r'provider\s+"oci"\s*\{', _re2.MULTILINE)
+        for tf_file in sorted(wdir.glob("*.tf")):
+            if tf_file.name == "provider.tf":
+                continue
+            content = tf_file.read_text()
+            # Find each provider "oci" block and extract alias + region
+            blocks_to_remove = []
+            for m in provider_block_re.finditer(content):
+                start = m.start()
+                # Find matching closing brace
+                depth = 0
+                end = start
+                for ci in range(m.end() - 1, len(content)):
+                    if content[ci] == '{':
+                        depth += 1
+                    elif content[ci] == '}':
+                        depth -= 1
+                        if depth == 0:
+                            end = ci + 1
+                            break
+                block = content[start:end]
+                alias_m = _re2.search(r'alias\s*=\s*"([^"]+)"', block)
+                region_m = _re2.search(r'region\s*=\s*(.+)', block)
+                if alias_m:
+                    alias_name = alias_m.group(1)
+                    region_ref = region_m.group(1).strip().rstrip("}").strip() if region_m else '"unknown"'
+                    alias_blocks.append((alias_name, region_ref))
+                    blocks_to_remove.append((start, end))
+            # Remove model-generated provider blocks (we centralize in provider.tf)
+            if blocks_to_remove:
+                new_content = content
+                for s, e in reversed(blocks_to_remove):
+                    new_content = new_content[:s] + new_content[e:]
+                # Also remove leading comments right before removed blocks
+                new_content = _re2.sub(r'\n(//[^\n]*\n){1,5}\s*\n', '\n\n', new_content)
+                new_content = new_content.strip()
+                if new_content:
+                    tf_file.write_text(new_content + "\n")
+                else:
+                    tf_file.unlink()  # Remove empty file
+
+        # Scan all .tf files for variable declarations with region-like defaults
+        # so we can map alias providers to the correct variable reference
+        var_region_map = {}  # variable_name -> default_value
+        var_re = _re2.compile(r'variable\s+"([^"]*region[^"]*)"\s*\{', _re2.IGNORECASE)
+        for tf_file in sorted(wdir.glob("*.tf")):
+            if tf_file.name in ("provider.tf", "terraform.tfvars"):
+                continue
+            content = tf_file.read_text()
+            for vm in var_re.finditer(content):
+                var_region_map[vm.group(1)] = f'var.{vm.group(1)}'
+
+        # Check if variable "region" is declared — use it for primary provider
+        has_region_var = "region" in var_region_map
+
+        # Also scan for provider refs in resource blocks (provider = oci.xxx)
+        for tf_file in sorted(wdir.glob("*.tf")):
+            if tf_file.name == "provider.tf":
+                continue
+            content = tf_file.read_text()
+            for ref_m in _re2.finditer(r'provider\s*=\s*oci\.(\w+)', content):
+                ref_alias = ref_m.group(1)
+                if ref_alias not in [a[0] for a in alias_blocks]:
+                    # Try to find a matching region variable for this alias
+                    # e.g. alias "mad_3" might match var.region_secondary
+                    region_ref = 'var.region'  # fallback to primary region var
+                    for vname, vref in var_region_map.items():
+                        if vname != "region":  # prefer non-primary region vars for aliases
+                            region_ref = vref
+                            break
+                    alias_blocks.append((ref_alias, region_ref))
+
+        passphrase = oci_cfg.get('pass_phrase', '')
+        cred_block = f'''  tenancy_ocid     = "{oci_cfg.get('tenancy', '')}"
+  user_ocid        = "{oci_cfg.get('user', '')}"
+  fingerprint      = "{oci_cfg.get('fingerprint', '')}"
+  private_key_path = "{oci_cfg.get('key_file', '')}"'''
+        if passphrase:
+            cred_block += f'\n  private_key_password = "{passphrase}"'
+
+        # Primary region: MUST use var.region — model is required to declare it
+        with db() as c:
+            oci_row = c.execute("SELECT region FROM oci_configs WHERE id=?", (ws["oci_config_id"],)).fetchone()
+        primary_region = oci_row["region"] if oci_row else oci_cfg.get("region", "")
+        if not has_region_var:
+            error_msg = (
+                f'ERRO: O código Terraform não declarou variable "region". '
+                f'Isso é obrigatório para garantir que os recursos sejam provisionados na região correta. '
+                f'Adicione no variables.tf: variable "region" {{ type = string; default = "{primary_region}" }}'
+            )
+            log.warning(f"Workspace {wid}: missing variable 'region' — blocking plan")
+            with db() as c:
+                c.execute("UPDATE terraform_workspaces SET status='failed', plan_output=?, error=?, updated_at=datetime('now') WHERE id=?",
+                          (error_msg, error_msg, wid))
+            return
+        region_value = 'var.region'
+
+        provider_tf = f'''terraform {{
+  required_providers {{
+    oci = {{
+      source = "oracle/oci"
+    }}
+  }}
+}}
+
+provider "oci" {{
+{cred_block}
+  region           = {region_value}
+}}
+'''
+        # Add alias providers with same credentials
+        seen_aliases = set()
+        for alias_name, region_ref in alias_blocks:
+            if alias_name in seen_aliases:
+                continue
+            seen_aliases.add(alias_name)
+            provider_tf += f'''
+provider "oci" {{
+  alias            = "{alias_name}"
+{cred_block}
+  region           = {region_ref}
+}}
+'''
+        (wdir / "provider.tf").write_text(provider_tf)
+
+        # Auto-generate terraform.tfvars — scan declared variables and provide values from OCI config
+        with db() as c:
+            oci_row = c.execute("SELECT compartment_id, region, ssh_public_key FROM oci_configs WHERE id=?", (ws["oci_config_id"],)).fetchone()
+        comp_id = ws["compartment_id"] if ws["compartment_id"] else (_safe_dec(oci_row["compartment_id"]) if oci_row and oci_row["compartment_id"] else "")
+        try:
+            ssh_pub_key = oci_row["ssh_public_key"] if oci_row else ""
+        except (IndexError, KeyError):
+            ssh_pub_key = ""
+        ssh_pub_key = ssh_pub_key or ""
+
+        # Scan all declared variables in .tf files
+        declared_vars = set()
+        for tf_file in sorted(wdir.glob("*.tf")):
+            if tf_file.name in ("provider.tf", "terraform.tfvars"):
+                continue
+            content = tf_file.read_text()
+            for vm in _re2.finditer(r'variable\s+"([^"]+)"', content):
+                declared_vars.add(vm.group(1))
+
+        # Map known variable names to OCI config values
+        var_values = {"compartment_id": comp_id}
+        oci_var_map = {
+            "tenancy_ocid": oci_cfg.get("tenancy", ""),
+            "tenancy_id": oci_cfg.get("tenancy", ""),
+            "user_ocid": oci_cfg.get("user", ""),
+            "fingerprint": oci_cfg.get("fingerprint", ""),
+            "private_key_path": oci_cfg.get("key_file", ""),
+            "ssh_public_key": ssh_pub_key,
+            "ssh_authorized_keys": ssh_pub_key,
+        }
+        # NOTE: "region" is intentionally NOT included — Terraform uses the default from variable declarations
+        for vname, vval in oci_var_map.items():
+            if vname in declared_vars and vval:
+                var_values[vname] = vval
+
+        tfvars_lines = [f'{k} = "{v}"' for k, v in var_values.items()]
+        (wdir / "terraform.tfvars").write_text("\n".join(tfvars_lines) + "\n")
+
+        output_lines = []
+
+        def _update_output(text):
+            output_lines.append(text)
+            with db() as c:
+                c.execute(f"UPDATE terraform_workspaces SET {status_col}=?, updated_at=datetime('now') WHERE id=?",
+                          ("\n".join(output_lines[-200:]), wid))
+
+        # terraform init
+        _update_output("$ terraform init ...")
+        proc_init = await asyncio.create_subprocess_exec(
+            "terraform", f"-chdir={wdir}", "init", "-no-color", "-input=false",
+            stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT)
+        while True:
+            line = await proc_init.stdout.readline()
+            if not line:
+                break
+            _update_output(line.decode(errors="replace").rstrip())
+        await proc_init.wait()
+        if proc_init.returncode != 0:
+            _update_output(f"\n❌ terraform init failed (exit {proc_init.returncode})")
+            with db() as c:
+                c.execute("UPDATE terraform_workspaces SET status='failed', error=?, updated_at=datetime('now') WHERE id=?",
+                          ("terraform init failed", wid))
+            return
+
+        # terraform action
+        _update_output(f"\n$ terraform {action} ...")
+        cmd = ["terraform", f"-chdir={wdir}", action, "-no-color", "-input=false"]
+        if action in ("apply", "destroy"):
+            cmd.append("-auto-approve")
+
+        proc = await asyncio.create_subprocess_exec(
+            *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT)
+        _running_terraform[wid] = proc
+
+        while True:
+            line = await proc.stdout.readline()
+            if not line:
+                break
+            _update_output(line.decode(errors="replace").rstrip())
+        await proc.wait()
+        _running_terraform.pop(wid, None)
+
+        if proc.returncode == 0:
+            output_lines.append(f"\n✅ terraform {action} completed successfully")
+            full_output = "\n".join(output_lines)
+            with db() as c:
+                c.execute(f"UPDATE terraform_workspaces SET {status_col}=?, status=?, updated_at=datetime('now') WHERE id=?",
+                          (full_output, final_ok, wid))
+            _audit(user["id"], user["username"], f"terraform_{action}", wid, f"status={final_ok}")
+        else:
+            output_lines.append(f"\n❌ terraform {action} failed (exit {proc.returncode})")
+            full_output = "\n".join(output_lines)
+            with db() as c:
+                c.execute(f"UPDATE terraform_workspaces SET {status_col}=?, status='failed', error=?, updated_at=datetime('now') WHERE id=?",
+                          (full_output, f"terraform {action} failed (exit {proc.returncode})", wid))
+
+    except Exception as e:
+        log.error(f"Terraform exec error: {e}")
+        with db() as c:
+            c.execute("UPDATE terraform_workspaces SET status='failed', error=?, updated_at=datetime('now') WHERE id=?",
+                      (str(e)[:500], wid))
+
+
+# ── Audit ─────────────────────────────────────────────────────────────────────