A-Team-Security-Infra-Agent-Engineer/backend/services/genai.py

"""GenAI service — model calls, embeddings, vector search, memory compaction."""
import os, json, uuid, time, base64, re, hashlib
from datetime import datetime, timedelta
from typing import Optional, List, Dict, Any
from fastapi import HTTPException

from config import (
    APP_SECRET, DATA, OCI_DIR, WALLET_DIR, log,
    COMPACT_TOKEN_THRESHOLD, COMPACT_KEEP_RECENT, COMPACT_SUMMARY_MAX_TOKENS,
    COMPACT_MIN_MESSAGES, GENAI_MODELS, EMBEDDING_MODELS, GENAI_REGIONS,
    _chat_executor,
)
from database import db
from auth.crypto import _enc, _dec, _safe_dec
from auth.jwt_auth import _chat_log

def _estimate_tokens(text: str) -> int:
    return len(text) // 4

def _estimate_history_tokens(history: list) -> int:
    return sum(_estimate_tokens(h["content"]) for h in history)

def _should_compact(history: list) -> bool:
    if len(history) < COMPACT_MIN_MESSAGES:
        return False
    return _estimate_history_tokens(history) > COMPACT_TOKEN_THRESHOLD

def _generate_summary(genai_cfg: dict, messages_to_summarize: list) -> str:
    """Use the same GenAI model to summarize older conversation messages."""
    conversation_text = ""
    for m in messages_to_summarize:
        role_label = "Usuário" if m["role"] == "user" else "Assistente"
        # Truncate very long messages in the summary input
        content = m["content"][:3000] if len(m["content"]) > 3000 else m["content"]
        conversation_text += f"{role_label}: {content}\n\n"

    summary_prompt = (
        "Resuma a conversa abaixo de forma compacta (máximo 2-3 parágrafos curtos) que capture:\n"
        "- Tópicos discutidos e perguntas do usuário\n"
        "- Decisões tomadas, conclusões e resultados importantes\n"
        "- Resultados de ferramentas/tools executadas (checks PASS/FAIL, scores, recursos encontrados)\n"
        "- Contexto OCI/CIS relevante (compartments, regiões, config_ids usados)\n"
        "- Dados numéricos importantes (scores, contagens, OCIDs mencionados)\n\n"
        "Seja conciso mas preserve TODA informação acionável e dados concretos. "
        "Não inclua saudações ou formalidades.\n\n"
        "CONVERSA:\n" + conversation_text
    )

    summary_cfg = dict(genai_cfg)
    summary_cfg["system_prompt"] = "Você é um sumarizador. Responda apenas com o resumo."
    summary_cfg["max_tokens"] = COMPACT_SUMMARY_MAX_TOKENS

    try:
        text, _, _ = _call_genai(summary_cfg, summary_prompt, history=None, tools=None)
        return text.strip()
    except Exception as e:
        log.warning(f"Summary generation failed: {e}")
        return ""

def _compact_history(session_id: str, user_id: str, genai_cfg: dict, history: list) -> list:
    """Compact conversation history by summarizing older messages."""
    if len(history) <= COMPACT_KEEP_RECENT:
        return history

    messages_to_summarize = history[:-COMPACT_KEEP_RECENT]
    recent_messages = history[-COMPACT_KEEP_RECENT:]

    # Check for existing summary
    with db() as c:
        row = c.execute(
            "SELECT summary, messages_compacted FROM chat_summaries "
            "WHERE session_id=? ORDER BY created_at DESC LIMIT 1",
            (session_id,)
        ).fetchone()
        if row:
            existing_summary = row["summary"]
            prev_compacted = row["messages_compacted"]
            # Reuse if no new messages to summarize
            if len(messages_to_summarize) <= prev_compacted:
                return [{"role": "assistant", "content": f"[Resumo da conversa anterior: {existing_summary}]"}] + recent_messages
        else:
            existing_summary = None

    # Include existing summary as prefix for incremental compaction
    if existing_summary:
        messages_to_summarize = [{"role": "assistant", "content": f"[Resumo anterior: {existing_summary}]"}] + messages_to_summarize

    summary_text = _generate_summary(genai_cfg, messages_to_summarize)

    if not summary_text:
        # Fallback: truncate keeping recent messages that fit in budget
        truncated = []
        budget = 6000
        for m in reversed(history):
            t = _estimate_tokens(m["content"])
            if budget - t < 0:
                break
            truncated.insert(0, m)
            budget -= t
        return truncated

    # Save summary to DB
    actual_count = len(messages_to_summarize) - (1 if existing_summary else 0)
    with db() as c:
        last_msg = c.execute(
            "SELECT id FROM chat_messages WHERE session_id=? AND role IN ('user','assistant') "
            "ORDER BY created_at ASC LIMIT 1 OFFSET ?",
            (session_id, max(0, actual_count - 1))
        ).fetchone()
        last_id = last_msg["id"] if last_msg else "unknown"
        c.execute(
            "INSERT INTO chat_summaries (id,session_id,user_id,summary,messages_compacted,up_to_message_id) VALUES (?,?,?,?,?,?)",
            (str(uuid.uuid4()), session_id, user_id, summary_text, actual_count, last_id)
        )

    log.info(f"Generated summary for session {session_id}: {len(summary_text)} chars, {actual_count} msgs compacted")
    return [{"role": "assistant", "content": f"[Resumo da conversa anterior: {summary_text}]"}] + recent_messages

# ── GenAI Call ─────────────────────────────────────────────────────────────────

def _call_genai(gc: dict, message: str, history: list = None, tools: list = None,
                tool_results_cohere: list = None,
                extra_messages: list = None,
                attachments: list = None) -> tuple:
    """
    Call OCI Generative AI with optional tool use support.
    Returns (text, tool_calls, tool_calls_raw) tuple.
    tool_calls is a list of dicts or None. tool_calls_raw is the raw OCI SDK objects (for Generic format continuations).
    extra_messages: list of raw OCI SDK message objects to append after the user message (for tool use loop accumulation).
    """
    import oci

    # Load OCI config from stored credentials (same as ~/.oci/config)
    config_path = str(OCI_DIR / gc["oci_config_id"] / "config")
    config = oci.config.from_file(config_path, "DEFAULT")

    # Service endpoint - built from region
    endpoint = gc["endpoint"]

    # Create inference client with retry strategy and timeout
    generative_ai_inference_client = oci.generative_ai_inference.GenerativeAiInferenceClient(
        config=config,
        service_endpoint=endpoint,
        retry_strategy=oci.retry.NoneRetryStrategy(),
        timeout=(10, 600)
    )

    # System prompt
    system_prompt = gc.get("system_prompt", "")

    # Build ChatDetails
    chat_detail = oci.generative_ai_inference.models.ChatDetails()

    # Determine API format from model catalog
    model_info = GENAI_MODELS.get(gc["model_id"], {})
    api_format = model_info.get("api_format", "GENERIC")

    if api_format == "COHERE":
        # ── Cohere models (CohereChatRequest) ──
        chat_request = oci.generative_ai_inference.models.CohereChatRequest()
        chat_request.api_format = oci.generative_ai_inference.models.BaseChatRequest.API_FORMAT_COHERE
        if system_prompt:
            chat_request.preamble_override = system_prompt
        chat_request.message = message
        cohere_max = model_info.get("max_tokens", 4096)
        chat_request.max_tokens = min(int(gc.get("max_tokens", 4096)), cohere_max)
        chat_request.temperature = float(gc.get("temperature", 1))
        chat_request.frequency_penalty = float(gc.get("frequency_penalty", 0))
        chat_request.presence_penalty = float(gc.get("presence_penalty", 0))
        chat_request.top_p = float(gc.get("top_p", 0.95))
        chat_request.top_k = int(gc.get("top_k", 1))
        if history:
            chat_history = []
            for h in history:
                entry = oci.generative_ai_inference.models.CohereMessage()
                entry.role = "USER" if h["role"] == "user" else "CHATBOT"
                entry.message = h["content"]
                chat_history.append(entry)
            chat_request.chat_history = chat_history
        # Tool use support for Cohere
        if tools:
            cohere_tools = []
            for t in tools:
                props = t.get("input_schema", {}).get("properties", {})
                required = t.get("input_schema", {}).get("required", [])
                param_defs = {}
                for k, v in props.items():
                    pd = oci.generative_ai_inference.models.CohereParameterDefinition()
                    pd.type = v.get("type", "str")
                    pd.description = v.get("description", "")
                    pd.is_required = k in required
                    param_defs[k] = pd
                ct = oci.generative_ai_inference.models.CohereTool()
                ct.name = t["name"]
                ct.description = t.get("description", "")
                ct.parameter_definitions = param_defs if param_defs else None
                cohere_tools.append(ct)
            chat_request.tools = cohere_tools
        if tool_results_cohere:
            chat_request.tool_results = tool_results_cohere
    else:
        # ── Generic format (Meta Llama, Google, xAI, OpenAI) ──
        chat_request = oci.generative_ai_inference.models.GenericChatRequest()
        chat_request.api_format = oci.generative_ai_inference.models.BaseChatRequest.API_FORMAT_GENERIC
        provider = model_info.get("provider", "")
        is_openai = provider == "openai"
        is_reasoning = model_info.get("reasoning", False)
        # Clamp max_tokens to model limit
        model_max = model_info.get("max_tokens", 16384)
        requested_tokens = int(gc.get("max_tokens", 6000))
        clamped_tokens = min(requested_tokens, model_max)

        # ── Parameter matrix per provider ──
        # OpenAI reasoning (o3, o4-mini): max_completion_tokens + reasoning_effort only
        if is_reasoning:
            chat_request.max_completion_tokens = clamped_tokens
            re = gc.get("reasoning_effort")
            if re and hasattr(chat_request, "reasoning_effort"):
                chat_request.reasoning_effort = re.upper() if isinstance(re, str) else re
            # Explicitly unset unsupported params
            chat_request.temperature = None
            chat_request.top_p = None
            chat_request.frequency_penalty = None
            chat_request.presence_penalty = None
        # OpenAI standard (GPT-4.1, GPT-5.x, GPT-4o): max_completion_tokens, temperature, top_p
        # freq/pres penalty only for models with "penalties":True (GPT-4.1, GPT-4.1-mini, GPT-4o)
        elif is_openai:
            chat_request.max_completion_tokens = clamped_tokens
            chat_request.temperature = float(gc.get("temperature", 1))
            chat_request.top_p = float(gc.get("top_p", 0.95))
            if model_info.get("penalties"):
                chat_request.frequency_penalty = float(gc.get("frequency_penalty", 0))
                chat_request.presence_penalty = float(gc.get("presence_penalty", 0))
            else:
                chat_request.frequency_penalty = None
                chat_request.presence_penalty = None
        # xAI Grok: max_tokens, temperature, top_p (no top_k, no freq/pres penalty)
        elif provider == "xai":
            chat_request.max_tokens = clamped_tokens
            chat_request.temperature = float(gc.get("temperature", 1))
            chat_request.top_p = float(gc.get("top_p", 0.95))
            # Explicitly unset unsupported params to prevent SDK serialization
            chat_request.frequency_penalty = None
            chat_request.presence_penalty = None
        # Meta Llama: max_tokens, temperature, top_p, top_k (no freq/pres penalty)
        # Google Gemini: max_tokens, temperature, top_p, top_k (no freq/pres penalty)
        else:
            chat_request.max_tokens = clamped_tokens
            chat_request.temperature = float(gc.get("temperature", 1))
            chat_request.top_p = float(gc.get("top_p", 0.95))
            chat_request.top_k = int(gc.get("top_k", 1))
            # Explicitly unset unsupported params to prevent SDK serialization
            chat_request.frequency_penalty = None
            chat_request.presence_penalty = None

        # Log applied parameters
        params_log = f"provider={provider}, reasoning={is_reasoning}"
        params_log += f", max_tokens={getattr(chat_request, 'max_completion_tokens', None) or getattr(chat_request, 'max_tokens', None)}"
        params_log += f", temp={getattr(chat_request, 'temperature', None)}, top_p={getattr(chat_request, 'top_p', None)}"
        params_log += f", top_k={getattr(chat_request, 'top_k', None)}"
        params_log += f", freq_pen={getattr(chat_request, 'frequency_penalty', None)}, pres_pen={getattr(chat_request, 'presence_penalty', None)}"
        if is_reasoning:
            params_log += f", reasoning_effort={getattr(chat_request, 'reasoning_effort', None)}"
        log.info(f"GenAI params: {params_log}")

        messages = []
        if system_prompt:
            sys_content = oci.generative_ai_inference.models.TextContent()
            sys_content.text = system_prompt
            sys_msg = oci.generative_ai_inference.models.Message()
            sys_msg.role = "SYSTEM"
            sys_msg.content = [sys_content]
            messages.append(sys_msg)
        if history:
            for h in history:
                content = oci.generative_ai_inference.models.TextContent()
                content.text = h["content"]
                msg = oci.generative_ai_inference.models.Message()
                msg.role = "USER" if h["role"] == "user" else "ASSISTANT"
                msg.content = [content]
                messages.append(msg)

        # Current user message (with optional multimodal attachments)
        content_parts = []
        if attachments:
            for att in attachments:
                if att["type"] == "image":
                    img_url = oci.generative_ai_inference.models.ImageUrl()
                    img_url.url = att["data_uri"]
                    img_url.detail = "AUTO"
                    img_content = oci.generative_ai_inference.models.ImageContent()
                    img_content.image_url = img_url
                    content_parts.append(img_content)
                elif att["type"] == "document":
                    doc_url = oci.generative_ai_inference.models.DocumentUrl()
                    doc_url.url = att["data_uri"]
                    doc_content = oci.generative_ai_inference.models.DocumentContent()
                    doc_content.document_url = doc_url
                    content_parts.append(doc_content)
        text_content = oci.generative_ai_inference.models.TextContent()
        text_content.text = message
        content_parts.append(text_content)
        user_message = oci.generative_ai_inference.models.Message()
        user_message.role = "USER"
        user_message.content = content_parts
        messages.append(user_message)

        # Append accumulated tool use loop messages (assistant+tool_calls → tool_results → ...)
        if extra_messages:
            messages.extend(extra_messages)

        chat_request.messages = messages

        # Tool definitions for Generic format
        if tools:
            generic_tools = []
            for t in tools:
                fd = oci.generative_ai_inference.models.FunctionDefinition()
                fd.name = t["name"]
                fd.description = t.get("description", "")
                fd.parameters = t.get("input_schema")
                generic_tools.append(fd)
            chat_request.tools = generic_tools

    # Serving mode - resolve OCID: explicit model_ocid > catalog ocid for region > short model_id
    model_ref = gc.get("model_ocid") or ""
    if not model_ref:
        region = gc.get("genai_region", "")
        ocids = model_info.get("ocids", {})
        model_ref = ocids.get(region) or gc["model_id"]
    log.info(f"GenAI call: model_id={gc.get('model_id')}, model_ref={model_ref[:60]}...")
    if gc.get("serving_type") == "DEDICATED" and gc.get("dedicated_endpoint_id"):
        chat_detail.serving_mode = oci.generative_ai_inference.models.DedicatedServingMode(
            endpoint_id=gc["dedicated_endpoint_id"]
        )
    else:
        chat_detail.serving_mode = oci.generative_ai_inference.models.OnDemandServingMode(
            model_id=model_ref
        )

    chat_detail.chat_request = chat_request
    chat_detail.compartment_id = gc["compartment_id"]

    # Execute
    chat_response = generative_ai_inference_client.chat(chat_detail)

    # Extract text and tool_calls from response
    resp = chat_response.data.chat_response
    if api_format == "COHERE":
        text = resp.text if hasattr(resp, 'text') else ""
        tool_calls = None
        if hasattr(resp, 'tool_calls') and resp.tool_calls:
            tool_calls = []
            for tc in resp.tool_calls:
                tool_calls.append({
                    "id": tc.name,  # Cohere uses name as identifier
                    "name": tc.name,
                    "arguments": tc.parameters if isinstance(tc.parameters, dict) else {}
                })
        return (text or "", tool_calls, None)
    else:
        text = ""
        tool_calls = None
        tool_calls_raw = None  # raw content list from assistant message for continuation
        if hasattr(resp, 'choices') and resp.choices:
            choice = resp.choices[0]
            if hasattr(choice, 'message') and choice.message:
                # Check for tool calls
                if hasattr(choice.message, 'tool_calls') and choice.message.tool_calls:
                    tool_calls = []
                    for tc in choice.message.tool_calls:
                        args = {}
                        if hasattr(tc, 'arguments') and tc.arguments:
                            try: args = json.loads(tc.arguments) if isinstance(tc.arguments, str) else tc.arguments
                            except Exception as e: log.warning(f"Failed to parse tool call arguments: {e}"); args = {}
                        tool_calls.append({
                            "id": tc.id if hasattr(tc, 'id') else tc.name,
                            "name": tc.name if hasattr(tc, 'name') else "",
                            "arguments": args
                        })
                    # Preserve raw tool_calls for building assistant message in next iteration
                    tool_calls_raw = choice.message.tool_calls
                # Extract text from content
                if hasattr(choice.message, 'content') and choice.message.content:
                    contents = choice.message.content
                    if isinstance(contents, str):
                        text = contents
                    elif isinstance(contents, list):
                        text_parts = []
                        for c in contents:
                            if isinstance(c, str):
                                text_parts.append(c)
                            elif hasattr(c, 'text') and c.text:
                                text_parts.append(c.text)
                            elif isinstance(c, dict) and c.get('text'):
                                text_parts.append(c['text'])
                        text = "\n".join(text_parts)
                    else:
                        text = str(contents)
        # Fallback: check reasoning_content (some models like GPT-5.2 use this)
        if not text and hasattr(resp, 'choices') and resp.choices:
            choice = resp.choices[0]
            if hasattr(choice, 'message') and choice.message:
                msg_obj = choice.message
                if hasattr(msg_obj, 'reasoning_content') and msg_obj.reasoning_content:
                    rc = msg_obj.reasoning_content
                    if isinstance(rc, str):
                        text = rc
                    elif isinstance(rc, list):
                        text_parts = []
                        for c in rc:
                            if isinstance(c, str):
                                text_parts.append(c)
                            elif hasattr(c, 'text') and c.text:
                                text_parts.append(c.text)
                            elif isinstance(c, dict) and c.get('text'):
                                text_parts.append(c['text'])
                        text = "\n".join(text_parts)
                    else:
                        text = str(rc)
                    if text:
                        log.info(f"GenAI: extracted text from reasoning_content ({len(text)} chars)")
            # Last resort: try choice.text
            if not text:
                if hasattr(choice, 'text') and choice.text:
                    text = choice.text
        if not text and not tool_calls:
            raw_content = getattr(getattr(resp.choices[0], 'message', None), 'content', None) if hasattr(resp, 'choices') and resp.choices else None
            # Also dump message attrs for debugging
            msg_attrs = []
            if hasattr(resp, 'choices') and resp.choices and hasattr(resp.choices[0], 'message') and resp.choices[0].message:
                msg_attrs = [a for a in dir(resp.choices[0].message) if not a.startswith('_')]
            reasoning = getattr(resp.choices[0].message, 'reasoning_content', None) if hasattr(resp, 'choices') and resp.choices and hasattr(resp.choices[0], 'message') else None
            refusal = getattr(resp.choices[0].message, 'refusal', None) if hasattr(resp, 'choices') and resp.choices and hasattr(resp.choices[0], 'message') else None
            finish_reason = getattr(resp.choices[0], 'finish_reason', None) if hasattr(resp, 'choices') and resp.choices else None
            choice_attrs = [a for a in dir(resp.choices[0]) if not a.startswith('_')] if hasattr(resp, 'choices') and resp.choices else []
            log.warning(f"GenAI returned empty response. model={gc.get('model_id')}, finish_reason={finish_reason}, content_repr: {repr(raw_content)[:500]}, reasoning: {repr(reasoning)[:500] if reasoning else None}, refusal: {repr(refusal)[:200] if refusal else None}, choice_attrs: {choice_attrs}")
        return (text or "", tool_calls, tool_calls_raw)

# ── RAG Helpers ───────────────────────────────────────────────────────────────
RAG_CONTEXT_TEMPLATE = """--- CONTEXTO RECUPERADO (Vector Search) ---
{context}
--- FIM DO CONTEXTO ---

Pergunta do usuário: {question}"""

RAG_DEFAULT_SYSTEM_PROMPT = """Você é um assistente RAG especializado em Oracle Cloud Infrastructure (OCI).

### Escopo e restrições
- Responda **SOMENTE** perguntas relacionadas a Oracle Cloud (OCI), CIS Benchmark, CIS Report e itens de inventário OCI (IAM/AssetManagement/Networking/StorageBlock/FileStorageService/Objectstore/Compute/LoggingandMonitoring).
- Se a pergunta não for desse escopo, recuse educadamente e peça uma pergunta dentro do tema.

### Bases vetorizadas disponíveis (contexto recuperado automaticamente)

**Tabelas de findings (dados do scan da tenancy):**
- **summaryreportcsvvector**: resumo geral do CIS report (compliance scores por seção)
- **identityandaccess**: findings de IAM (users, policies, MFA, API keys)
- **networking**: findings de rede (security lists, NSGs, VCNs)
- **computeinstances**: findings de compute (instances, metadata, boot)
- **loggingandmonitoring**: findings de logging (alarms, events, notifications)
- **objectstorage**: findings de Object Storage (buckets, visibility, encryption)
- **storageblockvolume**: findings de Block Volume (encryption, CMK)
- **filestorageservice**: findings de File Storage (encryption, CMK)
- **assetmanagement**: findings de Asset Management (compartments, tagging)

**Tabelas de referência (conhecimento genérico):**
- **cisrecom**: recomendações oficiais do CIS Benchmark — contém passos detalhados de remediação, auditoria e rationale para cada controle CIS
- **engineerknowledgebase**: base de conhecimento complementar (blogs, documentações, PDFs)

### Quando usar RAG (dados armazenados) vs MCP Tools (tempo real)

**Use RAG (contexto recuperado abaixo) quando o usuário:**
- Perguntar sobre **relatórios já gerados**, resultados de scans anteriores, histórico
- Quiser saber **como corrigir** um problema (remediação do cisrecom)
- Pedir **resumo de compliance**, scores, comparações entre datas
- Perguntar sobre **documentação, boas práticas, referências** (engineerknowledgebase)
- Palavras-chave: "no último scan", "no report", "segundo o CIS", "como corrigir", "remediação", "recomendação"

**Use MCP Tools (scan em tempo real) quando o usuário:**
- Pedir para **verificar agora**, **validar agora**, **checar o estado atual**
- Quiser dados **atualizados/em tempo real** da tenancy
- Palavras-chave: "verifica agora", "valida agora", "estado atual", "scan", "checa", "como está hoje"
- Se tiver dúvida se os dados do RAG estão desatualizados

**Se não tiver certeza:** pergunte ao usuário se quer consultar os dados armazenados (mais rápido) ou verificar em tempo real (scan ao vivo).

### Hierarquia de fontes RAG (IMPORTANTE)
1. Para **identificar problemas e status**: use as tabelas de findings (identityandaccess, networking, etc.)
2. Para **como corrigir (remediação)**: use SEMPRE a tabela **cisrecom** como fonte principal — ela contém os passos oficiais do CIS
3. Para **informações complementares**: use **engineerknowledgebase** como plus, se tiver algo relevante a acrescentar
4. Para **resumo de compliance**: use **summaryreportcsvvector**
- Se houver conflito entre fontes, priorize: findings (dados reais) > cisrecom (recomendações oficiais) > engineerknowledgebase (complementar)

### Tratamento temporal (datas de extração)
- Cada documento possui um campo **date** no cabeçalho indicando quando os dados foram coletados.
- **Por padrão, priorize SEMPRE os dados mais recentes** (data mais nova).
- Se o usuário pedir comparação ou evolução, apresente os dados de ambas as datas lado a lado, indicando claramente qual é o mais recente e qual é o anterior.
- Informe a data de extração nas respostas para que o usuário saiba de quando são os dados.

### Regras de fidelidade
- Responda **somente** com informações suportadas por evidências do contexto recuperado.
- Se não houver evidência suficiente: **"Não encontrei nas minhas bases"** e peça dados para refinar (Recommendation #, seção, recurso, OCID, etc.).
- **Não invente** páginas, comandos, políticas, valores ou números.
- Use somente **1–3 linhas** de evidência por item para manter respostas compactas.
- **Sempre informe a data de extração** dos dados apresentados.

### Formato de resposta
- **Tenancy:** <tenancy>
- **Data de extração:** <date>
- **Recomendação <número> — <título>**
  - **Seção/Capítulo:** …
  - **Nível/Tipo:** Manual/Automated (se disponível)
  - **O que isso exige (CIS):** breve explicação
  - **Recursos afetados:** listar recursos non-compliant com nome/OCID (se disponíveis no contexto)
  - **Como corrigir (remediação):** passos COMPLETOS e DETALHADOS
    - Passos via Console OCI (se estiverem nas evidências)
    - Passos via OCI CLI com comandos completos em code blocks (```bash)
    - Passos via API (se disponíveis)
    - Observações/alertas importantes
  - **Como auditar:** como verificar se a correção foi aplicada
  - **Fontes:** tabela e data de extração de cada informação usada

### Regras de formatação
- Comandos CLI e policies devem ser apresentados em **code blocks** (```bash ou ```hcl)
- Listar TODOS os passos de remediação disponíveis no contexto — nunca resumir ou omitir passos
- Se a remediação tem passos via Console E via CLI, apresentar AMBOS
- Usar tabelas markdown para listar recursos afetados quando houver múltiplos
- Ser completo na remediação mas conciso nas explicações"""

CONSULT_SYSTEM_PROMPT = """Você é um consultor de dados vetorizados. Sua função é responder perguntas com base nos documentos recuperados do banco vetorial.

### Regras
- Responda **somente** com informações presentes nos documentos fornecidos.
- Se não encontrar informação relevante, diga claramente.
- Seja direto e objetivo. Apresente os dados de forma organizada.
- Use markdown para formatação (negrito, listas, tabelas quando apropriado).
- Cite de qual tabela/fonte veio cada informação.
- Não invente dados que não estejam nos documentos."""

TF_DEFAULT_SYSTEM_PROMPT = """Você gera somente Terraform HCL para Oracle Cloud Infrastructure (OCI) usando o provider oracle/oci.
Se pedirem AWS, Azure, GCP ou outro provider, recuse educadamente.

### Regras
- Gere código production-ready, com variáveis, defaults sensatos, tags e outputs úteis. Evite ficar solicitando ao usuário para ficar interagindo a todo momento, entregue o que ele precisa e faça o mínimo de interações para entender o que o usuário precisa. Explique de forma clara o que foi gerado e o que foi arrumado para que fosse possível gerar o código do terraform.
- Use sempre a sintaxe mais recente do provider OCI.
- Não inclua `terraform { required_providers }` nem `provider "oci"` principal.
- Em multi-região, use provider aliases para regiões adicionais e `provider = oci.alias` nos recursos.
- Para novas gerações, gere sempre múltiplos arquivos: no mínimo `variables.tf`, 1 arquivo de recursos e `outputs.tf`.
- Cada bloco deve começar com `// filename: nome.tf`.
- Nunca declare o mesmo recurso (tipo + nome) em mais de um arquivo.
- Use `var.compartment_id` nos recursos e declare sempre `variable "region"` em `variables.tf`.
- Se houver "RECURSOS OCI EXISTENTES NO COMPARTMENT", reutilize recursos com data sources sempre que possível.
- Se houver "ARQUIVOS TERRAFORM ATUAIS NO WORKSPACE", gere **SOMENTE os arquivos que precisam de correção**. Arquivos sem erro NÃO devem ser regenerados. Mantenha os mesmos nomes de arquivo. Se o erro está em `networking.tf`, gere APENAS `// filename: networking.tf` com o conteúdo corrigido. NUNCA regenere `variables.tf`, `outputs.tf` ou outros arquivos que não têm erro, a menos que o erro exija alteração neles (ex: variável faltando).
- Sempre responda com blocos `hcl`, depois **Resource Plan** (`+` criar, `~` reutilizar), uma breve explicação e uma seção `✅ Validação`.
- Valide: referências cruzadas, CIDRs, route tables, security lists, dependências, segurança e variáveis.
- Use apenas resource types válidos do OCI. Exemplos corretos:
  - `oci_network_firewall_network_firewall`
  - `oci_network_firewall_network_firewall_policy`
  - `oci_core_drg_attachment`
  - `oci_core_drg_attachment_management`
  - `oci_core_drg_route_table`
  - `oci_core_drg_route_distribution`
  - `oci_core_drg_route_distribution_statement`
  - `oci_core_drg_route_table_route_rule`
  - `oci_core_remote_peering_connection`
- Nunca invente recursos inexistentes.
- Use sempre HCL multi-line, nunca blocos inline com múltiplos argumentos.

### Correção de erros (IMPORTANTE)
- Quando o usuário envia um erro de plan/apply com arquivos existentes, gere SOMENTE o(s) arquivo(s) que corrigem o erro.
- Exemplo: se o erro é em `service_gateways.tf`, retorne APENAS `// filename: service_gateways.tf` com o conteúdo corrigido.
- Se a correção exige criar/alterar uma variável, inclua também `// filename: variables.tf` apenas com as variáveis novas/alteradas + todas existentes.
- NUNCA regenere todos os arquivos do zero. Isso descarta o trabalho anterior e cria loops infinitos de correção.
- Preserve os nomes de arquivo exatos. Não renomeie `drg_attachments_and_routes.tf` para `drg.tf`.

### Regras críticas
- `oci_core_drg_route_distribution_statement` deve usar `match_criteria`.
- Remote peering: APENAS UM LADO do par define `peer_id` e `peer_region_name`. O outro lado é criado SEM `peer_id`. Nunca gere dois RPCs apontando um para o outro (causa Cycle error). Exemplo: RPC_mad1 (sem peer_id) e RPC_mad3 (com peer_id = RPC_mad1.id, peer_region_name = var.region).
- Associação de route table com subnet deve ser via `route_table_id` na subnet ou `oci_core_route_table_attachment`.
- `oci_network_firewall_network_firewall` exige `network_firewall_policy_id`.
- DRG attachments: use `oci_core_drg_attachment` para VCN attachments (tipo padrão). Use `oci_core_drg_attachment_management` SOMENTE para REMOTE_PEERING_CONNECTION (único tipo suportado para management). Nunca duplique resource type+name entre arquivos (ex: dois `oci_core_drg_attachment.vcn_app_x`). Sempre associe `drg_route_table_id` nos VCN attachments para garantir roteamento correto.

### Proibições absolutas — módulos e variáveis
- NUNCA use `module` blocks. Este workspace é flat (um único diretório). Não existem subdiretórios `modules/`. Toda a infraestrutura deve ser definida diretamente com `resource` e `data` blocks.
- NUNCA declare a mesma `variable` em mais de um arquivo. Todas as variáveis devem estar em `variables.tf` e SOMENTE lá. Os outros arquivos (.tf) apenas referenciam `var.nome`.
- Antes de gerar, verifique se uma variável já foi declarada. Se já existe em `variables.tf`, NÃO redeclare.
"""

# ── Terraform OCI Resource Reference (generated at build time, updatable at runtime) ──
_TF_RESOURCE_REF_PATH = "/data/oci_tf_resource_reference.txt"
_tf_resource_ref_cache: dict = {"content": None, "mtime": 0}


def _populate_tf_valid_types():
    """Parse the OCI TF resource reference file and populate tf_valid_types table in SQLite."""
    p = Path(_TF_RESOURCE_REF_PATH)
    if not p.exists():
        return 0
    content = p.read_text()
    import re as _re
    entries = []
    current_category = ""
    for line in content.split('\n'):
        if line.startswith('## '):
            current_category = line[3:].strip()
            continue
        # Resource lines: - **oci_xxx**  required: ...  blocks: ...
        m = _re.match(r'^- \*\*(\w+)\*\*\s*(.*)', line)
        if m:
            name = m.group(1)
            rest = m.group(2).strip()
            required = ""
            blocks = ""
            # Split by "blocks:" to separate required from blocks
            blk_parts = rest.split('blocks:')
            if blk_parts[0].strip().startswith('required:'):
                required = blk_parts[0].replace('required:', '').strip()
            if len(blk_parts) > 1:
                blocks = blk_parts[1].strip()
            entries.append((name, 'resource', current_category, required, blocks))
            continue
        # Data source lines (in "All Resource Types" or "All Data Sources" section): - oci_xxx
        m2 = _re.match(r'^- (\w+)\s*$', line)
        if m2:
            name = m2.group(1)
            if current_category.startswith('All Data'):
                entries.append((name, 'data', '', '', ''))
            elif current_category.startswith('All Resource'):
                entries.append((name, 'resource', '', '', ''))
    if not entries:
        return 0
    with db() as c:
        c.execute("DELETE FROM tf_valid_types")
        # Insert detailed entries first (with required_args), then bulk lists with IGNORE to not overwrite
        detailed = [e for e in entries if e[3] or e[4]]  # has required_args or blocks
        bulk = [e for e in entries if not e[3] and not e[4]]
        if detailed:
            c.executemany(
                "INSERT OR REPLACE INTO tf_valid_types (name, kind, category, required_args, blocks) VALUES (?,?,?,?,?)",
                detailed)
        if bulk:
            c.executemany(
                "INSERT OR IGNORE INTO tf_valid_types (name, kind, category, required_args, blocks) VALUES (?,?,?,?,?)",
                bulk)
    log.info(f"Populated tf_valid_types: {len(entries)} entries")
    return len(entries)


def _validate_tf_resource_types(tf_code: str) -> list:
    """Validate resource/data types in generated TF code against the tf_valid_types table.
    Returns list of dicts with invalid types and suggestions."""
    import re as _re
    from difflib import get_close_matches
    # Extract only resource type declarations (data sources are read-only and may not be in the schema)
    used_types = set()
    for m in _re.finditer(r'resource\s+"(\w+)"', tf_code):
        used_types.add(m.group(1))
    if not used_types:
        return []
    # Load valid types from DB
    with db() as c:
        rows = c.execute("SELECT name, kind, required_args FROM tf_valid_types").fetchall()
    valid_names = {r["name"] for r in rows}
    valid_required = {r["name"]: r["required_args"] for r in rows}
    if not valid_names:
        return []  # No reference loaded
    # Check each used type
    errors = []
    for t in sorted(used_types):
        if t not in valid_names:
            # Find closest matches
            suggestions = get_close_matches(t, valid_names, n=3, cutoff=0.6)
            errors.append({
                "type": t,
                "suggestions": suggestions,
                "message": f"Resource type '{t}' NÃO EXISTE no provider oracle/oci."
            })
    return errors

def _load_tf_resource_reference() -> str:
    """Load the OCI Terraform resource reference file, cached by mtime."""
    try:
        p = Path(_TF_RESOURCE_REF_PATH)
        if not p.exists():
            return ""
        mtime = p.stat().st_mtime
        if _tf_resource_ref_cache["content"] and _tf_resource_ref_cache["mtime"] == mtime:
            return _tf_resource_ref_cache["content"]
        content = p.read_text()
        _tf_resource_ref_cache["content"] = content
        _tf_resource_ref_cache["mtime"] = mtime
        log.info(f"Loaded TF resource reference: {len(content)} chars, {content.count(chr(10))} lines")
        return content
    except Exception as e:
        log.warning(f"Failed to load TF resource reference: {e}")
        return ""

def _regenerate_tf_reference() -> dict:
    """Regenerate the OCI Terraform resource reference file."""
    try:
        result = subprocess.run(
            ["python", "/app/gen_tf_reference.py"],
            capture_output=True, text=True, timeout=300
        )
        _tf_resource_ref_cache["content"] = None  # invalidate cache
        if result.returncode == 0:
            ref = _load_tf_resource_reference()
            return {"ok": True, "output": result.stdout.strip(), "lines": ref.count('\n')}
        return {"ok": False, "error": result.stderr.strip()}
    except Exception as e:
        return {"ok": False, "error": str(e)}

# ── Terraform Resource Docs (on-demand from GitHub, cached in SQLite) ──
_TF_DOC_BASE_URL = "https://raw.githubusercontent.com/oracle/terraform-provider-oci/master/website/docs/r/"


def _fetch_tf_resource_doc(slug: str) -> dict | None:
    """Fetch a single resource doc from GitHub, parse Example Usage + Argument Reference, cache in SQLite."""
    import re as _re
    # Check cache first
    with db() as c:
        row = c.execute("SELECT example_usage, argument_ref FROM tf_resource_docs WHERE slug=?", (slug,)).fetchone()
    if row and (row["example_usage"] or row["argument_ref"]):
        return {"slug": slug, "example": row["example_usage"], "args": row["argument_ref"]}
    # Fetch from GitHub
    try:
        import urllib.request
        url = f"{_TF_DOC_BASE_URL}{slug}.html.markdown"
        req = urllib.request.Request(url, headers={"User-Agent": "oci-cis-agent/2.1"})
        with urllib.request.urlopen(req, timeout=15) as resp:
            if resp.status != 200:
                return None
            md = resp.read().decode("utf-8")
        # Extract Example Usage section
        example = ""
        m = _re.search(r'## Example Usage\s*\n(.*?)(?=\n## )', md, _re.DOTALL)
        if m:
            example = m.group(1).strip()
        # Extract Argument Reference section
        args = ""
        m2 = _re.search(r'## Argument Reference\s*\n(.*?)(?=\n## )', md, _re.DOTALL)
        if m2:
            args = m2.group(1).strip()
        # Cache in SQLite
        subcategory = ""
        m3 = _re.search(r'subcategory:\s*"([^"]+)"', md)
        if m3:
            subcategory = m3.group(1)
        with db() as c:
            c.execute(
                "INSERT OR REPLACE INTO tf_resource_docs (slug, subcategory, example_usage, argument_ref) VALUES (?,?,?,?)",
                (slug, subcategory, example, args))
        log.info(f"Fetched TF doc: {slug} (example={len(example)} chars, args={len(args)} chars)")
        return {"slug": slug, "example": example, "args": args}
    except Exception as e:
        log.warning(f"Failed to fetch TF doc for {slug}: {e}")
        return None


def _get_tf_docs_for_resources(resource_types: list[str]) -> str:
    """Given a list of oci_xxx resource types, fetch their docs and build a context string."""
    docs_parts = []
    for rtype in resource_types[:10]:  # limit to 10 to keep context manageable
        # Convert oci_core_vcn -> core_vcn
        slug = rtype.replace("oci_", "", 1) if rtype.startswith("oci_") else rtype
        doc = _fetch_tf_resource_doc(slug)
        if doc and (doc["example"] or doc["args"]):
            part = f"#### {rtype}\n"
            if doc["example"]:
                part += f"**Example Usage:**\n{doc['example']}\n\n"
            if doc["args"]:
                part += f"**Arguments:**\n{doc['args']}\n"
            docs_parts.append(part)
    if not docs_parts:
        return ""
    return "### Documentação Oficial dos Recursos (registry.terraform.io)\n" + \
           "Use exatamente os argumentos e estrutura mostrados nos exemplos abaixo.\n\n" + \
           "\n---\n".join(docs_parts)


def _detect_tf_resource_types(text: str) -> list[str]:
    """Detect OCI resource types mentioned or implied in a user message + conversation."""
    import re as _re
    # Direct mentions of oci_ resource types
    explicit = set(_re.findall(r'\boci_\w+', text))
    # Keyword-to-resource mapping for common infra requests
    keyword_map = {
        'vcn': ['oci_core_vcn', 'oci_core_subnet', 'oci_core_internet_gateway', 'oci_core_nat_gateway',
                'oci_core_route_table', 'oci_core_security_list'],
        'subnet': ['oci_core_subnet'],
        'instance': ['oci_core_instance', 'oci_core_instance_configuration'],
        'compute': ['oci_core_instance'],
        'load.?balancer': ['oci_load_balancer_load_balancer', 'oci_load_balancer_backend_set',
                           'oci_load_balancer_listener'],
        'nlb': ['oci_network_load_balancer_network_load_balancer', 'oci_network_load_balancer_backend_set',
                'oci_network_load_balancer_listener'],
        'firewall': ['oci_network_firewall_network_firewall', 'oci_network_firewall_network_firewall_policy'],
        'drg': ['oci_core_drg', 'oci_core_drg_attachment', 'oci_core_drg_route_table'],
        'peering|rpc': ['oci_core_remote_peering_connection'],
        'bucket|object.?storage': ['oci_objectstorage_bucket', 'oci_objectstorage_namespace_metadata'],
        'autonomous|adb': ['oci_database_autonomous_database'],
        'db.?system': ['oci_database_db_system'],
        'mysql': ['oci_mysql_mysql_db_system'],
        'oke|kubernetes|cluster': ['oci_containerengine_cluster', 'oci_containerengine_node_pool'],
        'dns': ['oci_dns_zone', 'oci_dns_record'],
        'vault|kms': ['oci_kms_vault', 'oci_kms_key'],
        'waf': ['oci_waf_web_app_firewall', 'oci_waf_web_app_firewall_policy'],
        'api.?gateway': ['oci_apigateway_gateway', 'oci_apigateway_deployment'],
        'function|serverless': ['oci_functions_application', 'oci_functions_function'],
        'nsg': ['oci_core_network_security_group', 'oci_core_network_security_group_security_rule'],
        'bastion': ['oci_bastion_bastion', 'oci_bastion_session'],
        'file.?storage': ['oci_file_storage_file_system', 'oci_file_storage_mount_target',
                          'oci_file_storage_export'],
        'block.?volume': ['oci_core_volume', 'oci_core_volume_attachment'],
        'policy|iam': ['oci_identity_policy', 'oci_identity_compartment', 'oci_identity_group',
                       'oci_identity_dynamic_group'],
    }
    text_lower = text.lower()
    for pattern, resources in keyword_map.items():
        if _re.search(pattern, text_lower):
            explicit.update(resources)
    return sorted(explicit)


def _get_adb_connection(cfg: dict):
    """Create an oracledb connection from an adb_vector_configs row."""
    import oracledb
    params = {"user": cfg["username"], "password": _dec(cfg["password_enc"]), "dsn": cfg["dsn"]}
    if cfg["use_mtls"] and cfg.get("wallet_dir"):
        params["config_dir"] = cfg["wallet_dir"]
        params["wallet_location"] = cfg["wallet_dir"]
        params["wallet_password"] = _dec(cfg["wallet_password_enc"]) if cfg.get("wallet_password_enc") else ""
    params["tcp_connect_timeout"] = 15  # 15s connection timeout
    return oracledb.connect(**params)

def _resolve_embed_config(oci_config_id: str = None, genai_cfg: dict = None, user_id: str = None) -> dict:
    """Resolve embedding config from genai_cfg or oci_config. Scoped to user_id when provided.
    Returns dict with oci_config_id, endpoint, genai_region, compartment_id."""
    if genai_cfg:
        return genai_cfg
    if oci_config_id:
        with db() as c:
            # Try linked genai config first (scoped to user if provided)
            if user_id:
                gc = c.execute("SELECT * FROM genai_configs WHERE oci_config_id=? AND user_id=? ORDER BY is_default DESC, created_at DESC",
                               (oci_config_id, user_id)).fetchone()
            else:
                gc = c.execute("SELECT * FROM genai_configs WHERE oci_config_id=? ORDER BY is_default DESC, created_at DESC",
                               (oci_config_id,)).fetchone()
            if gc: return dict(gc)
            # Fallback: build from oci_config directly
            oc = c.execute("SELECT * FROM oci_configs WHERE id=?", (oci_config_id,)).fetchone()
            if oc:
                return {
                    "oci_config_id": oc["id"],
                    "genai_region": oc["region"],
                    "endpoint": f"https://inference.generativeai.{oc['region']}.oci.oraclecloud.com",
                    "compartment_id": _safe_dec(dict(oc).get("compartment_id") or oc["tenancy_ocid"]),
                }
    # Last resort: user's own genai/oci config (never cross-user)
    with db() as c:
        if user_id:
            gc = c.execute("SELECT * FROM genai_configs WHERE user_id=? ORDER BY is_default DESC, created_at DESC", (user_id,)).fetchone()
        else:
            gc = c.execute("SELECT * FROM genai_configs ORDER BY is_default DESC, created_at DESC").fetchone()
        if gc: return dict(gc)
        if user_id:
            oc = c.execute("SELECT * FROM oci_configs WHERE user_id=? ORDER BY created_at DESC", (user_id,)).fetchone()
        else:
            oc = c.execute("SELECT * FROM oci_configs ORDER BY created_at DESC").fetchone()
        if oc:
            return {
                "oci_config_id": oc["id"],
                "genai_region": oc["region"],
                "endpoint": f"https://inference.generativeai.{oc['region']}.oci.oraclecloud.com",
                "compartment_id": _safe_dec(dict(oc).get("compartment_id") or oc["tenancy_ocid"]),
            }
    raise HTTPException(400, "Nenhuma credencial OCI/GenAI configurada para gerar embeddings.")

def _embed_text(text: str, genai_cfg: dict, embedding_model_id: str, max_chars: int = 8000) -> list:
    """Generate embedding using OCI GenAI embed endpoint."""
    import oci
    # Truncate text to fit model token limit (~8192 tokens)
    if len(text) > max_chars:
        text = text[:max_chars]
    config_path = str(OCI_DIR / genai_cfg["oci_config_id"] / "config")
    config = oci.config.from_file(config_path, "DEFAULT")
    endpoint = genai_cfg.get("endpoint") or f"https://inference.generativeai.{genai_cfg.get('genai_region','us-ashburn-1')}.oci.oraclecloud.com"
    client = oci.generative_ai_inference.GenerativeAiInferenceClient(
        config=config, service_endpoint=endpoint,
        retry_strategy=oci.retry.NoneRetryStrategy(), timeout=(10, 120)
    )
    embed_detail = oci.generative_ai_inference.models.EmbedTextDetails()
    embed_detail.inputs = [text]
    emb_info = EMBEDDING_MODELS.get(embedding_model_id, {})
    region = genai_cfg.get("genai_region", "")
    emb_ref = emb_info.get("ocids", {}).get(region) or embedding_model_id
    embed_detail.serving_mode = oci.generative_ai_inference.models.OnDemandServingMode(model_id=emb_ref)
    embed_detail.compartment_id = genai_cfg.get("compartment_id", "")
    embed_detail.truncate = "END"
    embed_detail.input_type = "SEARCH_QUERY"
    response = client.embed_text(embed_detail)
    return response.data.embeddings[0]

_DIM_TO_MODEL = {1536: "openai.text-embedding-3-small", 3072: "openai.text-embedding-3-large"}

def _get_table_embedding_dim(cfg: dict, table_name: str) -> int:
    """Detect the embedding dimension of a table by sampling one row, or from column definition if empty."""
    conn = _get_adb_connection(cfg)
    try:
        cur = conn.cursor()
        # Try from data first
        cur.execute(f'SELECT VECTOR_DIMENSION_COUNT(EMBEDDING) FROM "{table_name}" FETCH FIRST 1 ROWS ONLY')
        row = cur.fetchone()
        if row and row[0]:
            cur.close()
            return int(row[0])
        # Table is empty — check column definition from DDL
        try:
            cur.execute(f"""SELECT DBMS_METADATA.GET_DDL('TABLE', :1) FROM DUAL""", [table_name])
            ddl_row = cur.fetchone()
            cur.close()
            if ddl_row and ddl_row[0]:
                import re
                m = re.search(r'VECTOR\((\d+)', str(ddl_row[0]))
                if m:
                    return int(m.group(1))
        except Exception:
            pass
        return 0
    except Exception as e:
        log.warning(f"Failed to get embedding dimension for table '{table_name}': {e}")
        return 0
    finally:
        conn.close()

_GLOBAL_TABLES = {"cisrecom", "engineerknowledgebase"}  # Tables without tenancy filter (generic knowledge)

def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name: str = None, tenancy: str = None, text_filter: str = None, user_id: str = None) -> list:
    """Search ADB vector store using cosine similarity. Returns top-K documents.
    If tenancy is provided and table is not global, filters by tenancy.
    If user_id is provided and table is not global, filters by user_id (legacy docs without user_id are included).
    If text_filter is provided, also filters TEXT content with LIKE."""
    import array
    table_name = table_name or cfg.get("table_name", "")
    conn = _get_adb_connection(cfg)
    try:
        conn.call_timeout = 30000  # 30s query timeout (milliseconds)
        cur = conn.cursor()
        vec = array.array('f', query_embedding)
        limit = int(top_k)
        is_non_global = table_name.lower() not in _GLOBAL_TABLES
        use_tenancy_filter = tenancy and is_non_global
        use_user_filter = user_id and is_non_global
        # Build WHERE clauses
        conditions = []
        params = [vec]
        param_idx = 2
        if use_tenancy_filter:
            conditions.append(f"JSON_VALUE(METADATA, '$.tenancy') = :{param_idx}")
            params.append(tenancy)
            param_idx += 1
        if use_user_filter:
            conditions.append(f"(JSON_VALUE(METADATA, '$.user_id') = :{param_idx} OR JSON_VALUE(METADATA, '$.user_id') IS NULL)")
            params.append(user_id)
            param_idx += 1
        if text_filter:
            conditions.append(f"TEXT LIKE :{param_idx}")
            params.append(f"%{text_filter}%")
            param_idx += 1
        where = f"WHERE {' AND '.join(conditions)}" if conditions else ""
        cur.execute(f"""
            SELECT ID, TEXT, METADATA,
                   VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance
            FROM "{table_name}"
            {where}
            ORDER BY distance ASC
            FETCH FIRST {limit} ROWS ONLY
        """, params)
        results = []
        for row in cur:
            content = row[1]
            if hasattr(content, 'read'):
                content = content.read()
            results.append({
                "id": row[0], "content": content or "",
                "metadata": row[2], "distance": float(row[3])
            })
        cur.close()
        return results
    finally:
        conn.close()

# ── Table relevance mapping for smart skip ──
_TABLE_KEYWORDS = {
    "identityandaccess": {"iam", "user", "users", "policy", "policies", "mfa", "api key", "group", "password", "credential", "authentication", "identity", "access"},
    "networking": {"network", "vcn", "subnet", "security list", "nsg", "firewall", "route", "gateway", "nat", "internet", "port", "ingress", "egress", "cidr", "ip"},
    "computeinstances": {"compute", "instance", "vm", "boot", "metadata", "secure boot", "encryption"},
    "loggingandmonitoring": {"log", "logging", "monitor", "alarm", "event", "notification", "audit", "cloud guard", "flow"},
    "objectstorage": {"bucket", "object storage", "storage", "visibility", "public"},
    "storageblockvolume": {"block volume", "volume", "disk", "cmk", "encryption"},
    "filestorageservice": {"file storage", "file system", "mount", "nfs"},
    "assetmanagement": {"compartment", "tag", "asset", "resource"},
    "summaryreportcsvvector": {"summary", "score", "compliance", "report", "overview", "total"},
    "cisrecom": {"remediation", "fix", "correct", "resolve", "recommendation", "how to", "como corrigir", "remediação"},
    "engineerknowledgebase": {"documentation", "guide", "tutorial", "blog", "best practice", "knowledge"},
}

def _relevant_tables(query: str, tables: list[str]) -> list[str]:
    """Filter tables to only those relevant to the query. Always includes cisrecom and engineerknowledgebase."""
    q = query.lower()
    # If query mentions a CIS number, include all tables (the text filter handles precision)
    import re as _re
    if _re.search(r'cis\s*\d+\.\d+', q):
        return tables
    relevant = []
    for tbl in tables:
        tbl_lower = tbl.lower()
        # Always include global knowledge tables
        if tbl_lower in _GLOBAL_TABLES or tbl_lower == "summaryreportcsvvector":
            relevant.append(tbl)
            continue
        keywords = _TABLE_KEYWORDS.get(tbl_lower, set())
        if any(kw in q for kw in keywords):
            relevant.append(tbl)
    # If no specific table matched, search all (generic question)
    return relevant if len(relevant) > 2 else tables


def _vector_search_multi(cfg: dict, query_embedding: list, tables: list[str], top_k_per_table: int = 3,
                         tenancy: str = None, text_filter: str = None, user_id: str = None) -> list:
    """Search multiple tables using a SINGLE ADB connection. Returns all documents with source."""
    import array
    conn = _get_adb_connection(cfg)
    all_results = []
    try:
        conn.call_timeout = 30000
        vec = array.array('f', query_embedding)
        for table_name in tables:
            try:
                cur = conn.cursor()
                limit = int(top_k_per_table)
                is_non_global = table_name.lower() not in _GLOBAL_TABLES
                use_tenancy = tenancy and is_non_global
                use_user_filter = user_id and is_non_global
                conditions = []
                params = [vec]
                param_idx = 2
                if use_tenancy:
                    conditions.append(f"JSON_VALUE(METADATA, '$.tenancy') = :{param_idx}")
                    params.append(tenancy)
                    param_idx += 1
                if use_user_filter:
                    conditions.append(f"(JSON_VALUE(METADATA, '$.user_id') = :{param_idx} OR JSON_VALUE(METADATA, '$.user_id') IS NULL)")
                    params.append(user_id)
                    param_idx += 1
                tbl_filter = text_filter if (text_filter and is_non_global) else None
                if tbl_filter:
                    conditions.append(f"TEXT LIKE :{param_idx}")
                    params.append(f"%{tbl_filter}%")
                    param_idx += 1
                where = f"WHERE {' AND '.join(conditions)}" if conditions else ""
                cur.execute(f"""
                    SELECT ID, TEXT, METADATA,
                           VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance
                    FROM "{table_name}"
                    {where}
                    ORDER BY distance ASC
                    FETCH FIRST {limit} ROWS ONLY
                """, params)
                for row in cur:
                    content = row[1]
                    if hasattr(content, 'read'):
                        content = content.read()
                    all_results.append({
                        "id": row[0], "content": content or "",
                        "metadata": row[2], "distance": float(row[3]),
                        "source": table_name,
                    })
                cur.close()
            except Exception as e:
                log.warning(f"Multi-search failed for {table_name}: {str(e)[:120]}")
    finally:
        conn.close()
    return all_results


def _enrich_doc_content(doc: dict) -> str:
    """Extract the best text content from a document, checking metadata.text as fallback."""
    content = doc.get("content", "")
    meta = doc.get("metadata", "")
    if isinstance(meta, str) and meta:
        try:
            meta = json.loads(meta)
        except Exception as e:
            log.warning(f"Failed to parse document metadata JSON: {e}")
            meta = {}
    if isinstance(meta, dict):
        # If TEXT column is short/empty, use metadata.text instead
        meta_text = meta.get("text", "")
        if meta_text and len(meta_text) > len(content):
            header_parts = []
            if meta.get("recommendationNumber"):
                header_parts.append(f"Recommendation: {meta['recommendationNumber']}")
            if meta.get("chapter"):
                header_parts.append(f"Chapter: {meta['chapter']}")
            if meta.get("header"):
                header_parts.append(meta["header"])
            header = " | ".join(header_parts)
            content = (header + "\n" + meta_text) if header else meta_text
        # Also enrich with section/tenancy info if available
        elif meta.get("section") or meta.get("tenancy"):
            extra = []
            if meta.get("tenancy"): extra.append(f"Tenancy: {meta['tenancy']}")
            if meta.get("section"): extra.append(f"Section: {meta['section']}")
            if extra:
                content = " | ".join(extra) + "\n" + content
    return content

def _build_rag_context(documents: list, max_total_chars: int = 12000) -> str:
    """Format retrieved documents into a context string for the LLM prompt.
    Includes extract_date from metadata for temporal awareness.
    Limits total context to max_total_chars to prevent token overflow."""
    if not documents:
        return ""
    parts = []
    total = 0
    max_per_doc = max_total_chars // min(len(documents), 8)
    for i, doc in enumerate(documents, 1):
        source = doc.get("source", "unknown")
        dist = doc.get("distance", 0)
        # Extract date from metadata for temporal context
        extract_date = ""
        meta_raw = doc.get("metadata", "")
        if isinstance(meta_raw, str) and meta_raw:
            try:
                meta_obj = json.loads(meta_raw)
                extract_date = meta_obj.get("extract_date", "") or meta_obj.get("report_date", "")
            except Exception:
                pass
        content = _enrich_doc_content(doc)
        if len(content) > max_per_doc:
            content = content[:max_per_doc] + "..."
        date_tag = f" | date: {extract_date}" if extract_date else ""
        part = f"[Doc {i} | {source}{date_tag} | relevance: {1 - dist:.2f}]\n{content}"
        if total + len(part) > max_total_chars:
            break
        parts.append(part)
        total += len(part)
    return "\n\n---\n\n".join(parts)

def _get_active_adb_configs(user_id: str) -> list[dict]:
    """Get all active ADB vector configs accessible to this user (own + global). No cross-user fallback."""
    with db() as c:
        rows = c.execute(
            "SELECT * FROM adb_vector_configs WHERE is_active=1 AND (user_id=? OR is_global=1) ORDER BY is_global DESC, created_at DESC",
            (user_id,)
        ).fetchall()
    return [dict(r) for r in rows]

def _get_tables_for_config(adb_config_id: str, active_only: bool = True) -> list[dict]:
    """Get all registered vector tables for an ADB config."""
    with db() as c:
        sql = "SELECT * FROM adb_vector_tables WHERE adb_config_id=?"
        if active_only:
            sql += " AND is_active=1"
        sql += " ORDER BY created_at ASC"
        rows = c.execute(sql, (adb_config_id,)).fetchall()
    return [dict(r) for r in rows]

# ── MCP Servers ───────────────────────────────────────────────────────────────