"""GenAI service — model calls, embeddings, vector search, memory compaction.""" import os, json, uuid, time, base64, re, hashlib from datetime import datetime, timedelta from typing import Optional, List, Dict, Any from fastapi import HTTPException from config import ( APP_SECRET, DATA, OCI_DIR, WALLET_DIR, log, COMPACT_TOKEN_THRESHOLD, COMPACT_KEEP_RECENT, COMPACT_SUMMARY_MAX_TOKENS, COMPACT_MIN_MESSAGES, GENAI_MODELS, EMBEDDING_MODELS, GENAI_REGIONS, _chat_executor, ) from database import db from auth.crypto import _enc, _dec, _safe_dec from auth.jwt_auth import _chat_log def _estimate_tokens(text: str) -> int: return len(text) // 4 def _estimate_history_tokens(history: list) -> int: return sum(_estimate_tokens(h["content"]) for h in history) def _should_compact(history: list) -> bool: if len(history) < COMPACT_MIN_MESSAGES: return False return _estimate_history_tokens(history) > COMPACT_TOKEN_THRESHOLD def _generate_summary(genai_cfg: dict, messages_to_summarize: list) -> str: """Use the same GenAI model to summarize older conversation messages.""" conversation_text = "" for m in messages_to_summarize: role_label = "Usuário" if m["role"] == "user" else "Assistente" # Truncate very long messages in the summary input content = m["content"][:3000] if len(m["content"]) > 3000 else m["content"] conversation_text += f"{role_label}: {content}\n\n" summary_prompt = ( "Resuma a conversa abaixo de forma compacta (máximo 2-3 parágrafos curtos) que capture:\n" "- Tópicos discutidos e perguntas do usuário\n" "- Decisões tomadas, conclusões e resultados importantes\n" "- Resultados de ferramentas/tools executadas (checks PASS/FAIL, scores, recursos encontrados)\n" "- Contexto OCI/CIS relevante (compartments, regiões, config_ids usados)\n" "- Dados numéricos importantes (scores, contagens, OCIDs mencionados)\n\n" "Seja conciso mas preserve TODA informação acionável e dados concretos. " "Não inclua saudações ou formalidades.\n\n" "CONVERSA:\n" + conversation_text ) summary_cfg = dict(genai_cfg) summary_cfg["system_prompt"] = "Você é um sumarizador. Responda apenas com o resumo." summary_cfg["max_tokens"] = COMPACT_SUMMARY_MAX_TOKENS try: text, _, _ = _call_genai(summary_cfg, summary_prompt, history=None, tools=None) return text.strip() except Exception as e: log.warning(f"Summary generation failed: {e}") return "" def _compact_history(session_id: str, user_id: str, genai_cfg: dict, history: list) -> list: """Compact conversation history by summarizing older messages.""" if len(history) <= COMPACT_KEEP_RECENT: return history messages_to_summarize = history[:-COMPACT_KEEP_RECENT] recent_messages = history[-COMPACT_KEEP_RECENT:] # Check for existing summary with db() as c: row = c.execute( "SELECT summary, messages_compacted FROM chat_summaries " "WHERE session_id=? ORDER BY created_at DESC LIMIT 1", (session_id,) ).fetchone() if row: existing_summary = row["summary"] prev_compacted = row["messages_compacted"] # Reuse if no new messages to summarize if len(messages_to_summarize) <= prev_compacted: return [{"role": "assistant", "content": f"[Resumo da conversa anterior: {existing_summary}]"}] + recent_messages else: existing_summary = None # Include existing summary as prefix for incremental compaction if existing_summary: messages_to_summarize = [{"role": "assistant", "content": f"[Resumo anterior: {existing_summary}]"}] + messages_to_summarize summary_text = _generate_summary(genai_cfg, messages_to_summarize) if not summary_text: # Fallback: truncate keeping recent messages that fit in budget truncated = [] budget = 6000 for m in reversed(history): t = _estimate_tokens(m["content"]) if budget - t < 0: break truncated.insert(0, m) budget -= t return truncated # Save summary to DB actual_count = len(messages_to_summarize) - (1 if existing_summary else 0) with db() as c: last_msg = c.execute( "SELECT id FROM chat_messages WHERE session_id=? AND role IN ('user','assistant') " "ORDER BY created_at ASC LIMIT 1 OFFSET ?", (session_id, max(0, actual_count - 1)) ).fetchone() last_id = last_msg["id"] if last_msg else "unknown" c.execute( "INSERT INTO chat_summaries (id,session_id,user_id,summary,messages_compacted,up_to_message_id) VALUES (?,?,?,?,?,?)", (str(uuid.uuid4()), session_id, user_id, summary_text, actual_count, last_id) ) log.info(f"Generated summary for session {session_id}: {len(summary_text)} chars, {actual_count} msgs compacted") return [{"role": "assistant", "content": f"[Resumo da conversa anterior: {summary_text}]"}] + recent_messages # ── GenAI Call ───────────────────────────────────────────────────────────────── def _call_genai(gc: dict, message: str, history: list = None, tools: list = None, tool_results_cohere: list = None, extra_messages: list = None, attachments: list = None) -> tuple: """ Call OCI Generative AI with optional tool use support. Returns (text, tool_calls, tool_calls_raw) tuple. tool_calls is a list of dicts or None. tool_calls_raw is the raw OCI SDK objects (for Generic format continuations). extra_messages: list of raw OCI SDK message objects to append after the user message (for tool use loop accumulation). """ import oci # Load OCI config from stored credentials (same as ~/.oci/config) config_path = str(OCI_DIR / gc["oci_config_id"] / "config") config = oci.config.from_file(config_path, "DEFAULT") # Service endpoint - built from region endpoint = gc["endpoint"] # Create inference client with retry strategy and timeout generative_ai_inference_client = oci.generative_ai_inference.GenerativeAiInferenceClient( config=config, service_endpoint=endpoint, retry_strategy=oci.retry.NoneRetryStrategy(), timeout=(10, 600) ) # System prompt system_prompt = gc.get("system_prompt", "") # Build ChatDetails chat_detail = oci.generative_ai_inference.models.ChatDetails() # Determine API format from model catalog model_info = GENAI_MODELS.get(gc["model_id"], {}) api_format = model_info.get("api_format", "GENERIC") if api_format == "COHERE": # ── Cohere models (CohereChatRequest) ── chat_request = oci.generative_ai_inference.models.CohereChatRequest() chat_request.api_format = oci.generative_ai_inference.models.BaseChatRequest.API_FORMAT_COHERE if system_prompt: chat_request.preamble_override = system_prompt chat_request.message = message cohere_max = model_info.get("max_tokens", 4096) chat_request.max_tokens = min(int(gc.get("max_tokens", 4096)), cohere_max) chat_request.temperature = float(gc.get("temperature", 1)) chat_request.frequency_penalty = float(gc.get("frequency_penalty", 0)) chat_request.presence_penalty = float(gc.get("presence_penalty", 0)) chat_request.top_p = float(gc.get("top_p", 0.95)) chat_request.top_k = int(gc.get("top_k", 1)) if history: chat_history = [] for h in history: entry = oci.generative_ai_inference.models.CohereMessage() entry.role = "USER" if h["role"] == "user" else "CHATBOT" entry.message = h["content"] chat_history.append(entry) chat_request.chat_history = chat_history # Tool use support for Cohere if tools: cohere_tools = [] for t in tools: props = t.get("input_schema", {}).get("properties", {}) required = t.get("input_schema", {}).get("required", []) param_defs = {} for k, v in props.items(): pd = oci.generative_ai_inference.models.CohereParameterDefinition() pd.type = v.get("type", "str") pd.description = v.get("description", "") pd.is_required = k in required param_defs[k] = pd ct = oci.generative_ai_inference.models.CohereTool() ct.name = t["name"] ct.description = t.get("description", "") ct.parameter_definitions = param_defs if param_defs else None cohere_tools.append(ct) chat_request.tools = cohere_tools if tool_results_cohere: chat_request.tool_results = tool_results_cohere else: # ── Generic format (Meta Llama, Google, xAI, OpenAI) ── chat_request = oci.generative_ai_inference.models.GenericChatRequest() chat_request.api_format = oci.generative_ai_inference.models.BaseChatRequest.API_FORMAT_GENERIC provider = model_info.get("provider", "") is_openai = provider == "openai" is_reasoning = model_info.get("reasoning", False) # Clamp max_tokens to model limit model_max = model_info.get("max_tokens", 16384) requested_tokens = int(gc.get("max_tokens", 6000)) clamped_tokens = min(requested_tokens, model_max) # ── Parameter matrix per provider ── # OpenAI reasoning (o3, o4-mini): max_completion_tokens + reasoning_effort only if is_reasoning: chat_request.max_completion_tokens = clamped_tokens re = gc.get("reasoning_effort") if re and hasattr(chat_request, "reasoning_effort"): chat_request.reasoning_effort = re.upper() if isinstance(re, str) else re # Explicitly unset unsupported params chat_request.temperature = None chat_request.top_p = None chat_request.frequency_penalty = None chat_request.presence_penalty = None # OpenAI standard (GPT-4.1, GPT-5.x, GPT-4o): max_completion_tokens, temperature, top_p # freq/pres penalty only for models with "penalties":True (GPT-4.1, GPT-4.1-mini, GPT-4o) elif is_openai: chat_request.max_completion_tokens = clamped_tokens chat_request.temperature = float(gc.get("temperature", 1)) chat_request.top_p = float(gc.get("top_p", 0.95)) if model_info.get("penalties"): chat_request.frequency_penalty = float(gc.get("frequency_penalty", 0)) chat_request.presence_penalty = float(gc.get("presence_penalty", 0)) else: chat_request.frequency_penalty = None chat_request.presence_penalty = None # xAI Grok: max_tokens, temperature, top_p (no top_k, no freq/pres penalty) elif provider == "xai": chat_request.max_tokens = clamped_tokens chat_request.temperature = float(gc.get("temperature", 1)) chat_request.top_p = float(gc.get("top_p", 0.95)) # Explicitly unset unsupported params to prevent SDK serialization chat_request.frequency_penalty = None chat_request.presence_penalty = None # Meta Llama: max_tokens, temperature, top_p, top_k (no freq/pres penalty) # Google Gemini: max_tokens, temperature, top_p, top_k (no freq/pres penalty) else: chat_request.max_tokens = clamped_tokens chat_request.temperature = float(gc.get("temperature", 1)) chat_request.top_p = float(gc.get("top_p", 0.95)) chat_request.top_k = int(gc.get("top_k", 1)) # Explicitly unset unsupported params to prevent SDK serialization chat_request.frequency_penalty = None chat_request.presence_penalty = None # Log applied parameters params_log = f"provider={provider}, reasoning={is_reasoning}" params_log += f", max_tokens={getattr(chat_request, 'max_completion_tokens', None) or getattr(chat_request, 'max_tokens', None)}" params_log += f", temp={getattr(chat_request, 'temperature', None)}, top_p={getattr(chat_request, 'top_p', None)}" params_log += f", top_k={getattr(chat_request, 'top_k', None)}" params_log += f", freq_pen={getattr(chat_request, 'frequency_penalty', None)}, pres_pen={getattr(chat_request, 'presence_penalty', None)}" if is_reasoning: params_log += f", reasoning_effort={getattr(chat_request, 'reasoning_effort', None)}" log.info(f"GenAI params: {params_log}") messages = [] if system_prompt: sys_content = oci.generative_ai_inference.models.TextContent() sys_content.text = system_prompt sys_msg = oci.generative_ai_inference.models.Message() sys_msg.role = "SYSTEM" sys_msg.content = [sys_content] messages.append(sys_msg) if history: for h in history: content = oci.generative_ai_inference.models.TextContent() content.text = h["content"] msg = oci.generative_ai_inference.models.Message() msg.role = "USER" if h["role"] == "user" else "ASSISTANT" msg.content = [content] messages.append(msg) # Current user message (with optional multimodal attachments) content_parts = [] if attachments: for att in attachments: if att["type"] == "image": img_url = oci.generative_ai_inference.models.ImageUrl() img_url.url = att["data_uri"] img_url.detail = "AUTO" img_content = oci.generative_ai_inference.models.ImageContent() img_content.image_url = img_url content_parts.append(img_content) elif att["type"] == "document": doc_url = oci.generative_ai_inference.models.DocumentUrl() doc_url.url = att["data_uri"] doc_content = oci.generative_ai_inference.models.DocumentContent() doc_content.document_url = doc_url content_parts.append(doc_content) text_content = oci.generative_ai_inference.models.TextContent() text_content.text = message content_parts.append(text_content) user_message = oci.generative_ai_inference.models.Message() user_message.role = "USER" user_message.content = content_parts messages.append(user_message) # Append accumulated tool use loop messages (assistant+tool_calls → tool_results → ...) if extra_messages: messages.extend(extra_messages) chat_request.messages = messages # Tool definitions for Generic format if tools: generic_tools = [] for t in tools: fd = oci.generative_ai_inference.models.FunctionDefinition() fd.name = t["name"] fd.description = t.get("description", "") fd.parameters = t.get("input_schema") generic_tools.append(fd) chat_request.tools = generic_tools # Serving mode - resolve OCID: explicit model_ocid > catalog ocid for region > short model_id model_ref = gc.get("model_ocid") or "" if not model_ref: region = gc.get("genai_region", "") ocids = model_info.get("ocids", {}) model_ref = ocids.get(region) or gc["model_id"] log.info(f"GenAI call: model_id={gc.get('model_id')}, model_ref={model_ref[:60]}...") if gc.get("serving_type") == "DEDICATED" and gc.get("dedicated_endpoint_id"): chat_detail.serving_mode = oci.generative_ai_inference.models.DedicatedServingMode( endpoint_id=gc["dedicated_endpoint_id"] ) else: chat_detail.serving_mode = oci.generative_ai_inference.models.OnDemandServingMode( model_id=model_ref ) chat_detail.chat_request = chat_request chat_detail.compartment_id = gc["compartment_id"] # Execute chat_response = generative_ai_inference_client.chat(chat_detail) # Extract text and tool_calls from response resp = chat_response.data.chat_response if api_format == "COHERE": text = resp.text if hasattr(resp, 'text') else "" tool_calls = None if hasattr(resp, 'tool_calls') and resp.tool_calls: tool_calls = [] for tc in resp.tool_calls: tool_calls.append({ "id": tc.name, # Cohere uses name as identifier "name": tc.name, "arguments": tc.parameters if isinstance(tc.parameters, dict) else {} }) return (text or "", tool_calls, None) else: text = "" tool_calls = None tool_calls_raw = None # raw content list from assistant message for continuation if hasattr(resp, 'choices') and resp.choices: choice = resp.choices[0] if hasattr(choice, 'message') and choice.message: # Check for tool calls if hasattr(choice.message, 'tool_calls') and choice.message.tool_calls: tool_calls = [] for tc in choice.message.tool_calls: args = {} if hasattr(tc, 'arguments') and tc.arguments: try: args = json.loads(tc.arguments) if isinstance(tc.arguments, str) else tc.arguments except Exception as e: log.warning(f"Failed to parse tool call arguments: {e}"); args = {} tool_calls.append({ "id": tc.id if hasattr(tc, 'id') else tc.name, "name": tc.name if hasattr(tc, 'name') else "", "arguments": args }) # Preserve raw tool_calls for building assistant message in next iteration tool_calls_raw = choice.message.tool_calls # Extract text from content if hasattr(choice.message, 'content') and choice.message.content: contents = choice.message.content if isinstance(contents, str): text = contents elif isinstance(contents, list): text_parts = [] for c in contents: if isinstance(c, str): text_parts.append(c) elif hasattr(c, 'text') and c.text: text_parts.append(c.text) elif isinstance(c, dict) and c.get('text'): text_parts.append(c['text']) text = "\n".join(text_parts) else: text = str(contents) # Fallback: check reasoning_content (some models like GPT-5.2 use this) if not text and hasattr(resp, 'choices') and resp.choices: choice = resp.choices[0] if hasattr(choice, 'message') and choice.message: msg_obj = choice.message if hasattr(msg_obj, 'reasoning_content') and msg_obj.reasoning_content: rc = msg_obj.reasoning_content if isinstance(rc, str): text = rc elif isinstance(rc, list): text_parts = [] for c in rc: if isinstance(c, str): text_parts.append(c) elif hasattr(c, 'text') and c.text: text_parts.append(c.text) elif isinstance(c, dict) and c.get('text'): text_parts.append(c['text']) text = "\n".join(text_parts) else: text = str(rc) if text: log.info(f"GenAI: extracted text from reasoning_content ({len(text)} chars)") # Last resort: try choice.text if not text: if hasattr(choice, 'text') and choice.text: text = choice.text if not text and not tool_calls: raw_content = getattr(getattr(resp.choices[0], 'message', None), 'content', None) if hasattr(resp, 'choices') and resp.choices else None # Also dump message attrs for debugging msg_attrs = [] if hasattr(resp, 'choices') and resp.choices and hasattr(resp.choices[0], 'message') and resp.choices[0].message: msg_attrs = [a for a in dir(resp.choices[0].message) if not a.startswith('_')] reasoning = getattr(resp.choices[0].message, 'reasoning_content', None) if hasattr(resp, 'choices') and resp.choices and hasattr(resp.choices[0], 'message') else None refusal = getattr(resp.choices[0].message, 'refusal', None) if hasattr(resp, 'choices') and resp.choices and hasattr(resp.choices[0], 'message') else None finish_reason = getattr(resp.choices[0], 'finish_reason', None) if hasattr(resp, 'choices') and resp.choices else None choice_attrs = [a for a in dir(resp.choices[0]) if not a.startswith('_')] if hasattr(resp, 'choices') and resp.choices else [] log.warning(f"GenAI returned empty response. model={gc.get('model_id')}, finish_reason={finish_reason}, content_repr: {repr(raw_content)[:500]}, reasoning: {repr(reasoning)[:500] if reasoning else None}, refusal: {repr(refusal)[:200] if refusal else None}, choice_attrs: {choice_attrs}") return (text or "", tool_calls, tool_calls_raw) # ── RAG Helpers ─────────────────────────────────────────────────────────────── RAG_CONTEXT_TEMPLATE = """--- CONTEXTO RECUPERADO (Vector Search) --- {context} --- FIM DO CONTEXTO --- Pergunta do usuário: {question}""" RAG_DEFAULT_SYSTEM_PROMPT = """Você é um assistente RAG especializado em Oracle Cloud Infrastructure (OCI). ### Escopo e restrições - Responda **SOMENTE** perguntas relacionadas a Oracle Cloud (OCI), CIS Benchmark, CIS Report e itens de inventário OCI (IAM/AssetManagement/Networking/StorageBlock/FileStorageService/Objectstore/Compute/LoggingandMonitoring). - Se a pergunta não for desse escopo, recuse educadamente e peça uma pergunta dentro do tema. ### Bases vetorizadas disponíveis (contexto recuperado automaticamente) **Tabelas de findings (dados do scan da tenancy):** - **summaryreportcsvvector**: resumo geral do CIS report (compliance scores por seção) - **identityandaccess**: findings de IAM (users, policies, MFA, API keys) - **networking**: findings de rede (security lists, NSGs, VCNs) - **computeinstances**: findings de compute (instances, metadata, boot) - **loggingandmonitoring**: findings de logging (alarms, events, notifications) - **objectstorage**: findings de Object Storage (buckets, visibility, encryption) - **storageblockvolume**: findings de Block Volume (encryption, CMK) - **filestorageservice**: findings de File Storage (encryption, CMK) - **assetmanagement**: findings de Asset Management (compartments, tagging) **Tabelas de referência (conhecimento genérico):** - **cisrecom**: recomendações oficiais do CIS Benchmark — contém passos detalhados de remediação, auditoria e rationale para cada controle CIS - **engineerknowledgebase**: base de conhecimento complementar (blogs, documentações, PDFs) ### Quando usar RAG (dados armazenados) vs MCP Tools (tempo real) **Use RAG (contexto recuperado abaixo) quando o usuário:** - Perguntar sobre **relatórios já gerados**, resultados de scans anteriores, histórico - Quiser saber **como corrigir** um problema (remediação do cisrecom) - Pedir **resumo de compliance**, scores, comparações entre datas - Perguntar sobre **documentação, boas práticas, referências** (engineerknowledgebase) - Palavras-chave: "no último scan", "no report", "segundo o CIS", "como corrigir", "remediação", "recomendação" **Use MCP Tools (scan em tempo real) quando o usuário:** - Pedir para **verificar agora**, **validar agora**, **checar o estado atual** - Quiser dados **atualizados/em tempo real** da tenancy - Palavras-chave: "verifica agora", "valida agora", "estado atual", "scan", "checa", "como está hoje" - Se tiver dúvida se os dados do RAG estão desatualizados **Se não tiver certeza:** pergunte ao usuário se quer consultar os dados armazenados (mais rápido) ou verificar em tempo real (scan ao vivo). ### Hierarquia de fontes RAG (IMPORTANTE) 1. Para **identificar problemas e status**: use as tabelas de findings (identityandaccess, networking, etc.) 2. Para **como corrigir (remediação)**: use SEMPRE a tabela **cisrecom** como fonte principal — ela contém os passos oficiais do CIS 3. Para **informações complementares**: use **engineerknowledgebase** como plus, se tiver algo relevante a acrescentar 4. Para **resumo de compliance**: use **summaryreportcsvvector** - Se houver conflito entre fontes, priorize: findings (dados reais) > cisrecom (recomendações oficiais) > engineerknowledgebase (complementar) ### Tratamento temporal (datas de extração) - Cada documento possui um campo **date** no cabeçalho indicando quando os dados foram coletados. - **Por padrão, priorize SEMPRE os dados mais recentes** (data mais nova). - Se o usuário pedir comparação ou evolução, apresente os dados de ambas as datas lado a lado, indicando claramente qual é o mais recente e qual é o anterior. - Informe a data de extração nas respostas para que o usuário saiba de quando são os dados. ### Regras de fidelidade - Responda **somente** com informações suportadas por evidências do contexto recuperado. - Se não houver evidência suficiente: **"Não encontrei nas minhas bases"** e peça dados para refinar (Recommendation #, seção, recurso, OCID, etc.). - **Não invente** páginas, comandos, políticas, valores ou números. - Use somente **1–3 linhas** de evidência por item para manter respostas compactas. - **Sempre informe a data de extração** dos dados apresentados. ### Formato de resposta - **Tenancy:** - **Data de extração:** - **Recomendação — ** - **Seção/Capítulo:** … - **Nível/Tipo:** Manual/Automated (se disponível) - **O que isso exige (CIS):** breve explicação - **Recursos afetados:** listar recursos non-compliant com nome/OCID (se disponíveis no contexto) - **Como corrigir (remediação):** passos COMPLETOS e DETALHADOS - Passos via Console OCI (se estiverem nas evidências) - Passos via OCI CLI com comandos completos em code blocks (```bash) - Passos via API (se disponíveis) - Observações/alertas importantes - **Como auditar:** como verificar se a correção foi aplicada - **Fontes:** tabela e data de extração de cada informação usada ### Regras de formatação - Comandos CLI e policies devem ser apresentados em **code blocks** (```bash ou ```hcl) - Listar TODOS os passos de remediação disponíveis no contexto — nunca resumir ou omitir passos - Se a remediação tem passos via Console E via CLI, apresentar AMBOS - Usar tabelas markdown para listar recursos afetados quando houver múltiplos - Ser completo na remediação mas conciso nas explicações""" CONSULT_SYSTEM_PROMPT = """Você é um consultor de dados vetorizados. Sua função é responder perguntas com base nos documentos recuperados do banco vetorial. ### Regras - Responda **somente** com informações presentes nos documentos fornecidos. - Se não encontrar informação relevante, diga claramente. - Seja direto e objetivo. Apresente os dados de forma organizada. - Use markdown para formatação (negrito, listas, tabelas quando apropriado). - Cite de qual tabela/fonte veio cada informação. - Não invente dados que não estejam nos documentos.""" TF_DEFAULT_SYSTEM_PROMPT = """Você gera somente Terraform HCL para Oracle Cloud Infrastructure (OCI) usando o provider oracle/oci. Se pedirem AWS, Azure, GCP ou outro provider, recuse educadamente. ### Regras - Gere código production-ready, com variáveis, defaults sensatos, tags e outputs úteis. Evite ficar solicitando ao usuário para ficar interagindo a todo momento, entregue o que ele precisa e faça o mínimo de interações para entender o que o usuário precisa. Explique de forma clara o que foi gerado e o que foi arrumado para que fosse possível gerar o código do terraform. - Use sempre a sintaxe mais recente do provider OCI. - Não inclua `terraform { required_providers }` nem `provider "oci"` principal. - Em multi-região, use provider aliases para regiões adicionais e `provider = oci.alias` nos recursos. - Para novas gerações, gere sempre múltiplos arquivos: no mínimo `variables.tf`, 1 arquivo de recursos e `outputs.tf`. - Cada bloco deve começar com `// filename: nome.tf`. - Nunca declare o mesmo recurso (tipo + nome) em mais de um arquivo. - Use `var.compartment_id` nos recursos e declare sempre `variable "region"` em `variables.tf`. - Se houver "RECURSOS OCI EXISTENTES NO COMPARTMENT", reutilize recursos com data sources sempre que possível. - Se houver "ARQUIVOS TERRAFORM ATUAIS NO WORKSPACE", gere **SOMENTE os arquivos que precisam de correção**. Arquivos sem erro NÃO devem ser regenerados. Mantenha os mesmos nomes de arquivo. Se o erro está em `networking.tf`, gere APENAS `// filename: networking.tf` com o conteúdo corrigido. NUNCA regenere `variables.tf`, `outputs.tf` ou outros arquivos que não têm erro, a menos que o erro exija alteração neles (ex: variável faltando). - Sempre responda com blocos `hcl`, depois **Resource Plan** (`+` criar, `~` reutilizar), uma breve explicação e uma seção `✅ Validação`. - Valide: referências cruzadas, CIDRs, route tables, security lists, dependências, segurança e variáveis. - Use apenas resource types válidos do OCI. Exemplos corretos: - `oci_network_firewall_network_firewall` - `oci_network_firewall_network_firewall_policy` - `oci_core_drg_attachment` - `oci_core_drg_attachment_management` - `oci_core_drg_route_table` - `oci_core_drg_route_distribution` - `oci_core_drg_route_distribution_statement` - `oci_core_drg_route_table_route_rule` - `oci_core_remote_peering_connection` - Nunca invente recursos inexistentes. - Use sempre HCL multi-line, nunca blocos inline com múltiplos argumentos. ### Correção de erros (IMPORTANTE) - Quando o usuário envia um erro de plan/apply com arquivos existentes, gere SOMENTE o(s) arquivo(s) que corrigem o erro. - Exemplo: se o erro é em `service_gateways.tf`, retorne APENAS `// filename: service_gateways.tf` com o conteúdo corrigido. - Se a correção exige criar/alterar uma variável, inclua também `// filename: variables.tf` apenas com as variáveis novas/alteradas + todas existentes. - NUNCA regenere todos os arquivos do zero. Isso descarta o trabalho anterior e cria loops infinitos de correção. - Preserve os nomes de arquivo exatos. Não renomeie `drg_attachments_and_routes.tf` para `drg.tf`. ### Regras críticas - `oci_core_drg_route_distribution_statement` deve usar `match_criteria`. - Remote peering: APENAS UM LADO do par define `peer_id` e `peer_region_name`. O outro lado é criado SEM `peer_id`. Nunca gere dois RPCs apontando um para o outro (causa Cycle error). Exemplo: RPC_mad1 (sem peer_id) e RPC_mad3 (com peer_id = RPC_mad1.id, peer_region_name = var.region). - Associação de route table com subnet deve ser via `route_table_id` na subnet ou `oci_core_route_table_attachment`. - `oci_network_firewall_network_firewall` exige `network_firewall_policy_id`. - DRG attachments: use `oci_core_drg_attachment` para VCN attachments (tipo padrão). Use `oci_core_drg_attachment_management` SOMENTE para REMOTE_PEERING_CONNECTION (único tipo suportado para management). Nunca duplique resource type+name entre arquivos (ex: dois `oci_core_drg_attachment.vcn_app_x`). Sempre associe `drg_route_table_id` nos VCN attachments para garantir roteamento correto. ### Proibições absolutas — módulos e variáveis - NUNCA use `module` blocks. Este workspace é flat (um único diretório). Não existem subdiretórios `modules/`. Toda a infraestrutura deve ser definida diretamente com `resource` e `data` blocks. - NUNCA declare a mesma `variable` em mais de um arquivo. Todas as variáveis devem estar em `variables.tf` e SOMENTE lá. Os outros arquivos (.tf) apenas referenciam `var.nome`. - Antes de gerar, verifique se uma variável já foi declarada. Se já existe em `variables.tf`, NÃO redeclare. """ # ── Terraform OCI Resource Reference (generated at build time, updatable at runtime) ── _TF_RESOURCE_REF_PATH = "/data/oci_tf_resource_reference.txt" _tf_resource_ref_cache: dict = {"content": None, "mtime": 0} def _populate_tf_valid_types(): """Parse the OCI TF resource reference file and populate tf_valid_types table in SQLite.""" p = Path(_TF_RESOURCE_REF_PATH) if not p.exists(): return 0 content = p.read_text() import re as _re entries = [] current_category = "" for line in content.split('\n'): if line.startswith('## '): current_category = line[3:].strip() continue # Resource lines: - **oci_xxx** required: ... blocks: ... m = _re.match(r'^- \*\*(\w+)\*\*\s*(.*)', line) if m: name = m.group(1) rest = m.group(2).strip() required = "" blocks = "" # Split by "blocks:" to separate required from blocks blk_parts = rest.split('blocks:') if blk_parts[0].strip().startswith('required:'): required = blk_parts[0].replace('required:', '').strip() if len(blk_parts) > 1: blocks = blk_parts[1].strip() entries.append((name, 'resource', current_category, required, blocks)) continue # Data source lines (in "All Resource Types" or "All Data Sources" section): - oci_xxx m2 = _re.match(r'^- (\w+)\s*$', line) if m2: name = m2.group(1) if current_category.startswith('All Data'): entries.append((name, 'data', '', '', '')) elif current_category.startswith('All Resource'): entries.append((name, 'resource', '', '', '')) if not entries: return 0 with db() as c: c.execute("DELETE FROM tf_valid_types") # Insert detailed entries first (with required_args), then bulk lists with IGNORE to not overwrite detailed = [e for e in entries if e[3] or e[4]] # has required_args or blocks bulk = [e for e in entries if not e[3] and not e[4]] if detailed: c.executemany( "INSERT OR REPLACE INTO tf_valid_types (name, kind, category, required_args, blocks) VALUES (?,?,?,?,?)", detailed) if bulk: c.executemany( "INSERT OR IGNORE INTO tf_valid_types (name, kind, category, required_args, blocks) VALUES (?,?,?,?,?)", bulk) log.info(f"Populated tf_valid_types: {len(entries)} entries") return len(entries) def _validate_tf_resource_types(tf_code: str) -> list: """Validate resource/data types in generated TF code against the tf_valid_types table. Returns list of dicts with invalid types and suggestions.""" import re as _re from difflib import get_close_matches # Extract only resource type declarations (data sources are read-only and may not be in the schema) used_types = set() for m in _re.finditer(r'resource\s+"(\w+)"', tf_code): used_types.add(m.group(1)) if not used_types: return [] # Load valid types from DB with db() as c: rows = c.execute("SELECT name, kind, required_args FROM tf_valid_types").fetchall() valid_names = {r["name"] for r in rows} valid_required = {r["name"]: r["required_args"] for r in rows} if not valid_names: return [] # No reference loaded # Check each used type errors = [] for t in sorted(used_types): if t not in valid_names: # Find closest matches suggestions = get_close_matches(t, valid_names, n=3, cutoff=0.6) errors.append({ "type": t, "suggestions": suggestions, "message": f"Resource type '{t}' NÃO EXISTE no provider oracle/oci." }) return errors def _load_tf_resource_reference() -> str: """Load the OCI Terraform resource reference file, cached by mtime.""" try: p = Path(_TF_RESOURCE_REF_PATH) if not p.exists(): return "" mtime = p.stat().st_mtime if _tf_resource_ref_cache["content"] and _tf_resource_ref_cache["mtime"] == mtime: return _tf_resource_ref_cache["content"] content = p.read_text() _tf_resource_ref_cache["content"] = content _tf_resource_ref_cache["mtime"] = mtime log.info(f"Loaded TF resource reference: {len(content)} chars, {content.count(chr(10))} lines") return content except Exception as e: log.warning(f"Failed to load TF resource reference: {e}") return "" def _regenerate_tf_reference() -> dict: """Regenerate the OCI Terraform resource reference file.""" try: result = subprocess.run( ["python", "/app/gen_tf_reference.py"], capture_output=True, text=True, timeout=300 ) _tf_resource_ref_cache["content"] = None # invalidate cache if result.returncode == 0: ref = _load_tf_resource_reference() return {"ok": True, "output": result.stdout.strip(), "lines": ref.count('\n')} return {"ok": False, "error": result.stderr.strip()} except Exception as e: return {"ok": False, "error": str(e)} # ── Terraform Resource Docs (on-demand from GitHub, cached in SQLite) ── _TF_DOC_BASE_URL = "https://raw.githubusercontent.com/oracle/terraform-provider-oci/master/website/docs/r/" def _fetch_tf_resource_doc(slug: str) -> dict | None: """Fetch a single resource doc from GitHub, parse Example Usage + Argument Reference, cache in SQLite.""" import re as _re # Check cache first with db() as c: row = c.execute("SELECT example_usage, argument_ref FROM tf_resource_docs WHERE slug=?", (slug,)).fetchone() if row and (row["example_usage"] or row["argument_ref"]): return {"slug": slug, "example": row["example_usage"], "args": row["argument_ref"]} # Fetch from GitHub try: import urllib.request url = f"{_TF_DOC_BASE_URL}{slug}.html.markdown" req = urllib.request.Request(url, headers={"User-Agent": "oci-cis-agent/2.1"}) with urllib.request.urlopen(req, timeout=15) as resp: if resp.status != 200: return None md = resp.read().decode("utf-8") # Extract Example Usage section example = "" m = _re.search(r'## Example Usage\s*\n(.*?)(?=\n## )', md, _re.DOTALL) if m: example = m.group(1).strip() # Extract Argument Reference section args = "" m2 = _re.search(r'## Argument Reference\s*\n(.*?)(?=\n## )', md, _re.DOTALL) if m2: args = m2.group(1).strip() # Cache in SQLite subcategory = "" m3 = _re.search(r'subcategory:\s*"([^"]+)"', md) if m3: subcategory = m3.group(1) with db() as c: c.execute( "INSERT OR REPLACE INTO tf_resource_docs (slug, subcategory, example_usage, argument_ref) VALUES (?,?,?,?)", (slug, subcategory, example, args)) log.info(f"Fetched TF doc: {slug} (example={len(example)} chars, args={len(args)} chars)") return {"slug": slug, "example": example, "args": args} except Exception as e: log.warning(f"Failed to fetch TF doc for {slug}: {e}") return None def _get_tf_docs_for_resources(resource_types: list[str]) -> str: """Given a list of oci_xxx resource types, fetch their docs and build a context string.""" docs_parts = [] for rtype in resource_types[:10]: # limit to 10 to keep context manageable # Convert oci_core_vcn -> core_vcn slug = rtype.replace("oci_", "", 1) if rtype.startswith("oci_") else rtype doc = _fetch_tf_resource_doc(slug) if doc and (doc["example"] or doc["args"]): part = f"#### {rtype}\n" if doc["example"]: part += f"**Example Usage:**\n{doc['example']}\n\n" if doc["args"]: part += f"**Arguments:**\n{doc['args']}\n" docs_parts.append(part) if not docs_parts: return "" return "### Documentação Oficial dos Recursos (registry.terraform.io)\n" + \ "Use exatamente os argumentos e estrutura mostrados nos exemplos abaixo.\n\n" + \ "\n---\n".join(docs_parts) def _detect_tf_resource_types(text: str) -> list[str]: """Detect OCI resource types mentioned or implied in a user message + conversation.""" import re as _re # Direct mentions of oci_ resource types explicit = set(_re.findall(r'\boci_\w+', text)) # Keyword-to-resource mapping for common infra requests keyword_map = { 'vcn': ['oci_core_vcn', 'oci_core_subnet', 'oci_core_internet_gateway', 'oci_core_nat_gateway', 'oci_core_route_table', 'oci_core_security_list'], 'subnet': ['oci_core_subnet'], 'instance': ['oci_core_instance', 'oci_core_instance_configuration'], 'compute': ['oci_core_instance'], 'load.?balancer': ['oci_load_balancer_load_balancer', 'oci_load_balancer_backend_set', 'oci_load_balancer_listener'], 'nlb': ['oci_network_load_balancer_network_load_balancer', 'oci_network_load_balancer_backend_set', 'oci_network_load_balancer_listener'], 'firewall': ['oci_network_firewall_network_firewall', 'oci_network_firewall_network_firewall_policy'], 'drg': ['oci_core_drg', 'oci_core_drg_attachment', 'oci_core_drg_route_table'], 'peering|rpc': ['oci_core_remote_peering_connection'], 'bucket|object.?storage': ['oci_objectstorage_bucket', 'oci_objectstorage_namespace_metadata'], 'autonomous|adb': ['oci_database_autonomous_database'], 'db.?system': ['oci_database_db_system'], 'mysql': ['oci_mysql_mysql_db_system'], 'oke|kubernetes|cluster': ['oci_containerengine_cluster', 'oci_containerengine_node_pool'], 'dns': ['oci_dns_zone', 'oci_dns_record'], 'vault|kms': ['oci_kms_vault', 'oci_kms_key'], 'waf': ['oci_waf_web_app_firewall', 'oci_waf_web_app_firewall_policy'], 'api.?gateway': ['oci_apigateway_gateway', 'oci_apigateway_deployment'], 'function|serverless': ['oci_functions_application', 'oci_functions_function'], 'nsg': ['oci_core_network_security_group', 'oci_core_network_security_group_security_rule'], 'bastion': ['oci_bastion_bastion', 'oci_bastion_session'], 'file.?storage': ['oci_file_storage_file_system', 'oci_file_storage_mount_target', 'oci_file_storage_export'], 'block.?volume': ['oci_core_volume', 'oci_core_volume_attachment'], 'policy|iam': ['oci_identity_policy', 'oci_identity_compartment', 'oci_identity_group', 'oci_identity_dynamic_group'], } text_lower = text.lower() for pattern, resources in keyword_map.items(): if _re.search(pattern, text_lower): explicit.update(resources) return sorted(explicit) def _get_adb_connection(cfg: dict): """Create an oracledb connection from an adb_vector_configs row.""" import oracledb params = {"user": cfg["username"], "password": _dec(cfg["password_enc"]), "dsn": cfg["dsn"]} if cfg["use_mtls"] and cfg.get("wallet_dir"): params["config_dir"] = cfg["wallet_dir"] params["wallet_location"] = cfg["wallet_dir"] params["wallet_password"] = _dec(cfg["wallet_password_enc"]) if cfg.get("wallet_password_enc") else "" params["tcp_connect_timeout"] = 15 # 15s connection timeout return oracledb.connect(**params) def _resolve_embed_config(oci_config_id: str = None, genai_cfg: dict = None, user_id: str = None) -> dict: """Resolve embedding config from genai_cfg or oci_config. Scoped to user_id when provided. Returns dict with oci_config_id, endpoint, genai_region, compartment_id.""" if genai_cfg: return genai_cfg if oci_config_id: with db() as c: # Try linked genai config first (scoped to user if provided) if user_id: gc = c.execute("SELECT * FROM genai_configs WHERE oci_config_id=? AND user_id=? ORDER BY is_default DESC, created_at DESC", (oci_config_id, user_id)).fetchone() else: gc = c.execute("SELECT * FROM genai_configs WHERE oci_config_id=? ORDER BY is_default DESC, created_at DESC", (oci_config_id,)).fetchone() if gc: return dict(gc) # Fallback: build from oci_config directly oc = c.execute("SELECT * FROM oci_configs WHERE id=?", (oci_config_id,)).fetchone() if oc: return { "oci_config_id": oc["id"], "genai_region": oc["region"], "endpoint": f"https://inference.generativeai.{oc['region']}.oci.oraclecloud.com", "compartment_id": _safe_dec(dict(oc).get("compartment_id") or oc["tenancy_ocid"]), } # Last resort: user's own genai/oci config (never cross-user) with db() as c: if user_id: gc = c.execute("SELECT * FROM genai_configs WHERE user_id=? ORDER BY is_default DESC, created_at DESC", (user_id,)).fetchone() else: gc = c.execute("SELECT * FROM genai_configs ORDER BY is_default DESC, created_at DESC").fetchone() if gc: return dict(gc) if user_id: oc = c.execute("SELECT * FROM oci_configs WHERE user_id=? ORDER BY created_at DESC", (user_id,)).fetchone() else: oc = c.execute("SELECT * FROM oci_configs ORDER BY created_at DESC").fetchone() if oc: return { "oci_config_id": oc["id"], "genai_region": oc["region"], "endpoint": f"https://inference.generativeai.{oc['region']}.oci.oraclecloud.com", "compartment_id": _safe_dec(dict(oc).get("compartment_id") or oc["tenancy_ocid"]), } raise HTTPException(400, "Nenhuma credencial OCI/GenAI configurada para gerar embeddings.") def _embed_text(text: str, genai_cfg: dict, embedding_model_id: str, max_chars: int = 8000) -> list: """Generate embedding using OCI GenAI embed endpoint.""" import oci # Truncate text to fit model token limit (~8192 tokens) if len(text) > max_chars: text = text[:max_chars] config_path = str(OCI_DIR / genai_cfg["oci_config_id"] / "config") config = oci.config.from_file(config_path, "DEFAULT") endpoint = genai_cfg.get("endpoint") or f"https://inference.generativeai.{genai_cfg.get('genai_region','us-ashburn-1')}.oci.oraclecloud.com" client = oci.generative_ai_inference.GenerativeAiInferenceClient( config=config, service_endpoint=endpoint, retry_strategy=oci.retry.NoneRetryStrategy(), timeout=(10, 120) ) embed_detail = oci.generative_ai_inference.models.EmbedTextDetails() embed_detail.inputs = [text] emb_info = EMBEDDING_MODELS.get(embedding_model_id, {}) region = genai_cfg.get("genai_region", "") emb_ref = emb_info.get("ocids", {}).get(region) or embedding_model_id embed_detail.serving_mode = oci.generative_ai_inference.models.OnDemandServingMode(model_id=emb_ref) embed_detail.compartment_id = genai_cfg.get("compartment_id", "") embed_detail.truncate = "END" embed_detail.input_type = "SEARCH_QUERY" response = client.embed_text(embed_detail) return response.data.embeddings[0] _DIM_TO_MODEL = {1536: "openai.text-embedding-3-small", 3072: "openai.text-embedding-3-large"} def _get_table_embedding_dim(cfg: dict, table_name: str) -> int: """Detect the embedding dimension of a table by sampling one row, or from column definition if empty.""" conn = _get_adb_connection(cfg) try: cur = conn.cursor() # Try from data first cur.execute(f'SELECT VECTOR_DIMENSION_COUNT(EMBEDDING) FROM "{table_name}" FETCH FIRST 1 ROWS ONLY') row = cur.fetchone() if row and row[0]: cur.close() return int(row[0]) # Table is empty — check column definition from DDL try: cur.execute(f"""SELECT DBMS_METADATA.GET_DDL('TABLE', :1) FROM DUAL""", [table_name]) ddl_row = cur.fetchone() cur.close() if ddl_row and ddl_row[0]: import re m = re.search(r'VECTOR\((\d+)', str(ddl_row[0])) if m: return int(m.group(1)) except Exception: pass return 0 except Exception as e: log.warning(f"Failed to get embedding dimension for table '{table_name}': {e}") return 0 finally: conn.close() _GLOBAL_TABLES = {"cisrecom", "engineerknowledgebase"} # Tables without tenancy filter (generic knowledge) def _vector_search(cfg: dict, query_embedding: list, top_k: int = 5, table_name: str = None, tenancy: str = None, text_filter: str = None, user_id: str = None) -> list: """Search ADB vector store using cosine similarity. Returns top-K documents. If tenancy is provided and table is not global, filters by tenancy. If user_id is provided and table is not global, filters by user_id (legacy docs without user_id are included). If text_filter is provided, also filters TEXT content with LIKE.""" import array table_name = table_name or cfg.get("table_name", "") conn = _get_adb_connection(cfg) try: conn.call_timeout = 30000 # 30s query timeout (milliseconds) cur = conn.cursor() vec = array.array('f', query_embedding) limit = int(top_k) is_non_global = table_name.lower() not in _GLOBAL_TABLES use_tenancy_filter = tenancy and is_non_global use_user_filter = user_id and is_non_global # Build WHERE clauses conditions = [] params = [vec] param_idx = 2 if use_tenancy_filter: conditions.append(f"JSON_VALUE(METADATA, '$.tenancy') = :{param_idx}") params.append(tenancy) param_idx += 1 if use_user_filter: conditions.append(f"(JSON_VALUE(METADATA, '$.user_id') = :{param_idx} OR JSON_VALUE(METADATA, '$.user_id') IS NULL)") params.append(user_id) param_idx += 1 if text_filter: conditions.append(f"TEXT LIKE :{param_idx}") params.append(f"%{text_filter}%") param_idx += 1 where = f"WHERE {' AND '.join(conditions)}" if conditions else "" cur.execute(f""" SELECT ID, TEXT, METADATA, VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance FROM "{table_name}" {where} ORDER BY distance ASC FETCH FIRST {limit} ROWS ONLY """, params) results = [] for row in cur: content = row[1] if hasattr(content, 'read'): content = content.read() results.append({ "id": row[0], "content": content or "", "metadata": row[2], "distance": float(row[3]) }) cur.close() return results finally: conn.close() # ── Table relevance mapping for smart skip ── _TABLE_KEYWORDS = { "identityandaccess": {"iam", "user", "users", "policy", "policies", "mfa", "api key", "group", "password", "credential", "authentication", "identity", "access"}, "networking": {"network", "vcn", "subnet", "security list", "nsg", "firewall", "route", "gateway", "nat", "internet", "port", "ingress", "egress", "cidr", "ip"}, "computeinstances": {"compute", "instance", "vm", "boot", "metadata", "secure boot", "encryption"}, "loggingandmonitoring": {"log", "logging", "monitor", "alarm", "event", "notification", "audit", "cloud guard", "flow"}, "objectstorage": {"bucket", "object storage", "storage", "visibility", "public"}, "storageblockvolume": {"block volume", "volume", "disk", "cmk", "encryption"}, "filestorageservice": {"file storage", "file system", "mount", "nfs"}, "assetmanagement": {"compartment", "tag", "asset", "resource"}, "summaryreportcsvvector": {"summary", "score", "compliance", "report", "overview", "total"}, "cisrecom": {"remediation", "fix", "correct", "resolve", "recommendation", "how to", "como corrigir", "remediação"}, "engineerknowledgebase": {"documentation", "guide", "tutorial", "blog", "best practice", "knowledge"}, } def _relevant_tables(query: str, tables: list[str]) -> list[str]: """Filter tables to only those relevant to the query. Always includes cisrecom and engineerknowledgebase.""" q = query.lower() # If query mentions a CIS number, include all tables (the text filter handles precision) import re as _re if _re.search(r'cis\s*\d+\.\d+', q): return tables relevant = [] for tbl in tables: tbl_lower = tbl.lower() # Always include global knowledge tables if tbl_lower in _GLOBAL_TABLES or tbl_lower == "summaryreportcsvvector": relevant.append(tbl) continue keywords = _TABLE_KEYWORDS.get(tbl_lower, set()) if any(kw in q for kw in keywords): relevant.append(tbl) # If no specific table matched, search all (generic question) return relevant if len(relevant) > 2 else tables def _vector_search_multi(cfg: dict, query_embedding: list, tables: list[str], top_k_per_table: int = 3, tenancy: str = None, text_filter: str = None, user_id: str = None) -> list: """Search multiple tables using a SINGLE ADB connection. Returns all documents with source.""" import array conn = _get_adb_connection(cfg) all_results = [] try: conn.call_timeout = 30000 vec = array.array('f', query_embedding) for table_name in tables: try: cur = conn.cursor() limit = int(top_k_per_table) is_non_global = table_name.lower() not in _GLOBAL_TABLES use_tenancy = tenancy and is_non_global use_user_filter = user_id and is_non_global conditions = [] params = [vec] param_idx = 2 if use_tenancy: conditions.append(f"JSON_VALUE(METADATA, '$.tenancy') = :{param_idx}") params.append(tenancy) param_idx += 1 if use_user_filter: conditions.append(f"(JSON_VALUE(METADATA, '$.user_id') = :{param_idx} OR JSON_VALUE(METADATA, '$.user_id') IS NULL)") params.append(user_id) param_idx += 1 tbl_filter = text_filter if (text_filter and is_non_global) else None if tbl_filter: conditions.append(f"TEXT LIKE :{param_idx}") params.append(f"%{tbl_filter}%") param_idx += 1 where = f"WHERE {' AND '.join(conditions)}" if conditions else "" cur.execute(f""" SELECT ID, TEXT, METADATA, VECTOR_DISTANCE(EMBEDDING, :1, COSINE) AS distance FROM "{table_name}" {where} ORDER BY distance ASC FETCH FIRST {limit} ROWS ONLY """, params) for row in cur: content = row[1] if hasattr(content, 'read'): content = content.read() all_results.append({ "id": row[0], "content": content or "", "metadata": row[2], "distance": float(row[3]), "source": table_name, }) cur.close() except Exception as e: log.warning(f"Multi-search failed for {table_name}: {str(e)[:120]}") finally: conn.close() return all_results def _enrich_doc_content(doc: dict) -> str: """Extract the best text content from a document, checking metadata.text as fallback.""" content = doc.get("content", "") meta = doc.get("metadata", "") if isinstance(meta, str) and meta: try: meta = json.loads(meta) except Exception as e: log.warning(f"Failed to parse document metadata JSON: {e}") meta = {} if isinstance(meta, dict): # If TEXT column is short/empty, use metadata.text instead meta_text = meta.get("text", "") if meta_text and len(meta_text) > len(content): header_parts = [] if meta.get("recommendationNumber"): header_parts.append(f"Recommendation: {meta['recommendationNumber']}") if meta.get("chapter"): header_parts.append(f"Chapter: {meta['chapter']}") if meta.get("header"): header_parts.append(meta["header"]) header = " | ".join(header_parts) content = (header + "\n" + meta_text) if header else meta_text # Also enrich with section/tenancy info if available elif meta.get("section") or meta.get("tenancy"): extra = [] if meta.get("tenancy"): extra.append(f"Tenancy: {meta['tenancy']}") if meta.get("section"): extra.append(f"Section: {meta['section']}") if extra: content = " | ".join(extra) + "\n" + content return content def _build_rag_context(documents: list, max_total_chars: int = 12000) -> str: """Format retrieved documents into a context string for the LLM prompt. Includes extract_date from metadata for temporal awareness. Limits total context to max_total_chars to prevent token overflow.""" if not documents: return "" parts = [] total = 0 max_per_doc = max_total_chars // min(len(documents), 8) for i, doc in enumerate(documents, 1): source = doc.get("source", "unknown") dist = doc.get("distance", 0) # Extract date from metadata for temporal context extract_date = "" meta_raw = doc.get("metadata", "") if isinstance(meta_raw, str) and meta_raw: try: meta_obj = json.loads(meta_raw) extract_date = meta_obj.get("extract_date", "") or meta_obj.get("report_date", "") except Exception: pass content = _enrich_doc_content(doc) if len(content) > max_per_doc: content = content[:max_per_doc] + "..." date_tag = f" | date: {extract_date}" if extract_date else "" part = f"[Doc {i} | {source}{date_tag} | relevance: {1 - dist:.2f}]\n{content}" if total + len(part) > max_total_chars: break parts.append(part) total += len(part) return "\n\n---\n\n".join(parts) def _get_active_adb_configs(user_id: str) -> list[dict]: """Get all active ADB vector configs accessible to this user (own + global). No cross-user fallback.""" with db() as c: rows = c.execute( "SELECT * FROM adb_vector_configs WHERE is_active=1 AND (user_id=? OR is_global=1) ORDER BY is_global DESC, created_at DESC", (user_id,) ).fetchall() return [dict(r) for r in rows] def _get_tables_for_config(adb_config_id: str, active_only: bool = True) -> list[dict]: """Get all registered vector tables for an ADB config.""" with db() as c: sql = "SELECT * FROM adb_vector_tables WHERE adb_config_id=?" if active_only: sql += " AND is_active=1" sql += " ORDER BY created_at ASC" rows = c.execute(sql, (adb_config_id,)).fetchall() return [dict(r) for r in rows] # ── MCP Servers ───────────────────────────────────────────────────────────────