1135e9d6a9
Fase 0 complete: extract all business logic, auth, database, and 176 endpoints from monolithic app.py into dedicated modules. Structure: - app.py: FastAPI hub (CORS, routers, startup/shutdown) - models.py: 13 Pydantic request models - utils.py: shared utilities (embed status, upload validation, process registries) - config.py: constants, env vars, model catalogs - auth/: crypto, jwt_auth, oidc, rate_limit - database/: db(), init_db(), schema DDL - services/: genai, compliance, chat, terraform, embeddings, cis_reports, mcp - routes/: 13 APIRouter modules (auth, users, oci_config, oci_explorer, genai, mcp, adb, embeddings, reports, chat, terraform, settings, cis_engine) Also: README updated with OCIR auth instructions for manual docker run. 86 tests passing.
377 lines
18 KiB
Python
377 lines
18 KiB
Python
"""Database layer — SQLite context manager and schema initialization."""
|
|
import sqlite3
|
|
import uuid
|
|
import secrets
|
|
from contextlib import contextmanager
|
|
|
|
from config import DB_PATH, TERRAFORM_DIR, log
|
|
from auth.crypto import _hash_pw
|
|
|
|
# ── Database ──────────────────────────────────────────────────────────────────
|
|
@contextmanager
|
|
def db():
|
|
conn = sqlite3.connect(str(DB_PATH), timeout=30, check_same_thread=False)
|
|
conn.row_factory = sqlite3.Row
|
|
conn.execute("PRAGMA journal_mode=WAL")
|
|
conn.execute("PRAGMA busy_timeout=30000")
|
|
conn.execute("PRAGMA foreign_keys=ON")
|
|
try:
|
|
yield conn
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
def init_db():
|
|
with db() as c:
|
|
c.executescript("""
|
|
CREATE TABLE IF NOT EXISTS users (
|
|
id TEXT PRIMARY KEY,
|
|
first_name TEXT NOT NULL,
|
|
last_name TEXT NOT NULL,
|
|
username TEXT UNIQUE NOT NULL,
|
|
email TEXT,
|
|
password_hash TEXT NOT NULL, role TEXT NOT NULL DEFAULT 'viewer',
|
|
mfa_secret TEXT, mfa_enabled INTEGER DEFAULT 0,
|
|
timezone TEXT DEFAULT 'America/Sao_Paulo',
|
|
is_active INTEGER DEFAULT 1,
|
|
created_at TEXT DEFAULT (datetime('now')),
|
|
last_login TEXT
|
|
);
|
|
CREATE TABLE IF NOT EXISTS sessions (
|
|
id TEXT PRIMARY KEY, user_id TEXT NOT NULL,
|
|
created_at TEXT DEFAULT (datetime('now')),
|
|
expires_at TEXT NOT NULL, is_active INTEGER DEFAULT 1
|
|
);
|
|
CREATE TABLE IF NOT EXISTS oci_configs (
|
|
id TEXT PRIMARY KEY, user_id TEXT NOT NULL,
|
|
tenancy_name TEXT NOT NULL, tenancy_ocid TEXT NOT NULL,
|
|
user_ocid TEXT NOT NULL, fingerprint TEXT NOT NULL,
|
|
region TEXT NOT NULL, key_file_path TEXT NOT NULL,
|
|
compartment_id TEXT,
|
|
created_at TEXT DEFAULT (datetime('now'))
|
|
);
|
|
CREATE TABLE IF NOT EXISTS genai_configs (
|
|
id TEXT PRIMARY KEY, user_id TEXT NOT NULL,
|
|
name TEXT NOT NULL DEFAULT 'default',
|
|
oci_config_id TEXT NOT NULL,
|
|
model_id TEXT NOT NULL,
|
|
model_ocid TEXT,
|
|
compartment_id TEXT NOT NULL,
|
|
genai_region TEXT NOT NULL,
|
|
endpoint TEXT NOT NULL,
|
|
serving_type TEXT DEFAULT 'ON_DEMAND',
|
|
dedicated_endpoint_id TEXT,
|
|
temperature REAL DEFAULT 1,
|
|
max_tokens INTEGER DEFAULT 6000,
|
|
top_p REAL DEFAULT 0.95,
|
|
top_k INTEGER DEFAULT 1,
|
|
frequency_penalty REAL DEFAULT 0,
|
|
presence_penalty REAL DEFAULT 0,
|
|
reasoning_effort TEXT,
|
|
is_default INTEGER DEFAULT 0,
|
|
system_prompt TEXT DEFAULT '',
|
|
created_at TEXT DEFAULT (datetime('now')),
|
|
FOREIGN KEY (oci_config_id) REFERENCES oci_configs(id)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS reports (
|
|
id TEXT PRIMARY KEY, user_id TEXT NOT NULL,
|
|
tenancy_name TEXT NOT NULL, config_id TEXT,
|
|
level INTEGER DEFAULT 2,
|
|
obp_checks INTEGER DEFAULT 0,
|
|
raw_data INTEGER DEFAULT 0,
|
|
redact_output INTEGER DEFAULT 0,
|
|
status TEXT DEFAULT 'pending', progress TEXT DEFAULT '',
|
|
html_path TEXT, json_path TEXT,
|
|
created_at TEXT DEFAULT (datetime('now')),
|
|
completed_at TEXT, error_msg TEXT
|
|
);
|
|
CREATE TABLE IF NOT EXISTS report_files (
|
|
id TEXT PRIMARY KEY,
|
|
report_id TEXT NOT NULL,
|
|
file_name TEXT NOT NULL,
|
|
file_path TEXT NOT NULL,
|
|
file_type TEXT NOT NULL,
|
|
file_category TEXT NOT NULL,
|
|
file_size INTEGER DEFAULT 0
|
|
);
|
|
CREATE TABLE IF NOT EXISTS adb_vector_configs (
|
|
id TEXT PRIMARY KEY, user_id TEXT NOT NULL,
|
|
config_name TEXT NOT NULL,
|
|
dsn TEXT NOT NULL,
|
|
username TEXT NOT NULL,
|
|
password_enc TEXT NOT NULL,
|
|
wallet_dir TEXT,
|
|
wallet_password_enc TEXT,
|
|
table_name TEXT DEFAULT '',
|
|
use_mtls INTEGER DEFAULT 1,
|
|
is_active INTEGER DEFAULT 1,
|
|
is_global INTEGER DEFAULT 0,
|
|
created_at TEXT DEFAULT (datetime('now'))
|
|
);
|
|
CREATE TABLE IF NOT EXISTS adb_vector_tables (
|
|
id TEXT PRIMARY KEY,
|
|
adb_config_id TEXT NOT NULL,
|
|
table_name TEXT NOT NULL,
|
|
description TEXT DEFAULT '',
|
|
is_active INTEGER DEFAULT 1,
|
|
created_at TEXT DEFAULT (datetime('now')),
|
|
FOREIGN KEY (adb_config_id) REFERENCES adb_vector_configs(id) ON DELETE CASCADE,
|
|
UNIQUE(adb_config_id, table_name)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS mcp_servers (
|
|
id TEXT PRIMARY KEY, user_id TEXT NOT NULL,
|
|
name TEXT NOT NULL, description TEXT,
|
|
server_type TEXT NOT NULL DEFAULT 'stdio',
|
|
command TEXT, args TEXT, env_vars TEXT,
|
|
url TEXT, module_path TEXT,
|
|
tools TEXT,
|
|
linked_adb_id TEXT,
|
|
is_active INTEGER DEFAULT 1,
|
|
is_global INTEGER DEFAULT 0,
|
|
created_at TEXT DEFAULT (datetime('now'))
|
|
);
|
|
CREATE TABLE IF NOT EXISTS chat_messages (
|
|
id TEXT PRIMARY KEY, session_id TEXT NOT NULL,
|
|
user_id TEXT NOT NULL, role TEXT NOT NULL,
|
|
content TEXT NOT NULL,
|
|
model_id TEXT,
|
|
status TEXT DEFAULT 'done',
|
|
created_at TEXT DEFAULT (datetime('now'))
|
|
);
|
|
CREATE TABLE IF NOT EXISTS chat_summaries (
|
|
id TEXT PRIMARY KEY,
|
|
session_id TEXT NOT NULL,
|
|
user_id TEXT NOT NULL,
|
|
summary TEXT NOT NULL,
|
|
messages_compacted INTEGER NOT NULL,
|
|
up_to_message_id TEXT NOT NULL,
|
|
created_at TEXT DEFAULT (datetime('now'))
|
|
);
|
|
CREATE TABLE IF NOT EXISTS audit_log (
|
|
id TEXT PRIMARY KEY, user_id TEXT, username TEXT,
|
|
action TEXT NOT NULL, resource TEXT, details TEXT,
|
|
ip TEXT, created_at TEXT DEFAULT (datetime('now'))
|
|
);
|
|
CREATE TABLE IF NOT EXISTS terminal_history (
|
|
id TEXT PRIMARY KEY, user_id TEXT NOT NULL,
|
|
oci_config_id TEXT NOT NULL, command TEXT NOT NULL,
|
|
exit_code INTEGER, output TEXT, error TEXT,
|
|
created_at TEXT DEFAULT (datetime('now'))
|
|
);
|
|
CREATE TABLE IF NOT EXISTS config_logs (
|
|
id TEXT PRIMARY KEY,
|
|
config_type TEXT NOT NULL,
|
|
config_id TEXT NOT NULL,
|
|
config_name TEXT,
|
|
severity TEXT NOT NULL,
|
|
action TEXT NOT NULL,
|
|
message TEXT NOT NULL,
|
|
user_id TEXT,
|
|
username TEXT,
|
|
created_at TEXT DEFAULT (datetime('now'))
|
|
);
|
|
CREATE TABLE IF NOT EXISTS app_settings (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT NOT NULL,
|
|
updated_at TEXT DEFAULT (datetime('now'))
|
|
);
|
|
CREATE TABLE IF NOT EXISTS system_prompts (
|
|
id TEXT PRIMARY KEY,
|
|
name TEXT NOT NULL,
|
|
agent TEXT NOT NULL DEFAULT 'chat',
|
|
content TEXT NOT NULL,
|
|
is_active INTEGER DEFAULT 0,
|
|
created_at TEXT DEFAULT (datetime('now'))
|
|
);
|
|
CREATE TABLE IF NOT EXISTS chat_logs (
|
|
id TEXT PRIMARY KEY,
|
|
session_id TEXT,
|
|
message_id TEXT,
|
|
user_id TEXT,
|
|
severity TEXT NOT NULL,
|
|
source TEXT NOT NULL,
|
|
action TEXT NOT NULL,
|
|
message TEXT NOT NULL,
|
|
created_at TEXT DEFAULT (datetime('now'))
|
|
);
|
|
CREATE TABLE IF NOT EXISTS chat_sessions (
|
|
id TEXT PRIMARY KEY,
|
|
user_id TEXT NOT NULL,
|
|
agent_type TEXT NOT NULL DEFAULT 'chat',
|
|
title TEXT DEFAULT '',
|
|
created_at TEXT DEFAULT (datetime('now')),
|
|
updated_at TEXT DEFAULT (datetime('now'))
|
|
);
|
|
CREATE TABLE IF NOT EXISTS terraform_workspaces (
|
|
id TEXT PRIMARY KEY, user_id TEXT NOT NULL,
|
|
session_id TEXT NOT NULL,
|
|
oci_config_id TEXT NOT NULL,
|
|
compartment_id TEXT,
|
|
name TEXT DEFAULT 'workspace',
|
|
tf_code TEXT,
|
|
status TEXT DEFAULT 'draft',
|
|
plan_output TEXT, apply_output TEXT, destroy_output TEXT,
|
|
error TEXT,
|
|
created_at TEXT DEFAULT (datetime('now')),
|
|
updated_at TEXT DEFAULT (datetime('now'))
|
|
);
|
|
CREATE TABLE IF NOT EXISTS explorer_counts_cache (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
oci_config_id TEXT NOT NULL,
|
|
compartment_id TEXT NOT NULL,
|
|
resource_type TEXT NOT NULL,
|
|
count INTEGER NOT NULL DEFAULT 0,
|
|
regions TEXT DEFAULT '',
|
|
updated_at TEXT DEFAULT (datetime('now')),
|
|
UNIQUE(oci_config_id, compartment_id, resource_type, regions)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS tf_valid_types (
|
|
name TEXT PRIMARY KEY,
|
|
kind TEXT NOT NULL DEFAULT 'resource',
|
|
category TEXT DEFAULT '',
|
|
required_args TEXT DEFAULT '',
|
|
blocks TEXT DEFAULT ''
|
|
);
|
|
CREATE TABLE IF NOT EXISTS tf_resource_docs (
|
|
slug TEXT PRIMARY KEY,
|
|
subcategory TEXT DEFAULT '',
|
|
example_usage TEXT DEFAULT '',
|
|
argument_ref TEXT DEFAULT '',
|
|
fetched_at TEXT DEFAULT (datetime('now'))
|
|
);
|
|
""")
|
|
c.execute("DELETE FROM config_logs WHERE created_at < datetime('now', '-30 days')")
|
|
c.execute("DELETE FROM chat_logs WHERE created_at < datetime('now', '-30 days')")
|
|
c.execute("DELETE FROM audit_log WHERE created_at < datetime('now', '-30 days')")
|
|
# ── Indexes for performance ──
|
|
for idx in [
|
|
"CREATE INDEX IF NOT EXISTS idx_chat_msg_session ON chat_messages(session_id)",
|
|
"CREATE INDEX IF NOT EXISTS idx_chat_msg_user ON chat_messages(user_id)",
|
|
"CREATE INDEX IF NOT EXISTS idx_chat_msg_status ON chat_messages(status)",
|
|
"CREATE INDEX IF NOT EXISTS idx_chat_sess_user ON chat_sessions(user_id)",
|
|
"CREATE INDEX IF NOT EXISTS idx_reports_user ON reports(user_id)",
|
|
"CREATE INDEX IF NOT EXISTS idx_reports_status ON reports(status)",
|
|
"CREATE INDEX IF NOT EXISTS idx_report_files_report ON report_files(report_id)",
|
|
"CREATE INDEX IF NOT EXISTS idx_oci_cfg_user ON oci_configs(user_id)",
|
|
"CREATE INDEX IF NOT EXISTS idx_genai_cfg_user ON genai_configs(user_id)",
|
|
"CREATE INDEX IF NOT EXISTS idx_mcp_srv_user ON mcp_servers(user_id)",
|
|
"CREATE INDEX IF NOT EXISTS idx_adb_cfg_user ON adb_vector_configs(user_id)",
|
|
"CREATE INDEX IF NOT EXISTS idx_adb_tables_cfg ON adb_vector_tables(adb_config_id)",
|
|
"CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id)",
|
|
"CREATE INDEX IF NOT EXISTS idx_tf_ws_user ON terraform_workspaces(user_id)",
|
|
"CREATE INDEX IF NOT EXISTS idx_tf_ws_session ON terraform_workspaces(session_id)",
|
|
"CREATE INDEX IF NOT EXISTS idx_audit_user ON audit_log(user_id)",
|
|
]:
|
|
c.execute(idx)
|
|
# ── Migrations ──
|
|
for col in ["system_prompt TEXT DEFAULT ''", "reasoning_effort TEXT"]:
|
|
try:
|
|
c.execute(f"ALTER TABLE genai_configs ADD COLUMN {col}")
|
|
except sqlite3.OperationalError:
|
|
pass
|
|
for col in ["genai_config_id TEXT", "embedding_model_id TEXT DEFAULT 'cohere.embed-v4.0'", "is_global INTEGER DEFAULT 0"]:
|
|
try:
|
|
c.execute(f"ALTER TABLE adb_vector_configs ADD COLUMN {col}")
|
|
except sqlite3.OperationalError:
|
|
pass
|
|
for col in ["is_global INTEGER DEFAULT 0"]:
|
|
try:
|
|
c.execute(f"ALTER TABLE mcp_servers ADD COLUMN {col}")
|
|
except sqlite3.OperationalError:
|
|
pass
|
|
for col in ["progress TEXT DEFAULT ''", "level INTEGER DEFAULT 2", "obp_checks INTEGER DEFAULT 0",
|
|
"raw_data INTEGER DEFAULT 0", "redact_output INTEGER DEFAULT 0", "worker_pid INTEGER"]:
|
|
try:
|
|
c.execute(f"ALTER TABLE reports ADD COLUMN {col}")
|
|
except sqlite3.OperationalError:
|
|
pass
|
|
for col in ["status TEXT DEFAULT 'done'"]:
|
|
try:
|
|
c.execute(f"ALTER TABLE chat_messages ADD COLUMN {col}")
|
|
except sqlite3.OperationalError:
|
|
pass
|
|
for col in ["compartment_id TEXT"]:
|
|
try:
|
|
c.execute(f"ALTER TABLE terraform_workspaces ADD COLUMN {col}")
|
|
except sqlite3.OperationalError:
|
|
pass
|
|
for col in ["ssh_public_key TEXT", "auth_type TEXT DEFAULT 'api_key'"]:
|
|
try:
|
|
c.execute(f"ALTER TABLE oci_configs ADD COLUMN {col}")
|
|
except sqlite3.OperationalError:
|
|
pass
|
|
for col in ["timezone TEXT DEFAULT 'America/Sao_Paulo'", "auth_provider TEXT DEFAULT 'local'", "oidc_subject TEXT", "must_change_password INTEGER DEFAULT 0"]:
|
|
try:
|
|
c.execute(f"ALTER TABLE users ADD COLUMN {col}")
|
|
except sqlite3.OperationalError:
|
|
pass
|
|
# Migrate legacy table_name from adb_vector_configs into adb_vector_tables
|
|
try:
|
|
for cfg_row in c.execute("SELECT id, table_name FROM adb_vector_configs WHERE table_name IS NOT NULL AND table_name != ''").fetchall():
|
|
if not c.execute("SELECT 1 FROM adb_vector_tables WHERE adb_config_id=? AND table_name=?", (cfg_row["id"], cfg_row["table_name"])).fetchone():
|
|
c.execute("INSERT INTO adb_vector_tables (id, adb_config_id, table_name, description) VALUES (?,?,?,?)",
|
|
(str(uuid.uuid4()), cfg_row["id"], cfg_row["table_name"], "Migrado automaticamente"))
|
|
except Exception as e:
|
|
log.warning(f"DB migration: adb_vector_tables backfill failed: {e}")
|
|
# Backfill chat_sessions from existing chat_messages
|
|
try:
|
|
c.execute("""INSERT OR IGNORE INTO chat_sessions (id, user_id, agent_type, title, created_at, updated_at)
|
|
SELECT cm.session_id, cm.user_id,
|
|
CASE WHEN tw.id IS NOT NULL THEN 'terraform' ELSE 'chat' END,
|
|
SUBSTR((SELECT content FROM chat_messages cm2 WHERE cm2.session_id=cm.session_id AND cm2.role='user' ORDER BY cm2.created_at ASC LIMIT 1), 1, 80),
|
|
MIN(cm.created_at), MAX(cm.created_at)
|
|
FROM chat_messages cm
|
|
LEFT JOIN terraform_workspaces tw ON tw.session_id = cm.session_id
|
|
WHERE cm.session_id NOT IN (SELECT id FROM chat_sessions)
|
|
GROUP BY cm.session_id""")
|
|
except Exception as e:
|
|
log.warning(f"DB migration: chat_sessions backfill failed: {e}")
|
|
# Migrate legacy base64 fields to Fernet encryption
|
|
_enc_migrations = [
|
|
("oci_configs", ["tenancy_ocid", "user_ocid", "fingerprint", "compartment_id"]),
|
|
("adb_vector_configs", ["password_enc", "wallet_password_enc"]),
|
|
]
|
|
for table, cols in _enc_migrations:
|
|
for col in cols:
|
|
try:
|
|
rows = c.execute(f"SELECT id, {col} FROM {table} WHERE {col} IS NOT NULL AND {col} != ''").fetchall()
|
|
for row in rows:
|
|
val = row[col]
|
|
if val and not val.startswith(_FERNET_PREFIX):
|
|
try:
|
|
plaintext = base64.b64decode(val.encode()).decode()
|
|
c.execute(f"UPDATE {table} SET {col}=? WHERE id=?", (_enc(plaintext), row["id"]))
|
|
except Exception:
|
|
pass
|
|
except Exception as e:
|
|
log.warning(f"DB migration: encrypt {table}.{col} failed: {e}")
|
|
# Seed default system prompts (late import to avoid circular dependency)
|
|
from services.genai import RAG_DEFAULT_SYSTEM_PROMPT, TF_DEFAULT_SYSTEM_PROMPT
|
|
if not c.execute("SELECT 1 FROM system_prompts WHERE agent='chat'").fetchone():
|
|
c.execute("INSERT INTO system_prompts (id,name,agent,content,is_active) VALUES (?,?,?,?,?)",
|
|
(str(uuid.uuid4()), "OCI CIS RAG Agent", "chat", RAG_DEFAULT_SYSTEM_PROMPT, 1))
|
|
tf_row = c.execute("SELECT id FROM system_prompts WHERE agent='terraform' AND is_active=1").fetchone()
|
|
if tf_row:
|
|
c.execute("UPDATE system_prompts SET content=? WHERE id=?", (TF_DEFAULT_SYSTEM_PROMPT, tf_row["id"]))
|
|
else:
|
|
c.execute("INSERT INTO system_prompts (id,name,agent,content,is_active) VALUES (?,?,?,?,?)",
|
|
(str(uuid.uuid4()), "OCI Terraform Agent", "terraform", TF_DEFAULT_SYSTEM_PROMPT, 1))
|
|
# Recover stuck terraform workspaces (interrupted by container restart)
|
|
stuck = c.execute("SELECT id, status FROM terraform_workspaces WHERE status IN ('planning','applying','destroying')").fetchall()
|
|
for ws in stuck:
|
|
prev = 'applied' if ws["status"] == 'destroying' else 'failed'
|
|
c.execute("UPDATE terraform_workspaces SET status=?, error='Interrompido por restart do container', updated_at=datetime('now') WHERE id=?", (prev, ws["id"]))
|
|
# Remove stale lock files
|
|
lock_file = TERRAFORM_DIR / ws["id"] / ".terraform.tfstate.lock.info"
|
|
if lock_file.exists():
|
|
lock_file.unlink()
|
|
log.info(f"Recovered stuck workspace {ws['id']}: {ws['status']} -> {prev}")
|
|
adm = c.execute("SELECT id FROM users WHERE username='admin'").fetchone()
|
|
if not adm:
|
|
c.execute(
|
|
"INSERT INTO users (id,first_name,last_name,username,email,password_hash,role,must_change_password) VALUES (?,?,?,?,?,?,?,?)",
|
|
(str(uuid.uuid4()), "Admin", "Sistema", "admin", "admin@local", _hash_pw("admin123"), "admin", 1)
|
|
)
|
|
log.info("Default admin created — username: admin / password: admin123")
|
|
|