refactor: decompose monolith — app.py 10,600→143 lines, 30+ modules
Fase 0 complete: extract all business logic, auth, database, and 176 endpoints from monolithic app.py into dedicated modules. Structure: - app.py: FastAPI hub (CORS, routers, startup/shutdown) - models.py: 13 Pydantic request models - utils.py: shared utilities (embed status, upload validation, process registries) - config.py: constants, env vars, model catalogs - auth/: crypto, jwt_auth, oidc, rate_limit - database/: db(), init_db(), schema DDL - services/: genai, compliance, chat, terraform, embeddings, cis_reports, mcp - routes/: 13 APIRouter modules (auth, users, oci_config, oci_explorer, genai, mcp, adb, embeddings, reports, chat, terraform, settings, cis_engine) Also: README updated with OCIR auth instructions for manual docker run. 86 tests passing.
This commit is contained in:
nogueiraguh
376
backend/database/__init__.py
Normal file
376
backend/database/__init__.py
Normal file
@@ -0,0 +1,376 @@
|
||||
"""Database layer — SQLite context manager and schema initialization."""
|
||||
import sqlite3
|
||||
import uuid
|
||||
import secrets
|
||||
from contextlib import contextmanager
|
||||
|
||||
from config import DB_PATH, TERRAFORM_DIR, log
|
||||
from auth.crypto import _hash_pw
|
||||
|
||||
# ── Database ──────────────────────────────────────────────────────────────────
|
||||
@contextmanager
|
||||
def db():
|
||||
conn = sqlite3.connect(str(DB_PATH), timeout=30, check_same_thread=False)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA busy_timeout=30000")
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
try:
|
||||
yield conn
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def init_db():
|
||||
with db() as c:
|
||||
c.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id TEXT PRIMARY KEY,
|
||||
first_name TEXT NOT NULL,
|
||||
last_name TEXT NOT NULL,
|
||||
username TEXT UNIQUE NOT NULL,
|
||||
email TEXT,
|
||||
password_hash TEXT NOT NULL, role TEXT NOT NULL DEFAULT 'viewer',
|
||||
mfa_secret TEXT, mfa_enabled INTEGER DEFAULT 0,
|
||||
timezone TEXT DEFAULT 'America/Sao_Paulo',
|
||||
is_active INTEGER DEFAULT 1,
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
last_login TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id TEXT PRIMARY KEY, user_id TEXT NOT NULL,
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
expires_at TEXT NOT NULL, is_active INTEGER DEFAULT 1
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS oci_configs (
|
||||
id TEXT PRIMARY KEY, user_id TEXT NOT NULL,
|
||||
tenancy_name TEXT NOT NULL, tenancy_ocid TEXT NOT NULL,
|
||||
user_ocid TEXT NOT NULL, fingerprint TEXT NOT NULL,
|
||||
region TEXT NOT NULL, key_file_path TEXT NOT NULL,
|
||||
compartment_id TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS genai_configs (
|
||||
id TEXT PRIMARY KEY, user_id TEXT NOT NULL,
|
||||
name TEXT NOT NULL DEFAULT 'default',
|
||||
oci_config_id TEXT NOT NULL,
|
||||
model_id TEXT NOT NULL,
|
||||
model_ocid TEXT,
|
||||
compartment_id TEXT NOT NULL,
|
||||
genai_region TEXT NOT NULL,
|
||||
endpoint TEXT NOT NULL,
|
||||
serving_type TEXT DEFAULT 'ON_DEMAND',
|
||||
dedicated_endpoint_id TEXT,
|
||||
temperature REAL DEFAULT 1,
|
||||
max_tokens INTEGER DEFAULT 6000,
|
||||
top_p REAL DEFAULT 0.95,
|
||||
top_k INTEGER DEFAULT 1,
|
||||
frequency_penalty REAL DEFAULT 0,
|
||||
presence_penalty REAL DEFAULT 0,
|
||||
reasoning_effort TEXT,
|
||||
is_default INTEGER DEFAULT 0,
|
||||
system_prompt TEXT DEFAULT '',
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
FOREIGN KEY (oci_config_id) REFERENCES oci_configs(id)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS reports (
|
||||
id TEXT PRIMARY KEY, user_id TEXT NOT NULL,
|
||||
tenancy_name TEXT NOT NULL, config_id TEXT,
|
||||
level INTEGER DEFAULT 2,
|
||||
obp_checks INTEGER DEFAULT 0,
|
||||
raw_data INTEGER DEFAULT 0,
|
||||
redact_output INTEGER DEFAULT 0,
|
||||
status TEXT DEFAULT 'pending', progress TEXT DEFAULT '',
|
||||
html_path TEXT, json_path TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
completed_at TEXT, error_msg TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS report_files (
|
||||
id TEXT PRIMARY KEY,
|
||||
report_id TEXT NOT NULL,
|
||||
file_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
file_type TEXT NOT NULL,
|
||||
file_category TEXT NOT NULL,
|
||||
file_size INTEGER DEFAULT 0
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS adb_vector_configs (
|
||||
id TEXT PRIMARY KEY, user_id TEXT NOT NULL,
|
||||
config_name TEXT NOT NULL,
|
||||
dsn TEXT NOT NULL,
|
||||
username TEXT NOT NULL,
|
||||
password_enc TEXT NOT NULL,
|
||||
wallet_dir TEXT,
|
||||
wallet_password_enc TEXT,
|
||||
table_name TEXT DEFAULT '',
|
||||
use_mtls INTEGER DEFAULT 1,
|
||||
is_active INTEGER DEFAULT 1,
|
||||
is_global INTEGER DEFAULT 0,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS adb_vector_tables (
|
||||
id TEXT PRIMARY KEY,
|
||||
adb_config_id TEXT NOT NULL,
|
||||
table_name TEXT NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
is_active INTEGER DEFAULT 1,
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
FOREIGN KEY (adb_config_id) REFERENCES adb_vector_configs(id) ON DELETE CASCADE,
|
||||
UNIQUE(adb_config_id, table_name)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS mcp_servers (
|
||||
id TEXT PRIMARY KEY, user_id TEXT NOT NULL,
|
||||
name TEXT NOT NULL, description TEXT,
|
||||
server_type TEXT NOT NULL DEFAULT 'stdio',
|
||||
command TEXT, args TEXT, env_vars TEXT,
|
||||
url TEXT, module_path TEXT,
|
||||
tools TEXT,
|
||||
linked_adb_id TEXT,
|
||||
is_active INTEGER DEFAULT 1,
|
||||
is_global INTEGER DEFAULT 0,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS chat_messages (
|
||||
id TEXT PRIMARY KEY, session_id TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL, role TEXT NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
model_id TEXT,
|
||||
status TEXT DEFAULT 'done',
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS chat_summaries (
|
||||
id TEXT PRIMARY KEY,
|
||||
session_id TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
summary TEXT NOT NULL,
|
||||
messages_compacted INTEGER NOT NULL,
|
||||
up_to_message_id TEXT NOT NULL,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS audit_log (
|
||||
id TEXT PRIMARY KEY, user_id TEXT, username TEXT,
|
||||
action TEXT NOT NULL, resource TEXT, details TEXT,
|
||||
ip TEXT, created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS terminal_history (
|
||||
id TEXT PRIMARY KEY, user_id TEXT NOT NULL,
|
||||
oci_config_id TEXT NOT NULL, command TEXT NOT NULL,
|
||||
exit_code INTEGER, output TEXT, error TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS config_logs (
|
||||
id TEXT PRIMARY KEY,
|
||||
config_type TEXT NOT NULL,
|
||||
config_id TEXT NOT NULL,
|
||||
config_name TEXT,
|
||||
severity TEXT NOT NULL,
|
||||
action TEXT NOT NULL,
|
||||
message TEXT NOT NULL,
|
||||
user_id TEXT,
|
||||
username TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS app_settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
updated_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS system_prompts (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
agent TEXT NOT NULL DEFAULT 'chat',
|
||||
content TEXT NOT NULL,
|
||||
is_active INTEGER DEFAULT 0,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS chat_logs (
|
||||
id TEXT PRIMARY KEY,
|
||||
session_id TEXT,
|
||||
message_id TEXT,
|
||||
user_id TEXT,
|
||||
severity TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
action TEXT NOT NULL,
|
||||
message TEXT NOT NULL,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS chat_sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
agent_type TEXT NOT NULL DEFAULT 'chat',
|
||||
title TEXT DEFAULT '',
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
updated_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS terraform_workspaces (
|
||||
id TEXT PRIMARY KEY, user_id TEXT NOT NULL,
|
||||
session_id TEXT NOT NULL,
|
||||
oci_config_id TEXT NOT NULL,
|
||||
compartment_id TEXT,
|
||||
name TEXT DEFAULT 'workspace',
|
||||
tf_code TEXT,
|
||||
status TEXT DEFAULT 'draft',
|
||||
plan_output TEXT, apply_output TEXT, destroy_output TEXT,
|
||||
error TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
updated_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS explorer_counts_cache (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
oci_config_id TEXT NOT NULL,
|
||||
compartment_id TEXT NOT NULL,
|
||||
resource_type TEXT NOT NULL,
|
||||
count INTEGER NOT NULL DEFAULT 0,
|
||||
regions TEXT DEFAULT '',
|
||||
updated_at TEXT DEFAULT (datetime('now')),
|
||||
UNIQUE(oci_config_id, compartment_id, resource_type, regions)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS tf_valid_types (
|
||||
name TEXT PRIMARY KEY,
|
||||
kind TEXT NOT NULL DEFAULT 'resource',
|
||||
category TEXT DEFAULT '',
|
||||
required_args TEXT DEFAULT '',
|
||||
blocks TEXT DEFAULT ''
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS tf_resource_docs (
|
||||
slug TEXT PRIMARY KEY,
|
||||
subcategory TEXT DEFAULT '',
|
||||
example_usage TEXT DEFAULT '',
|
||||
argument_ref TEXT DEFAULT '',
|
||||
fetched_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
""")
|
||||
c.execute("DELETE FROM config_logs WHERE created_at < datetime('now', '-30 days')")
|
||||
c.execute("DELETE FROM chat_logs WHERE created_at < datetime('now', '-30 days')")
|
||||
c.execute("DELETE FROM audit_log WHERE created_at < datetime('now', '-30 days')")
|
||||
# ── Indexes for performance ──
|
||||
for idx in [
|
||||
"CREATE INDEX IF NOT EXISTS idx_chat_msg_session ON chat_messages(session_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_chat_msg_user ON chat_messages(user_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_chat_msg_status ON chat_messages(status)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_chat_sess_user ON chat_sessions(user_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_reports_user ON reports(user_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_reports_status ON reports(status)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_report_files_report ON report_files(report_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_oci_cfg_user ON oci_configs(user_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_genai_cfg_user ON genai_configs(user_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_mcp_srv_user ON mcp_servers(user_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_adb_cfg_user ON adb_vector_configs(user_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_adb_tables_cfg ON adb_vector_tables(adb_config_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_tf_ws_user ON terraform_workspaces(user_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_tf_ws_session ON terraform_workspaces(session_id)",
|
||||
"CREATE INDEX IF NOT EXISTS idx_audit_user ON audit_log(user_id)",
|
||||
]:
|
||||
c.execute(idx)
|
||||
# ── Migrations ──
|
||||
for col in ["system_prompt TEXT DEFAULT ''", "reasoning_effort TEXT"]:
|
||||
try:
|
||||
c.execute(f"ALTER TABLE genai_configs ADD COLUMN {col}")
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
for col in ["genai_config_id TEXT", "embedding_model_id TEXT DEFAULT 'cohere.embed-v4.0'", "is_global INTEGER DEFAULT 0"]:
|
||||
try:
|
||||
c.execute(f"ALTER TABLE adb_vector_configs ADD COLUMN {col}")
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
for col in ["is_global INTEGER DEFAULT 0"]:
|
||||
try:
|
||||
c.execute(f"ALTER TABLE mcp_servers ADD COLUMN {col}")
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
for col in ["progress TEXT DEFAULT ''", "level INTEGER DEFAULT 2", "obp_checks INTEGER DEFAULT 0",
|
||||
"raw_data INTEGER DEFAULT 0", "redact_output INTEGER DEFAULT 0", "worker_pid INTEGER"]:
|
||||
try:
|
||||
c.execute(f"ALTER TABLE reports ADD COLUMN {col}")
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
for col in ["status TEXT DEFAULT 'done'"]:
|
||||
try:
|
||||
c.execute(f"ALTER TABLE chat_messages ADD COLUMN {col}")
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
for col in ["compartment_id TEXT"]:
|
||||
try:
|
||||
c.execute(f"ALTER TABLE terraform_workspaces ADD COLUMN {col}")
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
for col in ["ssh_public_key TEXT", "auth_type TEXT DEFAULT 'api_key'"]:
|
||||
try:
|
||||
c.execute(f"ALTER TABLE oci_configs ADD COLUMN {col}")
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
for col in ["timezone TEXT DEFAULT 'America/Sao_Paulo'", "auth_provider TEXT DEFAULT 'local'", "oidc_subject TEXT", "must_change_password INTEGER DEFAULT 0"]:
|
||||
try:
|
||||
c.execute(f"ALTER TABLE users ADD COLUMN {col}")
|
||||
except sqlite3.OperationalError:
|
||||
pass
|
||||
# Migrate legacy table_name from adb_vector_configs into adb_vector_tables
|
||||
try:
|
||||
for cfg_row in c.execute("SELECT id, table_name FROM adb_vector_configs WHERE table_name IS NOT NULL AND table_name != ''").fetchall():
|
||||
if not c.execute("SELECT 1 FROM adb_vector_tables WHERE adb_config_id=? AND table_name=?", (cfg_row["id"], cfg_row["table_name"])).fetchone():
|
||||
c.execute("INSERT INTO adb_vector_tables (id, adb_config_id, table_name, description) VALUES (?,?,?,?)",
|
||||
(str(uuid.uuid4()), cfg_row["id"], cfg_row["table_name"], "Migrado automaticamente"))
|
||||
except Exception as e:
|
||||
log.warning(f"DB migration: adb_vector_tables backfill failed: {e}")
|
||||
# Backfill chat_sessions from existing chat_messages
|
||||
try:
|
||||
c.execute("""INSERT OR IGNORE INTO chat_sessions (id, user_id, agent_type, title, created_at, updated_at)
|
||||
SELECT cm.session_id, cm.user_id,
|
||||
CASE WHEN tw.id IS NOT NULL THEN 'terraform' ELSE 'chat' END,
|
||||
SUBSTR((SELECT content FROM chat_messages cm2 WHERE cm2.session_id=cm.session_id AND cm2.role='user' ORDER BY cm2.created_at ASC LIMIT 1), 1, 80),
|
||||
MIN(cm.created_at), MAX(cm.created_at)
|
||||
FROM chat_messages cm
|
||||
LEFT JOIN terraform_workspaces tw ON tw.session_id = cm.session_id
|
||||
WHERE cm.session_id NOT IN (SELECT id FROM chat_sessions)
|
||||
GROUP BY cm.session_id""")
|
||||
except Exception as e:
|
||||
log.warning(f"DB migration: chat_sessions backfill failed: {e}")
|
||||
# Migrate legacy base64 fields to Fernet encryption
|
||||
_enc_migrations = [
|
||||
("oci_configs", ["tenancy_ocid", "user_ocid", "fingerprint", "compartment_id"]),
|
||||
("adb_vector_configs", ["password_enc", "wallet_password_enc"]),
|
||||
]
|
||||
for table, cols in _enc_migrations:
|
||||
for col in cols:
|
||||
try:
|
||||
rows = c.execute(f"SELECT id, {col} FROM {table} WHERE {col} IS NOT NULL AND {col} != ''").fetchall()
|
||||
for row in rows:
|
||||
val = row[col]
|
||||
if val and not val.startswith(_FERNET_PREFIX):
|
||||
try:
|
||||
plaintext = base64.b64decode(val.encode()).decode()
|
||||
c.execute(f"UPDATE {table} SET {col}=? WHERE id=?", (_enc(plaintext), row["id"]))
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
log.warning(f"DB migration: encrypt {table}.{col} failed: {e}")
|
||||
# Seed default system prompts (late import to avoid circular dependency)
|
||||
from services.genai import RAG_DEFAULT_SYSTEM_PROMPT, TF_DEFAULT_SYSTEM_PROMPT
|
||||
if not c.execute("SELECT 1 FROM system_prompts WHERE agent='chat'").fetchone():
|
||||
c.execute("INSERT INTO system_prompts (id,name,agent,content,is_active) VALUES (?,?,?,?,?)",
|
||||
(str(uuid.uuid4()), "OCI CIS RAG Agent", "chat", RAG_DEFAULT_SYSTEM_PROMPT, 1))
|
||||
tf_row = c.execute("SELECT id FROM system_prompts WHERE agent='terraform' AND is_active=1").fetchone()
|
||||
if tf_row:
|
||||
c.execute("UPDATE system_prompts SET content=? WHERE id=?", (TF_DEFAULT_SYSTEM_PROMPT, tf_row["id"]))
|
||||
else:
|
||||
c.execute("INSERT INTO system_prompts (id,name,agent,content,is_active) VALUES (?,?,?,?,?)",
|
||||
(str(uuid.uuid4()), "OCI Terraform Agent", "terraform", TF_DEFAULT_SYSTEM_PROMPT, 1))
|
||||
# Recover stuck terraform workspaces (interrupted by container restart)
|
||||
stuck = c.execute("SELECT id, status FROM terraform_workspaces WHERE status IN ('planning','applying','destroying')").fetchall()
|
||||
for ws in stuck:
|
||||
prev = 'applied' if ws["status"] == 'destroying' else 'failed'
|
||||
c.execute("UPDATE terraform_workspaces SET status=?, error='Interrompido por restart do container', updated_at=datetime('now') WHERE id=?", (prev, ws["id"]))
|
||||
# Remove stale lock files
|
||||
lock_file = TERRAFORM_DIR / ws["id"] / ".terraform.tfstate.lock.info"
|
||||
if lock_file.exists():
|
||||
lock_file.unlink()
|
||||
log.info(f"Recovered stuck workspace {ws['id']}: {ws['status']} -> {prev}")
|
||||
adm = c.execute("SELECT id FROM users WHERE username='admin'").fetchone()
|
||||
if not adm:
|
||||
c.execute(
|
||||
"INSERT INTO users (id,first_name,last_name,username,email,password_hash,role,must_change_password) VALUES (?,?,?,?,?,?,?,?)",
|
||||
(str(uuid.uuid4()), "Admin", "Sistema", "admin", "admin@local", _hash_pw("admin123"), "admin", 1)
|
||||
)
|
||||
log.info("Default admin created — username: admin / password: admin123")
|
||||
|
||||
Reference in New Issue
Block a user