refactor: decompose monolith — app.py 10,600→143 lines, 30+ modules
Fase 0 complete: extract all business logic, auth, database, and 176 endpoints from monolithic app.py into dedicated modules. Structure: - app.py: FastAPI hub (CORS, routers, startup/shutdown) - models.py: 13 Pydantic request models - utils.py: shared utilities (embed status, upload validation, process registries) - config.py: constants, env vars, model catalogs - auth/: crypto, jwt_auth, oidc, rate_limit - database/: db(), init_db(), schema DDL - services/: genai, compliance, chat, terraform, embeddings, cis_reports, mcp - routes/: 13 APIRouter modules (auth, users, oci_config, oci_explorer, genai, mcp, adb, embeddings, reports, chat, terraform, settings, cis_engine) Also: README updated with OCIR auth instructions for manual docker run. 86 tests passing.
This commit is contained in:
nogueiraguh
421
backend/services/terraform_exec.py
Normal file
421
backend/services/terraform_exec.py
Normal file
@@ -0,0 +1,421 @@
|
||||
"""Terraform execution — split monolith, write files, plan/apply/destroy."""
|
||||
import os, json, uuid, asyncio, subprocess, re
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
from config import TERRAFORM_DIR, OCI_DIR, log
|
||||
from database import db
|
||||
from auth.crypto import _safe_dec
|
||||
from auth.jwt_auth import _audit, _verify_config_access
|
||||
|
||||
def _split_tf_monolith(content: str) -> dict:
|
||||
"""Split a monolithic HCL string into multiple logical files by resource type.
|
||||
Returns dict of {filename: content} or None if splitting not needed."""
|
||||
import re as _re
|
||||
content = _fix_single_line_blocks(content)
|
||||
lines = content.split('\n')
|
||||
top_blocks = []
|
||||
preamble = []
|
||||
accum = []
|
||||
in_block = False
|
||||
b_depth = 0
|
||||
cur_header = ''
|
||||
|
||||
for line in lines:
|
||||
stripped = _re.sub(r'"[^"]*"', '', line)
|
||||
opens = stripped.count('{')
|
||||
closes = stripped.count('}')
|
||||
if not in_block:
|
||||
hdr = _re.match(r'^\s*(variable|output|resource|data|provider|module)\s+"', line) or \
|
||||
_re.match(r'^\s*(locals|terraform)\s*\{', line)
|
||||
if hdr:
|
||||
in_block = True
|
||||
cur_header = line
|
||||
accum = preamble + [line]
|
||||
preamble = []
|
||||
b_depth = opens - closes
|
||||
if b_depth <= 0:
|
||||
b_depth = 0
|
||||
if b_depth == 0 and opens > 0:
|
||||
top_blocks.append((cur_header, '\n'.join(accum)))
|
||||
in_block = False
|
||||
accum = []
|
||||
else:
|
||||
preamble.append(line)
|
||||
else:
|
||||
accum.append(line)
|
||||
b_depth += opens - closes
|
||||
if b_depth <= 0:
|
||||
b_depth = 0
|
||||
top_blocks.append((cur_header, '\n'.join(accum)))
|
||||
in_block = False
|
||||
accum = []
|
||||
cur_header = ''
|
||||
if accum:
|
||||
top_blocks.append((cur_header or '', '\n'.join(accum)))
|
||||
|
||||
if len(top_blocks) < 3:
|
||||
return None
|
||||
|
||||
cats = {'variables': [], 'outputs': [], 'networking': [], 'compute': [], 'database': [],
|
||||
'firewall': [], 'loadbalancer': [], 'storage': [], 'iam': [], 'drg': [],
|
||||
'data': [], 'providers': [], 'other': []}
|
||||
for header, body in top_blocks:
|
||||
h = header.strip()
|
||||
if h.startswith('variable '):
|
||||
cats['variables'].append(body)
|
||||
elif h.startswith('output '):
|
||||
cats['outputs'].append(body)
|
||||
elif h.startswith('provider '):
|
||||
cats['providers'].append(body)
|
||||
elif _re.search(r'resource\s+"oci_core_(drg|remote_peering|drg_route|drg_attachment)', h):
|
||||
cats['drg'].append(body)
|
||||
elif _re.search(r'resource\s+"oci_core_(vcn|subnet|internet_gateway|nat_gateway|service_gateway|route_table|security_list|network_security_group|dhcp_options|local_peering_gateway|virtual_circuit)', h):
|
||||
cats['networking'].append(body)
|
||||
elif _re.search(r'resource\s+"oci_core_instance', h):
|
||||
cats['compute'].append(body)
|
||||
elif _re.search(r'resource\s+"oci_(database|nosql|mysql|psql)', h) or _re.search(r'resource\s+"oci_core_autonomous', h):
|
||||
cats['database'].append(body)
|
||||
elif _re.search(r'resource\s+"oci_network_firewall', h):
|
||||
cats['firewall'].append(body)
|
||||
elif _re.search(r'resource\s+"oci_(load_balancer|network_load_balancer)', h):
|
||||
cats['loadbalancer'].append(body)
|
||||
elif _re.search(r'resource\s+"oci_(objectstorage|file_storage)', h):
|
||||
cats['storage'].append(body)
|
||||
elif _re.search(r'resource\s+"oci_identity', h):
|
||||
cats['iam'].append(body)
|
||||
elif h.startswith('data '):
|
||||
cats['data'].append(body)
|
||||
else:
|
||||
cats['other'].append(body)
|
||||
|
||||
result = {}
|
||||
mapping = [('variables.tf', 'variables'), ('providers.tf', 'providers'), ('networking.tf', 'networking'),
|
||||
('drg.tf', 'drg'), ('compute.tf', 'compute'), ('database.tf', 'database'),
|
||||
('firewall.tf', 'firewall'), ('loadbalancer.tf', 'loadbalancer'), ('storage.tf', 'storage'),
|
||||
('iam.tf', 'iam'), ('data.tf', 'data'), ('main.tf', 'other'), ('outputs.tf', 'outputs')]
|
||||
for fname, key in mapping:
|
||||
if cats[key]:
|
||||
result[fname] = '\n\n'.join(cats[key])
|
||||
return result if len(result) >= 3 else None
|
||||
|
||||
|
||||
def _write_tf_files(wdir: Path, tf_code: str):
|
||||
"""Parse tf_code for '// filename: xxx.tf' markers and write separate files.
|
||||
If only 1-2 large files, auto-split into multiple files by resource type."""
|
||||
import re as _re
|
||||
parts = _re.split(r'^//\s*filename:\s*(\S+)\s*$', tf_code, flags=_re.MULTILINE)
|
||||
|
||||
files = {}
|
||||
if len(parts) >= 3:
|
||||
if parts[0].strip():
|
||||
files['main.tf'] = parts[0].strip()
|
||||
for i in range(1, len(parts), 2):
|
||||
fname = parts[i].strip().replace("/", "_").replace("..", "_")
|
||||
content = parts[i + 1].strip() if i + 1 < len(parts) else ""
|
||||
if fname and content:
|
||||
files[fname] = content
|
||||
else:
|
||||
files['main.tf'] = tf_code
|
||||
|
||||
# Auto-split: if 1-2 large files, split by resource type
|
||||
if len(files) <= 2:
|
||||
all_content = '\n\n'.join(files.values())
|
||||
if len(all_content) > 2000:
|
||||
split = _split_tf_monolith(all_content)
|
||||
if split:
|
||||
files = split
|
||||
log.info(f"Backend auto-split monolithic TF into {len(files)} files: {list(files.keys())}")
|
||||
|
||||
# Deduplicate: if a resource appears in multiple files, keep only in the split file
|
||||
# This handles edge cases where monolithic + split files both exist
|
||||
if len(files) > 2:
|
||||
resource_locations = {} # "type.name" -> [(filename, line)]
|
||||
dupes_in = set()
|
||||
for fname, content in files.items():
|
||||
for m in _re.finditer(r'^resource\s+"(\S+)"\s+"(\S+)"', content, _re.MULTILINE):
|
||||
key = f'{m.group(1)}.{m.group(2)}'
|
||||
resource_locations.setdefault(key, []).append(fname)
|
||||
# Find files that contain ONLY duplicate resources (= monolithic leftovers)
|
||||
for key, fnames in resource_locations.items():
|
||||
if len(fnames) > 1:
|
||||
# The largest file is likely the monolithic one
|
||||
largest = max(fnames, key=lambda f: len(files.get(f, '')))
|
||||
dupes_in.add(largest)
|
||||
# Remove monolithic files that are fully duplicated
|
||||
for fname in dupes_in:
|
||||
if all(
|
||||
any(f2 != fname for f2 in resource_locations.get(key, []))
|
||||
for key, flist in resource_locations.items()
|
||||
if fname in flist
|
||||
) and len(files) - 1 >= 2:
|
||||
log.info(f"Removing duplicate monolithic file: {fname}")
|
||||
del files[fname]
|
||||
|
||||
# Fix single-line blocks in all files
|
||||
for fname in files:
|
||||
files[fname] = _fix_single_line_blocks(files[fname])
|
||||
|
||||
# Write files to disk
|
||||
for fname, content in files.items():
|
||||
(wdir / fname).write_text(content)
|
||||
|
||||
|
||||
async def _terraform_exec(wid: str, action: str, user: dict):
|
||||
"""Background: run terraform init + plan/apply/destroy in workspace dir."""
|
||||
log.info(f"Terraform {action} started: wid={wid}")
|
||||
status_col = f"{action}_output"
|
||||
final_ok = {"plan": "planned", "apply": "applied", "destroy": "destroyed"}[action]
|
||||
|
||||
try:
|
||||
with db() as c:
|
||||
ws = c.execute("SELECT * FROM terraform_workspaces WHERE id=?", (wid,)).fetchone()
|
||||
if not ws:
|
||||
return
|
||||
|
||||
wdir = TERRAFORM_DIR / wid
|
||||
wdir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Clean old .tf files (keep .terraform/, state, lock)
|
||||
for old_tf in wdir.glob("*.tf"):
|
||||
old_tf.unlink()
|
||||
|
||||
# Write .tf files — parse // filename: comments to split into separate files
|
||||
_write_tf_files(wdir, ws["tf_code"] or "")
|
||||
|
||||
# Auto-generate provider.tf from OCI config
|
||||
# Detect provider aliases declared by the model in generated files
|
||||
import re as _re2
|
||||
oci_cfg = _get_oci_config(ws["oci_config_id"])
|
||||
alias_blocks = [] # list of (alias_name, region_ref)
|
||||
|
||||
# Scan all .tf files for provider "oci" { alias = "..." ... region = ... }
|
||||
provider_block_re = _re2.compile(r'provider\s+"oci"\s*\{', _re2.MULTILINE)
|
||||
for tf_file in sorted(wdir.glob("*.tf")):
|
||||
if tf_file.name == "provider.tf":
|
||||
continue
|
||||
content = tf_file.read_text()
|
||||
# Find each provider "oci" block and extract alias + region
|
||||
blocks_to_remove = []
|
||||
for m in provider_block_re.finditer(content):
|
||||
start = m.start()
|
||||
# Find matching closing brace
|
||||
depth = 0
|
||||
end = start
|
||||
for ci in range(m.end() - 1, len(content)):
|
||||
if content[ci] == '{':
|
||||
depth += 1
|
||||
elif content[ci] == '}':
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
end = ci + 1
|
||||
break
|
||||
block = content[start:end]
|
||||
alias_m = _re2.search(r'alias\s*=\s*"([^"]+)"', block)
|
||||
region_m = _re2.search(r'region\s*=\s*(.+)', block)
|
||||
if alias_m:
|
||||
alias_name = alias_m.group(1)
|
||||
region_ref = region_m.group(1).strip().rstrip("}").strip() if region_m else '"unknown"'
|
||||
alias_blocks.append((alias_name, region_ref))
|
||||
blocks_to_remove.append((start, end))
|
||||
# Remove model-generated provider blocks (we centralize in provider.tf)
|
||||
if blocks_to_remove:
|
||||
new_content = content
|
||||
for s, e in reversed(blocks_to_remove):
|
||||
new_content = new_content[:s] + new_content[e:]
|
||||
# Also remove leading comments right before removed blocks
|
||||
new_content = _re2.sub(r'\n(//[^\n]*\n){1,5}\s*\n', '\n\n', new_content)
|
||||
new_content = new_content.strip()
|
||||
if new_content:
|
||||
tf_file.write_text(new_content + "\n")
|
||||
else:
|
||||
tf_file.unlink() # Remove empty file
|
||||
|
||||
# Scan all .tf files for variable declarations with region-like defaults
|
||||
# so we can map alias providers to the correct variable reference
|
||||
var_region_map = {} # variable_name -> default_value
|
||||
var_re = _re2.compile(r'variable\s+"([^"]*region[^"]*)"\s*\{', _re2.IGNORECASE)
|
||||
for tf_file in sorted(wdir.glob("*.tf")):
|
||||
if tf_file.name in ("provider.tf", "terraform.tfvars"):
|
||||
continue
|
||||
content = tf_file.read_text()
|
||||
for vm in var_re.finditer(content):
|
||||
var_region_map[vm.group(1)] = f'var.{vm.group(1)}'
|
||||
|
||||
# Check if variable "region" is declared — use it for primary provider
|
||||
has_region_var = "region" in var_region_map
|
||||
|
||||
# Also scan for provider refs in resource blocks (provider = oci.xxx)
|
||||
for tf_file in sorted(wdir.glob("*.tf")):
|
||||
if tf_file.name == "provider.tf":
|
||||
continue
|
||||
content = tf_file.read_text()
|
||||
for ref_m in _re2.finditer(r'provider\s*=\s*oci\.(\w+)', content):
|
||||
ref_alias = ref_m.group(1)
|
||||
if ref_alias not in [a[0] for a in alias_blocks]:
|
||||
# Try to find a matching region variable for this alias
|
||||
# e.g. alias "mad_3" might match var.region_secondary
|
||||
region_ref = 'var.region' # fallback to primary region var
|
||||
for vname, vref in var_region_map.items():
|
||||
if vname != "region": # prefer non-primary region vars for aliases
|
||||
region_ref = vref
|
||||
break
|
||||
alias_blocks.append((ref_alias, region_ref))
|
||||
|
||||
passphrase = oci_cfg.get('pass_phrase', '')
|
||||
cred_block = f''' tenancy_ocid = "{oci_cfg.get('tenancy', '')}"
|
||||
user_ocid = "{oci_cfg.get('user', '')}"
|
||||
fingerprint = "{oci_cfg.get('fingerprint', '')}"
|
||||
private_key_path = "{oci_cfg.get('key_file', '')}"'''
|
||||
if passphrase:
|
||||
cred_block += f'\n private_key_password = "{passphrase}"'
|
||||
|
||||
# Primary region: MUST use var.region — model is required to declare it
|
||||
with db() as c:
|
||||
oci_row = c.execute("SELECT region FROM oci_configs WHERE id=?", (ws["oci_config_id"],)).fetchone()
|
||||
primary_region = oci_row["region"] if oci_row else oci_cfg.get("region", "")
|
||||
if not has_region_var:
|
||||
error_msg = (
|
||||
f'ERRO: O código Terraform não declarou variable "region". '
|
||||
f'Isso é obrigatório para garantir que os recursos sejam provisionados na região correta. '
|
||||
f'Adicione no variables.tf: variable "region" {{ type = string; default = "{primary_region}" }}'
|
||||
)
|
||||
log.warning(f"Workspace {wid}: missing variable 'region' — blocking plan")
|
||||
with db() as c:
|
||||
c.execute("UPDATE terraform_workspaces SET status='failed', plan_output=?, error=?, updated_at=datetime('now') WHERE id=?",
|
||||
(error_msg, error_msg, wid))
|
||||
return
|
||||
region_value = 'var.region'
|
||||
|
||||
provider_tf = f'''terraform {{
|
||||
required_providers {{
|
||||
oci = {{
|
||||
source = "oracle/oci"
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
|
||||
provider "oci" {{
|
||||
{cred_block}
|
||||
region = {region_value}
|
||||
}}
|
||||
'''
|
||||
# Add alias providers with same credentials
|
||||
seen_aliases = set()
|
||||
for alias_name, region_ref in alias_blocks:
|
||||
if alias_name in seen_aliases:
|
||||
continue
|
||||
seen_aliases.add(alias_name)
|
||||
provider_tf += f'''
|
||||
provider "oci" {{
|
||||
alias = "{alias_name}"
|
||||
{cred_block}
|
||||
region = {region_ref}
|
||||
}}
|
||||
'''
|
||||
(wdir / "provider.tf").write_text(provider_tf)
|
||||
|
||||
# Auto-generate terraform.tfvars — scan declared variables and provide values from OCI config
|
||||
with db() as c:
|
||||
oci_row = c.execute("SELECT compartment_id, region, ssh_public_key FROM oci_configs WHERE id=?", (ws["oci_config_id"],)).fetchone()
|
||||
comp_id = ws["compartment_id"] if ws["compartment_id"] else (_safe_dec(oci_row["compartment_id"]) if oci_row and oci_row["compartment_id"] else "")
|
||||
try:
|
||||
ssh_pub_key = oci_row["ssh_public_key"] if oci_row else ""
|
||||
except (IndexError, KeyError):
|
||||
ssh_pub_key = ""
|
||||
ssh_pub_key = ssh_pub_key or ""
|
||||
|
||||
# Scan all declared variables in .tf files
|
||||
declared_vars = set()
|
||||
for tf_file in sorted(wdir.glob("*.tf")):
|
||||
if tf_file.name in ("provider.tf", "terraform.tfvars"):
|
||||
continue
|
||||
content = tf_file.read_text()
|
||||
for vm in _re2.finditer(r'variable\s+"([^"]+)"', content):
|
||||
declared_vars.add(vm.group(1))
|
||||
|
||||
# Map known variable names to OCI config values
|
||||
var_values = {"compartment_id": comp_id}
|
||||
oci_var_map = {
|
||||
"tenancy_ocid": oci_cfg.get("tenancy", ""),
|
||||
"tenancy_id": oci_cfg.get("tenancy", ""),
|
||||
"user_ocid": oci_cfg.get("user", ""),
|
||||
"fingerprint": oci_cfg.get("fingerprint", ""),
|
||||
"private_key_path": oci_cfg.get("key_file", ""),
|
||||
"ssh_public_key": ssh_pub_key,
|
||||
"ssh_authorized_keys": ssh_pub_key,
|
||||
}
|
||||
# NOTE: "region" is intentionally NOT included — Terraform uses the default from variable declarations
|
||||
for vname, vval in oci_var_map.items():
|
||||
if vname in declared_vars and vval:
|
||||
var_values[vname] = vval
|
||||
|
||||
tfvars_lines = [f'{k} = "{v}"' for k, v in var_values.items()]
|
||||
(wdir / "terraform.tfvars").write_text("\n".join(tfvars_lines) + "\n")
|
||||
|
||||
output_lines = []
|
||||
|
||||
def _update_output(text):
|
||||
output_lines.append(text)
|
||||
with db() as c:
|
||||
c.execute(f"UPDATE terraform_workspaces SET {status_col}=?, updated_at=datetime('now') WHERE id=?",
|
||||
("\n".join(output_lines[-200:]), wid))
|
||||
|
||||
# terraform init
|
||||
_update_output("$ terraform init ...")
|
||||
proc_init = await asyncio.create_subprocess_exec(
|
||||
"terraform", f"-chdir={wdir}", "init", "-no-color", "-input=false",
|
||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT)
|
||||
while True:
|
||||
line = await proc_init.stdout.readline()
|
||||
if not line:
|
||||
break
|
||||
_update_output(line.decode(errors="replace").rstrip())
|
||||
await proc_init.wait()
|
||||
if proc_init.returncode != 0:
|
||||
_update_output(f"\n❌ terraform init failed (exit {proc_init.returncode})")
|
||||
with db() as c:
|
||||
c.execute("UPDATE terraform_workspaces SET status='failed', error=?, updated_at=datetime('now') WHERE id=?",
|
||||
("terraform init failed", wid))
|
||||
return
|
||||
|
||||
# terraform action
|
||||
_update_output(f"\n$ terraform {action} ...")
|
||||
cmd = ["terraform", f"-chdir={wdir}", action, "-no-color", "-input=false"]
|
||||
if action in ("apply", "destroy"):
|
||||
cmd.append("-auto-approve")
|
||||
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT)
|
||||
_running_terraform[wid] = proc
|
||||
|
||||
while True:
|
||||
line = await proc.stdout.readline()
|
||||
if not line:
|
||||
break
|
||||
_update_output(line.decode(errors="replace").rstrip())
|
||||
await proc.wait()
|
||||
_running_terraform.pop(wid, None)
|
||||
|
||||
if proc.returncode == 0:
|
||||
output_lines.append(f"\n✅ terraform {action} completed successfully")
|
||||
full_output = "\n".join(output_lines)
|
||||
with db() as c:
|
||||
c.execute(f"UPDATE terraform_workspaces SET {status_col}=?, status=?, updated_at=datetime('now') WHERE id=?",
|
||||
(full_output, final_ok, wid))
|
||||
_audit(user["id"], user["username"], f"terraform_{action}", wid, f"status={final_ok}")
|
||||
else:
|
||||
output_lines.append(f"\n❌ terraform {action} failed (exit {proc.returncode})")
|
||||
full_output = "\n".join(output_lines)
|
||||
with db() as c:
|
||||
c.execute(f"UPDATE terraform_workspaces SET {status_col}=?, status='failed', error=?, updated_at=datetime('now') WHERE id=?",
|
||||
(full_output, f"terraform {action} failed (exit {proc.returncode})", wid))
|
||||
|
||||
except Exception as e:
|
||||
log.error(f"Terraform exec error: {e}")
|
||||
with db() as c:
|
||||
c.execute("UPDATE terraform_workspaces SET status='failed', error=?, updated_at=datetime('now') WHERE id=?",
|
||||
(str(e)[:500], wid))
|
||||
|
||||
|
||||
# ── Audit ─────────────────────────────────────────────────────────────────────
|
||||
Reference in New Issue
Block a user