GustavoNogueira/A-Team-Security-Infra-Agent-Engineer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Oracle CIS report engine, CIS engine auto-update, and granular report params

Browse Source 
Replace lightweight cis_runner.py with Oracle's official cis_reports.py engine
(6660 lines, 48 CIS + 11 OBP checks). Add granular execution parameters (Level,
OBP, Raw Data, Redact), per-report file storage with category browser, tenancy
filter in Downloads, and individual file download.

Add CIS Engine auto-update: check/download latest cis_reports.py from Oracle's
GitHub repo with automatic patch reapplication (admin UI card + 3 new endpoints).
Save engine version to app_settings on startup.

Update README to v1.8 with new features, API endpoints, and versioning table.
This commit is contained in:
nogueiraguh
2026-03-05 11:11:31 -03:00
parent 5038403d08
commit 973ff65989
5 changed files with 303 additions and 1435 deletions
Download Patch File Download Diff File Expand all files Collapse all files

215
backend/app.py
View File

@@ -262,13 +262,24 @@ def init_db():
CREATE TABLE IF NOT EXISTS reports (
id TEXT PRIMARY KEY, user_id TEXT NOT NULL,
tenancy_name TEXT NOT NULL, config_id TEXT,
mcp_server_id TEXT,
level INTEGER DEFAULT 2,
obp_checks INTEGER DEFAULT 0,
raw_data INTEGER DEFAULT 0,
redact_output INTEGER DEFAULT 0,
status TEXT DEFAULT 'pending', progress TEXT DEFAULT '',
report_data TEXT,
html_path TEXT, json_path TEXT,
created_at TEXT DEFAULT (datetime('now')),
completed_at TEXT, error_msg TEXT
);
CREATE TABLE IF NOT EXISTS report_files (
id TEXT PRIMARY KEY,
report_id TEXT NOT NULL,
file_name TEXT NOT NULL,
file_path TEXT NOT NULL,
file_type TEXT NOT NULL,
file_category TEXT NOT NULL,
file_size INTEGER DEFAULT 0
);
CREATE TABLE IF NOT EXISTS adb_vector_configs (
id TEXT PRIMARY KEY, user_id TEXT NOT NULL,
config_name TEXT NOT NULL,
@@ -362,7 +373,8 @@ def init_db():
c.execute(f"ALTER TABLE adb_vector_configs ADD COLUMN {col}")
except sqlite3.OperationalError:
pass
for col in ["progress TEXT DEFAULT ''"]:
for col in ["progress TEXT DEFAULT ''", "level INTEGER DEFAULT 2", "obp_checks INTEGER DEFAULT 0",
"raw_data INTEGER DEFAULT 0", "redact_output INTEGER DEFAULT 0"]:
try:
c.execute(f"ALTER TABLE reports ADD COLUMN {col}")
except sqlite3.OperationalError:
@@ -473,7 +485,8 @@ class ChatMsg(BaseModel):
frequency_penalty: Optional[float] = None; presence_penalty: Optional[float] = None
use_tools: Optional[bool] = True
class RunReportReq(BaseModel):
config_id: str; mcp_server_id: Optional[str] = None; regions: Optional[List[str]] = None
config_id: str; regions: Optional[List[str]] = None
level: int = 2; obp: bool = False; raw: bool = False; redact_output: bool = False
class GenAIConfigReq(BaseModel):
name: str = "default"
oci_config_id: str; model_id: str; model_ocid: Optional[str] = None
@@ -1974,38 +1987,50 @@ async def delete_embedding(vid: str, doc_id: str, table_name: str = Query(""), u
raise HTTPException(500, f"Erro ao deletar: {str(e)[:500]}")
# ── Reports ───────────────────────────────────────────────────────────────────
def _classify_report_file(fname: str) -> str:
"""Classify a report file into a category based on its filename."""
fl = fname.lower()
if "summary_report" in fl: return "summary"
if "error_report" in fl or "error" in fl and fl.endswith(".csv"): return "error"
if fl.startswith("obp_") and "findings" in fl: return "obp_finding"
if fl.startswith("obp_") and "best_practices" in fl: return "obp_best_practice"
if fl.startswith("obp_"): return "obp_finding"
if fl.startswith("raw_data_"): return "raw_data"
if fl.startswith("cis_"): return "cis_finding"
if "consolidated_report" in fl: return "consolidated"
if fl.endswith(".png"): return "diagram"
return "other"
@app.post("/api/reports/run")
async def run_report(req: RunReportReq, bg: BackgroundTasks, u=Depends(require("admin","user"))):
if req.level not in (1, 2): raise HTTPException(400, "Level must be 1 or 2")
with db() as c:
cfg = c.execute("SELECT * FROM oci_configs WHERE id=?",(req.config_id,)).fetchone()
if not cfg: raise HTTPException(404, "Config não encontrada")
rid = str(uuid.uuid4())
c.execute("INSERT INTO reports (id,user_id,tenancy_name,config_id,mcp_server_id,status) VALUES (?,?,?,?,?,?)",
(rid, u["id"], cfg["tenancy_name"], req.config_id, req.mcp_server_id, "running"))
bg.add_task(_exec_report, rid, dict(cfg), req.regions, req.mcp_server_id)
c.execute("INSERT INTO reports (id,user_id,tenancy_name,config_id,level,obp_checks,raw_data,redact_output,status) VALUES (?,?,?,?,?,?,?,?,?)",
(rid, u["id"], cfg["tenancy_name"], req.config_id, req.level, int(req.obp), int(req.raw), int(req.redact_output), "running"))
bg.add_task(_exec_report, rid, dict(cfg), req.regions, req.level, req.obp, req.raw, req.redact_output)
_audit(u["id"], u["username"], "run_report", rid)
return {"report_id": rid, "status": "running"}
async def _exec_report(rid, cfg, regions, mcp_server_id):
async def _exec_report(rid, cfg, regions, level=2, obp=False, raw=False, redact_output=False):
rdir = REPORTS / rid; rdir.mkdir(parents=True, exist_ok=True)
config_path = str(OCI_DIR / cfg["id"] / "config")
try:
cmd = ["python3", "-u", "/app/cis_runner.py", "--config", config_path,
"--output", str(rdir), "--tenancy-name", cfg["tenancy_name"]]
cmd = ["python3", "-u", "/app/cis_reports.py",
"-c", config_path,
"--report-directory", str(rdir),
"--level", str(level),
"--print-to-screen", "True",
"--report-summary-json"]
if regions: cmd += ["--regions", ",".join(regions)]
if mcp_server_id:
with db() as c:
mcp = c.execute("SELECT * FROM mcp_servers WHERE id=?",(mcp_server_id,)).fetchone()
if mcp:
if mcp["module_path"]: cmd += ["--mcp-module", mcp["module_path"]]
if mcp.get("linked_adb_id"):
adb_cfg = c.execute("SELECT * FROM adb_vector_configs WHERE id=?",(mcp["linked_adb_id"],)).fetchone()
if adb_cfg:
cmd += ["--adb-dsn", adb_cfg["dsn"], "--adb-user", adb_cfg["username"]]
if adb_cfg.get("wallet_dir"): cmd += ["--adb-wallet", adb_cfg["wallet_dir"]]
if obp: cmd.append("--obp")
if raw: cmd.append("--raw")
if redact_output: cmd.append("--redact-output")
proc = await asyncio.create_subprocess_exec(*cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
_running_reports[rid] = proc
# Read stdout line by line for real-time progress
progress_lines = []
try:
while True:
@@ -2017,27 +2042,41 @@ async def _exec_report(rid, cfg, regions, mcp_server_id):
progress_lines.append(text)
with db() as c:
c.execute("UPDATE reports SET progress=? WHERE id=?",
("\n".join(progress_lines[-30:]), rid))
("\n".join(progress_lines[-50:]), rid))
await proc.wait()
finally:
_running_reports.pop(rid, None)
stderr_data = await proc.stderr.read()
jp = rdir / "report.json"; hp = rdir / "report.html"
# Check if cancelled
with db() as c:
cur_status = c.execute("SELECT status FROM reports WHERE id=?", (rid,)).fetchone()
if cur_status and cur_status["status"] == "cancelled":
return # Already marked as cancelled by the cancel endpoint
with db() as c:
if proc.returncode == 0 and jp.exists():
c.execute("UPDATE reports SET status='completed',progress=?,report_data=?,html_path=?,json_path=?,completed_at=datetime('now') WHERE id=?",
("\n".join(progress_lines), jp.read_text()[:500_000], str(hp) if hp.exists() else None, str(jp), rid))
_config_log("oci", cfg["id"], cfg["tenancy_name"], "success", "report", f"Relatório concluído: {rid}")
else:
err = (stderr_data.decode(errors="replace") if stderr_data else "Unknown")[:2000]
return
if proc.returncode == 0:
# Scan output directory for all generated files
html_path = None; json_path = None
with db() as c:
for fpath in rdir.iterdir():
if fpath.is_file():
fname = fpath.name
ftype = fpath.suffix.lstrip(".")
category = _classify_report_file(fname)
fsize = fpath.stat().st_size
c.execute("INSERT INTO report_files (id,report_id,file_name,file_path,file_type,file_category,file_size) VALUES (?,?,?,?,?,?,?)",
(str(uuid.uuid4()), rid, fname, str(fpath), ftype, category, fsize))
if "summary_report" in fname and fname.endswith(".html"):
html_path = str(fpath)
elif "summary_report" in fname and fname.endswith(".json"):
json_path = str(fpath)
c.execute("UPDATE reports SET status='completed',progress=?,html_path=?,json_path=?,completed_at=datetime('now') WHERE id=?",
("\n".join(progress_lines), html_path, json_path, rid))
_config_log("oci", cfg["id"], cfg["tenancy_name"], "success", "report", f"Report completed: {rid}")
else:
err = (stderr_data.decode(errors="replace") if stderr_data else "Unknown")[:2000]
with db() as c:
c.execute("UPDATE reports SET status='failed',progress=?,error_msg=?,completed_at=datetime('now') WHERE id=?",
("\n".join(progress_lines), err, rid))
_config_log("oci", cfg["id"], cfg["tenancy_name"], "error", "report", err)
_config_log("oci", cfg["id"], cfg["tenancy_name"], "error", "report", err)
except Exception as e:
_running_reports.pop(rid, None)
with db() as c:
@@ -2047,7 +2086,7 @@ async def _exec_report(rid, cfg, regions, mcp_server_id):
@app.get("/api/reports")
async def list_reports(u=Depends(current_user)):
with db() as c:
q = "SELECT id,user_id,tenancy_name,status,progress,mcp_server_id,created_at,completed_at,error_msg FROM reports"
q = "SELECT id,user_id,tenancy_name,status,progress,level,obp_checks,raw_data,redact_output,created_at,completed_at,error_msg FROM reports"
rows = c.execute(q+" ORDER BY created_at DESC").fetchall() if u["role"]=="admin" \
else c.execute(q+" WHERE user_id=? ORDER BY created_at DESC",(u["id"],)).fetchall()
return [dict(r) for r in rows]
@@ -2065,7 +2104,7 @@ async def cancel_report(rid: str, u=Depends(require("admin", "user"))):
r = c.execute("SELECT status FROM reports WHERE id=?", (rid,)).fetchone()
if not r: raise HTTPException(404)
if r["status"] != "running":
raise HTTPException(400, "Relatório não está em execução")
raise HTTPException(400, "Report is not running")
proc = _running_reports.get(rid)
if proc:
try:
@@ -2078,7 +2117,7 @@ async def cancel_report(rid: str, u=Depends(require("admin", "user"))):
pass
_running_reports.pop(rid, None)
with db() as c:
c.execute("UPDATE reports SET status='cancelled',error_msg='Cancelado pelo usuário',completed_at=datetime('now') WHERE id=?", (rid,))
c.execute("UPDATE reports SET status='cancelled',error_msg='Cancelled by user',completed_at=datetime('now') WHERE id=?", (rid,))
_audit(u["id"], u["username"], "cancel_report", rid)
return {"ok": True, "status": "cancelled"}
@@ -2088,11 +2127,34 @@ async def get_report(rid, u=Depends(current_user)):
if not r: raise HTTPException(404)
if u["role"]!="admin" and r["user_id"]!=u["id"]: raise HTTPException(403)
d=dict(r)
if d.get("report_data"):
try: d["report_data"]=json.loads(d["report_data"])
except: pass
d.pop("report_data", None)
d.pop("mcp_server_id", None)
return d
@app.get("/api/reports/{rid}/files")
async def list_report_files(rid: str, u=Depends(current_user)):
with db() as c:
r = c.execute("SELECT user_id FROM reports WHERE id=?", (rid,)).fetchone()
if not r: raise HTTPException(404)
if u["role"] != "admin" and r["user_id"] != u["id"]: raise HTTPException(403)
files = c.execute(
"SELECT id,file_name,file_type,file_category,file_size FROM report_files WHERE report_id=? ORDER BY file_category,file_name",
(rid,)
).fetchall()
return [dict(f) for f in files]
@app.get("/api/reports/{rid}/files/{fid}/download")
async def download_report_file(rid: str, fid: str, u=Depends(current_user)):
with db() as c:
r = c.execute("SELECT user_id FROM reports WHERE id=?", (rid,)).fetchone()
if not r: raise HTTPException(404)
if u["role"] != "admin" and r["user_id"] != u["id"]: raise HTTPException(403)
f = c.execute("SELECT * FROM report_files WHERE id=? AND report_id=?", (fid, rid)).fetchone()
if not f: raise HTTPException(404)
p = Path(f["file_path"])
if not p.exists(): raise HTTPException(404, "File not found on disk")
return FileResponse(p, filename=f["file_name"])
@app.get("/api/reports/{rid}/html")
async def report_html(rid):
with db() as c: r=c.execute("SELECT html_path FROM reports WHERE id=?",(rid,)).fetchone()
@@ -2446,6 +2508,73 @@ async def delete_prompt(pid: str, u=Depends(require("admin"))):
c.execute("DELETE FROM system_prompts WHERE id=?", (pid,))
return {"ok": True}
# ── CIS Engine Version Management ─────────────────────────────────────────────
CIS_REPORTS_PATH = Path("/app/cis_reports.py")
CIS_GITHUB_RAW = "https://raw.githubusercontent.com/oci-landing-zones/oci-cis-landingzone-quickstart/main/scripts/cis_reports.py"
CIS_PATCHES = [
{"name": "region_none_check_1",
"original": "elif region.region_name in self.__regions_to_run_in or self.__run_in_all_regions:",
"patched": "elif self.__run_in_all_regions or (self.__regions_to_run_in and region.region_name in self.__regions_to_run_in):"},
{"name": "region_none_check_2",
"original": "if self.__home_region not in self.__regions_to_run_in:",
"patched": "if not self.__run_in_all_regions and self.__regions_to_run_in and self.__home_region not in self.__regions_to_run_in:"},
]
def _read_cis_version(content: str = None) -> dict:
if content is None:
if not CIS_REPORTS_PATH.exists(): return {"version": "unknown", "date": "unknown"}
content = CIS_REPORTS_PATH.read_text(encoding="utf-8", errors="replace")
ver = re.search(r'RELEASE_VERSION\s*=\s*["\']([^"\']+)["\']', content)
dt = re.search(r'UPDATED_DATE\s*=\s*["\']([^"\']+)["\']', content)
return {"version": ver.group(1) if ver else "unknown", "date": dt.group(1) if dt else "unknown"}
@app.get("/api/cis-engine/version")
async def cis_engine_version(u=Depends(current_user)):
info = _read_cis_version()
return {"version": info["version"], "updated_date": info["date"], "file_path": str(CIS_REPORTS_PATH)}
@app.get("/api/cis-engine/check-update")
async def cis_engine_check_update(u=Depends(require("admin"))):
import requests as req
local = _read_cis_version()
try:
resp = req.get(CIS_GITHUB_RAW, timeout=30)
resp.raise_for_status()
remote = _read_cis_version(resp.text)
except Exception as e:
raise HTTPException(502, f"Failed to check GitHub: {str(e)[:300]}")
return {
"local_version": local["version"], "local_date": local["date"],
"remote_version": remote["version"], "remote_date": remote["date"],
"update_available": remote["version"] != local["version"]
}
@app.post("/api/cis-engine/update")
async def cis_engine_update(u=Depends(require("admin"))):
import requests as req
old = _read_cis_version()
try:
resp = req.get(CIS_GITHUB_RAW, timeout=60)
resp.raise_for_status()
content = resp.text
except Exception as e:
raise HTTPException(502, f"Failed to download from GitHub: {str(e)[:300]}")
new = _read_cis_version(content)
# Apply patches
patches_applied = []
for p in CIS_PATCHES:
if p["original"] in content:
content = content.replace(p["original"], p["patched"])
patches_applied.append(p["name"])
CIS_REPORTS_PATH.write_text(content, encoding="utf-8")
# Save version info
with db() as c:
c.execute("INSERT INTO app_settings (key,value,updated_at) VALUES ('cis_engine_version',?,datetime('now')) ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at",
(new["version"],))
log.info(f"CIS engine updated: {old['version']} → {new['version']}, patches: {patches_applied}")
_audit(u["id"], u["username"], "cis_engine_update", None, f"{old['version']} → {new['version']}")
return {"ok": True, "old_version": old["version"], "new_version": new["version"], "patches_applied": patches_applied}
@app.get("/api/health")
async def health():
return {"status":"ok","ts":datetime.utcnow().isoformat(),"version":VERSION}
@@ -2471,6 +2600,16 @@ async def startup():
)
log.info(f"Auto-registered CIS Compliance Scanner MCP server (id={mid})")
# Save CIS engine version to app_settings
try:
vi = _read_cis_version()
with db() as c:
c.execute("INSERT INTO app_settings (key,value,updated_at) VALUES ('cis_engine_version',?,datetime('now')) ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at",
(vi["version"],))
log.info(f"CIS engine version: {vi['version']} ({vi['date']})")
except Exception as e:
log.warning(f"Could not read CIS engine version: {e}")
log.info(f"OCI CIS AI Agent v{VERSION} API started")
if __name__ == "__main__":