commit 2966439b107dd1cd2875e8b94692314129573ebb Author: Rodrigo Date: Thu May 28 14:56:51 2026 -0300 Initial OCI Data Safe ADB inventory tool diff --git a/.gitignore b/.gitignore new file mode 100755 index 0000000..e52351b --- /dev/null +++ b/.gitignore @@ -0,0 +1,35 @@ +# Python +__pycache__/ +*.py[cod] +*.pyo +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ + +# Virtual environments +.venv/ +venv/ +env/ + +# Local OCI/security files +.oci/ +*.pem +*.key + +# Generated reports and packages +*.csv +*.json +*.log +*.tmp +*.zip +output/ +tmp/ + +# OS/editor files +.DS_Store +Thumbs.db +.idea/ +.vscode/ + +# Local backups +backup_before_codex_fix/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100755 index 0000000..ef6a0ab --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,9 @@ +# Changelog + +## 0.1.0 + +- Initial inventory scripts for Autonomous Databases and Oracle Data Safe targets. +- Added consolidated report with `datasafe_registered=true/false`. +- Added split report generation for registered and not registered databases. +- Added region, multi-region, all-region, compartment, and compartment subtree scan options. +- Added progress bar, throttling retries, CSV/JSON output, and OCI SDK support. diff --git a/INSTRUCOES_USO.md b/INSTRUCOES_USO.md new file mode 100755 index 0000000..a923502 --- /dev/null +++ b/INSTRUCOES_USO.md @@ -0,0 +1,91 @@ +# Instrucoes de Uso + +## O que este pacote faz + +Gera um inventario de Autonomous Databases e indica se cada banco esta cadastrado no Oracle Data Safe. + +Script recomendado: + +```text +adb_datasafe_inventory.py +``` + +Saidas principais: + +- `adb_datasafe_inventory.csv`: inventario completo. +- `adb_in_datasafe.csv`: apenas ADBs cadastrados no Data Safe. +- `adb_not_in_datasafe.csv`: apenas ADBs nao cadastrados no Data Safe. + +## Linux + +```bash +cd oci-datasafe-adb-inventory +python3 -m pip install -r requirements.txt + +python3 ./adb_datasafe_inventory.py \ + --profile DEFAULT \ + --region sa-saopaulo-1 \ + --workers 3 \ + --retries 8 \ + --retry-base-sleep 5 \ + --split-outputs \ + --output ./adb_datasafe_inventory.csv +``` + +## Windows PowerShell + +```powershell +cd .\oci-datasafe-adb-inventory +python -m pip install -r .\requirements.txt + +python .\adb_datasafe_inventory.py ` + --profile DEFAULT ` + --region sa-saopaulo-1 ` + --workers 3 ` + --retries 8 ` + --retry-base-sleep 5 ` + --split-outputs ` + --output .\adb_datasafe_inventory.csv +``` + +Se `python` nao existir no Windows, use `py` no lugar. + +## OCI Cloud Shell + +```bash +unzip oci-datasafe-adb-inventory.zip +cd oci-datasafe-adb-inventory +python3 -m pip install --user -r requirements.txt + +python3 ./adb_datasafe_inventory.py \ + --region sa-saopaulo-1 \ + --workers 3 \ + --retries 8 \ + --retry-base-sleep 5 \ + --split-outputs \ + --output ./adb_datasafe_inventory.csv +``` + +## Opcoes uteis + +- Trocar regiao: `--region us-ashburn-1` +- Rodar mais de uma regiao: `--regions sa-saopaulo-1,us-ashburn-1` +- Gerar todas as regioes subscribed: `--all-regions` +- Rodar apenas em um compartment: `--compartment-id ocid1.compartment.oc1..aaaa` +- Rodar em um compartment e seus subcompartments: `--compartment-id ocid1.compartment.oc1..aaaa --compartment-subtree` +- Reduzir throttling: use `--workers 2` +- Desativar barra de progresso: `--no-progress` +- Gerar somente cadastrados: `--filter registered` +- Gerar somente nao cadastrados: `--filter not_registered` + +## Compactar para envio + +No Linux ou Cloud Shell: + +```bash +cd .. +zip -r oci-datasafe-adb-inventory.zip oci-datasafe-adb-inventory \ + -x "oci-datasafe-adb-inventory/*.csv" \ + -x "oci-datasafe-adb-inventory/*.json" \ + -x "oci-datasafe-adb-inventory/__pycache__/*" +``` diff --git a/INSTRUCOES_USO.pdf b/INSTRUCOES_USO.pdf new file mode 100755 index 0000000..c373e1d --- /dev/null +++ b/INSTRUCOES_USO.pdf @@ -0,0 +1,118 @@ +%PDF-1.4 +%“Œ‹ž ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R /F2 3 0 R /F3 4 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font +>> +endobj +4 0 obj +<< +/BaseFont /Courier /Encoding /WinAnsiEncoding /Name /F3 /Subtype /Type1 /Type /Font +>> +endobj +5 0 obj +<< +/Contents 11 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 10 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +6 0 obj +<< +/Contents 12 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 10 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +7 0 obj +<< +/Contents 13 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 10 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +8 0 obj +<< +/PageMode /UseNone /Pages 10 0 R /Type /Catalog +>> +endobj +9 0 obj +<< +/Author (\(anonymous\)) /CreationDate (D:20260527154328-03'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260527154328-03'00') /Producer (ReportLab PDF Library - \(opensource\)) + /Subject (\(unspecified\)) /Title (Instrucoes de Uso - OCI Data Safe ADB Inventory) /Trapped /False +>> +endobj +10 0 obj +<< +/Count 3 /Kids [ 5 0 R 6 0 R 7 0 R ] /Type /Pages +>> +endobj +11 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 767 +>> +stream +GauaP-&tQB>4D<)2CRI([pgi!'&&-pQu%!ONi;+VB@sS:YF=lf!(<\$PO9dn!QjYQ#L,O,.LBu.3Z*d5]tTti'VU_RFFi$&1Nemkfak&?QL+fOuR#EROHTYb>H%P'KZBP7GL$A+F'egoWkr%I,.c($+>TD_Ugh28[UUI>+\JCnW8cp_LJVFDnc0t:#kliqfTr?PI8)kDofS\`AM5Cp`LObeCVeS],qUIj@tq`V@#Ui3)<3RM#DXnAg(rWCL;V/fY-KQ]Z87lendstream +endobj +12 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 1192 +>> +stream +Gb!SkgMZ%0&:Ml+bY%uG=>HU=?+b+1c)+Sc5h!_A?NR9JL2Jd(e[ki(bPSjW!B&.Z;!W4p.j&6.5KZOr_:,u"ni7$cNk%M#_q1UOt(31=i#u;GcBH4<;"j4_2%`_u>hLI2jo9`!%O!4ic)Y&KUFTnP45CS>#i<3DA.g[;$Ve]T&eonouYS'.O\$C224F@['5@F&7O1=.,B&6h+j7Za%<@o?KYk\Io?Q1S;8^ip$=iq=80%1rgZB-;],YWKB^CcCPBEq'.grDB<%=prRM,I]:d@%#hqlPDI+d7&i?A/Ncie>6nk=/cl;[MVtEA!20$9IkIN&"d,f-[+)7\F%>=?sbVZ+L(-.oKbD/QS9q@RejbA!LEh0rMHpp0HdRg6I9>X\Z9UbV^WtO5aT4\\6h>[3WmqII1)A`[ca1anWKJ$[l[H>i/+oE@o@S5;V?iq"Y&<(=D=b1Qu-O^6kERgdL`Md*o$)drY@:4/_tt3?@&?Y3D+<:ZX[^S3B.m8o+q$9,YH7N2,h2b"Sr(/:`]#'mDR;[-tG$];ZNep8&a.bhKfF?`\aB6uCGiC:hBD'a["I4TMF^^KlK&&RM9$aYMlDn1q_qC?HAiI++6Gt"4?NWXmh1B5:ln:)JU@2c8:h64Wi9&X0\2S$g_Zq`*_T;Z2:pS>ee;]FHZ/EekQW44*+N(=a,KD4mRP9(VZ,bdkf8s97S_;esR4LI2*%4J4YJM(T(V2"5=X,\)@36ZEi;,m7MNEY/O'1rq0;7.]TC)T0KAn:DXM-t~>endstream +endobj +13 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 606 +>> +stream +Gat%_CMVZY(ko`f>t&oa;Frm,n\AM3&F):Df';LgW>W'A0."TR1en'-7P,_,A-=,$]>Gi-K^+DlQgI;%&@:S(?)d17P/1]Mjg0A\HW:C[BM_A.QaNI`J$r9YF^/8p7(tAZ`ap4hmQfkGGp/R79TA;E4(b*^+*Y#kLr"&lFtuo=RpZlk2V:M*'h:<55'r^;,Dt].CW^a@RR];-"h7WNEI>9]ZgCC1WL-68blFKkW.NL5U*&C5^fQF/tX=j$%R)TR%C=peDJpH6bUY+BU8u'#eS-KKI))8J/QnVd/doF=s+))Y9Vn9R=!PC9"\a>DKRd<^G?ld_%Pkdge?a_]nJP4;E'^2XTFh_,XWV*apsoY0Opr$UK0Zq.8@<*jpOf&]=4Jit9D&mI5`[%umPQeh`r%T5p4SlhPoj)SS04%/p94jWd=~>endstream +endobj +xref +0 14 +0000000000 65535 f +0000000061 00000 n +0000000112 00000 n +0000000219 00000 n +0000000331 00000 n +0000000436 00000 n +0000000641 00000 n +0000000846 00000 n +0000001051 00000 n +0000001120 00000 n +0000001434 00000 n +0000001506 00000 n +0000002364 00000 n +0000003648 00000 n +trailer +<< +/ID +[] +% ReportLab generated PDF document -- digest (opensource) + +/Info 9 0 R +/Root 8 0 R +/Size 14 +>> +startxref +4345 +%%EOF diff --git a/README.md b/README.md new file mode 100755 index 0000000..899b6b0 --- /dev/null +++ b/README.md @@ -0,0 +1,174 @@ +# OCI Data Safe ADB Inventory + +Inventory tool for Oracle Cloud Infrastructure Autonomous Databases and Oracle Data Safe target registration. + +The tool scans Autonomous Databases in an OCI tenancy, region, compartment, or compartment subtree, compares them with Oracle Data Safe target databases, and exports CSV or JSON reports showing whether each database is registered in Data Safe. + +## Features + +- Inventory Autonomous Databases by region, multiple regions, or all subscribed regions. +- Limit scans to a specific compartment or recursively scan a compartment subtree. +- Compare Autonomous Database OCIDs against Oracle Data Safe target databases. +- Export a consolidated report with `datasafe_registered=true/false`. +- Optionally generate separate reports for registered and not registered databases. +- Show terminal progress with processed compartments and running counts. +- Retry transient OCI API errors such as throttling (`429 TooManyRequests`). +- Support OCI SDK by default, with optional OCI CLI fallback. + +## Requirements + +- Python 3.9 or later. +- OCI Python SDK, installed from `requirements.txt`. +- An OCI config profile in `~/.oci/config`, or OCI instance/resource principal authentication. +- Permissions to list: + - IAM compartments + - Autonomous Databases + - Oracle Data Safe target databases + +Install dependencies: + +```bash +python3 -m pip install -r requirements.txt +``` + +On Windows, use `python` or `py` instead of `python3` depending on your installation. + +## Recommended Command + +Run a scan for one region and generate the consolidated report plus the two split reports: + +```bash +python3 ./adb_datasafe_inventory.py \ + --profile DEFAULT \ + --region sa-saopaulo-1 \ + --workers 3 \ + --retries 8 \ + --retry-base-sleep 5 \ + --split-outputs \ + --output ./adb_datasafe_inventory.csv +``` + +Generated files: + +- `adb_datasafe_inventory.csv`: consolidated inventory. +- `adb_in_datasafe.csv`: only Autonomous Databases registered in Data Safe. +- `adb_not_in_datasafe.csv`: only Autonomous Databases not registered in Data Safe. + +## Scan A Specific Compartment + +Scan only one compartment: + +```bash +python3 ./adb_datasafe_inventory.py \ + --profile DEFAULT \ + --region sa-saopaulo-1 \ + --compartment-id ocid1.compartment.oc1..example \ + --output ./adb_datasafe_inventory.csv +``` + +Scan one compartment and all of its subcompartments recursively: + +```bash +python3 ./adb_datasafe_inventory.py \ + --profile DEFAULT \ + --region sa-saopaulo-1 \ + --compartment-id ocid1.compartment.oc1..example \ + --compartment-subtree \ + --output ./adb_datasafe_inventory.csv +``` + +## Other Common Options + +Scan multiple regions: + +```bash +python3 ./adb_datasafe_inventory.py \ + --profile DEFAULT \ + --regions sa-saopaulo-1,us-ashburn-1 \ + --output ./adb_datasafe_inventory.csv +``` + +Scan all subscribed regions: + +```bash +python3 ./adb_datasafe_inventory.py \ + --profile DEFAULT \ + --all-regions \ + --output ./adb_datasafe_inventory_all_regions.csv +``` + +Generate only registered databases: + +```bash +python3 ./adb_datasafe_inventory.py \ + --profile DEFAULT \ + --region sa-saopaulo-1 \ + --filter registered \ + --output ./adb_in_datasafe.csv +``` + +Generate only databases not registered in Data Safe: + +```bash +python3 ./adb_datasafe_inventory.py \ + --profile DEFAULT \ + --region sa-saopaulo-1 \ + --filter not_registered \ + --output ./adb_not_in_datasafe.csv +``` + +Reduce API throttling: + +```bash +python3 ./adb_datasafe_inventory.py \ + --profile DEFAULT \ + --region sa-saopaulo-1 \ + --workers 2 \ + --retries 8 \ + --retry-base-sleep 5 \ + --output ./adb_datasafe_inventory.csv +``` + +## Output Columns + +The first columns in the CSV are: + +1. `datasafe_registered` +2. `adb_time_created` +3. `adb_compartment_name` +4. `adb_compartment_path` + +Additional columns include Autonomous Database identifiers, lifecycle state, workload, compartment OCID, Data Safe target information, and associated target OCIDs. + +## Scripts + +- `adb_datasafe_inventory.py`: recommended consolidated inventory script. +- `adb_in_datasafe.py`: legacy helper that exports only registered databases. +- `adb_not_in_datasafe.py`: legacy helper that exports only databases not registered in Data Safe. +- `oci_datasafe_inventory.py`: shared library used by the scripts. + +## OCI Cloud Shell + +In OCI Cloud Shell: + +```bash +unzip oci-datasafe-adb-inventory.zip +cd oci-datasafe-adb-inventory +python3 -m pip install --user -r requirements.txt + +python3 ./adb_datasafe_inventory.py \ + --region sa-saopaulo-1 \ + --workers 3 \ + --retries 8 \ + --retry-base-sleep 5 \ + --split-outputs \ + --output ./adb_datasafe_inventory.csv +``` + +## Notes + +- `--all-regions` can take a long time in tenancies with many compartments. +- Use `--workers` conservatively if the tenancy hits API throttling. +- Use `--no-progress` to disable the terminal progress bar. +- Use `--format json` to export JSON instead of CSV. +- By default, terminated Autonomous Databases and deleted Data Safe targets are excluded. diff --git a/adb_datasafe_inventory.py b/adb_datasafe_inventory.py new file mode 100755 index 0000000..f4da4cf --- /dev/null +++ b/adb_datasafe_inventory.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Consolidated Autonomous Database x Data Safe inventory.""" + +from __future__ import annotations + +import argparse +import sys + +from oci_datasafe_inventory import ( + ProgressBar, + add_common_args, + adb_to_base_row, + build_context, + enrich_missing_target_associations, + field, + get_regions, + list_all_adbs, + list_datasafe_targets, + print_warnings, + target_associated_adb_id, + target_to_base_row, + write_rows, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Gera inventario unico de Autonomous Databases com status de cadastro no Oracle Data Safe." + ) + add_common_args(parser) + parser.add_argument( + "--filter", + choices=["all", "registered", "not_registered"], + default="all", + help="Filtra a saida. Padrao: all.", + ) + parser.add_argument( + "--output", + default="adb_datasafe_inventory.csv", + help="Arquivo de saida. Padrao: adb_datasafe_inventory.csv", + ) + parser.add_argument( + "--split-outputs", + action="store_true", + help="Tambem gera adb_in_datasafe.csv e adb_not_in_datasafe.csv a partir do mesmo inventario.", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + context = build_context(args) + + if args.all_regions and not context.tenancy_id: + print("--all-regions exige --tenancy-id ou tenancy configurada no perfil OCI.", file=sys.stderr) + return 2 + + regions = get_regions( + context.client, + context.tenancy_id, + context.default_region, + args.all_regions, + args.regions, + ) # type: ignore[arg-type] + rows: list[dict[str, object]] = [] + all_errors: list[str] = [] + + for region in regions: + region_label = region or context.default_region or "perfil" + print(f"Regiao {region_label}: consultando Data Safe...", file=sys.stderr) + targets, target_errors = list_datasafe_targets( + context.client, + context.compartments, + root_compartment_id=context.tenancy_id, + region=region, + workers=args.workers, + retries=args.retries, + retry_base_sleep=args.retry_base_sleep, + include_deleted=args.include_deleted_targets, + ) + detail_errors = enrich_missing_target_associations( + context.client, + targets, + region=region, + workers=args.workers, + retries=args.retries, + retry_base_sleep=args.retry_base_sleep, + ) + target_by_adb_id = { + target_associated_adb_id(target): target + for target in targets + if target_associated_adb_id(target) + } + print( + f"Regiao {region_label}: {len(targets)} targets Data Safe encontrados; " + f"{len(target_by_adb_id)} com OCID de ADB associado; " + f"varrendo {len(context.compartments)} compartments...", + file=sys.stderr, + ) + + progress = ProgressBar( + f"Regiao {region_label}", + len(context.compartments), + enabled=not args.no_progress, + ) + stats = {"adbs": 0, "reg": 0, "nao_reg": 0, "erros": 0} + + def update_progress(compartment: dict[str, object], items: list[dict[str, object]], error: str | None) -> None: + stats["adbs"] += len(items) + stats["reg"] += sum(1 for adb in items if field(adb, "id") in target_by_adb_id) + stats["nao_reg"] += sum(1 for adb in items if field(adb, "id") not in target_by_adb_id) + if error: + stats["erros"] += 1 + progress.update( + current_label=str(field(compartment, "name", "display-name", default=field(compartment, "id"))), + stats=stats, + ) + + adbs, adb_errors = list_all_adbs( + context.client, + context.compartments, + region=region, + workers=args.workers, + retries=args.retries, + retry_base_sleep=args.retry_base_sleep, + include_terminated=args.include_terminated_adbs, + progress_callback=update_progress, + ) + progress.finish(stats=stats) + all_errors.extend(target_errors + detail_errors + adb_errors) + + for adb in sorted(adbs, key=lambda item: (field(item, "display-name"), field(item, "id"))): + adb_id = field(adb, "id") + target = target_by_adb_id.get(adb_id) + registered = target is not None + if args.filter == "registered" and not registered: + continue + if args.filter == "not_registered" and registered: + continue + + row = adb_to_base_row(adb, context.compartment_index, region=region) + row["datasafe_registered"] = str(registered).lower() + if target: + row.update(target_to_base_row(target, context.compartment_index)) + rows.append(row) + + write_rows(rows, args.output, args.format) + registered_rows = [row for row in rows if row.get("datasafe_registered") == "true"] + not_registered_rows = [row for row in rows if row.get("datasafe_registered") == "false"] + + if args.split_outputs: + write_rows(registered_rows, "adb_in_datasafe.csv", args.format) + write_rows(not_registered_rows, "adb_not_in_datasafe.csv", args.format) + + print(f"ADBs no inventario: {len(rows)}") + print(f"ADBs cadastrados no Data Safe: {len(registered_rows)}") + print(f"ADBs nao cadastrados no Data Safe: {len(not_registered_rows)}") + print(f"Arquivo gerado: {args.output}") + if args.split_outputs: + print("Arquivos adicionais gerados: adb_in_datasafe.csv, adb_not_in_datasafe.csv") + print_warnings("o levantamento", all_errors) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/adb_in_datasafe.py b/adb_in_datasafe.py new file mode 100755 index 0000000..21811ff --- /dev/null +++ b/adb_in_datasafe.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Report Autonomous Databases that are registered as Data Safe targets.""" + +from __future__ import annotations + +import argparse +import sys + +from oci_datasafe_inventory import ( + add_common_args, + adb_to_base_row, + build_context, + enrich_missing_target_associations, + field, + get_regions, + list_all_adbs, + list_datasafe_targets, + ProgressBar, + print_warnings, + target_associated_adb_id, + target_to_base_row, + write_rows, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Lista Autonomous Databases cadastrados no Oracle Data Safe e seus compartments." + ) + add_common_args(parser) + parser.add_argument( + "--output", + default="adb_in_datasafe.csv", + help="Arquivo de saida. Padrao: adb_in_datasafe.csv", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + context = build_context(args) + + if args.all_regions and not context.tenancy_id: + print("--all-regions exige --tenancy-id ou tenancy configurada no perfil OCI.", file=sys.stderr) + return 2 + + regions = get_regions( + context.client, + context.tenancy_id, + context.default_region, + args.all_regions, + args.regions, + ) # type: ignore[arg-type] + rows: list[dict[str, object]] = [] + all_errors: list[str] = [] + + for region in regions: + region_label = region or context.default_region or "perfil" + print( + f"Regiao {region_label}: consultando Data Safe...", + file=sys.stderr, + ) + targets, target_errors = list_datasafe_targets( + context.client, + context.compartments, + root_compartment_id=context.tenancy_id, + region=region, + workers=args.workers, + retries=args.retries, + retry_base_sleep=args.retry_base_sleep, + include_deleted=args.include_deleted_targets, + ) + detail_errors = enrich_missing_target_associations( + context.client, + targets, + region=region, + workers=args.workers, + retries=args.retries, + retry_base_sleep=args.retry_base_sleep, + ) + registered_adb_ids = { + target_associated_adb_id(target) + for target in targets + if target_associated_adb_id(target) + } + print( + f"Regiao {region_label}: {len(targets)} targets Data Safe encontrados; " + f"varrendo {len(context.compartments)} compartments...", + file=sys.stderr, + ) + + progress = ProgressBar( + f"Regiao {region_label}", + len(context.compartments), + enabled=not args.no_progress, + ) + stats = {"adbs": 0, "reg": 0, "nao_reg": 0, "erros": 0} + + def update_progress(compartment: dict[str, object], items: list[dict[str, object]], error: str | None) -> None: + stats["adbs"] += len(items) + stats["reg"] += sum(1 for adb in items if field(adb, "id") in registered_adb_ids) + stats["nao_reg"] += sum(1 for adb in items if field(adb, "id") not in registered_adb_ids) + if error: + stats["erros"] += 1 + progress.update( + current_label=str(field(compartment, "name", "display-name", default=field(compartment, "id"))), + stats=stats, + ) + + adbs, adb_errors = list_all_adbs( + context.client, + context.compartments, + region=region, + workers=args.workers, + retries=args.retries, + retry_base_sleep=args.retry_base_sleep, + include_terminated=args.include_terminated_adbs, + progress_callback=update_progress, + ) + progress.finish(stats=stats) + all_errors.extend(target_errors + detail_errors + adb_errors) + + adb_by_id = {field(adb, "id"): adb for adb in adbs if field(adb, "id")} + + for target in sorted(targets, key=lambda item: (field(item, "display-name"), field(item, "id"))): + adb_id = target_associated_adb_id(target) + adb = adb_by_id.get(adb_id) + if adb: + row = adb_to_base_row(adb, context.compartment_index, region=region) + row["adb_found_in_inventory"] = "true" + else: + row = { + "region": region or "", + "adb_display_name": "", + "adb_db_name": "", + "adb_ocid": adb_id, + "adb_lifecycle_state": "", + "adb_workload": "", + "adb_is_free_tier": "", + "adb_time_created": "", + "adb_compartment_name": "", + "adb_compartment_path": "", + "adb_compartment_ocid": "", + "adb_found_in_inventory": "false", + } + row.update(target_to_base_row(target, context.compartment_index)) + row["datasafe_registered"] = "true" + rows.append(row) + + write_rows(rows, args.output, args.format) + print(f"ADBs cadastrados no Data Safe: {len(rows)}") + print(f"Arquivo gerado: {args.output}") + print_warnings("o levantamento", all_errors) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/adb_not_in_datasafe.py b/adb_not_in_datasafe.py new file mode 100755 index 0000000..ae44b4c --- /dev/null +++ b/adb_not_in_datasafe.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Report Autonomous Databases that are not registered as Data Safe targets.""" + +from __future__ import annotations + +import argparse +import sys + +from oci_datasafe_inventory import ( + add_common_args, + adb_to_base_row, + build_context, + enrich_missing_target_associations, + field, + get_regions, + list_all_adbs, + list_datasafe_targets, + ProgressBar, + print_warnings, + target_associated_adb_id, + write_rows, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Lista Autonomous Databases que nao estao cadastrados no Oracle Data Safe." + ) + add_common_args(parser) + parser.add_argument( + "--output", + default="adb_not_in_datasafe.csv", + help="Arquivo de saida. Padrao: adb_not_in_datasafe.csv", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + context = build_context(args) + + if args.all_regions and not context.tenancy_id: + print("--all-regions exige --tenancy-id ou tenancy configurada no perfil OCI.", file=sys.stderr) + return 2 + + regions = get_regions( + context.client, + context.tenancy_id, + context.default_region, + args.all_regions, + args.regions, + ) # type: ignore[arg-type] + rows: list[dict[str, object]] = [] + all_errors: list[str] = [] + + for region in regions: + region_label = region or context.default_region or "perfil" + print( + f"Regiao {region_label}: consultando Data Safe...", + file=sys.stderr, + ) + targets, target_errors = list_datasafe_targets( + context.client, + context.compartments, + root_compartment_id=context.tenancy_id, + region=region, + workers=args.workers, + retries=args.retries, + retry_base_sleep=args.retry_base_sleep, + include_deleted=args.include_deleted_targets, + ) + detail_errors = enrich_missing_target_associations( + context.client, + targets, + region=region, + workers=args.workers, + retries=args.retries, + retry_base_sleep=args.retry_base_sleep, + ) + registered_adb_ids = { + target_associated_adb_id(target) + for target in targets + if target_associated_adb_id(target) + } + print( + f"Regiao {region_label}: {len(targets)} targets Data Safe encontrados; " + f"varrendo {len(context.compartments)} compartments...", + file=sys.stderr, + ) + + progress = ProgressBar( + f"Regiao {region_label}", + len(context.compartments), + enabled=not args.no_progress, + ) + stats = {"adbs": 0, "reg": 0, "nao_reg": 0, "erros": 0} + + def update_progress(compartment: dict[str, object], items: list[dict[str, object]], error: str | None) -> None: + stats["adbs"] += len(items) + stats["reg"] += sum(1 for adb in items if field(adb, "id") in registered_adb_ids) + stats["nao_reg"] += sum(1 for adb in items if field(adb, "id") not in registered_adb_ids) + if error: + stats["erros"] += 1 + progress.update( + current_label=str(field(compartment, "name", "display-name", default=field(compartment, "id"))), + stats=stats, + ) + + adbs, adb_errors = list_all_adbs( + context.client, + context.compartments, + region=region, + workers=args.workers, + retries=args.retries, + retry_base_sleep=args.retry_base_sleep, + include_terminated=args.include_terminated_adbs, + progress_callback=update_progress, + ) + progress.finish(stats=stats) + all_errors.extend(target_errors + detail_errors + adb_errors) + + for adb in sorted(adbs, key=lambda item: (field(item, "display-name"), field(item, "id"))): + if field(adb, "id") in registered_adb_ids: + continue + row = adb_to_base_row(adb, context.compartment_index, region=region) + row["datasafe_registered"] = "false" + rows.append(row) + + write_rows(rows, args.output, args.format) + print(f"ADBs nao cadastrados no Data Safe: {len(rows)}") + print(f"Arquivo gerado: {args.output}") + print_warnings("o levantamento", all_errors) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/oci_datasafe_inventory.py b/oci_datasafe_inventory.py new file mode 100755 index 0000000..d6005d4 --- /dev/null +++ b/oci_datasafe_inventory.py @@ -0,0 +1,984 @@ +#!/usr/bin/env python3 +"""Shared helpers for OCI Autonomous Database and Data Safe inventory scripts.""" + +from __future__ import annotations + +import argparse +import configparser +import concurrent.futures +import csv +import json +import os +import random +import shutil +import subprocess +import sys +import time +from dataclasses import dataclass +from datetime import date, datetime +from pathlib import Path +from typing import Any, Callable + + +ADB_TERMINAL_STATES = {"TERMINATED", "TERMINATING"} +DATASAFE_DELETED_STATES = {"DELETED", "DELETING"} +RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504} +PREFERRED_CSV_COLUMNS = [ + "datasafe_registered", + "adb_time_created", + "adb_compartment_name", + "adb_compartment_path", +] + + +class OciCliError(RuntimeError): + def __init__(self, cmd: list[str], returncode: int, stdout: str, stderr: str) -> None: + self.cmd = cmd + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + super().__init__(f"OCI CLI failed with exit code {returncode}: {' '.join(cmd)}") + + +@dataclass(frozen=True) +class InventoryContext: + client: Any + tenancy_id: str | None + default_region: str | None + compartments: list[dict[str, Any]] + compartment_index: dict[str, dict[str, Any]] + + +class OciCli: + def __init__( + self, + *, + profile: str | None = None, + config_file: str | None = None, + auth: str | None = None, + auth_purpose: str | None = None, + region: str | None = None, + verbose: bool = False, + ) -> None: + self.profile = profile + self.config_file = config_file + self.auth = auth + self.auth_purpose = auth_purpose + self.region = region + self.verbose = verbose + + def global_args(self, region: str | None = None) -> list[str]: + args: list[str] = [] + if self.profile: + args.extend(["--profile", self.profile]) + if self.config_file: + args.extend(["--config-file", self.config_file]) + if self.auth: + args.extend(["--auth", self.auth]) + if self.auth_purpose: + args.extend(["--auth-purpose", self.auth_purpose]) + selected_region = region or self.region + if selected_region: + args.extend(["--region", selected_region]) + return args + + def run_json(self, service_args: list[str], *, region: str | None = None) -> dict[str, Any]: + cmd = ["oci", *service_args, *self.global_args(region), "--output", "json"] + if self.verbose: + print("+ " + " ".join(cmd), file=sys.stderr) + try: + completed = subprocess.run(cmd, capture_output=True, text=True, check=False) + except FileNotFoundError as exc: + raise SystemExit("OCI CLI nao encontrado no PATH. Instale/configure o comando 'oci'.") from exc + if completed.returncode != 0: + raise OciCliError(cmd, completed.returncode, completed.stdout, completed.stderr) + stdout = completed.stdout.strip() + if not stdout: + return {} + try: + return json.loads(stdout) + except json.JSONDecodeError as exc: + raise RuntimeError(f"Resposta da OCI CLI nao e JSON valido para: {' '.join(cmd)}") from exc + + def list_compartments(self, tenancy_id: str | None) -> list[dict[str, Any]]: + base_args = [ + "iam", + "compartment", + "list", + "--compartment-id-in-subtree", + "true", + "--access-level", + "ANY", + "--lifecycle-state", + "ACTIVE", + "--all", + ] + if tenancy_id: + base_args.extend(["--compartment-id", tenancy_id]) + + try: + response = self.run_json([*base_args, "--include-root"]) + return item_list(response) + except OciCliError: + compartments = item_list(self.run_json(base_args)) + if tenancy_id and all(field(c, "id") != tenancy_id for c in compartments): + compartments.append(root_compartment_stub(tenancy_id)) + return compartments + + def list_region_subscriptions(self, tenancy_id: str) -> list[dict[str, Any]]: + return item_list(self.run_json(["iam", "region-subscription", "list", "--tenancy-id", tenancy_id])) + + def list_autonomous_databases(self, compartment_id: str, *, region: str | None) -> list[dict[str, Any]]: + response = self.run_json( + ["db", "autonomous-database", "list", "--compartment-id", compartment_id, "--all"], + region=region, + ) + return item_list(response) + + def list_target_databases_subtree(self, root_compartment_id: str, *, region: str | None) -> list[dict[str, Any]]: + response = self.run_json( + [ + "data-safe", + "target-database", + "list", + "--compartment-id", + root_compartment_id, + "--compartment-id-in-subtree", + "true", + "--access-level", + "ACCESSIBLE", + "--database-type", + "AUTONOMOUS_DATABASE", + "--all", + ], + region=region, + ) + return item_list(response) + + def list_target_databases_compartment(self, compartment_id: str, *, region: str | None) -> list[dict[str, Any]]: + response = self.run_json( + [ + "data-safe", + "target-database", + "list", + "--compartment-id", + compartment_id, + "--database-type", + "AUTONOMOUS_DATABASE", + "--all", + ], + region=region, + ) + return item_list(response) + + def get_target_database(self, target_database_id: str, *, region: str | None) -> dict[str, Any] | None: + response = self.run_json( + ["data-safe", "target-database", "get", "--target-database-id", target_database_id], + region=region, + ) + data = response.get("data") + return data if isinstance(data, dict) else None + + +class OciSdkClient: + def __init__( + self, + *, + profile: str | None = None, + config_file: str | None = None, + auth: str | None = None, + region: str | None = None, + verbose: bool = False, + ) -> None: + try: + import oci # type: ignore[import-not-found] + import oci.data_safe # type: ignore[import-not-found] + import oci.database # type: ignore[import-not-found] + import oci.identity # type: ignore[import-not-found] + except ImportError as exc: + raise SystemExit( + "Pacote 'oci' nao encontrado. Rode: pip install -r requirements.txt " + "ou use --engine cli para manter a versao baseada na OCI CLI." + ) from exc + + self.oci = oci + self.profile = profile or os.environ.get("OCI_CLI_PROFILE") or "DEFAULT" + self.config_file = config_file or str(Path.home() / ".oci" / "config") + self.auth = auth + self.region = region + self.verbose = verbose + self.signer = None + + if auth == "instance_principal": + self.config = {"region": region or os.environ.get("OCI_REGION", "")} + self.signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner() + elif auth == "resource_principal": + self.config = {"region": region or os.environ.get("OCI_RESOURCE_PRINCIPAL_REGION", "")} + self.signer = oci.auth.signers.get_resource_principals_signer() + elif auth in (None, "", "api_key"): + self.config = oci.config.from_file(file_location=self.config_file, profile_name=self.profile) + if region: + self.config["region"] = region + oci.config.validate_config(self.config) + else: + raise SystemExit( + f"--auth {auth!r} ainda nao e suportado pelo engine SDK. " + "Use --engine cli para esse modo de autenticacao." + ) + + def _config_for_region(self, region: str | None) -> dict[str, Any]: + config = dict(self.config) + selected_region = region or self.region or config.get("region") + if selected_region: + config["region"] = selected_region + if not config.get("region"): + raise RuntimeError("Regiao OCI nao definida. Informe --region ou configure region no perfil OCI.") + return config + + def _client(self, client_cls: Any, *, region: str | None = None) -> Any: + kwargs = {"signer": self.signer} if self.signer else {} + return client_cls(self._config_for_region(region), **kwargs) + + def _list_all(self, method: Any, *args: Any, **kwargs: Any) -> list[dict[str, Any]]: + response = self.oci.pagination.list_call_get_all_results(method, *args, **kwargs) + return [self.oci.util.to_dict(item) for item in response.data] + + def _to_dict(self, response: Any) -> dict[str, Any] | None: + if response.data is None: + return None + return self.oci.util.to_dict(response.data) + + def list_compartments(self, tenancy_id: str | None) -> list[dict[str, Any]]: + if not tenancy_id: + raise RuntimeError("Nao consegui resolver a tenancy. Informe --tenancy-id.") + identity = self._client(self.oci.identity.IdentityClient) + compartments = self._list_all( + identity.list_compartments, + tenancy_id, + compartment_id_in_subtree=True, + access_level="ANY", + lifecycle_state="ACTIVE", + ) + try: + tenancy = self._to_dict(identity.get_tenancy(tenancy_id)) or root_compartment_stub(tenancy_id) + tenancy["name"] = field(tenancy, "name", "display-name", default="root-tenancy") + tenancy["display-name"] = field(tenancy, "display-name", "name", default="root-tenancy") + tenancy["compartment-id"] = "" + compartments.append(tenancy) + except Exception: + compartments.append(root_compartment_stub(tenancy_id)) + return compartments + + def list_region_subscriptions(self, tenancy_id: str) -> list[dict[str, Any]]: + identity = self._client(self.oci.identity.IdentityClient) + return self._list_all(identity.list_region_subscriptions, tenancy_id) + + def list_autonomous_databases(self, compartment_id: str, *, region: str | None) -> list[dict[str, Any]]: + database = self._client(self.oci.database.DatabaseClient, region=region) + return self._list_all(database.list_autonomous_databases, compartment_id) + + def list_target_databases_subtree(self, root_compartment_id: str, *, region: str | None) -> list[dict[str, Any]]: + data_safe = self._client(self.oci.data_safe.DataSafeClient, region=region) + return self._list_all( + data_safe.list_target_databases, + root_compartment_id, + compartment_id_in_subtree=True, + access_level="ACCESSIBLE", + database_type="AUTONOMOUS_DATABASE", + ) + + def list_target_databases_compartment(self, compartment_id: str, *, region: str | None) -> list[dict[str, Any]]: + data_safe = self._client(self.oci.data_safe.DataSafeClient, region=region) + return self._list_all( + data_safe.list_target_databases, + compartment_id, + database_type="AUTONOMOUS_DATABASE", + ) + + def get_target_database(self, target_database_id: str, *, region: str | None) -> dict[str, Any] | None: + data_safe = self._client(self.oci.data_safe.DataSafeClient, region=region) + return self._to_dict(data_safe.get_target_database(target_database_id)) + + +def item_list(response: dict[str, Any]) -> list[dict[str, Any]]: + data = response.get("data", []) + if isinstance(data, list): + return data + if isinstance(data, dict): + items = data.get("items") + if isinstance(items, list): + return items + return [] + + +def root_compartment_stub(tenancy_id: str) -> dict[str, Any]: + return { + "id": tenancy_id, + "name": "root-tenancy", + "display-name": "root-tenancy", + "lifecycle-state": "ACTIVE", + "compartment-id": "", + } + + +def format_error(exc: Exception) -> str: + if isinstance(exc, OciCliError): + return exc.stderr.strip() or exc.stdout.strip() or str(exc.returncode) + status = getattr(exc, "status", None) + code = getattr(exc, "code", None) + message = getattr(exc, "message", None) + request_id = getattr(exc, "request_id", None) + if status or code or message: + parts = [str(part) for part in (status, code, message) if part] + if request_id: + parts.append(f"opc-request-id={request_id}") + return " - ".join(parts) + return str(exc) + + +def retry_after_seconds(exc: Exception) -> float | None: + headers = getattr(exc, "headers", None) + if not headers: + return None + value = None + if isinstance(headers, dict): + value = headers.get("retry-after") or headers.get("Retry-After") + elif hasattr(headers, "get"): + value = headers.get("retry-after") or headers.get("Retry-After") + if value is None: + return None + try: + return max(0.0, float(value)) + except (TypeError, ValueError): + return None + + +def is_retryable_error(exc: Exception) -> bool: + status = getattr(exc, "status", None) + if status in RETRYABLE_STATUS_CODES: + return True + if isinstance(exc, OciCliError): + text = f"{exc.stdout}\n{exc.stderr}" + return exc.returncode != 0 and ( + "TooManyRequests" in text + or "Too many requests" in text + or "ServiceUnavailable" in text + ) + return False + + +def run_with_retries( + operation: Any, + *, + retries: int, + base_sleep: float, + verbose: bool = False, +) -> Any: + attempts = max(1, retries + 1) + last_exc: Exception | None = None + for attempt in range(1, attempts + 1): + try: + return operation() + except Exception as exc: + last_exc = exc + if attempt >= attempts or not is_retryable_error(exc): + raise + retry_after = retry_after_seconds(exc) + sleep_seconds = retry_after if retry_after is not None else base_sleep * (2 ** (attempt - 1)) + sleep_seconds += random.uniform(0, min(1.0, base_sleep)) + if verbose: + print( + f"Retry {attempt}/{retries} apos erro temporario: {format_error(exc)}. " + f"Aguardando {sleep_seconds:.1f}s...", + file=sys.stderr, + ) + time.sleep(sleep_seconds) + if last_exc: + raise last_exc + raise RuntimeError("Falha inesperada no mecanismo de retry.") + + +class ProgressBar: + def __init__(self, label: str, total: int, *, enabled: bool = True, width: int = 28) -> None: + self.label = label + self.total = max(1, total) + self.enabled = enabled + self.width = width + self.current = 0 + self.started_at = time.monotonic() + self.stats: dict[str, Any] = {} + self.current_label = "" + self.last_len = 0 + + def update( + self, + *, + increment: int = 1, + current_label: str | None = None, + stats: dict[str, Any] | None = None, + ) -> None: + self.current = min(self.total, self.current + increment) + if current_label is not None: + self.current_label = current_label + if stats: + self.stats.update(stats) + self.render() + + def finish(self, *, stats: dict[str, Any] | None = None) -> None: + if stats: + self.stats.update(stats) + self.current = self.total + self.render(done=True) + + def render(self, *, done: bool = False) -> None: + if not self.enabled: + return + percent = self.current / self.total + filled = int(self.width * percent) + bar = "#" * filled + "-" * (self.width - filled) + elapsed = max(0.1, time.monotonic() - self.started_at) + rate = self.current / elapsed + stats_text = " ".join(f"{key}={value}" for key, value in self.stats.items()) + current_text = f" atual={self.current_label}" if self.current_label else "" + line = ( + f"{self.label} [{bar}] {percent:6.1%} " + f"{self.current}/{self.total} comps {rate:5.1f}/s {stats_text}{current_text}" + ) + columns = shutil.get_terminal_size((120, 20)).columns + if len(line) > columns - 1: + line = line[: columns - 4] + "..." + padding = " " * max(0, self.last_len - len(line)) + sys.stderr.write("\r" + line + padding) + sys.stderr.flush() + self.last_len = len(line) + if done: + sys.stderr.write("\n") + sys.stderr.flush() + self.last_len = 0 + + +def field(item: dict[str, Any] | None, *names: str, default: Any = "") -> Any: + if not item: + return default + for name in names: + candidates = { + name, + name.replace("_", "-"), + name.replace("-", "_"), + } + parts = name.replace("_", "-").split("-") + candidates.add(parts[0] + "".join(part.title() for part in parts[1:])) + for candidate in candidates: + if candidate in item and item[candidate] is not None: + return item[candidate] + return default + + +def read_oci_config_value(profile: str | None, config_file: str | None, key: str) -> str | None: + config_path = Path(config_file).expanduser() if config_file else Path.home() / ".oci" / "config" + if not config_path.exists(): + return None + + parser = configparser.ConfigParser() + parser.read(config_path) + section = profile or os.environ.get("OCI_CLI_PROFILE") or "DEFAULT" + if section == "DEFAULT" and key in parser.defaults(): + return parser.defaults().get(key) + if parser.has_section(section) and parser.has_option(section, key): + return parser.get(section, key) + if key in parser.defaults(): + return parser.defaults().get(key) + return None + + +def resolve_tenancy_id(args: argparse.Namespace) -> str | None: + return ( + args.tenancy_id + or os.environ.get("OCI_TENANCY") + or os.environ.get("OCI_CLI_TENANCY") + or read_oci_config_value(args.profile, args.config_file, "tenancy") + ) + + +def resolve_default_region(args: argparse.Namespace) -> str | None: + return args.region or read_oci_config_value(args.profile, args.config_file, "region") + + +def get_compartments(client: Any, tenancy_id: str | None) -> list[dict[str, Any]]: + compartments = client.list_compartments(tenancy_id) + + deduped: dict[str, dict[str, Any]] = {} + for compartment in compartments: + cid = field(compartment, "id") + state = str(field(compartment, "lifecycle-state", default="")).upper() + if cid and state != "DELETED": + deduped[cid] = compartment + return list(deduped.values()) + + +def filter_compartments_for_scan( + compartments: list[dict[str, Any]], + *, + compartment_id: str | None, + include_subtree: bool, +) -> list[dict[str, Any]]: + if not compartment_id: + return compartments + + by_id = {field(compartment, "id"): compartment for compartment in compartments if field(compartment, "id")} + if compartment_id not in by_id: + raise RuntimeError( + "Compartment informado em --compartment-id nao foi encontrado na lista acessivel. " + "Verifique o OCID e as permissoes do usuario/grupo." + ) + if not include_subtree: + return [by_id[compartment_id]] + + selected: list[dict[str, Any]] = [] + for compartment in compartments: + current = field(compartment, "id") + seen: set[str] = set() + while current and current not in seen: + seen.add(current) + if current == compartment_id: + selected.append(compartment) + break + current_item = by_id.get(current) + if not current_item: + break + current = field(current_item, "compartment-id", default="") + return selected + + +def build_compartment_index(compartments: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + index = {field(c, "id"): dict(c) for c in compartments if field(c, "id")} + for cid, compartment in index.items(): + compartment["path"] = compartment_path(cid, index) + compartment["resolved-name"] = field(compartment, "name", "display-name", default=cid) + return index + + +def compartment_path(compartment_id: str, index: dict[str, dict[str, Any]]) -> str: + names: list[str] = [] + seen: set[str] = set() + current = compartment_id + while current and current in index and current not in seen: + seen.add(current) + current_item = index[current] + names.append(field(current_item, "name", "display-name", default=current)) + current = field(current_item, "compartment-id", default="") + return "/".join(reversed(names)) + + +def get_regions( + client: Any, + tenancy_id: str, + default_region: str | None, + all_regions: bool, + selected_regions: str | None = None, +) -> list[str | None]: + if selected_regions: + regions = [region.strip() for region in selected_regions.split(",") if region.strip()] + if not regions: + raise RuntimeError("--regions foi informado, mas nenhuma regiao valida foi encontrada.") + return regions + if not all_regions: + return [default_region] + regions = [] + for item in client.list_region_subscriptions(tenancy_id): + region_name = field(item, "region-name") + status = str(field(item, "status", default="READY")).upper() + if region_name and status in {"READY", "ACTIVE"}: + regions.append(region_name) + return sorted(set(regions)) + + +def list_adbs_in_compartment( + client: Any, + compartment: dict[str, Any], + *, + region: str | None, + retries: int, + retry_base_sleep: float, +) -> tuple[list[dict[str, Any]], str | None]: + compartment_id = field(compartment, "id") + try: + adbs = run_with_retries( + lambda: client.list_autonomous_databases(compartment_id, region=region), + retries=retries, + base_sleep=retry_base_sleep, + verbose=getattr(client, "verbose", False), + ) + for adb in adbs: + adb["_inventory_region"] = region or "" + return adbs, None + except Exception as exc: + return [], f"{compartment_id}: {format_error(exc)}" + + +def list_all_adbs( + client: Any, + compartments: list[dict[str, Any]], + *, + region: str | None, + workers: int, + retries: int, + retry_base_sleep: float, + include_terminated: bool, + progress_callback: Callable[[dict[str, Any], list[dict[str, Any]], str | None], None] | None = None, +) -> tuple[list[dict[str, Any]], list[str]]: + adbs: list[dict[str, Any]] = [] + errors: list[str] = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, workers)) as executor: + futures = { + executor.submit( + list_adbs_in_compartment, + client, + compartment, + region=region, + retries=retries, + retry_base_sleep=retry_base_sleep, + ): compartment + for compartment in compartments + } + for future in concurrent.futures.as_completed(futures): + items, error = future.result() + filtered_items: list[dict[str, Any]] = [] + if error: + errors.append(error) + for adb in items: + state = str(field(adb, "lifecycle-state", default="")).upper() + if include_terminated or state not in ADB_TERMINAL_STATES: + filtered_items.append(adb) + adbs.extend(filtered_items) + if progress_callback: + progress_callback(futures[future], filtered_items, error) + return adbs, sorted(errors) + + +def list_datasafe_targets_in_subtree( + client: Any, + root_compartment_id: str, + *, + region: str | None, +) -> list[dict[str, Any]]: + return client.list_target_databases_subtree(root_compartment_id, region=region) + + +def list_datasafe_targets_per_compartment( + client: Any, + compartments: list[dict[str, Any]], + *, + region: str | None, + workers: int, + retries: int, + retry_base_sleep: float, +) -> tuple[list[dict[str, Any]], list[str]]: + targets: list[dict[str, Any]] = [] + errors: list[str] = [] + + def worker(compartment: dict[str, Any]) -> tuple[list[dict[str, Any]], str | None]: + compartment_id = field(compartment, "id") + try: + return run_with_retries( + lambda: client.list_target_databases_compartment(compartment_id, region=region), + retries=retries, + base_sleep=retry_base_sleep, + verbose=getattr(client, "verbose", False), + ), None + except Exception as exc: + return [], f"{compartment_id}: {format_error(exc)}" + + with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, workers)) as executor: + futures = [executor.submit(worker, compartment) for compartment in compartments] + for future in concurrent.futures.as_completed(futures): + items, error = future.result() + if error: + errors.append(error) + targets.extend(items) + return targets, sorted(errors) + + +def list_datasafe_targets( + client: Any, + compartments: list[dict[str, Any]], + *, + root_compartment_id: str | None, + region: str | None, + workers: int, + retries: int, + retry_base_sleep: float, + include_deleted: bool, +) -> tuple[list[dict[str, Any]], list[str]]: + errors: list[str] = [] + try: + if root_compartment_id: + targets = run_with_retries( + lambda: list_datasafe_targets_in_subtree(client, root_compartment_id, region=region), + retries=retries, + base_sleep=retry_base_sleep, + verbose=getattr(client, "verbose", False), + ) + else: + targets, errors = list_datasafe_targets_per_compartment( + client, + compartments, + region=region, + workers=workers, + retries=retries, + retry_base_sleep=retry_base_sleep, + ) + except Exception as exc: + errors.append(f"subtree Data Safe: {format_error(exc)}") + targets, per_compartment_errors = list_datasafe_targets_per_compartment( + client, + compartments, + region=region, + workers=workers, + retries=retries, + retry_base_sleep=retry_base_sleep, + ) + errors.extend(per_compartment_errors) + + filtered = [] + for target in targets: + target["_inventory_region"] = region or "" + state = str(field(target, "lifecycle-state", default="")).upper() + if include_deleted or state not in DATASAFE_DELETED_STATES: + filtered.append(target) + return filtered, sorted(errors) + + +def target_associated_adb_id(target: dict[str, Any]) -> str: + associated_resource_ids = field( + target, + "associated-resource-ids", + "associated_resource_ids", + "associatedResourceIds", + default=[], + ) + if isinstance(associated_resource_ids, list): + for resource_id in associated_resource_ids: + if isinstance(resource_id, str) and resource_id.startswith("ocid1.autonomousdatabase."): + return resource_id + for resource_id in associated_resource_ids: + if isinstance(resource_id, str) and resource_id.startswith("ocid1.autonomous"): + return resource_id + + direct = field( + target, + "associated-resource-id", + "associated_resource_id", + "associatedResourceId", + "autonomous-database-id", + "autonomous_database_id", + "autonomousDatabaseId", + default="", + ) + if isinstance(direct, str) and direct: + return direct + details = field(target, "database-details", "databaseDetails", default={}) + if isinstance(details, dict): + return field( + details, + "autonomous-database-id", + "autonomous_database_id", + "autonomousDatabaseId", + default="", + ) + return "" + + +def enrich_missing_target_associations( + client: Any, + targets: list[dict[str, Any]], + *, + region: str | None, + workers: int, + retries: int, + retry_base_sleep: float, +) -> list[str]: + errors: list[str] = [] + missing = [target for target in targets if not target_associated_adb_id(target) and field(target, "id")] + if not missing: + return errors + + def worker(target: dict[str, Any]) -> tuple[dict[str, Any] | None, str | None]: + target_id = field(target, "id") + try: + return run_with_retries( + lambda: client.get_target_database(target_id, region=region), + retries=retries, + base_sleep=retry_base_sleep, + verbose=getattr(client, "verbose", False), + ), None + except Exception as exc: + return None, f"{target_id}: {format_error(exc)}" + + by_id = {field(target, "id"): target for target in targets} + with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, workers)) as executor: + futures = [executor.submit(worker, target) for target in missing] + for future in concurrent.futures.as_completed(futures): + detail, error = future.result() + if error: + errors.append(error) + if detail and field(detail, "id") in by_id: + by_id[field(detail, "id")].update(detail) + return sorted(errors) + + +def adb_to_base_row( + adb: dict[str, Any], + compartment_index: dict[str, dict[str, Any]], + *, + region: str | None = None, +) -> dict[str, Any]: + compartment_id = field(adb, "compartment-id") + compartment = compartment_index.get(compartment_id, {}) + return { + "region": region or field(adb, "_inventory_region"), + "adb_display_name": field(adb, "display-name"), + "adb_db_name": field(adb, "db-name"), + "adb_ocid": field(adb, "id"), + "adb_lifecycle_state": field(adb, "lifecycle-state"), + "adb_workload": field(adb, "db-workload"), + "adb_is_free_tier": field(adb, "is-free-tier"), + "adb_time_created": field(adb, "time-created"), + "adb_compartment_name": field(compartment, "resolved-name", "name", "display-name"), + "adb_compartment_path": field(compartment, "path"), + "adb_compartment_ocid": compartment_id, + } + + +def target_to_base_row( + target: dict[str, Any], + compartment_index: dict[str, dict[str, Any]], + *, + prefix: str = "datasafe_target", +) -> dict[str, Any]: + compartment_id = field(target, "compartment-id") + compartment = compartment_index.get(compartment_id, {}) + return { + f"{prefix}_display_name": field(target, "display-name"), + f"{prefix}_ocid": field(target, "id"), + f"{prefix}_lifecycle_state": field(target, "lifecycle-state"), + f"{prefix}_database_type": field(target, "database-type"), + f"{prefix}_infrastructure_type": field(target, "infrastructure-type"), + f"{prefix}_time_created": field(target, "time-created"), + f"{prefix}_compartment_name": field(compartment, "resolved-name", "name", "display-name"), + f"{prefix}_compartment_path": field(compartment, "path"), + f"{prefix}_compartment_ocid": compartment_id, + f"{prefix}_associated_adb_ocid": target_associated_adb_id(target), + } + + +def normalize_output_value(value: Any) -> Any: + if isinstance(value, (datetime, date)): + return value.isoformat() + if isinstance(value, (list, tuple)): + return json.dumps([normalize_output_value(item) for item in value], ensure_ascii=False) + if isinstance(value, dict): + return json.dumps({key: normalize_output_value(item) for key, item in value.items()}, ensure_ascii=False) + return value + + +def write_rows(rows: list[dict[str, Any]], output_path: str, output_format: str) -> None: + path = Path(output_path) + path.parent.mkdir(parents=True, exist_ok=True) + normalized_rows = [ + {key: normalize_output_value(value) for key, value in row.items()} + for row in rows + ] + if output_format == "json": + path.write_text(json.dumps(normalized_rows, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return + + fieldnames: list[str] = [] + for column in PREFERRED_CSV_COLUMNS: + if any(column in row for row in normalized_rows): + fieldnames.append(column) + for row in normalized_rows: + for key in row: + if key not in fieldnames: + fieldnames.append(key) + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(normalized_rows) + + +def add_common_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--tenancy-id", help="OCID da tenancy/root compartment. Se omitido, tenta ler do perfil OCI.") + parser.add_argument("--profile", help="Perfil da OCI CLI. Ex.: DEFAULT, PROD.") + parser.add_argument("--config-file", help="Caminho para arquivo de configuracao OCI CLI.") + parser.add_argument("--auth", help="Metodo de autenticacao OCI CLI. Ex.: instance_principal, resource_principal, security_token.") + parser.add_argument("--auth-purpose", help="Auth purpose da OCI CLI, quando aplicavel.") + parser.add_argument("--engine", choices=["sdk", "cli"], default="sdk", help="Motor de consulta. Padrao: sdk, mais rapido que chamar a OCI CLI repetidamente.") + parser.add_argument("--region", help="Regiao OCI. Se omitida, usa a regiao do perfil OCI CLI.") + parser.add_argument("--regions", help="Lista de regioes separadas por virgula. Ex.: sa-saopaulo-1,us-ashburn-1.") + parser.add_argument("--all-regions", action="store_true", help="Executa em todas as regioes subscribed da tenancy.") + parser.add_argument("--compartment-id", help="Limita a varredura de ADBs a um compartment especifico.") + parser.add_argument("--compartment-subtree", action="store_true", help="Com --compartment-id, inclui subcompartments na varredura de ADBs.") + parser.add_argument("--workers", type=int, default=4, help="Quantidade de chamadas paralelas por compartment. Padrao: 4.") + parser.add_argument("--retries", type=int, default=6, help="Tentativas adicionais para erros temporarios como 429/5xx. Padrao: 6.") + parser.add_argument("--retry-base-sleep", type=float, default=2.0, help="Espera inicial em segundos para backoff exponencial. Padrao: 2.0.") + parser.add_argument("--include-terminated-adbs", action="store_true", help="Inclui ADBs TERMINATED/TERMINATING.") + parser.add_argument("--include-deleted-targets", action="store_true", help="Inclui targets Data Safe DELETED/DELETING.") + parser.add_argument("--format", choices=["csv", "json"], default="csv", help="Formato de saida. Padrao: csv.") + parser.add_argument("--no-progress", action="store_true", help="Desativa a barra de progresso no terminal.") + parser.add_argument("--verbose", action="store_true", help="Mostra comandos OCI CLI executados.") + + +def build_context(args: argparse.Namespace) -> InventoryContext: + default_region = resolve_default_region(args) + if args.engine == "cli": + client: Any = OciCli( + profile=args.profile, + config_file=args.config_file, + auth=args.auth, + auth_purpose=args.auth_purpose, + region=args.region, + verbose=args.verbose, + ) + else: + client = OciSdkClient( + profile=args.profile, + config_file=args.config_file, + auth=args.auth, + region=args.region, + verbose=args.verbose, + ) + tenancy_id = resolve_tenancy_id(args) + if not tenancy_id: + config = getattr(client, "config", {}) + if isinstance(config, dict): + tenancy_id = config.get("tenancy") + all_compartments = get_compartments(client, tenancy_id) + compartment_index = build_compartment_index(all_compartments) + compartments = filter_compartments_for_scan( + all_compartments, + compartment_id=args.compartment_id, + include_subtree=args.compartment_subtree, + ) + if not tenancy_id: + root_candidates = [ + cid + for cid, item in compartment_index.items() + if not field(item, "compartment-id", default="") + ] + tenancy_id = root_candidates[0] if root_candidates else None + return InventoryContext( + client=client, + tenancy_id=tenancy_id, + default_region=default_region, + compartments=compartments, + compartment_index=compartment_index, + ) + + +def print_warnings(title: str, errors: list[str]) -> None: + if not errors: + return + print(f"\nAvisos durante {title}:", file=sys.stderr) + for error in errors: + print(f" - {error}", file=sys.stderr) diff --git a/requirements.txt b/requirements.txt new file mode 100755 index 0000000..e3c4783 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +# Default engine used by the scripts. +oci>=2.0.0 + +# Optional fallback if you run with --engine cli: +# oci-cli