Initial OCI Data Safe ADB inventory tool
This commit is contained in:
984
oci_datasafe_inventory.py
Executable file
984
oci_datasafe_inventory.py
Executable file
@@ -0,0 +1,984 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared helpers for OCI Autonomous Database and Data Safe inventory scripts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import configparser
|
||||
import concurrent.futures
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
ADB_TERMINAL_STATES = {"TERMINATED", "TERMINATING"}
|
||||
DATASAFE_DELETED_STATES = {"DELETED", "DELETING"}
|
||||
RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504}
|
||||
PREFERRED_CSV_COLUMNS = [
|
||||
"datasafe_registered",
|
||||
"adb_time_created",
|
||||
"adb_compartment_name",
|
||||
"adb_compartment_path",
|
||||
]
|
||||
|
||||
|
||||
class OciCliError(RuntimeError):
|
||||
def __init__(self, cmd: list[str], returncode: int, stdout: str, stderr: str) -> None:
|
||||
self.cmd = cmd
|
||||
self.returncode = returncode
|
||||
self.stdout = stdout
|
||||
self.stderr = stderr
|
||||
super().__init__(f"OCI CLI failed with exit code {returncode}: {' '.join(cmd)}")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InventoryContext:
|
||||
client: Any
|
||||
tenancy_id: str | None
|
||||
default_region: str | None
|
||||
compartments: list[dict[str, Any]]
|
||||
compartment_index: dict[str, dict[str, Any]]
|
||||
|
||||
|
||||
class OciCli:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
profile: str | None = None,
|
||||
config_file: str | None = None,
|
||||
auth: str | None = None,
|
||||
auth_purpose: str | None = None,
|
||||
region: str | None = None,
|
||||
verbose: bool = False,
|
||||
) -> None:
|
||||
self.profile = profile
|
||||
self.config_file = config_file
|
||||
self.auth = auth
|
||||
self.auth_purpose = auth_purpose
|
||||
self.region = region
|
||||
self.verbose = verbose
|
||||
|
||||
def global_args(self, region: str | None = None) -> list[str]:
|
||||
args: list[str] = []
|
||||
if self.profile:
|
||||
args.extend(["--profile", self.profile])
|
||||
if self.config_file:
|
||||
args.extend(["--config-file", self.config_file])
|
||||
if self.auth:
|
||||
args.extend(["--auth", self.auth])
|
||||
if self.auth_purpose:
|
||||
args.extend(["--auth-purpose", self.auth_purpose])
|
||||
selected_region = region or self.region
|
||||
if selected_region:
|
||||
args.extend(["--region", selected_region])
|
||||
return args
|
||||
|
||||
def run_json(self, service_args: list[str], *, region: str | None = None) -> dict[str, Any]:
|
||||
cmd = ["oci", *service_args, *self.global_args(region), "--output", "json"]
|
||||
if self.verbose:
|
||||
print("+ " + " ".join(cmd), file=sys.stderr)
|
||||
try:
|
||||
completed = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||||
except FileNotFoundError as exc:
|
||||
raise SystemExit("OCI CLI nao encontrado no PATH. Instale/configure o comando 'oci'.") from exc
|
||||
if completed.returncode != 0:
|
||||
raise OciCliError(cmd, completed.returncode, completed.stdout, completed.stderr)
|
||||
stdout = completed.stdout.strip()
|
||||
if not stdout:
|
||||
return {}
|
||||
try:
|
||||
return json.loads(stdout)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(f"Resposta da OCI CLI nao e JSON valido para: {' '.join(cmd)}") from exc
|
||||
|
||||
def list_compartments(self, tenancy_id: str | None) -> list[dict[str, Any]]:
|
||||
base_args = [
|
||||
"iam",
|
||||
"compartment",
|
||||
"list",
|
||||
"--compartment-id-in-subtree",
|
||||
"true",
|
||||
"--access-level",
|
||||
"ANY",
|
||||
"--lifecycle-state",
|
||||
"ACTIVE",
|
||||
"--all",
|
||||
]
|
||||
if tenancy_id:
|
||||
base_args.extend(["--compartment-id", tenancy_id])
|
||||
|
||||
try:
|
||||
response = self.run_json([*base_args, "--include-root"])
|
||||
return item_list(response)
|
||||
except OciCliError:
|
||||
compartments = item_list(self.run_json(base_args))
|
||||
if tenancy_id and all(field(c, "id") != tenancy_id for c in compartments):
|
||||
compartments.append(root_compartment_stub(tenancy_id))
|
||||
return compartments
|
||||
|
||||
def list_region_subscriptions(self, tenancy_id: str) -> list[dict[str, Any]]:
|
||||
return item_list(self.run_json(["iam", "region-subscription", "list", "--tenancy-id", tenancy_id]))
|
||||
|
||||
def list_autonomous_databases(self, compartment_id: str, *, region: str | None) -> list[dict[str, Any]]:
|
||||
response = self.run_json(
|
||||
["db", "autonomous-database", "list", "--compartment-id", compartment_id, "--all"],
|
||||
region=region,
|
||||
)
|
||||
return item_list(response)
|
||||
|
||||
def list_target_databases_subtree(self, root_compartment_id: str, *, region: str | None) -> list[dict[str, Any]]:
|
||||
response = self.run_json(
|
||||
[
|
||||
"data-safe",
|
||||
"target-database",
|
||||
"list",
|
||||
"--compartment-id",
|
||||
root_compartment_id,
|
||||
"--compartment-id-in-subtree",
|
||||
"true",
|
||||
"--access-level",
|
||||
"ACCESSIBLE",
|
||||
"--database-type",
|
||||
"AUTONOMOUS_DATABASE",
|
||||
"--all",
|
||||
],
|
||||
region=region,
|
||||
)
|
||||
return item_list(response)
|
||||
|
||||
def list_target_databases_compartment(self, compartment_id: str, *, region: str | None) -> list[dict[str, Any]]:
|
||||
response = self.run_json(
|
||||
[
|
||||
"data-safe",
|
||||
"target-database",
|
||||
"list",
|
||||
"--compartment-id",
|
||||
compartment_id,
|
||||
"--database-type",
|
||||
"AUTONOMOUS_DATABASE",
|
||||
"--all",
|
||||
],
|
||||
region=region,
|
||||
)
|
||||
return item_list(response)
|
||||
|
||||
def get_target_database(self, target_database_id: str, *, region: str | None) -> dict[str, Any] | None:
|
||||
response = self.run_json(
|
||||
["data-safe", "target-database", "get", "--target-database-id", target_database_id],
|
||||
region=region,
|
||||
)
|
||||
data = response.get("data")
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
|
||||
class OciSdkClient:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
profile: str | None = None,
|
||||
config_file: str | None = None,
|
||||
auth: str | None = None,
|
||||
region: str | None = None,
|
||||
verbose: bool = False,
|
||||
) -> None:
|
||||
try:
|
||||
import oci # type: ignore[import-not-found]
|
||||
import oci.data_safe # type: ignore[import-not-found]
|
||||
import oci.database # type: ignore[import-not-found]
|
||||
import oci.identity # type: ignore[import-not-found]
|
||||
except ImportError as exc:
|
||||
raise SystemExit(
|
||||
"Pacote 'oci' nao encontrado. Rode: pip install -r requirements.txt "
|
||||
"ou use --engine cli para manter a versao baseada na OCI CLI."
|
||||
) from exc
|
||||
|
||||
self.oci = oci
|
||||
self.profile = profile or os.environ.get("OCI_CLI_PROFILE") or "DEFAULT"
|
||||
self.config_file = config_file or str(Path.home() / ".oci" / "config")
|
||||
self.auth = auth
|
||||
self.region = region
|
||||
self.verbose = verbose
|
||||
self.signer = None
|
||||
|
||||
if auth == "instance_principal":
|
||||
self.config = {"region": region or os.environ.get("OCI_REGION", "")}
|
||||
self.signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
|
||||
elif auth == "resource_principal":
|
||||
self.config = {"region": region or os.environ.get("OCI_RESOURCE_PRINCIPAL_REGION", "")}
|
||||
self.signer = oci.auth.signers.get_resource_principals_signer()
|
||||
elif auth in (None, "", "api_key"):
|
||||
self.config = oci.config.from_file(file_location=self.config_file, profile_name=self.profile)
|
||||
if region:
|
||||
self.config["region"] = region
|
||||
oci.config.validate_config(self.config)
|
||||
else:
|
||||
raise SystemExit(
|
||||
f"--auth {auth!r} ainda nao e suportado pelo engine SDK. "
|
||||
"Use --engine cli para esse modo de autenticacao."
|
||||
)
|
||||
|
||||
def _config_for_region(self, region: str | None) -> dict[str, Any]:
|
||||
config = dict(self.config)
|
||||
selected_region = region or self.region or config.get("region")
|
||||
if selected_region:
|
||||
config["region"] = selected_region
|
||||
if not config.get("region"):
|
||||
raise RuntimeError("Regiao OCI nao definida. Informe --region ou configure region no perfil OCI.")
|
||||
return config
|
||||
|
||||
def _client(self, client_cls: Any, *, region: str | None = None) -> Any:
|
||||
kwargs = {"signer": self.signer} if self.signer else {}
|
||||
return client_cls(self._config_for_region(region), **kwargs)
|
||||
|
||||
def _list_all(self, method: Any, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
|
||||
response = self.oci.pagination.list_call_get_all_results(method, *args, **kwargs)
|
||||
return [self.oci.util.to_dict(item) for item in response.data]
|
||||
|
||||
def _to_dict(self, response: Any) -> dict[str, Any] | None:
|
||||
if response.data is None:
|
||||
return None
|
||||
return self.oci.util.to_dict(response.data)
|
||||
|
||||
def list_compartments(self, tenancy_id: str | None) -> list[dict[str, Any]]:
|
||||
if not tenancy_id:
|
||||
raise RuntimeError("Nao consegui resolver a tenancy. Informe --tenancy-id.")
|
||||
identity = self._client(self.oci.identity.IdentityClient)
|
||||
compartments = self._list_all(
|
||||
identity.list_compartments,
|
||||
tenancy_id,
|
||||
compartment_id_in_subtree=True,
|
||||
access_level="ANY",
|
||||
lifecycle_state="ACTIVE",
|
||||
)
|
||||
try:
|
||||
tenancy = self._to_dict(identity.get_tenancy(tenancy_id)) or root_compartment_stub(tenancy_id)
|
||||
tenancy["name"] = field(tenancy, "name", "display-name", default="root-tenancy")
|
||||
tenancy["display-name"] = field(tenancy, "display-name", "name", default="root-tenancy")
|
||||
tenancy["compartment-id"] = ""
|
||||
compartments.append(tenancy)
|
||||
except Exception:
|
||||
compartments.append(root_compartment_stub(tenancy_id))
|
||||
return compartments
|
||||
|
||||
def list_region_subscriptions(self, tenancy_id: str) -> list[dict[str, Any]]:
|
||||
identity = self._client(self.oci.identity.IdentityClient)
|
||||
return self._list_all(identity.list_region_subscriptions, tenancy_id)
|
||||
|
||||
def list_autonomous_databases(self, compartment_id: str, *, region: str | None) -> list[dict[str, Any]]:
|
||||
database = self._client(self.oci.database.DatabaseClient, region=region)
|
||||
return self._list_all(database.list_autonomous_databases, compartment_id)
|
||||
|
||||
def list_target_databases_subtree(self, root_compartment_id: str, *, region: str | None) -> list[dict[str, Any]]:
|
||||
data_safe = self._client(self.oci.data_safe.DataSafeClient, region=region)
|
||||
return self._list_all(
|
||||
data_safe.list_target_databases,
|
||||
root_compartment_id,
|
||||
compartment_id_in_subtree=True,
|
||||
access_level="ACCESSIBLE",
|
||||
database_type="AUTONOMOUS_DATABASE",
|
||||
)
|
||||
|
||||
def list_target_databases_compartment(self, compartment_id: str, *, region: str | None) -> list[dict[str, Any]]:
|
||||
data_safe = self._client(self.oci.data_safe.DataSafeClient, region=region)
|
||||
return self._list_all(
|
||||
data_safe.list_target_databases,
|
||||
compartment_id,
|
||||
database_type="AUTONOMOUS_DATABASE",
|
||||
)
|
||||
|
||||
def get_target_database(self, target_database_id: str, *, region: str | None) -> dict[str, Any] | None:
|
||||
data_safe = self._client(self.oci.data_safe.DataSafeClient, region=region)
|
||||
return self._to_dict(data_safe.get_target_database(target_database_id))
|
||||
|
||||
|
||||
def item_list(response: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
data = response.get("data", [])
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
if isinstance(data, dict):
|
||||
items = data.get("items")
|
||||
if isinstance(items, list):
|
||||
return items
|
||||
return []
|
||||
|
||||
|
||||
def root_compartment_stub(tenancy_id: str) -> dict[str, Any]:
|
||||
return {
|
||||
"id": tenancy_id,
|
||||
"name": "root-tenancy",
|
||||
"display-name": "root-tenancy",
|
||||
"lifecycle-state": "ACTIVE",
|
||||
"compartment-id": "",
|
||||
}
|
||||
|
||||
|
||||
def format_error(exc: Exception) -> str:
|
||||
if isinstance(exc, OciCliError):
|
||||
return exc.stderr.strip() or exc.stdout.strip() or str(exc.returncode)
|
||||
status = getattr(exc, "status", None)
|
||||
code = getattr(exc, "code", None)
|
||||
message = getattr(exc, "message", None)
|
||||
request_id = getattr(exc, "request_id", None)
|
||||
if status or code or message:
|
||||
parts = [str(part) for part in (status, code, message) if part]
|
||||
if request_id:
|
||||
parts.append(f"opc-request-id={request_id}")
|
||||
return " - ".join(parts)
|
||||
return str(exc)
|
||||
|
||||
|
||||
def retry_after_seconds(exc: Exception) -> float | None:
|
||||
headers = getattr(exc, "headers", None)
|
||||
if not headers:
|
||||
return None
|
||||
value = None
|
||||
if isinstance(headers, dict):
|
||||
value = headers.get("retry-after") or headers.get("Retry-After")
|
||||
elif hasattr(headers, "get"):
|
||||
value = headers.get("retry-after") or headers.get("Retry-After")
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return max(0.0, float(value))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def is_retryable_error(exc: Exception) -> bool:
|
||||
status = getattr(exc, "status", None)
|
||||
if status in RETRYABLE_STATUS_CODES:
|
||||
return True
|
||||
if isinstance(exc, OciCliError):
|
||||
text = f"{exc.stdout}\n{exc.stderr}"
|
||||
return exc.returncode != 0 and (
|
||||
"TooManyRequests" in text
|
||||
or "Too many requests" in text
|
||||
or "ServiceUnavailable" in text
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def run_with_retries(
|
||||
operation: Any,
|
||||
*,
|
||||
retries: int,
|
||||
base_sleep: float,
|
||||
verbose: bool = False,
|
||||
) -> Any:
|
||||
attempts = max(1, retries + 1)
|
||||
last_exc: Exception | None = None
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
return operation()
|
||||
except Exception as exc:
|
||||
last_exc = exc
|
||||
if attempt >= attempts or not is_retryable_error(exc):
|
||||
raise
|
||||
retry_after = retry_after_seconds(exc)
|
||||
sleep_seconds = retry_after if retry_after is not None else base_sleep * (2 ** (attempt - 1))
|
||||
sleep_seconds += random.uniform(0, min(1.0, base_sleep))
|
||||
if verbose:
|
||||
print(
|
||||
f"Retry {attempt}/{retries} apos erro temporario: {format_error(exc)}. "
|
||||
f"Aguardando {sleep_seconds:.1f}s...",
|
||||
file=sys.stderr,
|
||||
)
|
||||
time.sleep(sleep_seconds)
|
||||
if last_exc:
|
||||
raise last_exc
|
||||
raise RuntimeError("Falha inesperada no mecanismo de retry.")
|
||||
|
||||
|
||||
class ProgressBar:
|
||||
def __init__(self, label: str, total: int, *, enabled: bool = True, width: int = 28) -> None:
|
||||
self.label = label
|
||||
self.total = max(1, total)
|
||||
self.enabled = enabled
|
||||
self.width = width
|
||||
self.current = 0
|
||||
self.started_at = time.monotonic()
|
||||
self.stats: dict[str, Any] = {}
|
||||
self.current_label = ""
|
||||
self.last_len = 0
|
||||
|
||||
def update(
|
||||
self,
|
||||
*,
|
||||
increment: int = 1,
|
||||
current_label: str | None = None,
|
||||
stats: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
self.current = min(self.total, self.current + increment)
|
||||
if current_label is not None:
|
||||
self.current_label = current_label
|
||||
if stats:
|
||||
self.stats.update(stats)
|
||||
self.render()
|
||||
|
||||
def finish(self, *, stats: dict[str, Any] | None = None) -> None:
|
||||
if stats:
|
||||
self.stats.update(stats)
|
||||
self.current = self.total
|
||||
self.render(done=True)
|
||||
|
||||
def render(self, *, done: bool = False) -> None:
|
||||
if not self.enabled:
|
||||
return
|
||||
percent = self.current / self.total
|
||||
filled = int(self.width * percent)
|
||||
bar = "#" * filled + "-" * (self.width - filled)
|
||||
elapsed = max(0.1, time.monotonic() - self.started_at)
|
||||
rate = self.current / elapsed
|
||||
stats_text = " ".join(f"{key}={value}" for key, value in self.stats.items())
|
||||
current_text = f" atual={self.current_label}" if self.current_label else ""
|
||||
line = (
|
||||
f"{self.label} [{bar}] {percent:6.1%} "
|
||||
f"{self.current}/{self.total} comps {rate:5.1f}/s {stats_text}{current_text}"
|
||||
)
|
||||
columns = shutil.get_terminal_size((120, 20)).columns
|
||||
if len(line) > columns - 1:
|
||||
line = line[: columns - 4] + "..."
|
||||
padding = " " * max(0, self.last_len - len(line))
|
||||
sys.stderr.write("\r" + line + padding)
|
||||
sys.stderr.flush()
|
||||
self.last_len = len(line)
|
||||
if done:
|
||||
sys.stderr.write("\n")
|
||||
sys.stderr.flush()
|
||||
self.last_len = 0
|
||||
|
||||
|
||||
def field(item: dict[str, Any] | None, *names: str, default: Any = "") -> Any:
|
||||
if not item:
|
||||
return default
|
||||
for name in names:
|
||||
candidates = {
|
||||
name,
|
||||
name.replace("_", "-"),
|
||||
name.replace("-", "_"),
|
||||
}
|
||||
parts = name.replace("_", "-").split("-")
|
||||
candidates.add(parts[0] + "".join(part.title() for part in parts[1:]))
|
||||
for candidate in candidates:
|
||||
if candidate in item and item[candidate] is not None:
|
||||
return item[candidate]
|
||||
return default
|
||||
|
||||
|
||||
def read_oci_config_value(profile: str | None, config_file: str | None, key: str) -> str | None:
|
||||
config_path = Path(config_file).expanduser() if config_file else Path.home() / ".oci" / "config"
|
||||
if not config_path.exists():
|
||||
return None
|
||||
|
||||
parser = configparser.ConfigParser()
|
||||
parser.read(config_path)
|
||||
section = profile or os.environ.get("OCI_CLI_PROFILE") or "DEFAULT"
|
||||
if section == "DEFAULT" and key in parser.defaults():
|
||||
return parser.defaults().get(key)
|
||||
if parser.has_section(section) and parser.has_option(section, key):
|
||||
return parser.get(section, key)
|
||||
if key in parser.defaults():
|
||||
return parser.defaults().get(key)
|
||||
return None
|
||||
|
||||
|
||||
def resolve_tenancy_id(args: argparse.Namespace) -> str | None:
|
||||
return (
|
||||
args.tenancy_id
|
||||
or os.environ.get("OCI_TENANCY")
|
||||
or os.environ.get("OCI_CLI_TENANCY")
|
||||
or read_oci_config_value(args.profile, args.config_file, "tenancy")
|
||||
)
|
||||
|
||||
|
||||
def resolve_default_region(args: argparse.Namespace) -> str | None:
|
||||
return args.region or read_oci_config_value(args.profile, args.config_file, "region")
|
||||
|
||||
|
||||
def get_compartments(client: Any, tenancy_id: str | None) -> list[dict[str, Any]]:
|
||||
compartments = client.list_compartments(tenancy_id)
|
||||
|
||||
deduped: dict[str, dict[str, Any]] = {}
|
||||
for compartment in compartments:
|
||||
cid = field(compartment, "id")
|
||||
state = str(field(compartment, "lifecycle-state", default="")).upper()
|
||||
if cid and state != "DELETED":
|
||||
deduped[cid] = compartment
|
||||
return list(deduped.values())
|
||||
|
||||
|
||||
def filter_compartments_for_scan(
|
||||
compartments: list[dict[str, Any]],
|
||||
*,
|
||||
compartment_id: str | None,
|
||||
include_subtree: bool,
|
||||
) -> list[dict[str, Any]]:
|
||||
if not compartment_id:
|
||||
return compartments
|
||||
|
||||
by_id = {field(compartment, "id"): compartment for compartment in compartments if field(compartment, "id")}
|
||||
if compartment_id not in by_id:
|
||||
raise RuntimeError(
|
||||
"Compartment informado em --compartment-id nao foi encontrado na lista acessivel. "
|
||||
"Verifique o OCID e as permissoes do usuario/grupo."
|
||||
)
|
||||
if not include_subtree:
|
||||
return [by_id[compartment_id]]
|
||||
|
||||
selected: list[dict[str, Any]] = []
|
||||
for compartment in compartments:
|
||||
current = field(compartment, "id")
|
||||
seen: set[str] = set()
|
||||
while current and current not in seen:
|
||||
seen.add(current)
|
||||
if current == compartment_id:
|
||||
selected.append(compartment)
|
||||
break
|
||||
current_item = by_id.get(current)
|
||||
if not current_item:
|
||||
break
|
||||
current = field(current_item, "compartment-id", default="")
|
||||
return selected
|
||||
|
||||
|
||||
def build_compartment_index(compartments: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
index = {field(c, "id"): dict(c) for c in compartments if field(c, "id")}
|
||||
for cid, compartment in index.items():
|
||||
compartment["path"] = compartment_path(cid, index)
|
||||
compartment["resolved-name"] = field(compartment, "name", "display-name", default=cid)
|
||||
return index
|
||||
|
||||
|
||||
def compartment_path(compartment_id: str, index: dict[str, dict[str, Any]]) -> str:
|
||||
names: list[str] = []
|
||||
seen: set[str] = set()
|
||||
current = compartment_id
|
||||
while current and current in index and current not in seen:
|
||||
seen.add(current)
|
||||
current_item = index[current]
|
||||
names.append(field(current_item, "name", "display-name", default=current))
|
||||
current = field(current_item, "compartment-id", default="")
|
||||
return "/".join(reversed(names))
|
||||
|
||||
|
||||
def get_regions(
|
||||
client: Any,
|
||||
tenancy_id: str,
|
||||
default_region: str | None,
|
||||
all_regions: bool,
|
||||
selected_regions: str | None = None,
|
||||
) -> list[str | None]:
|
||||
if selected_regions:
|
||||
regions = [region.strip() for region in selected_regions.split(",") if region.strip()]
|
||||
if not regions:
|
||||
raise RuntimeError("--regions foi informado, mas nenhuma regiao valida foi encontrada.")
|
||||
return regions
|
||||
if not all_regions:
|
||||
return [default_region]
|
||||
regions = []
|
||||
for item in client.list_region_subscriptions(tenancy_id):
|
||||
region_name = field(item, "region-name")
|
||||
status = str(field(item, "status", default="READY")).upper()
|
||||
if region_name and status in {"READY", "ACTIVE"}:
|
||||
regions.append(region_name)
|
||||
return sorted(set(regions))
|
||||
|
||||
|
||||
def list_adbs_in_compartment(
|
||||
client: Any,
|
||||
compartment: dict[str, Any],
|
||||
*,
|
||||
region: str | None,
|
||||
retries: int,
|
||||
retry_base_sleep: float,
|
||||
) -> tuple[list[dict[str, Any]], str | None]:
|
||||
compartment_id = field(compartment, "id")
|
||||
try:
|
||||
adbs = run_with_retries(
|
||||
lambda: client.list_autonomous_databases(compartment_id, region=region),
|
||||
retries=retries,
|
||||
base_sleep=retry_base_sleep,
|
||||
verbose=getattr(client, "verbose", False),
|
||||
)
|
||||
for adb in adbs:
|
||||
adb["_inventory_region"] = region or ""
|
||||
return adbs, None
|
||||
except Exception as exc:
|
||||
return [], f"{compartment_id}: {format_error(exc)}"
|
||||
|
||||
|
||||
def list_all_adbs(
|
||||
client: Any,
|
||||
compartments: list[dict[str, Any]],
|
||||
*,
|
||||
region: str | None,
|
||||
workers: int,
|
||||
retries: int,
|
||||
retry_base_sleep: float,
|
||||
include_terminated: bool,
|
||||
progress_callback: Callable[[dict[str, Any], list[dict[str, Any]], str | None], None] | None = None,
|
||||
) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
adbs: list[dict[str, Any]] = []
|
||||
errors: list[str] = []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, workers)) as executor:
|
||||
futures = {
|
||||
executor.submit(
|
||||
list_adbs_in_compartment,
|
||||
client,
|
||||
compartment,
|
||||
region=region,
|
||||
retries=retries,
|
||||
retry_base_sleep=retry_base_sleep,
|
||||
): compartment
|
||||
for compartment in compartments
|
||||
}
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
items, error = future.result()
|
||||
filtered_items: list[dict[str, Any]] = []
|
||||
if error:
|
||||
errors.append(error)
|
||||
for adb in items:
|
||||
state = str(field(adb, "lifecycle-state", default="")).upper()
|
||||
if include_terminated or state not in ADB_TERMINAL_STATES:
|
||||
filtered_items.append(adb)
|
||||
adbs.extend(filtered_items)
|
||||
if progress_callback:
|
||||
progress_callback(futures[future], filtered_items, error)
|
||||
return adbs, sorted(errors)
|
||||
|
||||
|
||||
def list_datasafe_targets_in_subtree(
|
||||
client: Any,
|
||||
root_compartment_id: str,
|
||||
*,
|
||||
region: str | None,
|
||||
) -> list[dict[str, Any]]:
|
||||
return client.list_target_databases_subtree(root_compartment_id, region=region)
|
||||
|
||||
|
||||
def list_datasafe_targets_per_compartment(
|
||||
client: Any,
|
||||
compartments: list[dict[str, Any]],
|
||||
*,
|
||||
region: str | None,
|
||||
workers: int,
|
||||
retries: int,
|
||||
retry_base_sleep: float,
|
||||
) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
targets: list[dict[str, Any]] = []
|
||||
errors: list[str] = []
|
||||
|
||||
def worker(compartment: dict[str, Any]) -> tuple[list[dict[str, Any]], str | None]:
|
||||
compartment_id = field(compartment, "id")
|
||||
try:
|
||||
return run_with_retries(
|
||||
lambda: client.list_target_databases_compartment(compartment_id, region=region),
|
||||
retries=retries,
|
||||
base_sleep=retry_base_sleep,
|
||||
verbose=getattr(client, "verbose", False),
|
||||
), None
|
||||
except Exception as exc:
|
||||
return [], f"{compartment_id}: {format_error(exc)}"
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, workers)) as executor:
|
||||
futures = [executor.submit(worker, compartment) for compartment in compartments]
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
items, error = future.result()
|
||||
if error:
|
||||
errors.append(error)
|
||||
targets.extend(items)
|
||||
return targets, sorted(errors)
|
||||
|
||||
|
||||
def list_datasafe_targets(
|
||||
client: Any,
|
||||
compartments: list[dict[str, Any]],
|
||||
*,
|
||||
root_compartment_id: str | None,
|
||||
region: str | None,
|
||||
workers: int,
|
||||
retries: int,
|
||||
retry_base_sleep: float,
|
||||
include_deleted: bool,
|
||||
) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
errors: list[str] = []
|
||||
try:
|
||||
if root_compartment_id:
|
||||
targets = run_with_retries(
|
||||
lambda: list_datasafe_targets_in_subtree(client, root_compartment_id, region=region),
|
||||
retries=retries,
|
||||
base_sleep=retry_base_sleep,
|
||||
verbose=getattr(client, "verbose", False),
|
||||
)
|
||||
else:
|
||||
targets, errors = list_datasafe_targets_per_compartment(
|
||||
client,
|
||||
compartments,
|
||||
region=region,
|
||||
workers=workers,
|
||||
retries=retries,
|
||||
retry_base_sleep=retry_base_sleep,
|
||||
)
|
||||
except Exception as exc:
|
||||
errors.append(f"subtree Data Safe: {format_error(exc)}")
|
||||
targets, per_compartment_errors = list_datasafe_targets_per_compartment(
|
||||
client,
|
||||
compartments,
|
||||
region=region,
|
||||
workers=workers,
|
||||
retries=retries,
|
||||
retry_base_sleep=retry_base_sleep,
|
||||
)
|
||||
errors.extend(per_compartment_errors)
|
||||
|
||||
filtered = []
|
||||
for target in targets:
|
||||
target["_inventory_region"] = region or ""
|
||||
state = str(field(target, "lifecycle-state", default="")).upper()
|
||||
if include_deleted or state not in DATASAFE_DELETED_STATES:
|
||||
filtered.append(target)
|
||||
return filtered, sorted(errors)
|
||||
|
||||
|
||||
def target_associated_adb_id(target: dict[str, Any]) -> str:
|
||||
associated_resource_ids = field(
|
||||
target,
|
||||
"associated-resource-ids",
|
||||
"associated_resource_ids",
|
||||
"associatedResourceIds",
|
||||
default=[],
|
||||
)
|
||||
if isinstance(associated_resource_ids, list):
|
||||
for resource_id in associated_resource_ids:
|
||||
if isinstance(resource_id, str) and resource_id.startswith("ocid1.autonomousdatabase."):
|
||||
return resource_id
|
||||
for resource_id in associated_resource_ids:
|
||||
if isinstance(resource_id, str) and resource_id.startswith("ocid1.autonomous"):
|
||||
return resource_id
|
||||
|
||||
direct = field(
|
||||
target,
|
||||
"associated-resource-id",
|
||||
"associated_resource_id",
|
||||
"associatedResourceId",
|
||||
"autonomous-database-id",
|
||||
"autonomous_database_id",
|
||||
"autonomousDatabaseId",
|
||||
default="",
|
||||
)
|
||||
if isinstance(direct, str) and direct:
|
||||
return direct
|
||||
details = field(target, "database-details", "databaseDetails", default={})
|
||||
if isinstance(details, dict):
|
||||
return field(
|
||||
details,
|
||||
"autonomous-database-id",
|
||||
"autonomous_database_id",
|
||||
"autonomousDatabaseId",
|
||||
default="",
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
def enrich_missing_target_associations(
|
||||
client: Any,
|
||||
targets: list[dict[str, Any]],
|
||||
*,
|
||||
region: str | None,
|
||||
workers: int,
|
||||
retries: int,
|
||||
retry_base_sleep: float,
|
||||
) -> list[str]:
|
||||
errors: list[str] = []
|
||||
missing = [target for target in targets if not target_associated_adb_id(target) and field(target, "id")]
|
||||
if not missing:
|
||||
return errors
|
||||
|
||||
def worker(target: dict[str, Any]) -> tuple[dict[str, Any] | None, str | None]:
|
||||
target_id = field(target, "id")
|
||||
try:
|
||||
return run_with_retries(
|
||||
lambda: client.get_target_database(target_id, region=region),
|
||||
retries=retries,
|
||||
base_sleep=retry_base_sleep,
|
||||
verbose=getattr(client, "verbose", False),
|
||||
), None
|
||||
except Exception as exc:
|
||||
return None, f"{target_id}: {format_error(exc)}"
|
||||
|
||||
by_id = {field(target, "id"): target for target in targets}
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, workers)) as executor:
|
||||
futures = [executor.submit(worker, target) for target in missing]
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
detail, error = future.result()
|
||||
if error:
|
||||
errors.append(error)
|
||||
if detail and field(detail, "id") in by_id:
|
||||
by_id[field(detail, "id")].update(detail)
|
||||
return sorted(errors)
|
||||
|
||||
|
||||
def adb_to_base_row(
|
||||
adb: dict[str, Any],
|
||||
compartment_index: dict[str, dict[str, Any]],
|
||||
*,
|
||||
region: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
compartment_id = field(adb, "compartment-id")
|
||||
compartment = compartment_index.get(compartment_id, {})
|
||||
return {
|
||||
"region": region or field(adb, "_inventory_region"),
|
||||
"adb_display_name": field(adb, "display-name"),
|
||||
"adb_db_name": field(adb, "db-name"),
|
||||
"adb_ocid": field(adb, "id"),
|
||||
"adb_lifecycle_state": field(adb, "lifecycle-state"),
|
||||
"adb_workload": field(adb, "db-workload"),
|
||||
"adb_is_free_tier": field(adb, "is-free-tier"),
|
||||
"adb_time_created": field(adb, "time-created"),
|
||||
"adb_compartment_name": field(compartment, "resolved-name", "name", "display-name"),
|
||||
"adb_compartment_path": field(compartment, "path"),
|
||||
"adb_compartment_ocid": compartment_id,
|
||||
}
|
||||
|
||||
|
||||
def target_to_base_row(
|
||||
target: dict[str, Any],
|
||||
compartment_index: dict[str, dict[str, Any]],
|
||||
*,
|
||||
prefix: str = "datasafe_target",
|
||||
) -> dict[str, Any]:
|
||||
compartment_id = field(target, "compartment-id")
|
||||
compartment = compartment_index.get(compartment_id, {})
|
||||
return {
|
||||
f"{prefix}_display_name": field(target, "display-name"),
|
||||
f"{prefix}_ocid": field(target, "id"),
|
||||
f"{prefix}_lifecycle_state": field(target, "lifecycle-state"),
|
||||
f"{prefix}_database_type": field(target, "database-type"),
|
||||
f"{prefix}_infrastructure_type": field(target, "infrastructure-type"),
|
||||
f"{prefix}_time_created": field(target, "time-created"),
|
||||
f"{prefix}_compartment_name": field(compartment, "resolved-name", "name", "display-name"),
|
||||
f"{prefix}_compartment_path": field(compartment, "path"),
|
||||
f"{prefix}_compartment_ocid": compartment_id,
|
||||
f"{prefix}_associated_adb_ocid": target_associated_adb_id(target),
|
||||
}
|
||||
|
||||
|
||||
def normalize_output_value(value: Any) -> Any:
|
||||
if isinstance(value, (datetime, date)):
|
||||
return value.isoformat()
|
||||
if isinstance(value, (list, tuple)):
|
||||
return json.dumps([normalize_output_value(item) for item in value], ensure_ascii=False)
|
||||
if isinstance(value, dict):
|
||||
return json.dumps({key: normalize_output_value(item) for key, item in value.items()}, ensure_ascii=False)
|
||||
return value
|
||||
|
||||
|
||||
def write_rows(rows: list[dict[str, Any]], output_path: str, output_format: str) -> None:
|
||||
path = Path(output_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
normalized_rows = [
|
||||
{key: normalize_output_value(value) for key, value in row.items()}
|
||||
for row in rows
|
||||
]
|
||||
if output_format == "json":
|
||||
path.write_text(json.dumps(normalized_rows, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
return
|
||||
|
||||
fieldnames: list[str] = []
|
||||
for column in PREFERRED_CSV_COLUMNS:
|
||||
if any(column in row for row in normalized_rows):
|
||||
fieldnames.append(column)
|
||||
for row in normalized_rows:
|
||||
for key in row:
|
||||
if key not in fieldnames:
|
||||
fieldnames.append(key)
|
||||
with path.open("w", newline="", encoding="utf-8") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(normalized_rows)
|
||||
|
||||
|
||||
def add_common_args(parser: argparse.ArgumentParser) -> None:
|
||||
parser.add_argument("--tenancy-id", help="OCID da tenancy/root compartment. Se omitido, tenta ler do perfil OCI.")
|
||||
parser.add_argument("--profile", help="Perfil da OCI CLI. Ex.: DEFAULT, PROD.")
|
||||
parser.add_argument("--config-file", help="Caminho para arquivo de configuracao OCI CLI.")
|
||||
parser.add_argument("--auth", help="Metodo de autenticacao OCI CLI. Ex.: instance_principal, resource_principal, security_token.")
|
||||
parser.add_argument("--auth-purpose", help="Auth purpose da OCI CLI, quando aplicavel.")
|
||||
parser.add_argument("--engine", choices=["sdk", "cli"], default="sdk", help="Motor de consulta. Padrao: sdk, mais rapido que chamar a OCI CLI repetidamente.")
|
||||
parser.add_argument("--region", help="Regiao OCI. Se omitida, usa a regiao do perfil OCI CLI.")
|
||||
parser.add_argument("--regions", help="Lista de regioes separadas por virgula. Ex.: sa-saopaulo-1,us-ashburn-1.")
|
||||
parser.add_argument("--all-regions", action="store_true", help="Executa em todas as regioes subscribed da tenancy.")
|
||||
parser.add_argument("--compartment-id", help="Limita a varredura de ADBs a um compartment especifico.")
|
||||
parser.add_argument("--compartment-subtree", action="store_true", help="Com --compartment-id, inclui subcompartments na varredura de ADBs.")
|
||||
parser.add_argument("--workers", type=int, default=4, help="Quantidade de chamadas paralelas por compartment. Padrao: 4.")
|
||||
parser.add_argument("--retries", type=int, default=6, help="Tentativas adicionais para erros temporarios como 429/5xx. Padrao: 6.")
|
||||
parser.add_argument("--retry-base-sleep", type=float, default=2.0, help="Espera inicial em segundos para backoff exponencial. Padrao: 2.0.")
|
||||
parser.add_argument("--include-terminated-adbs", action="store_true", help="Inclui ADBs TERMINATED/TERMINATING.")
|
||||
parser.add_argument("--include-deleted-targets", action="store_true", help="Inclui targets Data Safe DELETED/DELETING.")
|
||||
parser.add_argument("--format", choices=["csv", "json"], default="csv", help="Formato de saida. Padrao: csv.")
|
||||
parser.add_argument("--no-progress", action="store_true", help="Desativa a barra de progresso no terminal.")
|
||||
parser.add_argument("--verbose", action="store_true", help="Mostra comandos OCI CLI executados.")
|
||||
|
||||
|
||||
def build_context(args: argparse.Namespace) -> InventoryContext:
|
||||
default_region = resolve_default_region(args)
|
||||
if args.engine == "cli":
|
||||
client: Any = OciCli(
|
||||
profile=args.profile,
|
||||
config_file=args.config_file,
|
||||
auth=args.auth,
|
||||
auth_purpose=args.auth_purpose,
|
||||
region=args.region,
|
||||
verbose=args.verbose,
|
||||
)
|
||||
else:
|
||||
client = OciSdkClient(
|
||||
profile=args.profile,
|
||||
config_file=args.config_file,
|
||||
auth=args.auth,
|
||||
region=args.region,
|
||||
verbose=args.verbose,
|
||||
)
|
||||
tenancy_id = resolve_tenancy_id(args)
|
||||
if not tenancy_id:
|
||||
config = getattr(client, "config", {})
|
||||
if isinstance(config, dict):
|
||||
tenancy_id = config.get("tenancy")
|
||||
all_compartments = get_compartments(client, tenancy_id)
|
||||
compartment_index = build_compartment_index(all_compartments)
|
||||
compartments = filter_compartments_for_scan(
|
||||
all_compartments,
|
||||
compartment_id=args.compartment_id,
|
||||
include_subtree=args.compartment_subtree,
|
||||
)
|
||||
if not tenancy_id:
|
||||
root_candidates = [
|
||||
cid
|
||||
for cid, item in compartment_index.items()
|
||||
if not field(item, "compartment-id", default="")
|
||||
]
|
||||
tenancy_id = root_candidates[0] if root_candidates else None
|
||||
return InventoryContext(
|
||||
client=client,
|
||||
tenancy_id=tenancy_id,
|
||||
default_region=default_region,
|
||||
compartments=compartments,
|
||||
compartment_index=compartment_index,
|
||||
)
|
||||
|
||||
|
||||
def print_warnings(title: str, errors: list[str]) -> None:
|
||||
if not errors:
|
||||
return
|
||||
print(f"\nAvisos durante {title}:", file=sys.stderr)
|
||||
for error in errors:
|
||||
print(f" - {error}", file=sys.stderr)
|
||||
Reference in New Issue
Block a user