985 lines
37 KiB
Python
Executable File
985 lines
37 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Shared helpers for OCI Autonomous Database and Data Safe inventory scripts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import configparser
|
|
import concurrent.futures
|
|
import csv
|
|
import json
|
|
import os
|
|
import random
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from dataclasses import dataclass
|
|
from datetime import date, datetime
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
|
|
ADB_TERMINAL_STATES = {"TERMINATED", "TERMINATING"}
|
|
DATASAFE_DELETED_STATES = {"DELETED", "DELETING"}
|
|
RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504}
|
|
PREFERRED_CSV_COLUMNS = [
|
|
"datasafe_registered",
|
|
"adb_time_created",
|
|
"adb_compartment_name",
|
|
"adb_compartment_path",
|
|
]
|
|
|
|
|
|
class OciCliError(RuntimeError):
|
|
def __init__(self, cmd: list[str], returncode: int, stdout: str, stderr: str) -> None:
|
|
self.cmd = cmd
|
|
self.returncode = returncode
|
|
self.stdout = stdout
|
|
self.stderr = stderr
|
|
super().__init__(f"OCI CLI failed with exit code {returncode}: {' '.join(cmd)}")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class InventoryContext:
|
|
client: Any
|
|
tenancy_id: str | None
|
|
default_region: str | None
|
|
compartments: list[dict[str, Any]]
|
|
compartment_index: dict[str, dict[str, Any]]
|
|
|
|
|
|
class OciCli:
|
|
def __init__(
|
|
self,
|
|
*,
|
|
profile: str | None = None,
|
|
config_file: str | None = None,
|
|
auth: str | None = None,
|
|
auth_purpose: str | None = None,
|
|
region: str | None = None,
|
|
verbose: bool = False,
|
|
) -> None:
|
|
self.profile = profile
|
|
self.config_file = config_file
|
|
self.auth = auth
|
|
self.auth_purpose = auth_purpose
|
|
self.region = region
|
|
self.verbose = verbose
|
|
|
|
def global_args(self, region: str | None = None) -> list[str]:
|
|
args: list[str] = []
|
|
if self.profile:
|
|
args.extend(["--profile", self.profile])
|
|
if self.config_file:
|
|
args.extend(["--config-file", self.config_file])
|
|
if self.auth:
|
|
args.extend(["--auth", self.auth])
|
|
if self.auth_purpose:
|
|
args.extend(["--auth-purpose", self.auth_purpose])
|
|
selected_region = region or self.region
|
|
if selected_region:
|
|
args.extend(["--region", selected_region])
|
|
return args
|
|
|
|
def run_json(self, service_args: list[str], *, region: str | None = None) -> dict[str, Any]:
|
|
cmd = ["oci", *service_args, *self.global_args(region), "--output", "json"]
|
|
if self.verbose:
|
|
print("+ " + " ".join(cmd), file=sys.stderr)
|
|
try:
|
|
completed = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
|
except FileNotFoundError as exc:
|
|
raise SystemExit("OCI CLI nao encontrado no PATH. Instale/configure o comando 'oci'.") from exc
|
|
if completed.returncode != 0:
|
|
raise OciCliError(cmd, completed.returncode, completed.stdout, completed.stderr)
|
|
stdout = completed.stdout.strip()
|
|
if not stdout:
|
|
return {}
|
|
try:
|
|
return json.loads(stdout)
|
|
except json.JSONDecodeError as exc:
|
|
raise RuntimeError(f"Resposta da OCI CLI nao e JSON valido para: {' '.join(cmd)}") from exc
|
|
|
|
def list_compartments(self, tenancy_id: str | None) -> list[dict[str, Any]]:
|
|
base_args = [
|
|
"iam",
|
|
"compartment",
|
|
"list",
|
|
"--compartment-id-in-subtree",
|
|
"true",
|
|
"--access-level",
|
|
"ANY",
|
|
"--lifecycle-state",
|
|
"ACTIVE",
|
|
"--all",
|
|
]
|
|
if tenancy_id:
|
|
base_args.extend(["--compartment-id", tenancy_id])
|
|
|
|
try:
|
|
response = self.run_json([*base_args, "--include-root"])
|
|
return item_list(response)
|
|
except OciCliError:
|
|
compartments = item_list(self.run_json(base_args))
|
|
if tenancy_id and all(field(c, "id") != tenancy_id for c in compartments):
|
|
compartments.append(root_compartment_stub(tenancy_id))
|
|
return compartments
|
|
|
|
def list_region_subscriptions(self, tenancy_id: str) -> list[dict[str, Any]]:
|
|
return item_list(self.run_json(["iam", "region-subscription", "list", "--tenancy-id", tenancy_id]))
|
|
|
|
def list_autonomous_databases(self, compartment_id: str, *, region: str | None) -> list[dict[str, Any]]:
|
|
response = self.run_json(
|
|
["db", "autonomous-database", "list", "--compartment-id", compartment_id, "--all"],
|
|
region=region,
|
|
)
|
|
return item_list(response)
|
|
|
|
def list_target_databases_subtree(self, root_compartment_id: str, *, region: str | None) -> list[dict[str, Any]]:
|
|
response = self.run_json(
|
|
[
|
|
"data-safe",
|
|
"target-database",
|
|
"list",
|
|
"--compartment-id",
|
|
root_compartment_id,
|
|
"--compartment-id-in-subtree",
|
|
"true",
|
|
"--access-level",
|
|
"ACCESSIBLE",
|
|
"--database-type",
|
|
"AUTONOMOUS_DATABASE",
|
|
"--all",
|
|
],
|
|
region=region,
|
|
)
|
|
return item_list(response)
|
|
|
|
def list_target_databases_compartment(self, compartment_id: str, *, region: str | None) -> list[dict[str, Any]]:
|
|
response = self.run_json(
|
|
[
|
|
"data-safe",
|
|
"target-database",
|
|
"list",
|
|
"--compartment-id",
|
|
compartment_id,
|
|
"--database-type",
|
|
"AUTONOMOUS_DATABASE",
|
|
"--all",
|
|
],
|
|
region=region,
|
|
)
|
|
return item_list(response)
|
|
|
|
def get_target_database(self, target_database_id: str, *, region: str | None) -> dict[str, Any] | None:
|
|
response = self.run_json(
|
|
["data-safe", "target-database", "get", "--target-database-id", target_database_id],
|
|
region=region,
|
|
)
|
|
data = response.get("data")
|
|
return data if isinstance(data, dict) else None
|
|
|
|
|
|
class OciSdkClient:
|
|
def __init__(
|
|
self,
|
|
*,
|
|
profile: str | None = None,
|
|
config_file: str | None = None,
|
|
auth: str | None = None,
|
|
region: str | None = None,
|
|
verbose: bool = False,
|
|
) -> None:
|
|
try:
|
|
import oci # type: ignore[import-not-found]
|
|
import oci.data_safe # type: ignore[import-not-found]
|
|
import oci.database # type: ignore[import-not-found]
|
|
import oci.identity # type: ignore[import-not-found]
|
|
except ImportError as exc:
|
|
raise SystemExit(
|
|
"Pacote 'oci' nao encontrado. Rode: pip install -r requirements.txt "
|
|
"ou use --engine cli para manter a versao baseada na OCI CLI."
|
|
) from exc
|
|
|
|
self.oci = oci
|
|
self.profile = profile or os.environ.get("OCI_CLI_PROFILE") or "DEFAULT"
|
|
self.config_file = config_file or str(Path.home() / ".oci" / "config")
|
|
self.auth = auth
|
|
self.region = region
|
|
self.verbose = verbose
|
|
self.signer = None
|
|
|
|
if auth == "instance_principal":
|
|
self.config = {"region": region or os.environ.get("OCI_REGION", "")}
|
|
self.signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
|
|
elif auth == "resource_principal":
|
|
self.config = {"region": region or os.environ.get("OCI_RESOURCE_PRINCIPAL_REGION", "")}
|
|
self.signer = oci.auth.signers.get_resource_principals_signer()
|
|
elif auth in (None, "", "api_key"):
|
|
self.config = oci.config.from_file(file_location=self.config_file, profile_name=self.profile)
|
|
if region:
|
|
self.config["region"] = region
|
|
oci.config.validate_config(self.config)
|
|
else:
|
|
raise SystemExit(
|
|
f"--auth {auth!r} ainda nao e suportado pelo engine SDK. "
|
|
"Use --engine cli para esse modo de autenticacao."
|
|
)
|
|
|
|
def _config_for_region(self, region: str | None) -> dict[str, Any]:
|
|
config = dict(self.config)
|
|
selected_region = region or self.region or config.get("region")
|
|
if selected_region:
|
|
config["region"] = selected_region
|
|
if not config.get("region"):
|
|
raise RuntimeError("Regiao OCI nao definida. Informe --region ou configure region no perfil OCI.")
|
|
return config
|
|
|
|
def _client(self, client_cls: Any, *, region: str | None = None) -> Any:
|
|
kwargs = {"signer": self.signer} if self.signer else {}
|
|
return client_cls(self._config_for_region(region), **kwargs)
|
|
|
|
def _list_all(self, method: Any, *args: Any, **kwargs: Any) -> list[dict[str, Any]]:
|
|
response = self.oci.pagination.list_call_get_all_results(method, *args, **kwargs)
|
|
return [self.oci.util.to_dict(item) for item in response.data]
|
|
|
|
def _to_dict(self, response: Any) -> dict[str, Any] | None:
|
|
if response.data is None:
|
|
return None
|
|
return self.oci.util.to_dict(response.data)
|
|
|
|
def list_compartments(self, tenancy_id: str | None) -> list[dict[str, Any]]:
|
|
if not tenancy_id:
|
|
raise RuntimeError("Nao consegui resolver a tenancy. Informe --tenancy-id.")
|
|
identity = self._client(self.oci.identity.IdentityClient)
|
|
compartments = self._list_all(
|
|
identity.list_compartments,
|
|
tenancy_id,
|
|
compartment_id_in_subtree=True,
|
|
access_level="ANY",
|
|
lifecycle_state="ACTIVE",
|
|
)
|
|
try:
|
|
tenancy = self._to_dict(identity.get_tenancy(tenancy_id)) or root_compartment_stub(tenancy_id)
|
|
tenancy["name"] = field(tenancy, "name", "display-name", default="root-tenancy")
|
|
tenancy["display-name"] = field(tenancy, "display-name", "name", default="root-tenancy")
|
|
tenancy["compartment-id"] = ""
|
|
compartments.append(tenancy)
|
|
except Exception:
|
|
compartments.append(root_compartment_stub(tenancy_id))
|
|
return compartments
|
|
|
|
def list_region_subscriptions(self, tenancy_id: str) -> list[dict[str, Any]]:
|
|
identity = self._client(self.oci.identity.IdentityClient)
|
|
return self._list_all(identity.list_region_subscriptions, tenancy_id)
|
|
|
|
def list_autonomous_databases(self, compartment_id: str, *, region: str | None) -> list[dict[str, Any]]:
|
|
database = self._client(self.oci.database.DatabaseClient, region=region)
|
|
return self._list_all(database.list_autonomous_databases, compartment_id)
|
|
|
|
def list_target_databases_subtree(self, root_compartment_id: str, *, region: str | None) -> list[dict[str, Any]]:
|
|
data_safe = self._client(self.oci.data_safe.DataSafeClient, region=region)
|
|
return self._list_all(
|
|
data_safe.list_target_databases,
|
|
root_compartment_id,
|
|
compartment_id_in_subtree=True,
|
|
access_level="ACCESSIBLE",
|
|
database_type="AUTONOMOUS_DATABASE",
|
|
)
|
|
|
|
def list_target_databases_compartment(self, compartment_id: str, *, region: str | None) -> list[dict[str, Any]]:
|
|
data_safe = self._client(self.oci.data_safe.DataSafeClient, region=region)
|
|
return self._list_all(
|
|
data_safe.list_target_databases,
|
|
compartment_id,
|
|
database_type="AUTONOMOUS_DATABASE",
|
|
)
|
|
|
|
def get_target_database(self, target_database_id: str, *, region: str | None) -> dict[str, Any] | None:
|
|
data_safe = self._client(self.oci.data_safe.DataSafeClient, region=region)
|
|
return self._to_dict(data_safe.get_target_database(target_database_id))
|
|
|
|
|
|
def item_list(response: dict[str, Any]) -> list[dict[str, Any]]:
|
|
data = response.get("data", [])
|
|
if isinstance(data, list):
|
|
return data
|
|
if isinstance(data, dict):
|
|
items = data.get("items")
|
|
if isinstance(items, list):
|
|
return items
|
|
return []
|
|
|
|
|
|
def root_compartment_stub(tenancy_id: str) -> dict[str, Any]:
|
|
return {
|
|
"id": tenancy_id,
|
|
"name": "root-tenancy",
|
|
"display-name": "root-tenancy",
|
|
"lifecycle-state": "ACTIVE",
|
|
"compartment-id": "",
|
|
}
|
|
|
|
|
|
def format_error(exc: Exception) -> str:
|
|
if isinstance(exc, OciCliError):
|
|
return exc.stderr.strip() or exc.stdout.strip() or str(exc.returncode)
|
|
status = getattr(exc, "status", None)
|
|
code = getattr(exc, "code", None)
|
|
message = getattr(exc, "message", None)
|
|
request_id = getattr(exc, "request_id", None)
|
|
if status or code or message:
|
|
parts = [str(part) for part in (status, code, message) if part]
|
|
if request_id:
|
|
parts.append(f"opc-request-id={request_id}")
|
|
return " - ".join(parts)
|
|
return str(exc)
|
|
|
|
|
|
def retry_after_seconds(exc: Exception) -> float | None:
|
|
headers = getattr(exc, "headers", None)
|
|
if not headers:
|
|
return None
|
|
value = None
|
|
if isinstance(headers, dict):
|
|
value = headers.get("retry-after") or headers.get("Retry-After")
|
|
elif hasattr(headers, "get"):
|
|
value = headers.get("retry-after") or headers.get("Retry-After")
|
|
if value is None:
|
|
return None
|
|
try:
|
|
return max(0.0, float(value))
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def is_retryable_error(exc: Exception) -> bool:
|
|
status = getattr(exc, "status", None)
|
|
if status in RETRYABLE_STATUS_CODES:
|
|
return True
|
|
if isinstance(exc, OciCliError):
|
|
text = f"{exc.stdout}\n{exc.stderr}"
|
|
return exc.returncode != 0 and (
|
|
"TooManyRequests" in text
|
|
or "Too many requests" in text
|
|
or "ServiceUnavailable" in text
|
|
)
|
|
return False
|
|
|
|
|
|
def run_with_retries(
|
|
operation: Any,
|
|
*,
|
|
retries: int,
|
|
base_sleep: float,
|
|
verbose: bool = False,
|
|
) -> Any:
|
|
attempts = max(1, retries + 1)
|
|
last_exc: Exception | None = None
|
|
for attempt in range(1, attempts + 1):
|
|
try:
|
|
return operation()
|
|
except Exception as exc:
|
|
last_exc = exc
|
|
if attempt >= attempts or not is_retryable_error(exc):
|
|
raise
|
|
retry_after = retry_after_seconds(exc)
|
|
sleep_seconds = retry_after if retry_after is not None else base_sleep * (2 ** (attempt - 1))
|
|
sleep_seconds += random.uniform(0, min(1.0, base_sleep))
|
|
if verbose:
|
|
print(
|
|
f"Retry {attempt}/{retries} apos erro temporario: {format_error(exc)}. "
|
|
f"Aguardando {sleep_seconds:.1f}s...",
|
|
file=sys.stderr,
|
|
)
|
|
time.sleep(sleep_seconds)
|
|
if last_exc:
|
|
raise last_exc
|
|
raise RuntimeError("Falha inesperada no mecanismo de retry.")
|
|
|
|
|
|
class ProgressBar:
|
|
def __init__(self, label: str, total: int, *, enabled: bool = True, width: int = 28) -> None:
|
|
self.label = label
|
|
self.total = max(1, total)
|
|
self.enabled = enabled
|
|
self.width = width
|
|
self.current = 0
|
|
self.started_at = time.monotonic()
|
|
self.stats: dict[str, Any] = {}
|
|
self.current_label = ""
|
|
self.last_len = 0
|
|
|
|
def update(
|
|
self,
|
|
*,
|
|
increment: int = 1,
|
|
current_label: str | None = None,
|
|
stats: dict[str, Any] | None = None,
|
|
) -> None:
|
|
self.current = min(self.total, self.current + increment)
|
|
if current_label is not None:
|
|
self.current_label = current_label
|
|
if stats:
|
|
self.stats.update(stats)
|
|
self.render()
|
|
|
|
def finish(self, *, stats: dict[str, Any] | None = None) -> None:
|
|
if stats:
|
|
self.stats.update(stats)
|
|
self.current = self.total
|
|
self.render(done=True)
|
|
|
|
def render(self, *, done: bool = False) -> None:
|
|
if not self.enabled:
|
|
return
|
|
percent = self.current / self.total
|
|
filled = int(self.width * percent)
|
|
bar = "#" * filled + "-" * (self.width - filled)
|
|
elapsed = max(0.1, time.monotonic() - self.started_at)
|
|
rate = self.current / elapsed
|
|
stats_text = " ".join(f"{key}={value}" for key, value in self.stats.items())
|
|
current_text = f" atual={self.current_label}" if self.current_label else ""
|
|
line = (
|
|
f"{self.label} [{bar}] {percent:6.1%} "
|
|
f"{self.current}/{self.total} comps {rate:5.1f}/s {stats_text}{current_text}"
|
|
)
|
|
columns = shutil.get_terminal_size((120, 20)).columns
|
|
if len(line) > columns - 1:
|
|
line = line[: columns - 4] + "..."
|
|
padding = " " * max(0, self.last_len - len(line))
|
|
sys.stderr.write("\r" + line + padding)
|
|
sys.stderr.flush()
|
|
self.last_len = len(line)
|
|
if done:
|
|
sys.stderr.write("\n")
|
|
sys.stderr.flush()
|
|
self.last_len = 0
|
|
|
|
|
|
def field(item: dict[str, Any] | None, *names: str, default: Any = "") -> Any:
|
|
if not item:
|
|
return default
|
|
for name in names:
|
|
candidates = {
|
|
name,
|
|
name.replace("_", "-"),
|
|
name.replace("-", "_"),
|
|
}
|
|
parts = name.replace("_", "-").split("-")
|
|
candidates.add(parts[0] + "".join(part.title() for part in parts[1:]))
|
|
for candidate in candidates:
|
|
if candidate in item and item[candidate] is not None:
|
|
return item[candidate]
|
|
return default
|
|
|
|
|
|
def read_oci_config_value(profile: str | None, config_file: str | None, key: str) -> str | None:
|
|
config_path = Path(config_file).expanduser() if config_file else Path.home() / ".oci" / "config"
|
|
if not config_path.exists():
|
|
return None
|
|
|
|
parser = configparser.ConfigParser()
|
|
parser.read(config_path)
|
|
section = profile or os.environ.get("OCI_CLI_PROFILE") or "DEFAULT"
|
|
if section == "DEFAULT" and key in parser.defaults():
|
|
return parser.defaults().get(key)
|
|
if parser.has_section(section) and parser.has_option(section, key):
|
|
return parser.get(section, key)
|
|
if key in parser.defaults():
|
|
return parser.defaults().get(key)
|
|
return None
|
|
|
|
|
|
def resolve_tenancy_id(args: argparse.Namespace) -> str | None:
|
|
return (
|
|
args.tenancy_id
|
|
or os.environ.get("OCI_TENANCY")
|
|
or os.environ.get("OCI_CLI_TENANCY")
|
|
or read_oci_config_value(args.profile, args.config_file, "tenancy")
|
|
)
|
|
|
|
|
|
def resolve_default_region(args: argparse.Namespace) -> str | None:
|
|
return args.region or read_oci_config_value(args.profile, args.config_file, "region")
|
|
|
|
|
|
def get_compartments(client: Any, tenancy_id: str | None) -> list[dict[str, Any]]:
|
|
compartments = client.list_compartments(tenancy_id)
|
|
|
|
deduped: dict[str, dict[str, Any]] = {}
|
|
for compartment in compartments:
|
|
cid = field(compartment, "id")
|
|
state = str(field(compartment, "lifecycle-state", default="")).upper()
|
|
if cid and state != "DELETED":
|
|
deduped[cid] = compartment
|
|
return list(deduped.values())
|
|
|
|
|
|
def filter_compartments_for_scan(
|
|
compartments: list[dict[str, Any]],
|
|
*,
|
|
compartment_id: str | None,
|
|
include_subtree: bool,
|
|
) -> list[dict[str, Any]]:
|
|
if not compartment_id:
|
|
return compartments
|
|
|
|
by_id = {field(compartment, "id"): compartment for compartment in compartments if field(compartment, "id")}
|
|
if compartment_id not in by_id:
|
|
raise RuntimeError(
|
|
"Compartment informado em --compartment-id nao foi encontrado na lista acessivel. "
|
|
"Verifique o OCID e as permissoes do usuario/grupo."
|
|
)
|
|
if not include_subtree:
|
|
return [by_id[compartment_id]]
|
|
|
|
selected: list[dict[str, Any]] = []
|
|
for compartment in compartments:
|
|
current = field(compartment, "id")
|
|
seen: set[str] = set()
|
|
while current and current not in seen:
|
|
seen.add(current)
|
|
if current == compartment_id:
|
|
selected.append(compartment)
|
|
break
|
|
current_item = by_id.get(current)
|
|
if not current_item:
|
|
break
|
|
current = field(current_item, "compartment-id", default="")
|
|
return selected
|
|
|
|
|
|
def build_compartment_index(compartments: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
|
index = {field(c, "id"): dict(c) for c in compartments if field(c, "id")}
|
|
for cid, compartment in index.items():
|
|
compartment["path"] = compartment_path(cid, index)
|
|
compartment["resolved-name"] = field(compartment, "name", "display-name", default=cid)
|
|
return index
|
|
|
|
|
|
def compartment_path(compartment_id: str, index: dict[str, dict[str, Any]]) -> str:
|
|
names: list[str] = []
|
|
seen: set[str] = set()
|
|
current = compartment_id
|
|
while current and current in index and current not in seen:
|
|
seen.add(current)
|
|
current_item = index[current]
|
|
names.append(field(current_item, "name", "display-name", default=current))
|
|
current = field(current_item, "compartment-id", default="")
|
|
return "/".join(reversed(names))
|
|
|
|
|
|
def get_regions(
|
|
client: Any,
|
|
tenancy_id: str,
|
|
default_region: str | None,
|
|
all_regions: bool,
|
|
selected_regions: str | None = None,
|
|
) -> list[str | None]:
|
|
if selected_regions:
|
|
regions = [region.strip() for region in selected_regions.split(",") if region.strip()]
|
|
if not regions:
|
|
raise RuntimeError("--regions foi informado, mas nenhuma regiao valida foi encontrada.")
|
|
return regions
|
|
if not all_regions:
|
|
return [default_region]
|
|
regions = []
|
|
for item in client.list_region_subscriptions(tenancy_id):
|
|
region_name = field(item, "region-name")
|
|
status = str(field(item, "status", default="READY")).upper()
|
|
if region_name and status in {"READY", "ACTIVE"}:
|
|
regions.append(region_name)
|
|
return sorted(set(regions))
|
|
|
|
|
|
def list_adbs_in_compartment(
|
|
client: Any,
|
|
compartment: dict[str, Any],
|
|
*,
|
|
region: str | None,
|
|
retries: int,
|
|
retry_base_sleep: float,
|
|
) -> tuple[list[dict[str, Any]], str | None]:
|
|
compartment_id = field(compartment, "id")
|
|
try:
|
|
adbs = run_with_retries(
|
|
lambda: client.list_autonomous_databases(compartment_id, region=region),
|
|
retries=retries,
|
|
base_sleep=retry_base_sleep,
|
|
verbose=getattr(client, "verbose", False),
|
|
)
|
|
for adb in adbs:
|
|
adb["_inventory_region"] = region or ""
|
|
return adbs, None
|
|
except Exception as exc:
|
|
return [], f"{compartment_id}: {format_error(exc)}"
|
|
|
|
|
|
def list_all_adbs(
|
|
client: Any,
|
|
compartments: list[dict[str, Any]],
|
|
*,
|
|
region: str | None,
|
|
workers: int,
|
|
retries: int,
|
|
retry_base_sleep: float,
|
|
include_terminated: bool,
|
|
progress_callback: Callable[[dict[str, Any], list[dict[str, Any]], str | None], None] | None = None,
|
|
) -> tuple[list[dict[str, Any]], list[str]]:
|
|
adbs: list[dict[str, Any]] = []
|
|
errors: list[str] = []
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, workers)) as executor:
|
|
futures = {
|
|
executor.submit(
|
|
list_adbs_in_compartment,
|
|
client,
|
|
compartment,
|
|
region=region,
|
|
retries=retries,
|
|
retry_base_sleep=retry_base_sleep,
|
|
): compartment
|
|
for compartment in compartments
|
|
}
|
|
for future in concurrent.futures.as_completed(futures):
|
|
items, error = future.result()
|
|
filtered_items: list[dict[str, Any]] = []
|
|
if error:
|
|
errors.append(error)
|
|
for adb in items:
|
|
state = str(field(adb, "lifecycle-state", default="")).upper()
|
|
if include_terminated or state not in ADB_TERMINAL_STATES:
|
|
filtered_items.append(adb)
|
|
adbs.extend(filtered_items)
|
|
if progress_callback:
|
|
progress_callback(futures[future], filtered_items, error)
|
|
return adbs, sorted(errors)
|
|
|
|
|
|
def list_datasafe_targets_in_subtree(
|
|
client: Any,
|
|
root_compartment_id: str,
|
|
*,
|
|
region: str | None,
|
|
) -> list[dict[str, Any]]:
|
|
return client.list_target_databases_subtree(root_compartment_id, region=region)
|
|
|
|
|
|
def list_datasafe_targets_per_compartment(
|
|
client: Any,
|
|
compartments: list[dict[str, Any]],
|
|
*,
|
|
region: str | None,
|
|
workers: int,
|
|
retries: int,
|
|
retry_base_sleep: float,
|
|
) -> tuple[list[dict[str, Any]], list[str]]:
|
|
targets: list[dict[str, Any]] = []
|
|
errors: list[str] = []
|
|
|
|
def worker(compartment: dict[str, Any]) -> tuple[list[dict[str, Any]], str | None]:
|
|
compartment_id = field(compartment, "id")
|
|
try:
|
|
return run_with_retries(
|
|
lambda: client.list_target_databases_compartment(compartment_id, region=region),
|
|
retries=retries,
|
|
base_sleep=retry_base_sleep,
|
|
verbose=getattr(client, "verbose", False),
|
|
), None
|
|
except Exception as exc:
|
|
return [], f"{compartment_id}: {format_error(exc)}"
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, workers)) as executor:
|
|
futures = [executor.submit(worker, compartment) for compartment in compartments]
|
|
for future in concurrent.futures.as_completed(futures):
|
|
items, error = future.result()
|
|
if error:
|
|
errors.append(error)
|
|
targets.extend(items)
|
|
return targets, sorted(errors)
|
|
|
|
|
|
def list_datasafe_targets(
|
|
client: Any,
|
|
compartments: list[dict[str, Any]],
|
|
*,
|
|
root_compartment_id: str | None,
|
|
region: str | None,
|
|
workers: int,
|
|
retries: int,
|
|
retry_base_sleep: float,
|
|
include_deleted: bool,
|
|
) -> tuple[list[dict[str, Any]], list[str]]:
|
|
errors: list[str] = []
|
|
try:
|
|
if root_compartment_id:
|
|
targets = run_with_retries(
|
|
lambda: list_datasafe_targets_in_subtree(client, root_compartment_id, region=region),
|
|
retries=retries,
|
|
base_sleep=retry_base_sleep,
|
|
verbose=getattr(client, "verbose", False),
|
|
)
|
|
else:
|
|
targets, errors = list_datasafe_targets_per_compartment(
|
|
client,
|
|
compartments,
|
|
region=region,
|
|
workers=workers,
|
|
retries=retries,
|
|
retry_base_sleep=retry_base_sleep,
|
|
)
|
|
except Exception as exc:
|
|
errors.append(f"subtree Data Safe: {format_error(exc)}")
|
|
targets, per_compartment_errors = list_datasafe_targets_per_compartment(
|
|
client,
|
|
compartments,
|
|
region=region,
|
|
workers=workers,
|
|
retries=retries,
|
|
retry_base_sleep=retry_base_sleep,
|
|
)
|
|
errors.extend(per_compartment_errors)
|
|
|
|
filtered = []
|
|
for target in targets:
|
|
target["_inventory_region"] = region or ""
|
|
state = str(field(target, "lifecycle-state", default="")).upper()
|
|
if include_deleted or state not in DATASAFE_DELETED_STATES:
|
|
filtered.append(target)
|
|
return filtered, sorted(errors)
|
|
|
|
|
|
def target_associated_adb_id(target: dict[str, Any]) -> str:
|
|
associated_resource_ids = field(
|
|
target,
|
|
"associated-resource-ids",
|
|
"associated_resource_ids",
|
|
"associatedResourceIds",
|
|
default=[],
|
|
)
|
|
if isinstance(associated_resource_ids, list):
|
|
for resource_id in associated_resource_ids:
|
|
if isinstance(resource_id, str) and resource_id.startswith("ocid1.autonomousdatabase."):
|
|
return resource_id
|
|
for resource_id in associated_resource_ids:
|
|
if isinstance(resource_id, str) and resource_id.startswith("ocid1.autonomous"):
|
|
return resource_id
|
|
|
|
direct = field(
|
|
target,
|
|
"associated-resource-id",
|
|
"associated_resource_id",
|
|
"associatedResourceId",
|
|
"autonomous-database-id",
|
|
"autonomous_database_id",
|
|
"autonomousDatabaseId",
|
|
default="",
|
|
)
|
|
if isinstance(direct, str) and direct:
|
|
return direct
|
|
details = field(target, "database-details", "databaseDetails", default={})
|
|
if isinstance(details, dict):
|
|
return field(
|
|
details,
|
|
"autonomous-database-id",
|
|
"autonomous_database_id",
|
|
"autonomousDatabaseId",
|
|
default="",
|
|
)
|
|
return ""
|
|
|
|
|
|
def enrich_missing_target_associations(
|
|
client: Any,
|
|
targets: list[dict[str, Any]],
|
|
*,
|
|
region: str | None,
|
|
workers: int,
|
|
retries: int,
|
|
retry_base_sleep: float,
|
|
) -> list[str]:
|
|
errors: list[str] = []
|
|
missing = [target for target in targets if not target_associated_adb_id(target) and field(target, "id")]
|
|
if not missing:
|
|
return errors
|
|
|
|
def worker(target: dict[str, Any]) -> tuple[dict[str, Any] | None, str | None]:
|
|
target_id = field(target, "id")
|
|
try:
|
|
return run_with_retries(
|
|
lambda: client.get_target_database(target_id, region=region),
|
|
retries=retries,
|
|
base_sleep=retry_base_sleep,
|
|
verbose=getattr(client, "verbose", False),
|
|
), None
|
|
except Exception as exc:
|
|
return None, f"{target_id}: {format_error(exc)}"
|
|
|
|
by_id = {field(target, "id"): target for target in targets}
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, workers)) as executor:
|
|
futures = [executor.submit(worker, target) for target in missing]
|
|
for future in concurrent.futures.as_completed(futures):
|
|
detail, error = future.result()
|
|
if error:
|
|
errors.append(error)
|
|
if detail and field(detail, "id") in by_id:
|
|
by_id[field(detail, "id")].update(detail)
|
|
return sorted(errors)
|
|
|
|
|
|
def adb_to_base_row(
|
|
adb: dict[str, Any],
|
|
compartment_index: dict[str, dict[str, Any]],
|
|
*,
|
|
region: str | None = None,
|
|
) -> dict[str, Any]:
|
|
compartment_id = field(adb, "compartment-id")
|
|
compartment = compartment_index.get(compartment_id, {})
|
|
return {
|
|
"region": region or field(adb, "_inventory_region"),
|
|
"adb_display_name": field(adb, "display-name"),
|
|
"adb_db_name": field(adb, "db-name"),
|
|
"adb_ocid": field(adb, "id"),
|
|
"adb_lifecycle_state": field(adb, "lifecycle-state"),
|
|
"adb_workload": field(adb, "db-workload"),
|
|
"adb_is_free_tier": field(adb, "is-free-tier"),
|
|
"adb_time_created": field(adb, "time-created"),
|
|
"adb_compartment_name": field(compartment, "resolved-name", "name", "display-name"),
|
|
"adb_compartment_path": field(compartment, "path"),
|
|
"adb_compartment_ocid": compartment_id,
|
|
}
|
|
|
|
|
|
def target_to_base_row(
|
|
target: dict[str, Any],
|
|
compartment_index: dict[str, dict[str, Any]],
|
|
*,
|
|
prefix: str = "datasafe_target",
|
|
) -> dict[str, Any]:
|
|
compartment_id = field(target, "compartment-id")
|
|
compartment = compartment_index.get(compartment_id, {})
|
|
return {
|
|
f"{prefix}_display_name": field(target, "display-name"),
|
|
f"{prefix}_ocid": field(target, "id"),
|
|
f"{prefix}_lifecycle_state": field(target, "lifecycle-state"),
|
|
f"{prefix}_database_type": field(target, "database-type"),
|
|
f"{prefix}_infrastructure_type": field(target, "infrastructure-type"),
|
|
f"{prefix}_time_created": field(target, "time-created"),
|
|
f"{prefix}_compartment_name": field(compartment, "resolved-name", "name", "display-name"),
|
|
f"{prefix}_compartment_path": field(compartment, "path"),
|
|
f"{prefix}_compartment_ocid": compartment_id,
|
|
f"{prefix}_associated_adb_ocid": target_associated_adb_id(target),
|
|
}
|
|
|
|
|
|
def normalize_output_value(value: Any) -> Any:
|
|
if isinstance(value, (datetime, date)):
|
|
return value.isoformat()
|
|
if isinstance(value, (list, tuple)):
|
|
return json.dumps([normalize_output_value(item) for item in value], ensure_ascii=False)
|
|
if isinstance(value, dict):
|
|
return json.dumps({key: normalize_output_value(item) for key, item in value.items()}, ensure_ascii=False)
|
|
return value
|
|
|
|
|
|
def write_rows(rows: list[dict[str, Any]], output_path: str, output_format: str) -> None:
|
|
path = Path(output_path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
normalized_rows = [
|
|
{key: normalize_output_value(value) for key, value in row.items()}
|
|
for row in rows
|
|
]
|
|
if output_format == "json":
|
|
path.write_text(json.dumps(normalized_rows, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
return
|
|
|
|
fieldnames: list[str] = []
|
|
for column in PREFERRED_CSV_COLUMNS:
|
|
if any(column in row for row in normalized_rows):
|
|
fieldnames.append(column)
|
|
for row in normalized_rows:
|
|
for key in row:
|
|
if key not in fieldnames:
|
|
fieldnames.append(key)
|
|
with path.open("w", newline="", encoding="utf-8") as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=fieldnames)
|
|
writer.writeheader()
|
|
writer.writerows(normalized_rows)
|
|
|
|
|
|
def add_common_args(parser: argparse.ArgumentParser) -> None:
|
|
parser.add_argument("--tenancy-id", help="OCID da tenancy/root compartment. Se omitido, tenta ler do perfil OCI.")
|
|
parser.add_argument("--profile", help="Perfil da OCI CLI. Ex.: DEFAULT, PROD.")
|
|
parser.add_argument("--config-file", help="Caminho para arquivo de configuracao OCI CLI.")
|
|
parser.add_argument("--auth", help="Metodo de autenticacao OCI CLI. Ex.: instance_principal, resource_principal, security_token.")
|
|
parser.add_argument("--auth-purpose", help="Auth purpose da OCI CLI, quando aplicavel.")
|
|
parser.add_argument("--engine", choices=["sdk", "cli"], default="sdk", help="Motor de consulta. Padrao: sdk, mais rapido que chamar a OCI CLI repetidamente.")
|
|
parser.add_argument("--region", help="Regiao OCI. Se omitida, usa a regiao do perfil OCI CLI.")
|
|
parser.add_argument("--regions", help="Lista de regioes separadas por virgula. Ex.: sa-saopaulo-1,us-ashburn-1.")
|
|
parser.add_argument("--all-regions", action="store_true", help="Executa em todas as regioes subscribed da tenancy.")
|
|
parser.add_argument("--compartment-id", help="Limita a varredura de ADBs a um compartment especifico.")
|
|
parser.add_argument("--compartment-subtree", action="store_true", help="Com --compartment-id, inclui subcompartments na varredura de ADBs.")
|
|
parser.add_argument("--workers", type=int, default=4, help="Quantidade de chamadas paralelas por compartment. Padrao: 4.")
|
|
parser.add_argument("--retries", type=int, default=6, help="Tentativas adicionais para erros temporarios como 429/5xx. Padrao: 6.")
|
|
parser.add_argument("--retry-base-sleep", type=float, default=2.0, help="Espera inicial em segundos para backoff exponencial. Padrao: 2.0.")
|
|
parser.add_argument("--include-terminated-adbs", action="store_true", help="Inclui ADBs TERMINATED/TERMINATING.")
|
|
parser.add_argument("--include-deleted-targets", action="store_true", help="Inclui targets Data Safe DELETED/DELETING.")
|
|
parser.add_argument("--format", choices=["csv", "json"], default="csv", help="Formato de saida. Padrao: csv.")
|
|
parser.add_argument("--no-progress", action="store_true", help="Desativa a barra de progresso no terminal.")
|
|
parser.add_argument("--verbose", action="store_true", help="Mostra comandos OCI CLI executados.")
|
|
|
|
|
|
def build_context(args: argparse.Namespace) -> InventoryContext:
|
|
default_region = resolve_default_region(args)
|
|
if args.engine == "cli":
|
|
client: Any = OciCli(
|
|
profile=args.profile,
|
|
config_file=args.config_file,
|
|
auth=args.auth,
|
|
auth_purpose=args.auth_purpose,
|
|
region=args.region,
|
|
verbose=args.verbose,
|
|
)
|
|
else:
|
|
client = OciSdkClient(
|
|
profile=args.profile,
|
|
config_file=args.config_file,
|
|
auth=args.auth,
|
|
region=args.region,
|
|
verbose=args.verbose,
|
|
)
|
|
tenancy_id = resolve_tenancy_id(args)
|
|
if not tenancy_id:
|
|
config = getattr(client, "config", {})
|
|
if isinstance(config, dict):
|
|
tenancy_id = config.get("tenancy")
|
|
all_compartments = get_compartments(client, tenancy_id)
|
|
compartment_index = build_compartment_index(all_compartments)
|
|
compartments = filter_compartments_for_scan(
|
|
all_compartments,
|
|
compartment_id=args.compartment_id,
|
|
include_subtree=args.compartment_subtree,
|
|
)
|
|
if not tenancy_id:
|
|
root_candidates = [
|
|
cid
|
|
for cid, item in compartment_index.items()
|
|
if not field(item, "compartment-id", default="")
|
|
]
|
|
tenancy_id = root_candidates[0] if root_candidates else None
|
|
return InventoryContext(
|
|
client=client,
|
|
tenancy_id=tenancy_id,
|
|
default_region=default_region,
|
|
compartments=compartments,
|
|
compartment_index=compartment_index,
|
|
)
|
|
|
|
|
|
def print_warnings(title: str, errors: list[str]) -> None:
|
|
if not errors:
|
|
return
|
|
print(f"\nAvisos durante {title}:", file=sys.stderr)
|
|
for error in errors:
|
|
print(f" - {error}", file=sys.stderr)
|