forked from diegoecab/oci-deal-accelerator
initial version
This commit is contained in:
212
tests/test_feature_matrix.py
Normal file
212
tests/test_feature_matrix.py
Normal file
@@ -0,0 +1,212 @@
|
||||
"""
|
||||
Tests for the ADB Feature Compatibility Matrix YAML and CLI.
|
||||
|
||||
Matrix YAML: kb/compatibility/adb-feature-matrix.yaml
|
||||
CLI module: tools/feature_matrix_cli.py
|
||||
"""
|
||||
|
||||
import csv
|
||||
import io
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Path setup
|
||||
# ---------------------------------------------------------------------------
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
MATRIX_PATH = PROJECT_ROOT / "kb" / "compatibility" / "adb-feature-matrix.yaml"
|
||||
TOOLS_DIR = PROJECT_ROOT / "tools"
|
||||
|
||||
# Make the tools directory importable so we can import the CLI module.
|
||||
if str(TOOLS_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(TOOLS_DIR))
|
||||
|
||||
try:
|
||||
import feature_matrix_cli as cli
|
||||
|
||||
CLI_AVAILABLE = True
|
||||
except ImportError:
|
||||
CLI_AVAILABLE = False
|
||||
|
||||
VALID_STATUSES = {"GA", "GA_CAVEAT", "PREVIEW", "LIMITED", "NOT_AVAIL", "BROKEN", "UNTESTED"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
@pytest.fixture(scope="module")
|
||||
def matrix_data():
|
||||
"""Load the feature-matrix YAML once per module."""
|
||||
with open(MATRIX_PATH, "r") as fh:
|
||||
return yaml.safe_load(fh)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def features(matrix_data):
|
||||
"""Return the list of feature entries."""
|
||||
return matrix_data["features"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# YAML structure tests
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestMatrixYAML:
|
||||
def test_matrix_loads(self, matrix_data):
|
||||
"""YAML loads without errors and contains expected top-level keys."""
|
||||
assert matrix_data is not None
|
||||
assert "features" in matrix_data
|
||||
assert "versions" in matrix_data
|
||||
assert "deployment_types" in matrix_data
|
||||
|
||||
def test_all_features_have_at_least_one_deployment(self, features):
|
||||
"""Every feature must have a non-empty matrix (at least one deployment type)."""
|
||||
for feat in features:
|
||||
matrix = feat.get("matrix", {})
|
||||
assert matrix, (
|
||||
f"Feature '{feat['name']}' has an empty matrix — "
|
||||
"at least one deployment type must be present."
|
||||
)
|
||||
|
||||
def test_all_statuses_are_valid(self, features):
|
||||
"""Every status value across the entire matrix must be in the allowed set."""
|
||||
for feat in features:
|
||||
for deploy_id, versions in feat.get("matrix", {}).items():
|
||||
for ver, entry in versions.items():
|
||||
status = entry if isinstance(entry, str) else entry.get("status")
|
||||
assert status in VALID_STATUSES, (
|
||||
f"Feature '{feat['name']}', deployment '{deploy_id}', "
|
||||
f"version '{ver}' has invalid status '{status}'. "
|
||||
f"Allowed: {VALID_STATUSES}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI function tests — use internal helpers from feature_matrix_cli
|
||||
# ---------------------------------------------------------------------------
|
||||
requires_cli = pytest.mark.skipif(not CLI_AVAILABLE, reason="feature_matrix_cli not available")
|
||||
|
||||
|
||||
@requires_cli
|
||||
class TestMatrixCLICheck:
|
||||
def test_check_returns_correct_status(self):
|
||||
"""check('Auto Scaling', 'adb_s', '23ai') should contain 'GA_CAVEAT'."""
|
||||
data = cli.load_matrix(str(MATRIX_PATH))
|
||||
matches = cli.find_features(data, "Auto Scaling")
|
||||
assert len(matches) >= 1, "Expected at least one match for 'Auto Scaling'"
|
||||
feature = matches[0]
|
||||
cell = cli.get_cell(feature, "adb_s", "23ai")
|
||||
assert cell is not None, "Expected a cell for Auto Scaling / adb_s / 23ai"
|
||||
assert "GA_CAVEAT" in cell.get("status", ""), (
|
||||
f"Expected GA_CAVEAT for Auto Scaling / adb_s / 23ai, got: {cell}"
|
||||
)
|
||||
|
||||
def test_check_unknown_feature_returns_error(self):
|
||||
"""check with a nonexistent feature should return no matches."""
|
||||
data = cli.load_matrix(str(MATRIX_PATH))
|
||||
matches = cli.find_features(data, "Nonexistent Feature")
|
||||
assert len(matches) == 0, (
|
||||
f"Expected no matches for 'Nonexistent Feature', got {len(matches)}"
|
||||
)
|
||||
|
||||
|
||||
@requires_cli
|
||||
class TestMatrixCLICompare:
|
||||
def test_compare_returns_all_features(self, features):
|
||||
"""Compare output should cover every feature in the matrix."""
|
||||
data = cli.load_matrix(str(MATRIX_PATH))
|
||||
feature_names = {f["name"] for f in features}
|
||||
# Simulate what cmd_compare does: iterate all features and get cells
|
||||
compared = set()
|
||||
for feature in data["features"]:
|
||||
cell1 = cli.get_cell(feature, "adb_s", "23ai")
|
||||
cell2 = cli.get_cell(feature, "exacs", "23ai")
|
||||
# Both cells may be None (UNTESTED), but the feature is still compared
|
||||
compared.add(feature["name"])
|
||||
assert feature_names == compared, (
|
||||
f"Missing features in compare: {feature_names - compared}"
|
||||
)
|
||||
|
||||
|
||||
@requires_cli
|
||||
class TestMatrixCLIGaps:
|
||||
def test_gaps_finds_not_available(self):
|
||||
"""gaps('dbcs_ee', '23ai') should include features marked NOT_AVAIL."""
|
||||
data = cli.load_matrix(str(MATRIX_PATH))
|
||||
gap_statuses = {"NOT_AVAIL", "BROKEN", "LIMITED"}
|
||||
gaps = []
|
||||
for feature in data["features"]:
|
||||
cell = cli.get_cell(feature, "dbcs_ee", "23ai")
|
||||
if cell is not None:
|
||||
status = cell.get("status", "UNTESTED")
|
||||
if status in gap_statuses:
|
||||
gaps.append((feature["name"], status))
|
||||
|
||||
assert len(gaps) > 0, "Expected at least one gap for dbcs_ee / 23ai"
|
||||
not_avail_gaps = [g for g in gaps if g[1] == "NOT_AVAIL"]
|
||||
assert len(not_avail_gaps) > 0, (
|
||||
"Expected at least one NOT_AVAIL gap for dbcs_ee / 23ai"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Export tests
|
||||
# ---------------------------------------------------------------------------
|
||||
@requires_cli
|
||||
class TestMatrixCLIExport:
|
||||
def test_export_markdown_is_valid(self, matrix_data):
|
||||
"""Markdown export should contain a header row with | separators."""
|
||||
data = cli.load_matrix(str(MATRIX_PATH))
|
||||
versions = cli.get_version_ids(data)
|
||||
deployments = cli.get_deployment_ids(data)
|
||||
|
||||
columns = [(d, v) for d in deployments for v in versions]
|
||||
col_labels = [f"{d}/{v}" for d, v in columns]
|
||||
|
||||
# Capture the markdown output by calling the internal function
|
||||
captured = io.StringIO()
|
||||
old_stdout = sys.stdout
|
||||
sys.stdout = captured
|
||||
try:
|
||||
cli._export_markdown(data, columns, col_labels)
|
||||
finally:
|
||||
sys.stdout = old_stdout
|
||||
|
||||
content = captured.getvalue()
|
||||
lines = [ln for ln in content.splitlines() if ln.strip()]
|
||||
assert len(lines) >= 2, "Markdown export should have at least a header and separator row"
|
||||
# Header row must use pipe separators.
|
||||
assert "|" in lines[0], f"Header row missing '|' separators: {lines[0]}"
|
||||
# Second row is the --- separator row.
|
||||
assert "|" in lines[1], f"Separator row missing '|': {lines[1]}"
|
||||
|
||||
def test_export_csv_is_valid(self, matrix_data):
|
||||
"""CSV export should be parseable with csv.reader."""
|
||||
data = cli.load_matrix(str(MATRIX_PATH))
|
||||
versions = cli.get_version_ids(data)
|
||||
deployments = cli.get_deployment_ids(data)
|
||||
|
||||
columns = [(d, v) for d in deployments for v in versions]
|
||||
col_labels = [f"{d}/{v}" for d, v in columns]
|
||||
|
||||
# Capture the CSV output
|
||||
captured = io.StringIO()
|
||||
old_stdout = sys.stdout
|
||||
sys.stdout = captured
|
||||
try:
|
||||
cli._export_csv(data, columns, col_labels)
|
||||
finally:
|
||||
sys.stdout = old_stdout
|
||||
|
||||
content = captured.getvalue()
|
||||
reader = csv.reader(io.StringIO(content))
|
||||
rows = list(reader)
|
||||
assert len(rows) >= 2, "CSV export should have at least a header row and one data row"
|
||||
# All rows should have the same number of columns.
|
||||
col_count = len(rows[0])
|
||||
for i, row in enumerate(rows):
|
||||
assert len(row) == col_count, (
|
||||
f"Row {i} has {len(row)} columns, expected {col_count}"
|
||||
)
|
||||
374
tests/test_findings_tracker.py
Normal file
374
tests/test_findings_tracker.py
Normal file
@@ -0,0 +1,374 @@
|
||||
"""
|
||||
Tests for the Field Findings Tracker YAML and CLI.
|
||||
|
||||
Tracker YAML: kb/field-findings/tracker.yaml
|
||||
CLI module: tools/findings_cli.py
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Path setup
|
||||
# ---------------------------------------------------------------------------
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
TRACKER_PATH = PROJECT_ROOT / "kb" / "field-findings" / "tracker.yaml"
|
||||
TOOLS_DIR = PROJECT_ROOT / "tools"
|
||||
|
||||
if str(TOOLS_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(TOOLS_DIR))
|
||||
|
||||
try:
|
||||
import findings_cli as cli
|
||||
|
||||
CLI_AVAILABLE = True
|
||||
except ImportError:
|
||||
CLI_AVAILABLE = False
|
||||
|
||||
VALID_SEVERITIES = {"CRITICAL", "HIGH", "MEDIUM", "LOW", "INFO"}
|
||||
VALID_STATUSES = {"open", "resolved", "wontfix", "acknowledged", "monitoring"}
|
||||
VALID_CONFIDENCE = {"validated", "observed", "reported", "inferred"}
|
||||
ID_PATTERN = re.compile(r"^FF-\d{6}-\d{3}$")
|
||||
REQUIRED_FIELDS = {"id", "date", "summary", "severity", "status"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
@pytest.fixture(scope="module")
|
||||
def tracker_data():
|
||||
"""Load the tracker YAML once per module."""
|
||||
with open(TRACKER_PATH, "r") as fh:
|
||||
return yaml.safe_load(fh)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def findings(tracker_data):
|
||||
"""Return the list of findings."""
|
||||
return tracker_data["findings"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# YAML structure tests
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestTrackerYAML:
|
||||
def test_tracker_loads(self, tracker_data):
|
||||
"""YAML loads without errors and contains the findings key."""
|
||||
assert tracker_data is not None
|
||||
assert "findings" in tracker_data
|
||||
assert isinstance(tracker_data["findings"], list)
|
||||
assert len(tracker_data["findings"]) > 0
|
||||
|
||||
def test_all_findings_have_required_fields(self, findings):
|
||||
"""Each finding must have id, date, summary, severity, status."""
|
||||
for finding in findings:
|
||||
missing = REQUIRED_FIELDS - set(finding.keys())
|
||||
assert not missing, (
|
||||
f"Finding '{finding.get('id', '???')}' is missing required fields: {missing}"
|
||||
)
|
||||
|
||||
def test_all_findings_have_contributor_or_reported_by(self, findings):
|
||||
"""Each finding must have either a contributor block or reported_by."""
|
||||
for finding in findings:
|
||||
has_contributor = isinstance(finding.get("contributor"), dict)
|
||||
has_reported_by = bool(finding.get("reported_by"))
|
||||
assert has_contributor or has_reported_by, (
|
||||
f"Finding '{finding.get('id', '???')}' has neither contributor block nor reported_by"
|
||||
)
|
||||
|
||||
def test_contributor_blocks_have_required_fields(self, findings):
|
||||
"""Contributor blocks must have name, team, confidence."""
|
||||
for finding in findings:
|
||||
contributor = finding.get("contributor")
|
||||
if not isinstance(contributor, dict):
|
||||
continue
|
||||
assert contributor.get("name"), (
|
||||
f"Finding '{finding['id']}' contributor missing 'name'"
|
||||
)
|
||||
assert contributor.get("team"), (
|
||||
f"Finding '{finding['id']}' contributor missing 'team'"
|
||||
)
|
||||
assert contributor.get("confidence") in VALID_CONFIDENCE, (
|
||||
f"Finding '{finding['id']}' has invalid confidence "
|
||||
f"'{contributor.get('confidence')}'"
|
||||
)
|
||||
|
||||
def test_ids_are_unique(self, findings):
|
||||
"""No duplicate finding IDs."""
|
||||
ids = [f["id"] for f in findings]
|
||||
duplicates = [fid for fid in ids if ids.count(fid) > 1]
|
||||
assert not duplicates, f"Duplicate finding IDs: {set(duplicates)}"
|
||||
|
||||
def test_ids_follow_format(self, findings):
|
||||
"""All IDs must match FF-YYYYMM-NNN."""
|
||||
for finding in findings:
|
||||
fid = finding["id"]
|
||||
assert ID_PATTERN.match(fid), (
|
||||
f"Finding ID '{fid}' does not match expected format FF-YYYYMM-NNN"
|
||||
)
|
||||
|
||||
def test_severities_are_valid(self, findings):
|
||||
"""All severities must be in the allowed set."""
|
||||
for finding in findings:
|
||||
sev = finding["severity"]
|
||||
assert sev in VALID_SEVERITIES, (
|
||||
f"Finding '{finding['id']}' has invalid severity '{sev}'. "
|
||||
f"Allowed: {VALID_SEVERITIES}"
|
||||
)
|
||||
|
||||
def test_statuses_are_valid(self, findings):
|
||||
"""All statuses must be in the allowed set."""
|
||||
for finding in findings:
|
||||
status = finding["status"]
|
||||
assert status in VALID_STATUSES, (
|
||||
f"Finding '{finding['id']}' has invalid status '{status}'. "
|
||||
f"Allowed: {VALID_STATUSES}"
|
||||
)
|
||||
|
||||
def test_dates_are_valid(self, findings):
|
||||
"""All dates must parse as valid YYYY-MM-DD dates."""
|
||||
for finding in findings:
|
||||
date_val = finding["date"]
|
||||
# yaml.safe_load may auto-parse dates into datetime.date objects.
|
||||
if isinstance(date_val, str):
|
||||
try:
|
||||
datetime.strptime(date_val, "%Y-%m-%d")
|
||||
except ValueError:
|
||||
pytest.fail(
|
||||
f"Finding '{finding['id']}' has unparseable date '{date_val}'. "
|
||||
"Expected format: YYYY-MM-DD"
|
||||
)
|
||||
else:
|
||||
# datetime.date object returned by YAML parser is valid.
|
||||
assert hasattr(date_val, "year"), (
|
||||
f"Finding '{finding['id']}' date field is not a string or date: {date_val!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI function tests — use internal helpers from findings_cli
|
||||
# ---------------------------------------------------------------------------
|
||||
requires_cli = pytest.mark.skipif(not CLI_AVAILABLE, reason="findings_cli not available")
|
||||
|
||||
|
||||
@requires_cli
|
||||
class TestFindingsCLISearch:
|
||||
def test_search_finds_by_tag(self, findings):
|
||||
"""Search for 'dep' returns findings that have 'dep' in tags."""
|
||||
query = "dep"
|
||||
results = []
|
||||
for f in findings:
|
||||
searchable = " ".join([
|
||||
str(f.get("summary", "")),
|
||||
str(f.get("detail", "")),
|
||||
str(f.get("workaround", "")),
|
||||
" ".join(str(t) for t in f.get("tags", [])),
|
||||
]).lower()
|
||||
if query in searchable:
|
||||
results.append(f)
|
||||
|
||||
assert len(results) > 0, "Expected at least one finding with 'dep' in searchable fields"
|
||||
|
||||
def test_search_finds_by_text(self, findings):
|
||||
"""Search for 'maintenance' returns maintenance-related findings."""
|
||||
query = "maintenance"
|
||||
results = []
|
||||
for f in findings:
|
||||
searchable = " ".join([
|
||||
str(f.get("summary", "")),
|
||||
str(f.get("detail", "")),
|
||||
str(f.get("workaround", "")),
|
||||
" ".join(str(t) for t in f.get("tags", [])),
|
||||
]).lower()
|
||||
if query in searchable:
|
||||
results.append(f)
|
||||
|
||||
assert len(results) > 0, "Expected at least one finding related to 'maintenance'"
|
||||
|
||||
|
||||
@requires_cli
|
||||
class TestFindingsCLIFilter:
|
||||
def test_filter_by_severity(self, findings):
|
||||
"""Filter by severity HIGH returns only HIGH findings."""
|
||||
results = cli.filter_by_severity("HIGH")
|
||||
assert len(results) > 0, "Expected at least one HIGH severity finding"
|
||||
for r in results:
|
||||
assert r["severity"] == "HIGH", f"Expected severity HIGH, got '{r['severity']}'"
|
||||
|
||||
def test_filter_by_client(self):
|
||||
"""Filter by client 'Pepe' returns Pepe findings."""
|
||||
results = cli.filter_by_client("Pepe")
|
||||
assert len(results) > 0, "Expected at least one finding for client 'Pepe'"
|
||||
for r in results:
|
||||
client = cli._get_client(r)
|
||||
assert "Pepe" in client, f"Expected client containing 'Pepe', got '{client}'"
|
||||
|
||||
|
||||
@requires_cli
|
||||
class TestFindingsCLIAdd:
|
||||
def test_add_creates_valid_entry(self, tmp_path):
|
||||
"""Programmatic add produces an entry with all required fields."""
|
||||
tmp_tracker = tmp_path / "tracker.yaml"
|
||||
tmp_tracker.write_text(yaml.dump({"last_updated": "2026-03-14", "findings": []}))
|
||||
|
||||
entry = cli.add(
|
||||
tracker_path=str(tmp_tracker),
|
||||
name="Test User",
|
||||
team="Test Team",
|
||||
confidence="validated",
|
||||
client="Test Client",
|
||||
product="ADB-S",
|
||||
severity="LOW",
|
||||
category="gotcha",
|
||||
summary="Test finding for unit tests",
|
||||
detail="This is a test finding.",
|
||||
status="open",
|
||||
)
|
||||
|
||||
assert isinstance(entry, dict)
|
||||
missing = REQUIRED_FIELDS - set(entry.keys())
|
||||
assert not missing, f"Added entry missing fields: {missing}"
|
||||
assert entry["severity"] in VALID_SEVERITIES
|
||||
assert entry["status"] in VALID_STATUSES
|
||||
assert isinstance(entry.get("contributor"), dict)
|
||||
assert entry["contributor"]["name"] == "Test User"
|
||||
assert entry["contributor"]["team"] == "Test Team"
|
||||
assert entry["contributor"]["confidence"] == "validated"
|
||||
|
||||
def test_add_with_legacy_reported_by(self, tmp_path):
|
||||
"""Legacy reported_by parameter is supported for backward compat."""
|
||||
tmp_tracker = tmp_path / "tracker.yaml"
|
||||
tmp_tracker.write_text(yaml.dump({"last_updated": "2026-03-14", "findings": []}))
|
||||
|
||||
entry = cli.add(
|
||||
tracker_path=str(tmp_tracker),
|
||||
reported_by="Legacy User",
|
||||
product="ADB-S",
|
||||
severity="INFO",
|
||||
summary="Legacy add test",
|
||||
status="open",
|
||||
)
|
||||
|
||||
assert entry["contributor"]["name"] == "Legacy User"
|
||||
assert entry["reported_by"] == "Legacy User"
|
||||
|
||||
def test_auto_id_generation(self, tmp_path):
|
||||
"""Generated ID should use current year-month format (FF-YYYYMM-NNN)."""
|
||||
tmp_tracker = tmp_path / "tracker.yaml"
|
||||
tmp_tracker.write_text(yaml.dump({"last_updated": "2026-03-14", "findings": []}))
|
||||
|
||||
entry = cli.add(
|
||||
tracker_path=str(tmp_tracker),
|
||||
name="Test User",
|
||||
team="Test Team",
|
||||
product="ADB-S",
|
||||
severity="INFO",
|
||||
category="gotcha",
|
||||
summary="Auto-ID test",
|
||||
detail="Testing auto ID generation.",
|
||||
status="open",
|
||||
)
|
||||
|
||||
fid = entry["id"]
|
||||
assert ID_PATTERN.match(fid), f"Generated ID '{fid}' does not match FF-YYYYMM-NNN"
|
||||
now = datetime.now()
|
||||
expected_prefix = f"FF-{now.strftime('%Y%m')}-"
|
||||
assert fid.startswith(expected_prefix), (
|
||||
f"Generated ID '{fid}' does not start with '{expected_prefix}'"
|
||||
)
|
||||
assert fid.endswith("-001"), f"Expected first generated ID to end with '-001', got '{fid}'"
|
||||
|
||||
|
||||
@requires_cli
|
||||
class TestFindingsCLIConfirm:
|
||||
def test_confirm_adds_confirmation(self, tmp_path):
|
||||
"""Confirm command adds a confirmation entry to a finding."""
|
||||
tmp_tracker = tmp_path / "tracker.yaml"
|
||||
tmp_tracker.write_text(yaml.dump({"last_updated": "2026-03-14", "findings": []}))
|
||||
|
||||
# First add a finding
|
||||
entry = cli.add(
|
||||
tracker_path=str(tmp_tracker),
|
||||
name="Original Author",
|
||||
team="Team A",
|
||||
product="ADB-S",
|
||||
severity="HIGH",
|
||||
summary="Test finding to confirm",
|
||||
status="open",
|
||||
)
|
||||
|
||||
# Confirm it
|
||||
confirmation = cli.confirm(
|
||||
finding_id=entry["id"],
|
||||
name="Reviewer Name",
|
||||
team="Team B",
|
||||
note="Confirmed same behavior in my environment.",
|
||||
tracker_path=str(tmp_tracker),
|
||||
)
|
||||
|
||||
assert confirmation["name"] == "Reviewer Name"
|
||||
assert confirmation["team"] == "Team B"
|
||||
assert confirmation["note"] == "Confirmed same behavior in my environment."
|
||||
|
||||
# Verify it was persisted
|
||||
data = yaml.safe_load(tmp_tracker.read_text())
|
||||
finding = data["findings"][0]
|
||||
assert len(finding["confirmations"]) == 1
|
||||
assert finding["confirmations"][0]["name"] == "Reviewer Name"
|
||||
|
||||
def test_confirm_nonexistent_finding_raises(self, tmp_path):
|
||||
"""Confirming a nonexistent finding should raise LookupError."""
|
||||
tmp_tracker = tmp_path / "tracker.yaml"
|
||||
tmp_tracker.write_text(yaml.dump({"last_updated": "2026-03-14", "findings": []}))
|
||||
|
||||
with pytest.raises(LookupError):
|
||||
cli.confirm(
|
||||
finding_id="FF-999999-999",
|
||||
name="Test",
|
||||
team="Test",
|
||||
tracker_path=str(tmp_tracker),
|
||||
)
|
||||
|
||||
|
||||
@requires_cli
|
||||
class TestFindingsCLIStats:
|
||||
def test_stats_counts_correct(self, findings):
|
||||
"""Stats should return correct totals per category."""
|
||||
s = cli.stats()
|
||||
assert s["total"] == len(findings), (
|
||||
f"Stats total ({s['total']}) does not match findings count ({len(findings)})"
|
||||
)
|
||||
total_by_severity = sum(s["by_severity"].values())
|
||||
total_by_status = sum(s["by_status"].values())
|
||||
assert total_by_severity == len(findings), (
|
||||
f"Severity counts ({total_by_severity}) do not sum to total findings ({len(findings)})"
|
||||
)
|
||||
assert total_by_status == len(findings), (
|
||||
f"Status counts ({total_by_status}) do not sum to total findings ({len(findings)})"
|
||||
)
|
||||
|
||||
|
||||
@requires_cli
|
||||
class TestTagValidation:
|
||||
def test_valid_tags_pass(self):
|
||||
"""Known taxonomy tags should not produce warnings."""
|
||||
warnings = cli.validate_tags(["adb-s", "ha", "dr"])
|
||||
assert len(warnings) == 0
|
||||
|
||||
def test_unknown_tags_produce_warnings(self):
|
||||
"""Unknown tags should produce warning messages."""
|
||||
warnings = cli.validate_tags(["totally-fake-tag-xyz"])
|
||||
assert len(warnings) > 0
|
||||
assert "Unknown tag" in warnings[0]
|
||||
|
||||
def test_fuzzy_match_suggestion(self):
|
||||
"""Tags close to valid ones should suggest corrections."""
|
||||
warnings = cli.validate_tags(["adb_s"])
|
||||
# Should suggest 'adb-s'
|
||||
assert len(warnings) > 0
|
||||
assert "adb-s" in warnings[0]
|
||||
240
tests/test_kb_governance.py
Normal file
240
tests/test_kb_governance.py
Normal file
@@ -0,0 +1,240 @@
|
||||
"""
|
||||
Tests for KB governance tools: kb_linter.py, kb_cli.py, and config files.
|
||||
|
||||
Tests cover:
|
||||
- Config file structure validation
|
||||
- KB linter checks (contributor blocks, decay, tags, owners)
|
||||
- KB CLI functions (stats, search, owners)
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Path setup
|
||||
# ---------------------------------------------------------------------------
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
TOOLS_DIR = PROJECT_ROOT / "tools"
|
||||
CONFIG_DIR = PROJECT_ROOT / "config"
|
||||
KB_DIR = PROJECT_ROOT / "kb"
|
||||
|
||||
if str(TOOLS_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(TOOLS_DIR))
|
||||
|
||||
try:
|
||||
import kb_linter
|
||||
LINTER_AVAILABLE = True
|
||||
except ImportError:
|
||||
LINTER_AVAILABLE = False
|
||||
|
||||
try:
|
||||
import kb_cli
|
||||
CLI_AVAILABLE = True
|
||||
except ImportError:
|
||||
CLI_AVAILABLE = False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config file tests
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestGovernanceConfig:
|
||||
def test_governance_yaml_loads(self):
|
||||
"""kb-governance.yaml loads without errors."""
|
||||
path = CONFIG_DIR / "kb-governance.yaml"
|
||||
with open(path, "r") as fh:
|
||||
data = yaml.safe_load(fh)
|
||||
assert data is not None
|
||||
assert "contribution" in data
|
||||
assert "freshness" in data
|
||||
assert "confidence_decay" in data
|
||||
assert "review_triggers" in data
|
||||
|
||||
def test_governance_has_required_fields(self):
|
||||
"""Contribution config must list required fields."""
|
||||
path = CONFIG_DIR / "kb-governance.yaml"
|
||||
with open(path, "r") as fh:
|
||||
data = yaml.safe_load(fh)
|
||||
required = data["contribution"]["required_fields"]
|
||||
assert "name" in required
|
||||
assert "team" in required
|
||||
assert "confidence" in required
|
||||
|
||||
def test_governance_has_all_confidence_levels(self):
|
||||
"""Confidence decay must cover all declared levels."""
|
||||
path = CONFIG_DIR / "kb-governance.yaml"
|
||||
with open(path, "r") as fh:
|
||||
data = yaml.safe_load(fh)
|
||||
levels = data["contribution"]["confidence_levels"]
|
||||
decay = data["confidence_decay"]
|
||||
for level in levels:
|
||||
assert level in decay, f"Confidence level '{level}' not in decay config"
|
||||
assert "fresh" in decay[level]
|
||||
assert "stale" in decay[level]
|
||||
assert "expired" in decay[level]
|
||||
|
||||
|
||||
class TestOwnersConfig:
|
||||
def test_owners_yaml_loads(self):
|
||||
"""kb-owners.yaml loads and has domains."""
|
||||
path = CONFIG_DIR / "kb-owners.yaml"
|
||||
with open(path, "r") as fh:
|
||||
data = yaml.safe_load(fh)
|
||||
assert data is not None
|
||||
assert "domains" in data
|
||||
assert len(data["domains"]) > 0
|
||||
|
||||
def test_all_domains_have_area_and_owner(self):
|
||||
"""Each domain must have area and owner."""
|
||||
path = CONFIG_DIR / "kb-owners.yaml"
|
||||
with open(path, "r") as fh:
|
||||
data = yaml.safe_load(fh)
|
||||
for domain in data["domains"]:
|
||||
assert "area" in domain, f"Domain missing 'area': {domain}"
|
||||
assert "owner" in domain, f"Domain missing 'owner': {domain}"
|
||||
assert "name" in domain["owner"]
|
||||
|
||||
|
||||
class TestTagsConfig:
|
||||
def test_tags_yaml_loads(self):
|
||||
"""kb-tags.yaml loads and has taxonomy."""
|
||||
path = CONFIG_DIR / "kb-tags.yaml"
|
||||
with open(path, "r") as fh:
|
||||
data = yaml.safe_load(fh)
|
||||
assert data is not None
|
||||
assert "taxonomy" in data
|
||||
|
||||
def test_taxonomy_has_categories(self):
|
||||
"""Taxonomy must have products, versions, categories."""
|
||||
path = CONFIG_DIR / "kb-tags.yaml"
|
||||
with open(path, "r") as fh:
|
||||
data = yaml.safe_load(fh)
|
||||
taxonomy = data["taxonomy"]
|
||||
assert "products" in taxonomy
|
||||
assert "versions" in taxonomy
|
||||
assert "categories" in taxonomy
|
||||
assert len(taxonomy["products"]) > 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# KB Linter tests
|
||||
# ---------------------------------------------------------------------------
|
||||
requires_linter = pytest.mark.skipif(not LINTER_AVAILABLE, reason="kb_linter not available")
|
||||
|
||||
|
||||
@requires_linter
|
||||
class TestKBLinter:
|
||||
def test_contributor_blocks_check(self):
|
||||
"""Linter should check contributor blocks on real tracker."""
|
||||
issues = kb_linter.check_contributor_blocks()
|
||||
# All findings in the updated tracker should have contributor blocks
|
||||
# so there should be no issues
|
||||
assert isinstance(issues, list)
|
||||
|
||||
def test_confidence_decay_check(self):
|
||||
"""Decay check returns a list of decay reports."""
|
||||
results = kb_linter.check_confidence_decay()
|
||||
assert isinstance(results, list)
|
||||
assert len(results) > 0
|
||||
for r in results:
|
||||
assert "id" in r
|
||||
assert "status" in r
|
||||
assert r["status"] in {"FRESH", "STALE", "EXPIRED", "UNKNOWN"}
|
||||
|
||||
def test_tag_validation(self):
|
||||
"""Tag validation returns a list of issues."""
|
||||
issues = kb_linter.check_tags()
|
||||
assert isinstance(issues, list)
|
||||
# Some tags in the real tracker may not be in taxonomy
|
||||
# (like 'cli', 'maintenance-window', etc.)
|
||||
|
||||
def test_owner_check(self):
|
||||
"""Owner check returns issues for TBD owners."""
|
||||
issues = kb_linter.check_owners()
|
||||
assert isinstance(issues, list)
|
||||
# We know some owners are TBD
|
||||
tbd_issues = [i for i in issues if "TBD" in i]
|
||||
assert len(tbd_issues) > 0, "Expected TBD owner issues"
|
||||
|
||||
def test_freshness_check(self):
|
||||
"""Freshness check returns a list of file freshness reports."""
|
||||
results = kb_linter.check_freshness()
|
||||
assert isinstance(results, list)
|
||||
assert len(results) > 0
|
||||
for r in results:
|
||||
assert "file" in r
|
||||
assert "status" in r
|
||||
assert "age_days" in r
|
||||
|
||||
def test_contribution_stats(self):
|
||||
"""Contribution stats returns a dict of name -> count."""
|
||||
stats = kb_linter.contribution_stats()
|
||||
assert isinstance(stats, dict)
|
||||
assert len(stats) > 0
|
||||
assert "Diego Cabrera" in stats
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# KB CLI tests
|
||||
# ---------------------------------------------------------------------------
|
||||
requires_kb_cli = pytest.mark.skipif(not CLI_AVAILABLE, reason="kb_cli not available")
|
||||
|
||||
|
||||
@requires_kb_cli
|
||||
class TestKBCLI:
|
||||
def test_all_kb_files_returns_yaml_files(self):
|
||||
"""_all_kb_files should return YAML files from kb/ directory."""
|
||||
files = kb_cli._all_kb_files()
|
||||
assert len(files) > 0
|
||||
for f in files:
|
||||
assert f.endswith((".yaml", ".yml"))
|
||||
|
||||
def test_load_yaml_valid_file(self):
|
||||
"""_load_yaml should load a valid YAML file."""
|
||||
data = kb_cli._load_yaml(str(PROJECT_ROOT / "config" / "kb-governance.yaml"))
|
||||
assert data is not None
|
||||
assert "contribution" in data
|
||||
|
||||
def test_load_yaml_missing_file(self):
|
||||
"""_load_yaml should return None for missing files."""
|
||||
data = kb_cli._load_yaml("/nonexistent/file.yaml")
|
||||
assert data is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Service changelog tests
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestServiceChangelogs:
|
||||
"""Verify that service files have changelog sections."""
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def service_files(self):
|
||||
services_dir = KB_DIR / "services"
|
||||
return list(services_dir.glob("*.yaml"))
|
||||
|
||||
def test_service_files_exist(self, service_files):
|
||||
assert len(service_files) > 0
|
||||
|
||||
def test_service_files_have_changelog(self, service_files):
|
||||
"""Each service file should have a changelog section."""
|
||||
for filepath in service_files:
|
||||
with open(filepath, "r") as fh:
|
||||
data = yaml.safe_load(fh)
|
||||
assert "changelog" in data, (
|
||||
f"Service file '{filepath.name}' missing 'changelog' section"
|
||||
)
|
||||
assert len(data["changelog"]) > 0, (
|
||||
f"Service file '{filepath.name}' has empty changelog"
|
||||
)
|
||||
|
||||
def test_changelog_entries_have_required_fields(self, service_files):
|
||||
"""Changelog entries must have date, contributor, change."""
|
||||
for filepath in service_files:
|
||||
with open(filepath, "r") as fh:
|
||||
data = yaml.safe_load(fh)
|
||||
for entry in data.get("changelog", []):
|
||||
assert "date" in entry, f"Changelog entry in {filepath.name} missing 'date'"
|
||||
assert "contributor" in entry, f"Changelog entry in {filepath.name} missing 'contributor'"
|
||||
assert "change" in entry, f"Changelog entry in {filepath.name} missing 'change'"
|
||||
Reference in New Issue
Block a user