Improve SKU catalog hygiene and BOM accuracy

- refresh_sku_catalog.py: add --discover to report API SKUs missing from catalog, filtered to already-curated serviceCategory values. Auto-runs at end of --validate and --refresh so gaps surface on every maintenance pass. First run found 162 missing SKUs (Blackwell GPUs, X12 Ax compute, VMware reserved tiers, WebLogic-on-OKE, Analytics/OIC BYOL). - oci_bom_gen.py: fix Cost % column being blank. Back-fill was gated on formula detection but data rows write raw numbers, so the predicate never matched. Now tracks data_item_rows explicitly. - .gitea/workflows/sku-catalog-refresh.yaml: monthly automation (1st at 09:00 UTC). Auto-refresh prices → push branch + open PR; detect new SKUs → open issue with labels. Gated on secrets.GITEA_TOKEN. - kb/pricing/oci-sku-catalog.yaml: add B95714 / B95715 (Autonomous ATP Dedicated ECPU, LI + BYOL) — canonical SKUs for ADB-D pricing. - kb/services/compute.yaml: add X12 family (VM.Standard4.Ax.Flex, BM.Standard4.Ax.120), E6.Ax and A4.Ax variants. Verified against docs.oracle.com computeshapes.htm. Bump last_verified + changelog. - templates/bom-spec.yaml: document 730 hrs/month convention as default (= real annual billing / 12; 744 overstates by 1.92% on 12-month TCO). - Makefile: new 'make sku-discover' target. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-24 11:40:53 -03:00
parent a22711c065
commit 874d40d38a
7 changed files with 327 additions and 13 deletions
--- a/.gitea/workflows/sku-catalog-refresh.yaml
+++ b/.gitea/workflows/sku-catalog-refresh.yaml
@@ -0,0 +1,143 @@
+# Monthly SKU catalog refresh.
+#
+# Runs on the 1st of each month at 09:00 UTC and on manual dispatch.
+#
+# Does two things:
+#   1. `--refresh`: pulls latest PAYG prices from Oracle's public API,
+#      updates kb/pricing/oci-sku-catalog.yaml, and opens a PR if anything
+#      changed.
+#   2. `--discover`: reports SKUs present in the API but absent from the
+#      catalog (filtered to serviceCategory values already curated).
+#      If new SKUs are found, opens a Gitea issue listing them by category
+#      for manual review.
+#
+# Tokens:
+#   secrets.GITEA_TOKEN — must have repo:write + issues:write.
+#   If not set, the run logs the diff/discover output as an artifact and
+#   leaves the rest (branch push, PR, issue) as a no-op.
+name: SKU Catalog Refresh
+
+on:
+  schedule:
+    - cron: '0 9 1 * *'
+  workflow_dispatch:
+
+jobs:
+  refresh-and-discover:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y python3 python3-pip jq
+          python3 -m pip install --user requests pyyaml
+
+      - name: Refresh prices from Oracle API
+        id: refresh
+        run: |
+          python3 tools/refresh_sku_catalog.py --refresh --diff 2>&1 \
+            | tee /tmp/refresh.log
+          if git diff --quiet kb/pricing/oci-sku-catalog.yaml; then
+            echo "changed=false" >> $GITHUB_OUTPUT
+          else
+            echo "changed=true" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Discover missing SKUs
+        id: discover
+        run: |
+          python3 tools/refresh_sku_catalog.py --discover -v 2>&1 \
+            | tee /tmp/discover.log
+          NEW_COUNT=$(grep -oE '^[0-9]+ SKUs present in API' /tmp/discover.log \
+            | head -1 | awk '{print $1}' || echo "0")
+          echo "new_count=${NEW_COUNT:-0}" >> $GITHUB_OUTPUT
+
+      - name: Upload logs as artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: sku-refresh-logs
+          path: |
+            /tmp/refresh.log
+            /tmp/discover.log
+          retention-days: 90
+
+      - name: Open PR with price updates
+        if: steps.refresh.outputs.changed == 'true' && env.GITEA_TOKEN != ''
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+          GITEA_SERVER: ${{ github.server_url }}
+          GITEA_REPO: ${{ github.repository }}
+        run: |
+          DATE=$(date -u +%Y-%m-%d)
+          BRANCH="automation/sku-refresh-${DATE}"
+
+          git config user.name "sku-refresh-bot"
+          git config user.email "sku-refresh@automation.local"
+          git checkout -b "$BRANCH"
+          git add kb/pricing/oci-sku-catalog.yaml
+          git commit -m "chore(pricing): monthly SKU catalog refresh (${DATE})
+
+          Automated refresh from Oracle public pricing API.
+          See workflow artifact 'sku-refresh-logs' for the diff."
+
+          # Push using token auth
+          REMOTE=$(echo "$GITEA_SERVER/$GITEA_REPO.git" \
+            | sed "s#https://#https://sku-refresh-bot:${GITEA_TOKEN}@#")
+          git push "$REMOTE" "$BRANCH"
+
+          # Open PR via Gitea API
+          OWNER=$(echo "$GITEA_REPO" | cut -d/ -f1)
+          REPO=$(echo "$GITEA_REPO" | cut -d/ -f2)
+          curl -sS -X POST \
+            -H "Authorization: token ${GITEA_TOKEN}" \
+            -H "Content-Type: application/json" \
+            "${GITEA_SERVER}/api/v1/repos/${OWNER}/${REPO}/pulls" \
+            -d "$(jq -n \
+              --arg title "chore(pricing): monthly SKU refresh ${DATE}" \
+              --arg head "$BRANCH" \
+              --arg base "main" \
+              --arg body "$(printf 'Automated refresh — [refresh log artifact](../../actions).\n\n```\n%s\n```' \
+                "$(tail -40 /tmp/refresh.log)")" \
+              '{title:$title, head:$head, base:$base, body:$body}')"
+
+      - name: Open issue for new SKUs
+        if: steps.discover.outputs.new_count != '0' && env.GITEA_TOKEN != ''
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+          GITEA_SERVER: ${{ github.server_url }}
+          GITEA_REPO: ${{ github.repository }}
+          NEW_COUNT: ${{ steps.discover.outputs.new_count }}
+        run: |
+          DATE=$(date -u +%Y-%m-%d)
+          OWNER=$(echo "$GITEA_REPO" | cut -d/ -f1)
+          REPO=$(echo "$GITEA_REPO" | cut -d/ -f2)
+
+          BODY=$(printf '## %s new SKUs detected in Oracle API\n\n**Detected:** %s\n\nThese SKUs are present in the Oracle public pricing API but not yet in `kb/pricing/oci-sku-catalog.yaml`. Filter: serviceCategory values already represented in our catalog (so only relevant families show up).\n\n### Full listing\n\n```\n%s\n```\n\n### Next step\n\nReview and add relevant entries to `kb/pricing/oci-sku-catalog.yaml` under the appropriate category block. After merging, prices will be picked up automatically on the next monthly refresh.\n' \
+            "$NEW_COUNT" "$DATE" "$(cat /tmp/discover.log)")
+
+          curl -sS -X POST \
+            -H "Authorization: token ${GITEA_TOKEN}" \
+            -H "Content-Type: application/json" \
+            "${GITEA_SERVER}/api/v1/repos/${OWNER}/${REPO}/issues" \
+            -d "$(jq -n \
+              --arg title "SKU catalog: ${NEW_COUNT} new SKUs to review (${DATE})" \
+              --arg body "$BODY" \
+              '{title:$title, body:$body, labels:["automation","sku-catalog"]}')"
+
+      - name: Summary
+        if: always()
+        run: |
+          echo "## SKU Refresh Summary" >> $GITHUB_STEP_SUMMARY
+          echo "- Catalog changed: ${{ steps.refresh.outputs.changed }}" >> $GITHUB_STEP_SUMMARY
+          echo "- New SKUs discovered: ${{ steps.discover.outputs.new_count }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Token configured: ${{ env.GITEA_TOKEN != '' && 'yes' || 'no (PR/issue skipped)' }}" >> $GITHUB_STEP_SUMMARY
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
--- a/5
+++ b/5
@@ -1,6 +1,6 @@
 # OCI Deal Accelerator — Build Automation

-.PHONY: help install test validate example diagram deck full clean lint codex-package update-icons freshness freshness-refresh sync-skill
+.PHONY: help install test validate example diagram deck full clean lint codex-package update-icons freshness freshness-refresh sync-skill sku-discover

 # Use venv if present, otherwise find best available python3
 ifneq (,$(wildcard .venv/bin/python))
@@ -97,3 +97,6 @@ kb-check: ## KB freshness JSON (used by skill welcome-flow pre-flight)

 freshness-refresh: ## Run refresh tools for stale KB files that support automation
 	@$(PYTHON) tools/kb_freshness.py --auto-refresh
+
+sku-discover: ## Report SKUs present in Oracle API but missing from oci-sku-catalog.yaml
+	@$(PYTHON) tools/refresh_sku_catalog.py --discover
--- a/kb/pricing/oci-sku-catalog.yaml
+++ b/kb/pricing/oci-sku-catalog.yaml
@@ -560,6 +560,18 @@ categories:
      list_price_usd: 0.0807
      default_hours_units: 730
      billing_type: hourly
+    - sku: B95714
+      product: Oracle Autonomous AI Transaction Processing - Dedicated - ECPU
+      metric: ECPU Per Hour
+      list_price_usd: 0.336
+      default_hours_units: 730
+      billing_type: hourly
+    - sku: B95715
+      product: Oracle Autonomous AI Transaction Processing - Dedicated - ECPU - BYOL
+      metric: ECPU Per Hour
+      list_price_usd: 0.0807
+      default_hours_units: 730
+      billing_type: hourly
  exascale:
    display_name: Oracle Cloud Infrastructure - Database Exadata Exascale Infrastructure
    skus:
--- a/kb/services/compute.yaml
+++ b/kb/services/compute.yaml
@@ -1,8 +1,11 @@
 ---
-last_verified: 2026-03-14
+last_verified: 2026-04-24
 service: OCI Compute
 category: compute
 oci_color: "#4F7B6E"
+changelog:
+  - date: "2026-04-24"
+    change: "Added missing shapes surfaced by discover+WebFetch: VM/BM.Standard.E6.Ax, VM/BM.Standard.A4.Ax, and full X12 family (VM.Standard4.Ax.Flex 39 OCPU / 360 GB, BM.Standard4.Ax.120 latest-gen bare metal). Verified against docs.oracle.com/en-us/iaas/Content/Compute/References/computeshapes.htm."
 ---

 what: Virtual machines and bare metal instances. Flexible shapes allow custom
@@ -42,6 +45,24 @@ shape_families:
    max_ocpu: 18
    memory_per_ocpu_gb: "1-64 GB per OCPU"
    use_for: "High-frequency trading, HPC"
+  - name: VM.Standard.E6.Ax.Flex
+    processor: "AMD EPYC (Ampere-branded E6 variant)"
+    max_ocpu: 94
+    max_memory_gb: 712
+    network_gbps: 99
+    use_for: "E6 Ampere-variant general purpose"
+  - name: VM.Standard.A4.Ax.Flex
+    processor: "Arm (A4 Ampere-branded variant)"
+    max_ocpu: 45
+    max_memory_gb: 720
+    network_gbps: 100
+    use_for: "A4 Ampere-variant Arm workloads"
+  - name: VM.Standard4.Ax.Flex
+    processor: "AMD EPYC (X12 Ampere-branded)"
+    max_ocpu: 39
+    max_memory_gb: 360
+    network_gbps: 99
+    use_for: "X12-generation general purpose (latest)"
  - name: BM.Standard.E5
    processor: "AMD EPYC 9J14 (Genoa)"
    ocpus: 192
@@ -50,6 +71,30 @@ shape_families:
    processor: "AMD EPYC 9J45 (Turin)"
    ocpus: 256
    use_for: "Bare metal latest generation"
+  - name: BM.Standard.E6.Ax.192
+    processor: "AMD EPYC (E6 Ampere-branded variant)"
+    ocpus: 192
+    max_memory_gb: 1536
+    network_gbps: 200
+    use_for: "Bare metal E6 Ampere-variant"
+  - name: BM.Standard.A4.48
+    processor: "Arm (A4)"
+    ocpus: 48
+    max_memory_gb: 768
+    network_gbps: 100
+    use_for: "Bare metal A4 Arm general purpose"
+  - name: BM.Standard.A4.Ax.48
+    processor: "Arm (A4 Ampere-branded variant)"
+    ocpus: 48
+    max_memory_gb: 768
+    network_gbps: 100
+    use_for: "Bare metal A4 Ampere-variant Arm"
+  - name: BM.Standard4.Ax.120
+    processor: "AMD EPYC (X12 Ampere-branded)"
+    ocpus: 120
+    max_memory_gb: 1152
+    network_gbps: 200
+    use_for: "Bare metal X12 latest generation"
  - name: VM.GPU.A10.1
    processor: "NVIDIA A10"
    gpu_count: 1
--- a/templates/bom-spec.yaml
+++ b/templates/bom-spec.yaml
@@ -48,3 +48,4 @@ bom:
  notes:
    - "Pricing based on Ashburn region (OC1)"
    - "Discounts subject to Oracle approval"
+    - "Hours convention — 730 vs 744: this BOM uses 730 hrs/month (= 8,760 annual hrs / 12), which equals the actual yearly OCI billing for hourly SKUs. OCI Cost Estimator defaults to 744 hrs/month (worst-case 31-day month); extrapolated to 12 months, 744 overstates the real annual cost by 1.92%. For 12-month TCO/proposals, 730 is the accurate figure; 744 is only appropriate when quoting a single peak month as an upper bound."
--- a/tools/oci_bom_gen.py
+++ b/tools/oci_bom_gen.py
@@ -328,6 +328,7 @@ class OCIBomGenerator:

        data_start_row = row
        cat_subtotal_rows = []
+        data_item_rows = []  # line-item rows (excludes category headers/subtotals/totals)

        # Ensure categories present in items but missing from catalog order
        # (e.g., "other" from unknown SKUs) still render at the end.
@@ -418,6 +419,7 @@ class OCIBomGenerator:
                    ws.cell(row=row, column=COL_CONV_WO).number_format = '#,##0.00'
                    ws.cell(row=row, column=COL_CONV_W).number_format = '#,##0.00'

+                data_item_rows.append(row)
                row += 1

            # Category subtotal row
@@ -492,17 +494,15 @@ class OCIBomGenerator:

        # ── Cost proportion formulas (back-fill) ─────────────────
        # Cost % = line monthly w/ discount / total monthly w/ discount
-        for item_row in range(data_start_row, total_row):
-            cell_val = ws.cell(row=item_row, column=COL_M_W).value
-            if cell_val and str(cell_val).startswith("=") and "SUM" not in str(cell_val):
-                mw_l = get_column_letter(COL_M_W)
-                ws.cell(
-                    row=item_row, column=COL_PCT,
-                    value=f"=IF({mw_l}{total_row}=0,0,{mw_l}{item_row}/{mw_l}{total_row})",
-                )
-                ws.cell(row=item_row, column=COL_PCT).number_format = '0.0%'
-                ws.cell(row=item_row, column=COL_PCT).font = data_font
-                ws.cell(row=item_row, column=COL_PCT).border = thin_border
+        mw_l = get_column_letter(COL_M_W)
+        for item_row in data_item_rows:
+            ws.cell(
+                row=item_row, column=COL_PCT,
+                value=f"=IF({mw_l}{total_row}=0,0,{mw_l}{item_row}/{mw_l}{total_row})",
+            )
+            ws.cell(row=item_row, column=COL_PCT).number_format = '0.0%'
+            ws.cell(row=item_row, column=COL_PCT).font = data_font
+            ws.cell(row=item_row, column=COL_PCT).border = thin_border

        # ── Notes ─────────────────────────────────────────────────
        if self.notes:
--- a/tools/refresh_sku_catalog.py
+++ b/tools/refresh_sku_catalog.py
@@ -220,6 +220,13 @@ def refresh_catalog(verbose=False):
    print("  Not in API (possibly retired): {}".format(not_found))
    print("  Catalog saved: {}".format(CATALOG_PATH))

+    # Reverse-direction check: SKUs in API we haven't catalogued yet.
+    catalog_skus_after = {str(e["sku"]) for cat in raw.get("categories", {}).values()
+                         for e in cat.get("skus", [])}
+    stub_map = {s: {} for s in catalog_skus_after}
+    new_skus = discover_missing_skus(api_map, stub_map, verbose=verbose)
+    print_missing_skus(new_skus, limit=None if verbose else 20)
+

 def validate_catalog(verbose=False):
    """Compare current catalog prices against API and report differences."""
@@ -282,6 +289,104 @@ def validate_catalog(verbose=False):
        if len(missing) > 10:
            print("  ... and {} more".format(len(missing) - 10))

+    # Reverse-direction check: SKUs in API we haven't catalogued yet.
+    new_skus = discover_missing_skus(api_map, sku_map, verbose=verbose)
+    print_missing_skus(new_skus, limit=None if verbose else 20)
+
+
+def discover_missing_skus(api_map, catalog_sku_map, verbose=False):
+    """Find SKUs present in the API but missing from the catalog.
+
+    The filter is auto-derived: we collect every `serviceCategory` value for
+    SKUs already in the catalog and only report new SKUs whose serviceCategory
+    falls within that curated set. This keeps the report relevant — as new
+    categories are added to the catalog, the discovery scope expands.
+    """
+    curated_categories = set()
+    for sku in catalog_sku_map:
+        if sku in api_map:
+            sc = api_map[sku].get("serviceCategory", "") or ""
+            if sc:
+                curated_categories.add(sc)
+
+    if verbose:
+        print("  Curated serviceCategory values ({}): {}".format(
+            len(curated_categories), ", ".join(sorted(curated_categories))
+        ))
+
+    missing = []
+    for sku, product in api_map.items():
+        if sku in catalog_sku_map:
+            continue
+        sc = product.get("serviceCategory", "") or ""
+        if sc not in curated_categories:
+            continue
+        missing.append({
+            "sku": sku,
+            "product": product.get("displayName", ""),
+            "service_category": sc,
+            "metric": product.get("metricName", ""),
+            "price": extract_payg_price(product),
+        })
+    missing.sort(key=lambda x: (x["service_category"], x["sku"]))
+    return missing
+
+
+def print_missing_skus(missing, limit=None):
+    """Print a readable report of missing SKUs, grouped by serviceCategory."""
+    if not missing:
+        print("\nNo new SKUs discovered in curated categories. Catalog covers the API well.")
+        return
+
+    print("\n{} SKUs present in API but missing from catalog".format(len(missing)))
+    print("(filtered to serviceCategory values already represented in our catalog)\n")
+
+    by_cat = {}
+    for m in missing:
+        by_cat.setdefault(m["service_category"], []).append(m)
+
+    shown = 0
+    truncated = False
+    for cat, items in sorted(by_cat.items()):
+        print("  [{}]  ({} new)".format(cat, len(items)))
+        for m in items:
+            if limit is not None and shown >= limit:
+                truncated = True
+                break
+            price_str = "${:.4f}".format(m["price"]) if m["price"] else "    -"
+            print("    {:<10} {:>10}  {}".format(
+                m["sku"], price_str, m["product"][:70]
+            ))
+            shown += 1
+        if truncated:
+            break
+
+    if truncated:
+        print("\n  ... and {} more (re-run with -v to list all)".format(len(missing) - shown))
+
+    print("\nNext step: review and add relevant entries to kb/pricing/oci-sku-catalog.yaml")
+    print("           under the appropriate category block.")
+
+
+def discover_catalog(verbose=False):
+    """Stand-alone --discover entry point: report new API SKUs not yet in catalog."""
+    print("Fetching current products from Oracle API...")
+    products = fetch_all_products(currency=DEFAULT_CURRENCY, verbose=verbose)
+    api_map = {}
+    for p in products:
+        pn = p.get("partNumber", "")
+        if pn:
+            api_map[pn] = p
+
+    raw, sku_map = load_current_catalog()
+    if not raw:
+        print("Error: Could not load catalog at {}".format(CATALOG_PATH))
+        return 1
+
+    missing = discover_missing_skus(api_map, sku_map, verbose=verbose)
+    print_missing_skus(missing, limit=None if verbose else 40)
+    return 0
+

 def inspect_sku(part_number):
    """Fetch and display a single SKU from the API."""
@@ -571,6 +676,9 @@ def main():
                             "(currently: compute)")
    parser.add_argument("--validate", action="store_true",
                        help="Validate catalog prices against API")
+    parser.add_argument("--discover", action="store_true",
+                        help="Report SKUs present in API but missing from catalog "
+                             "(filtered to serviceCategory values already curated)")
    parser.add_argument("--sku", type=str,
                        help="Inspect a single SKU from the API")
    parser.add_argument("--dump", type=str, metavar="FILE",
@@ -589,6 +697,8 @@ def main():
        refresh_catalog(verbose=args.verbose or args.diff)
    elif args.validate:
        validate_catalog(verbose=args.verbose)
+    elif args.discover:
+        return discover_catalog(verbose=args.verbose)
    elif args.dump:
        dump_all(args.dump, verbose=args.verbose)
    else: