import fs from "node:fs/promises"; import path from "node:path"; import test from "node:test"; import assert from "node:assert/strict"; import { argValue, executionModes, flagsInArgs, runFetch, testsDir, withoutFlags, } from "./helpers/cli.mjs"; import { catalogMentionsIntent, documentedFlags, parseCurrentWorkloadDefaults, parseDefaultWeekYear, parsePrimaryCommercialSubjectArea, parsePrimaryCommercialSourceName, readSkillCatalogs, } from "./helpers/catalogs.mjs"; import { buildLiveExampleContext, placeholderKeysForItem, resolvePromptCase, } from "./helpers/examples.mjs"; const liveEnabled = process.env.KNOWLEDGE_ONE_VIEW_LIVE === "1"; const catalogs = await readSkillCatalogs(); const promptCases = JSON.parse( await fs.readFile(path.join(testsDir, "prompts", "sales-prompts.json"), "utf8") ); function resolveArgs(args, examples = {}, { allowUnresolved = true } = {}) { const weekYear = parseDefaultWeekYear(catalogs.scriptCatalog); return args .map((arg) => (arg === "catalog-default" ? String(weekYear) : arg)) .map((arg) => String(arg).replace(/\{\{([a-zA-Z0-9]+)\}\}/g, (match, key) => { if (examples[key]) return examples[key]; if (allowUnresolved) return match; throw new Error(`Missing live example value for {{${key}}}.`); })); } function resolvedCase(item, examples = {}, { allowUnresolved = true } = {}) { return resolvePromptCase(item, examples, { allowUnresolved }); } function caseArgs(item, examples = {}, options = {}) { return resolveArgs(resolvedCase(item, examples, options).args, examples, options); } function liveArgs(item, examples = {}, options = {}) { const resolved = resolvedCase(item, examples, options); return resolveArgs(resolved.liveArgs || withoutFlags(resolved.args, ["--render-sql"]), examples, options); } function promptNeedsInitialContext(prompt) { return /\b(manager|abaixo|hierarquia|owners abaixo)\b/i.test(prompt); } function promptNeedsWeekReference(prompt) { return /\bweek\b|\bwk\b|\bw\d+\b/i.test(prompt); } function globalParameterFlags(parameters) { return new Set((parameters || []).map((parameter) => parameter.cli)); } test("sales prompt commands are short, direct, and documented in references", () => { const flagsFromReferences = documentedFlags(catalogs); for (const item of promptCases) { const args = caseArgs(item); const modes = executionModes(args); assert.deepEqual(modes, [args.includes("--intent") ? "--intent" : "--query"], item.id); assert.equal(args.includes("--catalog"), false, `${item.id} should not start with discovery.`); assert.equal(args.includes("--check-auth"), false, `${item.id} should not add auth checks to the normal fetch.`); assert.ok(flagsInArgs(args).length <= 6, `${item.id} should remain easy for an agent to execute.`); for (const flag of flagsInArgs(args)) { assert.ok(flagsFromReferences.has(flag), `${item.id} uses undocumented flag ${flag}.`); } assert.ok(item.catalogRefs.includes("SKILL.md"), `${item.id} must start from SKILL.md.`); if (promptNeedsInitialContext(item.prompt)) { assert.ok( item.catalogRefs.includes("references/initial-context-catalog.md"), `${item.id} should point manager or hierarchy wording to initial-context-catalog.` ); } if (promptNeedsWeekReference(item.prompt)) { assert.ok( item.catalogRefs.includes("references/script-catalog.md#week-reference"), `${item.id} should point week wording to the reference week table.` ); } const intent = argValue(args, "--intent"); if (intent) { assert.equal(catalogMentionsIntent(catalogs, intent), true, `${item.id} intent is not mentioned in references.`); } } }); test("sales prompt people examples are resolved dynamically, not hard-coded", () => { const serializedPrompts = JSON.stringify(promptCases); assert.doesNotMatch(serializedPrompts, /resource\.manager(?:\.a|\.b)?@oracle\.com/); assert.doesNotMatch(serializedPrompts, /territory\.(?:owner|manager)@oracle\.com/); assert.doesNotMatch(serializedPrompts, /resource\.member\b/); const requiredPlaceholders = new Set([ "resourceManagerEmail", "resourceManagerEmailA", "resourceManagerEmailB", "resourceManagerEmailList", "territoryOwnerEmail", "territoryManagerEmail", "resourceMember", ]); const usedPlaceholders = new Set(promptCases.flatMap((item) => placeholderKeysForItem(item))); for (const placeholder of requiredPlaceholders) { assert.ok(usedPlaceholders.has(placeholder), `Expected {{${placeholder}}} in prompt test definitions.`); } }); test("intent prompt commands match the live CLI contract without local flag copies", async () => { const parameterCatalog = await runFetch(["--list-parameters"]); const globalFlags = globalParameterFlags(parameterCatalog.json.parameters); for (const item of promptCases) { const args = caseArgs(item); const intent = argValue(args, "--intent"); if (!intent) continue; const result = await runFetch(["--describe-intent", intent]); const intentFlags = new Set((result.json.intent.parameters || []).map((parameter) => parameter.cli)); assert.equal(result.json.intent.name, intent, item.id); assert.equal(catalogMentionsIntent(catalogs, intent), true, item.id); for (const flag of flagsInArgs(args)) { if (flag === "--intent") continue; assert.ok(globalFlags.has(flag), `${item.id} uses a flag missing from --list-parameters: ${flag}`); assert.ok( intentFlags.has(flag) || ["--limit"].includes(flag), `${item.id} uses a flag that is not part of the intent contract: ${flag}` ); } if (["workloads-by-owner-current", "workloads-by-owner-manager-current"].includes(intent)) { const expectedDefaults = parseCurrentWorkloadDefaults(catalogs, intent); assert.match(result.json.intent.description, /current/i); assert.ok( (result.json.intent.parameters || []).some( (parameter) => parameter.cli === "--quarter" && parameter.default === expectedDefaults.quarter ), `${item.id} current workload intent must keep the documented quarter default in the catalog.` ); assert.ok(catalogs.allText.includes(expectedDefaults.solution)); assert.ok(catalogs.allText.includes(expectedDefaults.revenueTypeGroup)); } if (intent === "workloads-by-manager-resource-close-window") { assert.match(result.json.intent.description, /Presales Involvement Aux - V1/); assert.match(result.json.intent.description, /does not fetch SR details from DV - SE Team/); assert.doesNotMatch(result.json.intent.description, /resolved SRs/i); } } }); test("ad hoc sales prompt renders an org-wide opportunity query", async () => { const item = promptCases.find((entry) => entry.id === "ad-hoc-query"); const args = caseArgs(item); const result = await runFetch(args); const selectAliases = argValue(args, "--select").split(",").map((value) => value.trim()); assert.equal(result.json.requestMode, "query"); assert.equal(result.json.scope.scopeMode, "org-wide"); assert.ok(catalogs.allText.includes(result.json.query.source), "Query source should be documented."); assert.match(result.json.expandedSql, new RegExp(escapeRegExp(parsePrimaryCommercialSubjectArea(catalogs.apiMap)))); assert.doesNotMatch(result.json.expandedSql, /SEC_ARIA_SI-SR-TEAM/); for (const alias of selectAliases) { assert.ok(catalogs.allText.includes(alias), `Query alias ${alias} should be documented in references.`); assert.ok(result.json.columns.includes(alias), `Rendered query should keep selected alias ${alias}.`); } }); test("org-wide query money aliases use the pipeline measure that Oracle DV accepts", async () => { const result = await runFetch([ "--query", "--select", "opportunityId,customerName,closeDate,bookingValue,workloadAmount", "--where", "closeDate >= DATE '2026-04-26' AND closeDate <= DATE '2026-05-02' AND opportunityStatus IN ('Open','Won','Won Pending') AND cluster = 'Brazil'", "--render-sql", ]); assert.equal(result.json.requestMode, "query"); assert.equal(result.json.scope.scopeMode, "org-wide"); assert.match(result.json.expandedSql, /"Columns"\."Pipeline" AS bookingValue/); assert.match(result.json.expandedSql, /"Columns"\."Pipeline" AS workloadAmount/); assert.doesNotMatch(result.json.expandedSql, /"Columns"\."Opportunity Value" AS bookingValue/); assert.doesNotMatch(result.json.expandedSql, /"Columns"\."Workload Amount" AS workloadAmount/); assert.match(result.json.expandedSql, /"Columns"\."Cluster" = 'Brazil'/); assert.match(result.json.expandedSql, /"Columns"\."Revenue Line Close Date" >= DATE '2026-04-26'/); }); test("CLI rejects invalid limit values before backend execution", async () => { for (const value of ["abc", "0", "-1", "1.5"]) { const result = await runFetch( [ "--render-sql", "--query", "--select", "opportunityId", "--limit", value, ], { expectCode: 1 } ); assert.match(result.stderr, new RegExp(`Unsupported --limit "${value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}"`)); assert.equal(result.stdout, ""); } }); test( "sales prompt commands run against Oracle DV when live mode is enabled", { skip: liveEnabled ? false : "set KNOWLEDGE_ONE_VIEW_LIVE=1 to run backend checks" }, async (t) => { assert.ok( process.env.ORACLE_SI_RUNTIME_HOME || process.env.ORACLE_SI_TEST_RUNTIME_HOME, "Set ORACLE_SI_RUNTIME_HOME or ORACLE_SI_TEST_RUNTIME_HOME for live DV tests." ); const sessionContext = await runFetch( ["--view", "data-quality-session-context", "--limit", "1"], { timeoutMs: 120000 } ); assert.ok(Array.isArray(sessionContext.json.records), sessionContext.stdout || sessionContext.stderr); assert.ok(sessionContext.json.apiMetrics.totalRowsPulledFromApi >= 0); assert.ok(sessionContext.args.includes("--no-cache"), "The live warm-up must run with --no-cache."); const liveExampleContext = await buildLiveExampleContext(catalogs); const failures = []; const tableRows = []; for (const item of promptCases.filter((entry) => entry.live !== false)) { const resolvedItem = { ...resolvedCase(item, liveExampleContext.examples, { allowUnresolved: false }), placeholderKeys: placeholderKeysForItem(item), }; let result = null; let checks = []; let commandError = null; try { result = await runFetch(liveArgs(item, liveExampleContext.examples, { allowUnresolved: false }), { timeoutMs: 180000 }); checks = evaluateLivePrompt(resolvedItem, result); } catch (error) { commandError = error; checks = [ { name: "command exited successfully", pass: false, actual: error.message, expected: "JSON payload with exit code 0", }, ]; } tableRows.push(buildPromptTableRow(resolvedItem, result, checks, commandError, liveExampleContext)); for (const check of checks) { if (!check.pass) { failures.push(`${item.id} :: ${check.name} :: expected ${formatValue(check.expected)}, got ${formatValue(check.actual)}`); } } } t.diagnostic(formatPromptResultsTable(tableRows)); assert.deepEqual(failures, [], failures.join("\n")); } ); function evaluateLivePrompt(item, result) { const payload = result.json; const expected = item.liveExpect || {}; const checks = []; checks.push({ name: "command uses --no-cache", pass: result.args.includes("--no-cache"), actual: result.args.includes("--no-cache"), expected: true, }); checks.push({ name: "records array exists", pass: Array.isArray(payload.records), actual: Array.isArray(payload.records), expected: true, }); checks.push({ name: "api rows metric exists", pass: typeof payload.apiMetrics?.totalRowsPulledFromApi === "number", actual: typeof payload.apiMetrics?.totalRowsPulledFromApi, expected: "number", }); const intent = argValue(caseArgs(item), "--intent"); checks.push( intent ? { name: "intent echoed by CLI", pass: payload.request?.intent === intent, actual: payload.request?.intent, expected: intent, } : { name: "query mode echoed by CLI", pass: payload.request?.mode === "query", actual: payload.request?.mode, expected: "query", } ); if (expected.scopeMode) { checks.push({ name: "scope mode", pass: payload.scope?.scopeMode === expected.scopeMode, actual: payload.scope?.scopeMode, expected: expected.scopeMode, }); } if (expected.dataSource === "primaryCommercial") { const expectedSource = parsePrimaryCommercialSourceName(catalogs.apiMap); checks.push({ name: "primary commercial data source from references/api-map.md", pass: payload.request?.dataSource === expectedSource, actual: payload.request?.dataSource, expected: expectedSource, }); } if (expected.allowFallback === false) { checks.push({ name: "no fallback for org-wide commercial prompt", pass: fallbackUsed(payload) === false, actual: fallbackUsed(payload), expected: false, }); } if (expected.noDvSeTeamQueries === true) { const dvSeTeamQueries = (payload.apiMetrics?.queries || []) .filter((query) => query?.source === "DV - SE Team") .map((query) => query.label); checks.push({ name: "no DV - SE Team execution for involvement prompt", pass: dvSeTeamQueries.length === 0, actual: dvSeTeamQueries, expected: [], }); } if (Number.isFinite(expected.minRecords)) { checks.push({ name: "minimum final records", pass: Array.isArray(payload.records) && payload.records.length >= expected.minRecords, actual: Array.isArray(payload.records) ? payload.records.length : null, expected: `>= ${expected.minRecords}`, }); } if (Number.isFinite(expected.minApiRows)) { checks.push({ name: "minimum raw API rows", pass: payload.apiMetrics?.totalRowsPulledFromApi >= expected.minApiRows, actual: payload.apiMetrics?.totalRowsPulledFromApi, expected: `>= ${expected.minApiRows}`, }); } return checks; } function buildPromptTableRow(item, result, checks, commandError = null, liveExampleContext = null) { const passed = checks.every((check) => check.pass); return { id: item.id, prompt: item.prompt, resultado: buildResultCell(result, checks, passed), trace: buildTraceCell(result, commandError, liveExampleContext, item), escopo: buildScopeCell(result), }; } function buildResultCell(result, checks, passed) { if (!result) { return `FAIL; ${checks.map((check) => `${check.name}: ${formatValue(check.actual)}`).join("; ")}`; } const records = Array.isArray(result.json?.records) ? result.json.records.length : "n/a"; const apiRows = result.json?.apiMetrics?.totalRowsPulledFromApi ?? "n/a"; const failedChecks = checks.filter((check) => !check.pass).map((check) => check.name); return [ passed ? "PASS" : "FAIL", `records=${records}`, `apiRows=${apiRows}`, failedChecks.length > 0 ? `falhas=${failedChecks.join(", ")}` : null, ] .filter(Boolean) .join("; "); } function buildTraceCell(result, commandError = null, liveExampleContext = null, item = null) { const resolverTrace = buildResolverTrace(liveExampleContext, item); if (!result) { return [ resolverTrace, `fetch-sales-intelligence.mjs ${formatArgs(commandError?.args || [])} -> erro: ${commandError?.message || "unknown error"}`, ].filter(Boolean).join("; "); } const payload = result.json; const commandTrace = `fetch-sales-intelligence.mjs ${formatArgs(result.args)} -> ok code=${result.code} durationMs=${Math.round(result.durationMs)}`; const source = payload?.request?.dataSource || payload?.scope?.baseSource || "unknown source"; if (fallbackUsed(payload)) { const primaryError = payload?.request?.primarySqlError || firstFallbackWarning(payload) || "primary source failed"; return [resolverTrace, `${commandTrace}; primary -> erro: ${primaryError}; fallback ${source} -> ok`] .filter(Boolean) .join("; "); } return [resolverTrace, `${commandTrace}; primary ${source} -> ok`].filter(Boolean).join("; "); } function buildScopeCell(result) { if (!result?.json) return "sem payload; escopo desconhecido"; const payload = result.json; const scopeMode = payload.scope?.scopeMode || payload.request?.scopeMode || "unknown"; const source = payload.request?.dataSource || payload.scope?.baseSource || "unknown source"; if (fallbackUsed(payload)) { return `${scopeMode}; caiu para fallback ${source}; ${scopeDisclosure(payload)}`; } return `${scopeMode}; sem fallback; fonte ${source}; ${scopeDisclosure(payload)}`; } function formatPromptResultsTable(rows) { const header = ["id", "prompt", "resultado", "trace da execucao do agente", "descricao escopo/fallback"]; const lines = [ `| ${header.join(" | ")} |`, `| ${header.map(() => "---").join(" | ")} |`, ...rows.map((row) => `| ${markdownCell(row.id)} | ${markdownCell(row.prompt)} | ${markdownCell(row.resultado)} | ${markdownCell(row.trace)} | ${markdownCell(row.escopo)} |` ), ]; return lines.join("\n"); } function formatArgs(args) { return (args || []) .map((arg) => (arg === "--runtime-home" ? "--runtime-home " : arg)) .filter((arg, index, list) => list[index - 1] !== "--runtime-home ") .join(" "); } function scopeDisclosure(payload) { return payload?.scope?.disclosureLines?.join(" ") || payload?.scope?.explanation || ""; } function buildResolverTrace(liveExampleContext, item) { const keys = new Set(item?.placeholderKeys || placeholderKeysForItem(item || {})); if (!keys.size || !liveExampleContext) return ""; const traces = []; for (const trace of liveExampleContext.trace || []) { if (!trace.profiles?.some((profile) => keys.has(profile))) continue; const relevantProfiles = trace.profiles .filter((profile) => keys.has(profile)) .map((profile) => `${profile}=${liveExampleContext.examples[profile]}`) .join(", "); traces.push(`${trace.script} ${formatArgs(trace.args)} -> ${trace.result}; selected ${relevantProfiles}`); } return traces.join("; "); } function firstFallbackWarning(payload) { return (payload?.warnings || []).find((warning) => /fell back|fallback/i.test(warning)); } function markdownCell(value) { return String(value ?? "") .replace(/\r?\n/g, "
") .replace(/\|/g, "\\|"); } function buildPromptDiagnostic(item, result, checks) { return { test: item.id, prompt: item.prompt, input: { args: result ? result.args : liveArgs(item), }, output: result ? { records: Array.isArray(result.json?.records) ? result.json.records.length : null, apiRows: result.json?.apiMetrics?.totalRowsPulledFromApi ?? null, scopeMode: result.json?.scope?.scopeMode ?? null, dataSource: result.json?.request?.dataSource ?? null, fallbackUsed: fallbackUsed(result.json), warnings: result.json?.warnings || [], } : null, evaluation: checks.map((check) => ({ name: check.name, passed: check.pass, expected: check.expected, actual: check.actual, })), }; } function fallbackUsed(payload) { const warningText = (payload?.warnings || []).join("\n"); return Boolean( payload?.request?.fallbackUsed || payload?.scope?.fallbackUsed || /fell back|fallback used/i.test(warningText) ); } function formatValue(value) { return typeof value === "string" ? value : JSON.stringify(value); } function escapeRegExp(value) { return String(value).replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); }