659 lines
25 KiB
JavaScript
659 lines
25 KiB
JavaScript
import fs from "node:fs/promises";
|
|
import path from "node:path";
|
|
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import {
|
|
argValue,
|
|
executionModes,
|
|
flagsInArgs,
|
|
runFetch,
|
|
testsDir,
|
|
withoutFlags,
|
|
} from "./helpers/cli.mjs";
|
|
import {
|
|
catalogMentionsIntent,
|
|
documentedFlags,
|
|
parseCurrentWorkloadDefaults,
|
|
parseDefaultWeekYear,
|
|
parsePrimaryCommercialSubjectArea,
|
|
parsePrimaryCommercialSourceName,
|
|
readSkillCatalogs,
|
|
} from "./helpers/catalogs.mjs";
|
|
import {
|
|
buildLiveExampleContext,
|
|
placeholderKeysForItem,
|
|
resolvePromptCase,
|
|
} from "./helpers/examples.mjs";
|
|
|
|
const liveEnabled = process.env.KNOWLEDGE_ONE_VIEW_LIVE === "1";
|
|
const catalogs = await readSkillCatalogs();
|
|
const promptCases = JSON.parse(
|
|
await fs.readFile(path.join(testsDir, "prompts", "sales-prompts.json"), "utf8")
|
|
);
|
|
|
|
function resolveArgs(args, examples = {}, { allowUnresolved = true } = {}) {
|
|
const weekYear = parseDefaultWeekYear(catalogs.scriptCatalog);
|
|
return args
|
|
.map((arg) => (arg === "catalog-default" ? String(weekYear) : arg))
|
|
.map((arg) => String(arg).replace(/\{\{([a-zA-Z0-9]+)\}\}/g, (match, key) => {
|
|
if (examples[key]) return examples[key];
|
|
if (allowUnresolved) return match;
|
|
throw new Error(`Missing live example value for {{${key}}}.`);
|
|
}));
|
|
}
|
|
|
|
function resolvedCase(item, examples = {}, { allowUnresolved = true } = {}) {
|
|
return resolvePromptCase(item, examples, { allowUnresolved });
|
|
}
|
|
|
|
function caseArgs(item, examples = {}, options = {}) {
|
|
return resolveArgs(resolvedCase(item, examples, options).args, examples, options);
|
|
}
|
|
|
|
function liveArgs(item, examples = {}, options = {}) {
|
|
const resolved = resolvedCase(item, examples, options);
|
|
return resolveArgs(resolved.liveArgs || withoutFlags(resolved.args, ["--render-sql"]), examples, options);
|
|
}
|
|
|
|
function promptNeedsInitialContext(prompt) {
|
|
return /\b(manager|abaixo|hierarquia|owners abaixo)\b/i.test(prompt);
|
|
}
|
|
|
|
function promptNeedsWeekReference(prompt) {
|
|
return /\bweek\b|\bwk\b|\bw\d+\b/i.test(prompt);
|
|
}
|
|
|
|
function globalParameterFlags(parameters) {
|
|
return new Set((parameters || []).map((parameter) => parameter.cli));
|
|
}
|
|
|
|
const expectedFocusedIntentCapabilities = {
|
|
"opportunities-by-owner-close-window": {
|
|
required: ["--owner-email"],
|
|
optional: ["--date", "--month", "--quarter", "--week", "--week-year", "--from-date", "--to-date", "--cluster", "--opty-status", "--limit"],
|
|
},
|
|
"opportunities-by-owner-manager-close-window": {
|
|
required: ["--owner-manager-email"],
|
|
optional: ["--date", "--month", "--quarter", "--week", "--week-year", "--from-date", "--to-date", "--cluster", "--opty-status", "--limit"],
|
|
},
|
|
"workloads-by-resource-close-window": {
|
|
required: ["--resource"],
|
|
optional: ["--date", "--month", "--quarter", "--week", "--week-year", "--from-date", "--to-date", "--cluster", "--forecast-type", "--limit"],
|
|
},
|
|
"workloads-by-owner-close-window": {
|
|
required: ["--owner-email"],
|
|
optional: ["--date", "--month", "--quarter", "--week", "--week-year", "--from-date", "--to-date", "--cluster", "--forecast-type", "--limit"],
|
|
},
|
|
"workloads-by-owner-manager-close-window": {
|
|
required: ["--owner-manager-email"],
|
|
optional: ["--date", "--month", "--quarter", "--week", "--week-year", "--from-date", "--to-date", "--cluster", "--forecast-type", "--limit"],
|
|
},
|
|
};
|
|
|
|
test("sales prompt commands are short, direct, and documented in references", () => {
|
|
const flagsFromReferences = documentedFlags(catalogs);
|
|
|
|
for (const item of promptCases) {
|
|
const args = caseArgs(item);
|
|
const modes = executionModes(args);
|
|
|
|
assert.deepEqual(modes, [args.includes("--intent") ? "--intent" : "--query"], item.id);
|
|
assert.equal(args.includes("--catalog"), false, `${item.id} should not start with discovery.`);
|
|
assert.equal(args.includes("--check-auth"), false, `${item.id} should not add auth checks to the normal fetch.`);
|
|
assert.ok(flagsInArgs(args).length <= 6, `${item.id} should remain easy for an agent to execute.`);
|
|
|
|
for (const flag of flagsInArgs(args)) {
|
|
assert.ok(flagsFromReferences.has(flag), `${item.id} uses undocumented flag ${flag}.`);
|
|
}
|
|
|
|
assert.ok(item.catalogRefs.includes("SKILL.md"), `${item.id} must start from SKILL.md.`);
|
|
if (promptNeedsInitialContext(item.prompt)) {
|
|
assert.ok(
|
|
item.catalogRefs.includes("references/initial-context-catalog.md"),
|
|
`${item.id} should point manager or hierarchy wording to initial-context-catalog.`
|
|
);
|
|
}
|
|
if (promptNeedsWeekReference(item.prompt)) {
|
|
assert.ok(
|
|
item.catalogRefs.includes("references/script-catalog.md#week-reference"),
|
|
`${item.id} should point week wording to the reference week table.`
|
|
);
|
|
}
|
|
|
|
const intent = argValue(args, "--intent");
|
|
if (intent) {
|
|
assert.equal(catalogMentionsIntent(catalogs, intent), true, `${item.id} intent is not mentioned in references.`);
|
|
}
|
|
}
|
|
});
|
|
|
|
test("sales prompt people examples are resolved dynamically, not hard-coded", () => {
|
|
const serializedPrompts = JSON.stringify(promptCases);
|
|
assert.doesNotMatch(serializedPrompts, /resource\.manager(?:\.a|\.b)?@oracle\.com/);
|
|
assert.doesNotMatch(serializedPrompts, /territory\.(?:owner|manager)@oracle\.com/);
|
|
assert.doesNotMatch(serializedPrompts, /resource\.member\b/);
|
|
|
|
const requiredPlaceholders = new Set([
|
|
"resourceManagerEmail",
|
|
"resourceManagerEmailA",
|
|
"resourceManagerEmailB",
|
|
"resourceManagerEmailList",
|
|
"territoryOwnerEmail",
|
|
"territoryManagerEmail",
|
|
"resourceMember",
|
|
]);
|
|
const usedPlaceholders = new Set(promptCases.flatMap((item) => placeholderKeysForItem(item)));
|
|
for (const placeholder of requiredPlaceholders) {
|
|
assert.ok(usedPlaceholders.has(placeholder), `Expected {{${placeholder}}} in prompt test definitions.`);
|
|
}
|
|
});
|
|
|
|
test("intent prompt commands match the live CLI contract without local flag copies", async () => {
|
|
const parameterCatalog = await runFetch(["--list-parameters"]);
|
|
const globalFlags = globalParameterFlags(parameterCatalog.json.parameters);
|
|
|
|
for (const item of promptCases) {
|
|
const args = caseArgs(item);
|
|
const intent = argValue(args, "--intent");
|
|
if (!intent) continue;
|
|
|
|
const result = await runFetch(["--describe-intent", intent]);
|
|
const intentFlags = new Set((result.json.intent.parameters || []).map((parameter) => parameter.cli));
|
|
|
|
assert.equal(result.json.intent.name, intent, item.id);
|
|
assert.equal(catalogMentionsIntent(catalogs, intent), true, item.id);
|
|
|
|
for (const flag of flagsInArgs(args)) {
|
|
if (flag === "--intent") continue;
|
|
assert.ok(globalFlags.has(flag), `${item.id} uses a flag missing from --list-parameters: ${flag}`);
|
|
assert.ok(
|
|
intentFlags.has(flag) || ["--limit"].includes(flag),
|
|
`${item.id} uses a flag that is not part of the intent contract: ${flag}`
|
|
);
|
|
}
|
|
|
|
if (["workloads-by-owner-current", "workloads-by-owner-manager-current"].includes(intent)) {
|
|
const expectedDefaults = parseCurrentWorkloadDefaults(catalogs, intent);
|
|
assert.match(result.json.intent.description, /current/i);
|
|
assert.ok(
|
|
(result.json.intent.parameters || []).some(
|
|
(parameter) => parameter.cli === "--quarter" && parameter.default === expectedDefaults.quarter
|
|
),
|
|
`${item.id} current workload intent must keep the documented quarter default in the catalog.`
|
|
);
|
|
assert.ok(catalogs.allText.includes(expectedDefaults.solution));
|
|
assert.ok(catalogs.allText.includes(expectedDefaults.revenueTypeGroup));
|
|
}
|
|
|
|
if (intent === "workloads-by-manager-resource-close-window") {
|
|
assert.match(result.json.intent.description, /Presales Involvement Aux - V1/);
|
|
assert.match(result.json.intent.description, /does not fetch SR details from DV - SE Team/);
|
|
assert.doesNotMatch(result.json.intent.description, /resolved SRs/i);
|
|
}
|
|
}
|
|
});
|
|
|
|
test("focused owner/resource close-window intents expose the expected offline contract", async () => {
|
|
for (const [intentName, expected] of Object.entries(expectedFocusedIntentCapabilities)) {
|
|
const result = await runFetch(["--describe-intent", intentName]);
|
|
const parameters = result.json.intent.parameters || [];
|
|
const flags = new Set(parameters.map((parameter) => parameter.cli));
|
|
const requiredFlags = new Set(
|
|
parameters.filter((parameter) => parameter.required).map((parameter) => parameter.cli)
|
|
);
|
|
|
|
assert.equal(result.json.intent.name, intentName);
|
|
assert.ok(catalogMentionsIntent(catalogs, intentName), `${intentName} must be documented.`);
|
|
|
|
for (const flag of expected.required) {
|
|
assert.ok(flags.has(flag), `${intentName} is missing ${flag}`);
|
|
assert.ok(requiredFlags.has(flag), `${intentName} must require ${flag}`);
|
|
}
|
|
|
|
for (const flag of expected.optional) {
|
|
assert.ok(flags.has(flag), `${intentName} is missing optional ${flag}`);
|
|
assert.equal(requiredFlags.has(flag), false, `${intentName} should not require optional ${flag}`);
|
|
}
|
|
|
|
assert.equal(flags.has("--country"), false, `${intentName} should use --cluster, not a new --country flag.`);
|
|
}
|
|
});
|
|
|
|
test("ad hoc sales prompt renders an org-wide opportunity query", async () => {
|
|
const item = promptCases.find((entry) => entry.id === "ad-hoc-query");
|
|
const args = caseArgs(item);
|
|
const result = await runFetch(args);
|
|
const selectAliases = argValue(args, "--select").split(",").map((value) => value.trim());
|
|
|
|
assert.equal(result.json.requestMode, "query");
|
|
assert.equal(result.json.scope.scopeMode, "org-wide");
|
|
assert.ok(catalogs.allText.includes(result.json.query.source), "Query source should be documented.");
|
|
assert.match(result.json.expandedSql, new RegExp(escapeRegExp(parsePrimaryCommercialSubjectArea(catalogs.apiMap))));
|
|
assert.doesNotMatch(result.json.expandedSql, /SEC_ARIA_SI-SR-TEAM/);
|
|
|
|
for (const alias of selectAliases) {
|
|
assert.ok(catalogs.allText.includes(alias), `Query alias ${alias} should be documented in references.`);
|
|
assert.ok(result.json.columns.includes(alias), `Rendered query should keep selected alias ${alias}.`);
|
|
}
|
|
});
|
|
|
|
test("org-wide query money aliases use the pipeline measure that Oracle DV accepts", async () => {
|
|
const result = await runFetch([
|
|
"--query",
|
|
"--select",
|
|
"opportunityId,customerName,closeDate,bookingValue,workloadAmount",
|
|
"--where",
|
|
"closeDate >= DATE '2026-04-26' AND closeDate <= DATE '2026-05-02' AND opportunityStatus IN ('Open','Won','Won Pending') AND cluster = 'Brazil'",
|
|
"--render-sql",
|
|
]);
|
|
|
|
assert.equal(result.json.requestMode, "query");
|
|
assert.equal(result.json.scope.scopeMode, "org-wide");
|
|
assert.match(result.json.expandedSql, /"Columns"\."Pipeline" AS bookingValue/);
|
|
assert.match(result.json.expandedSql, /"Columns"\."Pipeline" AS workloadAmount/);
|
|
assert.doesNotMatch(result.json.expandedSql, /"Columns"\."Opportunity Value" AS bookingValue/);
|
|
assert.doesNotMatch(result.json.expandedSql, /"Columns"\."Workload Amount" AS workloadAmount/);
|
|
assert.match(result.json.expandedSql, /"Columns"\."Cluster" = 'Brazil'/);
|
|
assert.match(result.json.expandedSql, /"Columns"\."Revenue Line Close Date" >= DATE '2026-04-26'/);
|
|
});
|
|
|
|
test("query mode keeps commercial auto-scope and SR auto-scope separated", async () => {
|
|
const commercial = await runFetch([
|
|
"--query",
|
|
"--select",
|
|
"opportunityId,customerName",
|
|
"--where",
|
|
"cluster = 'Brazil'",
|
|
"--render-sql",
|
|
]);
|
|
assert.equal(commercial.json.query.source, parsePrimaryCommercialSourceName(catalogs.apiMap));
|
|
assert.equal(commercial.json.scope.scopeMode, "org-wide");
|
|
assert.doesNotMatch(commercial.json.expandedSql, /SEC_ARIA_SI-SR-TEAM/);
|
|
|
|
const sr = await runFetch([
|
|
"--query",
|
|
"--select",
|
|
"srNumber,customerName",
|
|
"--render-sql",
|
|
]);
|
|
assert.equal(sr.json.query.source, "DV - SE Team");
|
|
assert.equal(sr.json.scope.scopeMode, "my-team-only");
|
|
assert.match(sr.json.expandedSql, /SEC_ARIA_SI-SR-TEAM/);
|
|
});
|
|
|
|
test("query mode honors my-team scope aliases for otherwise org-wide commercial fields", async () => {
|
|
for (const scopeAlias of ["team", "current-team"]) {
|
|
const result = await runFetch([
|
|
"--query",
|
|
"--select",
|
|
"opportunityId,customerName",
|
|
"--scope",
|
|
scopeAlias,
|
|
"--render-sql",
|
|
]);
|
|
|
|
assert.equal(result.json.query.requestedScope, "my-team-only", scopeAlias);
|
|
assert.equal(result.json.query.effectiveScope, "my-team-only", scopeAlias);
|
|
assert.equal(result.json.query.source, "DV - SE Team", scopeAlias);
|
|
assert.match(result.json.expandedSql, /SEC_ARIA_SI-SR-TEAM/, scopeAlias);
|
|
}
|
|
});
|
|
|
|
test("CLI rejects conflicting execution modes and unsupported scopes before backend execution", async () => {
|
|
const cases = [
|
|
{
|
|
args: ["--query", "--select", "opportunityId", "--sql", "select 1", "--render-sql"],
|
|
message: /Use either --query or --sql, not both/,
|
|
},
|
|
{
|
|
args: ["--query", "--select", "opportunityId", "--intent", "opportunities-default", "--render-sql"],
|
|
message: /Use either --query or --intent, not both/,
|
|
},
|
|
{
|
|
args: ["--query", "--select", "opportunityId", "--view", "opportunity-details", "--render-sql"],
|
|
message: /Use either --query or --view, not both/,
|
|
},
|
|
{
|
|
args: ["--query", "--select", "opportunityId", "--scope", "nope", "--render-sql"],
|
|
message: /Unsupported --scope "nope"/,
|
|
},
|
|
{
|
|
args: ["--render-sql"],
|
|
message: /Pass --sql or --query together with --render-sql/,
|
|
},
|
|
];
|
|
|
|
for (const item of cases) {
|
|
const result = await runFetch(item.args, { expectCode: 1 });
|
|
assert.match(result.stderr, item.message);
|
|
assert.equal(result.stdout, "");
|
|
}
|
|
});
|
|
|
|
test("CLI rejects invalid limit values before backend execution", async () => {
|
|
for (const value of ["abc", "0", "-1", "1.5"]) {
|
|
const result = await runFetch(
|
|
[
|
|
"--render-sql",
|
|
"--query",
|
|
"--select",
|
|
"opportunityId",
|
|
"--limit",
|
|
value,
|
|
],
|
|
{ expectCode: 1 }
|
|
);
|
|
assert.match(result.stderr, new RegExp(`Unsupported --limit "${value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}"`));
|
|
assert.equal(result.stdout, "");
|
|
}
|
|
});
|
|
|
|
test(
|
|
"sales prompt commands run against Oracle DV when live mode is enabled",
|
|
{ skip: liveEnabled ? false : "set KNOWLEDGE_ONE_VIEW_LIVE=1 to run backend checks" },
|
|
async (t) => {
|
|
assert.ok(
|
|
process.env.ORACLE_SI_RUNTIME_HOME || process.env.ORACLE_SI_TEST_RUNTIME_HOME,
|
|
"Set ORACLE_SI_RUNTIME_HOME or ORACLE_SI_TEST_RUNTIME_HOME for live DV tests."
|
|
);
|
|
|
|
const sessionContext = await runFetch(
|
|
["--view", "data-quality-session-context", "--limit", "1"],
|
|
{ timeoutMs: 120000 }
|
|
);
|
|
assert.ok(Array.isArray(sessionContext.json.records), sessionContext.stdout || sessionContext.stderr);
|
|
assert.ok(sessionContext.json.apiMetrics.totalRowsPulledFromApi >= 0);
|
|
assert.ok(sessionContext.args.includes("--no-cache"), "The live warm-up must run with --no-cache.");
|
|
|
|
const liveExampleContext = await buildLiveExampleContext(catalogs);
|
|
const failures = [];
|
|
const tableRows = [];
|
|
|
|
for (const item of promptCases.filter((entry) => entry.live !== false)) {
|
|
const resolvedItem = {
|
|
...resolvedCase(item, liveExampleContext.examples, { allowUnresolved: false }),
|
|
placeholderKeys: placeholderKeysForItem(item),
|
|
};
|
|
let result = null;
|
|
let checks = [];
|
|
let commandError = null;
|
|
try {
|
|
result = await runFetch(liveArgs(item, liveExampleContext.examples, { allowUnresolved: false }), { timeoutMs: 180000 });
|
|
checks = evaluateLivePrompt(resolvedItem, result);
|
|
} catch (error) {
|
|
commandError = error;
|
|
checks = [
|
|
{
|
|
name: "command exited successfully",
|
|
pass: false,
|
|
actual: error.message,
|
|
expected: "JSON payload with exit code 0",
|
|
},
|
|
];
|
|
}
|
|
tableRows.push(buildPromptTableRow(resolvedItem, result, checks, commandError, liveExampleContext));
|
|
|
|
for (const check of checks) {
|
|
if (!check.pass) {
|
|
failures.push(`${item.id} :: ${check.name} :: expected ${formatValue(check.expected)}, got ${formatValue(check.actual)}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
t.diagnostic(formatPromptResultsTable(tableRows));
|
|
assert.deepEqual(failures, [], failures.join("\n"));
|
|
}
|
|
);
|
|
|
|
function evaluateLivePrompt(item, result) {
|
|
const payload = result.json;
|
|
const expected = item.liveExpect || {};
|
|
const checks = [];
|
|
|
|
checks.push({
|
|
name: "command uses --no-cache",
|
|
pass: result.args.includes("--no-cache"),
|
|
actual: result.args.includes("--no-cache"),
|
|
expected: true,
|
|
});
|
|
checks.push({
|
|
name: "records array exists",
|
|
pass: Array.isArray(payload.records),
|
|
actual: Array.isArray(payload.records),
|
|
expected: true,
|
|
});
|
|
checks.push({
|
|
name: "api rows metric exists",
|
|
pass: typeof payload.apiMetrics?.totalRowsPulledFromApi === "number",
|
|
actual: typeof payload.apiMetrics?.totalRowsPulledFromApi,
|
|
expected: "number",
|
|
});
|
|
|
|
const intent = argValue(caseArgs(item), "--intent");
|
|
checks.push(
|
|
intent
|
|
? {
|
|
name: "intent echoed by CLI",
|
|
pass: payload.request?.intent === intent,
|
|
actual: payload.request?.intent,
|
|
expected: intent,
|
|
}
|
|
: {
|
|
name: "query mode echoed by CLI",
|
|
pass: payload.request?.mode === "query",
|
|
actual: payload.request?.mode,
|
|
expected: "query",
|
|
}
|
|
);
|
|
|
|
if (expected.scopeMode) {
|
|
checks.push({
|
|
name: "scope mode",
|
|
pass: payload.scope?.scopeMode === expected.scopeMode,
|
|
actual: payload.scope?.scopeMode,
|
|
expected: expected.scopeMode,
|
|
});
|
|
}
|
|
|
|
if (expected.dataSource === "primaryCommercial") {
|
|
const expectedSource = parsePrimaryCommercialSourceName(catalogs.apiMap);
|
|
checks.push({
|
|
name: "primary commercial data source from references/api-map.md",
|
|
pass: payload.request?.dataSource === expectedSource,
|
|
actual: payload.request?.dataSource,
|
|
expected: expectedSource,
|
|
});
|
|
}
|
|
|
|
if (expected.allowFallback === false) {
|
|
checks.push({
|
|
name: "no fallback for org-wide commercial prompt",
|
|
pass: fallbackUsed(payload) === false,
|
|
actual: fallbackUsed(payload),
|
|
expected: false,
|
|
});
|
|
}
|
|
|
|
if (expected.noDvSeTeamQueries === true) {
|
|
const dvSeTeamQueries = (payload.apiMetrics?.queries || [])
|
|
.filter((query) => query?.source === "DV - SE Team")
|
|
.map((query) => query.label);
|
|
checks.push({
|
|
name: "no DV - SE Team execution for involvement prompt",
|
|
pass: dvSeTeamQueries.length === 0,
|
|
actual: dvSeTeamQueries,
|
|
expected: [],
|
|
});
|
|
}
|
|
|
|
if (Number.isFinite(expected.minRecords)) {
|
|
checks.push({
|
|
name: "minimum final records",
|
|
pass: Array.isArray(payload.records) && payload.records.length >= expected.minRecords,
|
|
actual: Array.isArray(payload.records) ? payload.records.length : null,
|
|
expected: `>= ${expected.minRecords}`,
|
|
});
|
|
}
|
|
|
|
if (Number.isFinite(expected.minApiRows)) {
|
|
checks.push({
|
|
name: "minimum raw API rows",
|
|
pass: payload.apiMetrics?.totalRowsPulledFromApi >= expected.minApiRows,
|
|
actual: payload.apiMetrics?.totalRowsPulledFromApi,
|
|
expected: `>= ${expected.minApiRows}`,
|
|
});
|
|
}
|
|
|
|
return checks;
|
|
}
|
|
|
|
function buildPromptTableRow(item, result, checks, commandError = null, liveExampleContext = null) {
|
|
const passed = checks.every((check) => check.pass);
|
|
return {
|
|
id: item.id,
|
|
prompt: item.prompt,
|
|
resultado: buildResultCell(result, checks, passed),
|
|
trace: buildTraceCell(result, commandError, liveExampleContext, item),
|
|
escopo: buildScopeCell(result),
|
|
};
|
|
}
|
|
|
|
function buildResultCell(result, checks, passed) {
|
|
if (!result) {
|
|
return `FAIL; ${checks.map((check) => `${check.name}: ${formatValue(check.actual)}`).join("; ")}`;
|
|
}
|
|
const records = Array.isArray(result.json?.records) ? result.json.records.length : "n/a";
|
|
const apiRows = result.json?.apiMetrics?.totalRowsPulledFromApi ?? "n/a";
|
|
const failedChecks = checks.filter((check) => !check.pass).map((check) => check.name);
|
|
return [
|
|
passed ? "PASS" : "FAIL",
|
|
`records=${records}`,
|
|
`apiRows=${apiRows}`,
|
|
failedChecks.length > 0 ? `falhas=${failedChecks.join(", ")}` : null,
|
|
]
|
|
.filter(Boolean)
|
|
.join("; ");
|
|
}
|
|
|
|
function buildTraceCell(result, commandError = null, liveExampleContext = null, item = null) {
|
|
const resolverTrace = buildResolverTrace(liveExampleContext, item);
|
|
if (!result) {
|
|
return [
|
|
resolverTrace,
|
|
`fetch-sales-intelligence.mjs ${formatArgs(commandError?.args || [])} -> erro: ${commandError?.message || "unknown error"}`,
|
|
].filter(Boolean).join("; ");
|
|
}
|
|
const payload = result.json;
|
|
const commandTrace = `fetch-sales-intelligence.mjs ${formatArgs(result.args)} -> ok code=${result.code} durationMs=${Math.round(result.durationMs)}`;
|
|
const source = payload?.request?.dataSource || payload?.scope?.baseSource || "unknown source";
|
|
if (fallbackUsed(payload)) {
|
|
const primaryError = payload?.request?.primarySqlError || firstFallbackWarning(payload) || "primary source failed";
|
|
return [resolverTrace, `${commandTrace}; primary -> erro: ${primaryError}; fallback ${source} -> ok`]
|
|
.filter(Boolean)
|
|
.join("; ");
|
|
}
|
|
return [resolverTrace, `${commandTrace}; primary ${source} -> ok`].filter(Boolean).join("; ");
|
|
}
|
|
|
|
function buildScopeCell(result) {
|
|
if (!result?.json) return "sem payload; escopo desconhecido";
|
|
const payload = result.json;
|
|
const scopeMode = payload.scope?.scopeMode || payload.request?.scopeMode || "unknown";
|
|
const source = payload.request?.dataSource || payload.scope?.baseSource || "unknown source";
|
|
if (fallbackUsed(payload)) {
|
|
return `${scopeMode}; caiu para fallback ${source}; ${scopeDisclosure(payload)}`;
|
|
}
|
|
return `${scopeMode}; sem fallback; fonte ${source}; ${scopeDisclosure(payload)}`;
|
|
}
|
|
|
|
function formatPromptResultsTable(rows) {
|
|
const header = ["id", "prompt", "resultado", "trace da execucao do agente", "descricao escopo/fallback"];
|
|
const lines = [
|
|
`| ${header.join(" | ")} |`,
|
|
`| ${header.map(() => "---").join(" | ")} |`,
|
|
...rows.map((row) =>
|
|
`| ${markdownCell(row.id)} | ${markdownCell(row.prompt)} | ${markdownCell(row.resultado)} | ${markdownCell(row.trace)} | ${markdownCell(row.escopo)} |`
|
|
),
|
|
];
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function formatArgs(args) {
|
|
return (args || [])
|
|
.map((arg) => (arg === "--runtime-home" ? "--runtime-home <test-runtime>" : arg))
|
|
.filter((arg, index, list) => list[index - 1] !== "--runtime-home <test-runtime>")
|
|
.join(" ");
|
|
}
|
|
|
|
function scopeDisclosure(payload) {
|
|
return payload?.scope?.disclosureLines?.join(" ") || payload?.scope?.explanation || "";
|
|
}
|
|
|
|
function buildResolverTrace(liveExampleContext, item) {
|
|
const keys = new Set(item?.placeholderKeys || placeholderKeysForItem(item || {}));
|
|
if (!keys.size || !liveExampleContext) return "";
|
|
const traces = [];
|
|
for (const trace of liveExampleContext.trace || []) {
|
|
if (!trace.profiles?.some((profile) => keys.has(profile))) continue;
|
|
const relevantProfiles = trace.profiles
|
|
.filter((profile) => keys.has(profile))
|
|
.map((profile) => `${profile}=${liveExampleContext.examples[profile]}`)
|
|
.join(", ");
|
|
traces.push(`${trace.script} ${formatArgs(trace.args)} -> ${trace.result}; selected ${relevantProfiles}`);
|
|
}
|
|
return traces.join("; ");
|
|
}
|
|
|
|
function firstFallbackWarning(payload) {
|
|
return (payload?.warnings || []).find((warning) => /fell back|fallback/i.test(warning));
|
|
}
|
|
|
|
function markdownCell(value) {
|
|
return String(value ?? "")
|
|
.replace(/\r?\n/g, "<br>")
|
|
.replace(/\|/g, "\\|");
|
|
}
|
|
|
|
function buildPromptDiagnostic(item, result, checks) {
|
|
return {
|
|
test: item.id,
|
|
prompt: item.prompt,
|
|
input: {
|
|
args: result ? result.args : liveArgs(item),
|
|
},
|
|
output: result
|
|
? {
|
|
records: Array.isArray(result.json?.records) ? result.json.records.length : null,
|
|
apiRows: result.json?.apiMetrics?.totalRowsPulledFromApi ?? null,
|
|
scopeMode: result.json?.scope?.scopeMode ?? null,
|
|
dataSource: result.json?.request?.dataSource ?? null,
|
|
fallbackUsed: fallbackUsed(result.json),
|
|
warnings: result.json?.warnings || [],
|
|
}
|
|
: null,
|
|
evaluation: checks.map((check) => ({
|
|
name: check.name,
|
|
passed: check.pass,
|
|
expected: check.expected,
|
|
actual: check.actual,
|
|
})),
|
|
};
|
|
}
|
|
|
|
function fallbackUsed(payload) {
|
|
const warningText = (payload?.warnings || []).join("\n");
|
|
return Boolean(
|
|
payload?.request?.fallbackUsed ||
|
|
payload?.scope?.fallbackUsed ||
|
|
/fell back|fallback used/i.test(warningText)
|
|
);
|
|
}
|
|
|
|
function formatValue(value) {
|
|
return typeof value === "string" ? value : JSON.stringify(value);
|
|
}
|
|
|
|
function escapeRegExp(value) {
|
|
return String(value).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
}
|