diff --git a/ts/packages/benchmarks/src/translationBench/synthesizer/generationCandidate.ts b/ts/packages/benchmarks/src/translationBench/synthesizer/generationCandidate.ts index d96ae1589..ffdfc807e 100644 --- a/ts/packages/benchmarks/src/translationBench/synthesizer/generationCandidate.ts +++ b/ts/packages/benchmarks/src/translationBench/synthesizer/generationCandidate.ts @@ -289,13 +289,18 @@ function stripEmptyGoldPlaceholdersFromActions( actions: TranslationBenchBenchmarkAction[], ): TranslationBenchBenchmarkAction[] { return actions.map((action) => { + if (action.parameters === undefined) { + return action; + } const { parameters } = stripEmptyGoldPlaceholders(action.parameters); if (parameters === action.parameters) { return action; } if (parameters === undefined) { - const { parameters: _drop, ...rest } = action; - return rest; + // Keep parameters:{} when nested empties strip to nothing. Schemas + // like code.getSelection / desktop.ListThemes require the key + // (empty object type); dropping it fails validateAction. + return { ...action, parameters: {} }; } return { ...action, parameters }; }); diff --git a/ts/packages/benchmarks/test/translationBench.keepEmptyParams.spec.ts b/ts/packages/benchmarks/test/translationBench.keepEmptyParams.spec.ts new file mode 100644 index 000000000..2323993b2 --- /dev/null +++ b/ts/packages/benchmarks/test/translationBench.keepEmptyParams.spec.ts @@ -0,0 +1,164 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "@jest/globals"; + +import { stripEmptyGoldPlaceholdersFromCandidate } from "../src/translationBench/synthesizer/generationCandidate.js"; + +describe("keep empty parameters after gold placeholder strip", () => { + it("keeps parameters:{} when nested empties strip to nothing", () => { + const stripped = stripEmptyGoldPlaceholdersFromCandidate({ + seed: { + utterance: "get the current selection", + expectedActions: [ + { + schemaName: "code", + actionName: "getSelection", + parameters: { + unused: "", + nested: { flag: null }, + }, + }, + ], + order: "strict", + }, + genCases: [ + { + id: "pos-1", + role: "positive", + utterance: "what text is selected", + expectedActions: [ + { + schemaName: "code", + actionName: "getSelection", + parameters: { unused: " " }, + }, + ], + order: "strict", + dimensions: { variation: "paraphrase" }, + }, + { + id: "neg-1", + role: "negative", + utterance: "leave my editor alone", + expectedActions: [], + order: "strict", + dimensions: { negativeKind: "pure_refusal" }, + }, + ], + }); + + expect(stripped.seed.expectedActions[0]).toEqual({ + schemaName: "code", + actionName: "getSelection", + parameters: {}, + }); + expect(stripped.genCases[0]!.expectedActions[0]).toEqual({ + schemaName: "code", + actionName: "getSelection", + parameters: {}, + }); + // Negatives are not rewritten by the strip helper. + expect(stripped.genCases[1]!.expectedActions).toEqual([]); + }); + + it("does not invent parameters when gold omitted the key", () => { + const stripped = stripEmptyGoldPlaceholdersFromCandidate({ + seed: { + utterance: "list themes", + expectedActions: [ + { + schemaName: "desktop", + actionName: "ListThemes", + }, + ], + order: "strict", + }, + genCases: [ + { + id: "pos-1", + role: "positive", + utterance: "show desktop themes", + expectedActions: [ + { + schemaName: "desktop", + actionName: "ListThemes", + }, + ], + order: "strict", + dimensions: { variation: "paraphrase" }, + }, + { + id: "neg-1", + role: "negative", + utterance: "do not list themes", + expectedActions: [], + order: "strict", + dimensions: { negativeKind: "pure_refusal" }, + }, + ], + }); + + expect(stripped.seed.expectedActions[0]).toEqual({ + schemaName: "desktop", + actionName: "ListThemes", + }); + expect(stripped.seed.expectedActions[0]).not.toHaveProperty( + "parameters", + ); + }); + + it("still strips empty nested fields while keeping non-empty siblings", () => { + const stripped = stripEmptyGoldPlaceholdersFromCandidate({ + seed: { + utterance: "open apple.com", + expectedActions: [ + { + schemaName: "browser", + actionName: "openWebPage", + parameters: { + site: "apple.com", + tab: "", + extras: {}, + }, + }, + ], + order: "strict", + }, + genCases: [ + { + id: "pos-1", + role: "positive", + utterance: "navigate to apple.com", + expectedActions: [ + { + schemaName: "browser", + actionName: "openWebPage", + parameters: { + site: "apple.com", + tab: null, + }, + }, + ], + order: "strict", + dimensions: { variation: "paraphrase" }, + }, + { + id: "neg-1", + role: "negative", + utterance: "leave browser alone", + expectedActions: [], + order: "strict", + dimensions: { negativeKind: "pure_refusal" }, + }, + ], + }); + + expect(stripped.seed.expectedActions[0]!.parameters).toEqual({ + site: "apple.com", + }); + expect(stripped.genCases[0]!.expectedActions[0]!.parameters).toEqual({ + site: "apple.com", + }); + }); +});