From 8b725ab24e35f790a0903a5c582f629536cc9bec Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Thu, 13 Aug 2026 01:00:18 -0700 Subject: [PATCH 1/2] Keep empty parameters object after gold placeholder strip When nested empties strip to nothing, retain parameters:{} so schemas that require an empty object still pass validateAction. --- .../synthesizer/generationCandidate.ts | 9 +- .../translationBench.keepEmptyParams.spec.ts | 164 ++++++++++++++++++ 2 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 ts/packages/benchmarks/test/translationBench.keepEmptyParams.spec.ts diff --git a/ts/packages/benchmarks/src/translationBench/synthesizer/generationCandidate.ts b/ts/packages/benchmarks/src/translationBench/synthesizer/generationCandidate.ts index d96ae15891..ffdfc807e6 100644 --- a/ts/packages/benchmarks/src/translationBench/synthesizer/generationCandidate.ts +++ b/ts/packages/benchmarks/src/translationBench/synthesizer/generationCandidate.ts @@ -289,13 +289,18 @@ function stripEmptyGoldPlaceholdersFromActions( actions: TranslationBenchBenchmarkAction[], ): TranslationBenchBenchmarkAction[] { return actions.map((action) => { + if (action.parameters === undefined) { + return action; + } const { parameters } = stripEmptyGoldPlaceholders(action.parameters); if (parameters === action.parameters) { return action; } if (parameters === undefined) { - const { parameters: _drop, ...rest } = action; - return rest; + // Keep parameters:{} when nested empties strip to nothing. Schemas + // like code.getSelection / desktop.ListThemes require the key + // (empty object type); dropping it fails validateAction. + return { ...action, parameters: {} }; } return { ...action, parameters }; }); diff --git a/ts/packages/benchmarks/test/translationBench.keepEmptyParams.spec.ts b/ts/packages/benchmarks/test/translationBench.keepEmptyParams.spec.ts new file mode 100644 index 0000000000..2323993b2d --- /dev/null +++ b/ts/packages/benchmarks/test/translationBench.keepEmptyParams.spec.ts @@ -0,0 +1,164 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, expect, it } from "@jest/globals"; + +import { stripEmptyGoldPlaceholdersFromCandidate } from "../src/translationBench/synthesizer/generationCandidate.js"; + +describe("keep empty parameters after gold placeholder strip", () => { + it("keeps parameters:{} when nested empties strip to nothing", () => { + const stripped = stripEmptyGoldPlaceholdersFromCandidate({ + seed: { + utterance: "get the current selection", + expectedActions: [ + { + schemaName: "code", + actionName: "getSelection", + parameters: { + unused: "", + nested: { flag: null }, + }, + }, + ], + order: "strict", + }, + genCases: [ + { + id: "pos-1", + role: "positive", + utterance: "what text is selected", + expectedActions: [ + { + schemaName: "code", + actionName: "getSelection", + parameters: { unused: " " }, + }, + ], + order: "strict", + dimensions: { variation: "paraphrase" }, + }, + { + id: "neg-1", + role: "negative", + utterance: "leave my editor alone", + expectedActions: [], + order: "strict", + dimensions: { negativeKind: "pure_refusal" }, + }, + ], + }); + + expect(stripped.seed.expectedActions[0]).toEqual({ + schemaName: "code", + actionName: "getSelection", + parameters: {}, + }); + expect(stripped.genCases[0]!.expectedActions[0]).toEqual({ + schemaName: "code", + actionName: "getSelection", + parameters: {}, + }); + // Negatives are not rewritten by the strip helper. + expect(stripped.genCases[1]!.expectedActions).toEqual([]); + }); + + it("does not invent parameters when gold omitted the key", () => { + const stripped = stripEmptyGoldPlaceholdersFromCandidate({ + seed: { + utterance: "list themes", + expectedActions: [ + { + schemaName: "desktop", + actionName: "ListThemes", + }, + ], + order: "strict", + }, + genCases: [ + { + id: "pos-1", + role: "positive", + utterance: "show desktop themes", + expectedActions: [ + { + schemaName: "desktop", + actionName: "ListThemes", + }, + ], + order: "strict", + dimensions: { variation: "paraphrase" }, + }, + { + id: "neg-1", + role: "negative", + utterance: "do not list themes", + expectedActions: [], + order: "strict", + dimensions: { negativeKind: "pure_refusal" }, + }, + ], + }); + + expect(stripped.seed.expectedActions[0]).toEqual({ + schemaName: "desktop", + actionName: "ListThemes", + }); + expect(stripped.seed.expectedActions[0]).not.toHaveProperty( + "parameters", + ); + }); + + it("still strips empty nested fields while keeping non-empty siblings", () => { + const stripped = stripEmptyGoldPlaceholdersFromCandidate({ + seed: { + utterance: "open apple.com", + expectedActions: [ + { + schemaName: "browser", + actionName: "openWebPage", + parameters: { + site: "apple.com", + tab: "", + extras: {}, + }, + }, + ], + order: "strict", + }, + genCases: [ + { + id: "pos-1", + role: "positive", + utterance: "navigate to apple.com", + expectedActions: [ + { + schemaName: "browser", + actionName: "openWebPage", + parameters: { + site: "apple.com", + tab: null, + }, + }, + ], + order: "strict", + dimensions: { variation: "paraphrase" }, + }, + { + id: "neg-1", + role: "negative", + utterance: "leave browser alone", + expectedActions: [], + order: "strict", + dimensions: { negativeKind: "pure_refusal" }, + }, + ], + }); + + expect(stripped.seed.expectedActions[0]!.parameters).toEqual({ + site: "apple.com", + }); + expect(stripped.genCases[0]!.expectedActions[0]!.parameters).toEqual({ + site: "apple.com", + }); + }); +}); From 5885b2959178bca3a1ac4c62b77f9f0a23e2937b Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Thu, 13 Aug 2026 01:12:19 -0700 Subject: [PATCH 2/2] ci: retrigger after electron install flake