From 8790f83d9e52408de3201e661a5ad438d899676d Mon Sep 17 00:00:00 2001 From: Priyanka Tiwari Date: Wed, 4 Nov 2026 15:52:35 +0530 Subject: [PATCH] Publish auto-triage summaries when labels are queued first gh-aw publishes safe outputs in queue order, so the triage validator rejected the whole batch whenever the agent queued its Waiting for Author label change before the summary comment. Seven recent issues lost their completed triage summaries this way. Move label operations after the single summary before validation, then write the reordered batch back to the file that safe-output publication reads. Rejected batches are left unchanged and still block publication. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/tests/README.md | 7 +- .../tests/validate-triage-output.test.cjs | 150 +++++++++++++++++- .github/scripts/validate-triage-output.cjs | 55 ++++++- .github/workflows/issue-triage.lock.yml | 10 +- .github/workflows/issue-triage.md | 8 +- 5 files changed, 216 insertions(+), 14 deletions(-) diff --git a/.github/scripts/tests/README.md b/.github/scripts/tests/README.md index be9ecd3a93..b737cb0130 100644 --- a/.github/scripts/tests/README.md +++ b/.github/scripts/tests/README.md @@ -46,8 +46,11 @@ They also reject partial safe-output batches with recorded errors while allowing C# generic type syntax in completed summaries. Draft prefixes such as `TODO:` and `TBD:` are rejected in required fields and list items; ordinary mentions of those markers within completed prose remain valid. -Coverage also includes multiline template placeholders, label operations queued -before the summary, and contradictory no-op/incomplete results. +Coverage also includes multiline template placeholders and contradictory +no-op/incomplete results. Label operations queued before the summary are moved +after it, because safe outputs publish in queue order; the reordered batch must +still pass validation before it is written back to `GH_AW_AGENT_OUTPUT` for +publication, and the compiled workflow is checked for that wiring. Comparison operators in completed prose remain valid; angle-bracket detection targets known template fields and instructions rather than arbitrary bracketed text. Generic arguments such as `Task` and `List`, including nested diff --git a/.github/scripts/tests/validate-triage-output.test.cjs b/.github/scripts/tests/validate-triage-output.test.cjs index b2d593e6b7..8726017780 100644 --- a/.github/scripts/tests/validate-triage-output.test.cjs +++ b/.github/scripts/tests/validate-triage-output.test.cjs @@ -3,8 +3,16 @@ // See the LICENSE file in the project root for more information. const assert = require('node:assert/strict'); +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); const { test } = require('node:test'); -const { validateTriageOutput } = require('../validate-triage-output.cjs'); +const { + normalizeTriageOutput, + validateTriageOutput, + validateTriageOutputFile, +} = require('../validate-triage-output.cjs'); const summary = `## \u{1F50D} Triage Summary @@ -61,6 +69,146 @@ test('rejects label operations before the summary', () => { } }); +// The agent queued its label change first in the failed runs for #4855, #4879, #4904, #4940, +// #4953, #4965, and #4969, so the gate rejected otherwise complete summaries. +const labelFirst = (type) => { + const body = type === 'add_labels' ? summary : + summary.replace('Missing: SQL Server version', 'All required environment details provided'); + return { + items: [{ type, labels: [waitingLabel] }, { type: 'add_comment', body }], + errors: [], + }; +}; + +test('queues a label operation recorded before the summary after it', () => { + for (const type of ['add_labels', 'remove_labels']) { + // Arrange + const value = labelFirst(type); + const queued = structuredClone(value); + + // Act + const normalized = normalizeTriageOutput(value); + + // Assert + // The summary is published first, and the reordered batch passes the gate. + assert.deepEqual(normalized.items, [queued.items[1], queued.items[0]]); + assert.deepEqual(normalized.errors, []); + validateTriageOutput(normalized); + // The queued batch is not mutated. + assert.deepEqual(value, queued); + } +}); + +test('leaves batches without one summary or an early label operation unchanged', () => { + const label = { type: 'add_labels', labels: [waitingLabel] }; + const comment = output(summary).items[0]; + for (const value of [ + null, + {}, + { items: null }, + { items: [null, comment] }, + output(summary), + { items: [{ type: 'noop', reason: 'No new information' }] }, + { items: [label] }, + { items: [label, comment, comment] }, + ]) { + // Act + const normalized = normalizeTriageOutput(value); + + // Assert + assert.equal(normalized, value); + } +}); + +test('still rejects invalid batches after queueing labels after the summary', () => { + const label = { type: 'add_labels', labels: [waitingLabel] }; + const comment = output(summary).items[0]; + for (const [items, message] of [ + [[label, label, comment], /exactly one/], + [[label, { type: 'add_comment', body: 'test simple body' }], /Triage Summary/], + [[{ type: 'remove_labels', labels: [waitingLabel] }, comment], /add_labels/], + [[{ ...label, labels: ['Unrelated label'] }, comment], /label/i], + ]) { + // Act and Assert + assert.throws(() => validateTriageOutput(normalizeTriageOutput({ items })), message); + } +}); + +test('rewrites the queued batch file only after a reordered batch passes validation', () => { + // Arrange + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'triage-output-')); + const filename = path.join(directory, 'agent_output.json'); + try { + for (const type of ['add_labels', 'remove_labels']) { + // Act + const value = labelFirst(type); + fs.writeFileSync(filename, JSON.stringify(value)); + const reordered = validateTriageOutputFile(filename); + + // Assert + assert.equal(reordered, true); + assert.deepEqual(JSON.parse(fs.readFileSync(filename, 'utf8')).items, + [value.items[1], value.items[0]]); + } + + // An already ordered batch is validated without rewriting the file. + const ordered = JSON.stringify(output(summary), null, 2); + fs.writeFileSync(filename, ordered); + assert.equal(validateTriageOutputFile(filename), false); + assert.equal(fs.readFileSync(filename, 'utf8'), ordered); + + // A rejected batch leaves the queued file unchanged. + const invalid = JSON.stringify({ + items: [{ type: 'add_labels', labels: [waitingLabel] }, + { type: 'add_comment', body: 'test simple body' }], + }); + fs.writeFileSync(filename, invalid); + assert.throws(() => validateTriageOutputFile(filename), /Triage Summary/); + assert.equal(fs.readFileSync(filename, 'utf8'), invalid); + } finally { + // Cleanup + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +test('command line validation rewrites the batch that safe outputs publish', () => { + // Arrange + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'triage-output-')); + const filename = path.join(directory, 'agent_output.json'); + const value = labelFirst('add_labels'); + fs.writeFileSync(filename, JSON.stringify(value)); + try { + // Act + const result = spawnSync(process.execPath, + [path.join(__dirname, '..', 'validate-triage-output.cjs')], + { env: { ...process.env, GH_AW_AGENT_OUTPUT: filename }, encoding: 'utf8' }); + + // Assert + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /Queued label changes after the triage summary/); + assert.deepEqual(JSON.parse(fs.readFileSync(filename, 'utf8')).items, + [value.items[1], value.items[0]]); + } finally { + // Cleanup + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +test('the compiled workflow validates and rewrites the batch before publishing it', () => { + // Arrange + const workflow = fs.readFileSync( + path.join(__dirname, '..', '..', 'workflows', 'issue-triage.lock.yml'), 'utf8'); + + // Act + const validate = workflow.indexOf('name: Validate triage output before publishing'); + const rewrite = workflow.indexOf('validateTriageOutputFile(filename)', validate); + const publish = workflow.indexOf('name: Process Safe Outputs', validate); + + // Assert + assert.ok(validate >= 0 && rewrite > validate && publish > rewrite); + assert.doesNotMatch(workflow.slice(validate, publish), /validateTriageOutput\(JSON\.parse/); +}); + test('preserves explicit no-op and failure reporting without a comment', () => { for (const type of ['noop', 'report_incomplete', 'missing_tool', 'missing_data']) { validateTriageOutput({ items: [{ type, reason: 'Required context unavailable' }] }); diff --git a/.github/scripts/validate-triage-output.cjs b/.github/scripts/validate-triage-output.cjs index 4cc0fb1049..5e966a3030 100644 --- a/.github/scripts/validate-triage-output.cjs +++ b/.github/scripts/validate-triage-output.cjs @@ -2,10 +2,13 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -// Pre-publication gate for issue-triage safe outputs: reject incomplete summaries -// and partial batches before any queued comment or label change reaches GitHub. +// Pre-publication gate for issue-triage safe outputs: queue label changes after the +// summary, then reject incomplete summaries and partial batches before any queued +// comment or label change reaches GitHub. const fs = require('node:fs'); +const LABEL_TYPES = ['add_labels', 'remove_labels']; + const TEMPLATE_PLACEHOLDER_PATTERN = /<([^<>]+)>/g; const TEMPLATE_FIELD_PATTERN = /^(?:list|component|(?:the\s+)?affected\s+component|missing\s+details)$/i; @@ -51,6 +54,35 @@ function meaningful(value) { !/^(?:todo|tbd|test(?: message| simple body)?(?: please ignore)?|placeholder)$/i.test(text); } +// gh-aw publishes safe outputs in queue order, and the agent may queue its label change before +// the summary. Move label changes after a single summary so validation judges the batch that +// will be published; other batches are returned unchanged for validation to judge. +function normalizeTriageOutput(output) { + const items = output?.items; + if (!Array.isArray(items) || items.some(item => !item || typeof item.type !== 'string')) { + return output; + } + const comments = items.filter(item => item.type === 'add_comment'); + if (comments.length !== 1) { + return output; + } + const commentIndex = items.indexOf(comments[0]); + const before = items.slice(0, commentIndex); + const labels = before.filter(item => LABEL_TYPES.includes(item.type)); + if (labels.length === 0) { + return output; + } + return { + ...output, + items: [ + ...before.filter(item => !LABEL_TYPES.includes(item.type)), + comments[0], + ...labels, + ...items.slice(commentIndex + 1), + ], + }; +} + function validateTriageOutput(output) { if (!output || !Array.isArray(output.items) || output.items.length === 0 || output.items.some(item => !item || typeof item.type !== 'string')) { @@ -148,14 +180,29 @@ function validateTriageOutput(output) { } } +// Validates the queued batch in place. Safe-output publication reads the same file, so a +// reordered batch is written back only after it passes validation. Returns true if reordered. +function validateTriageOutputFile(filename) { + const queued = JSON.parse(fs.readFileSync(filename, 'utf8')); + const output = normalizeTriageOutput(queued); + validateTriageOutput(output); + if (output === queued) { + return false; + } + fs.writeFileSync(filename, JSON.stringify(output)); + return true; +} + if (require.main === module) { // gh-aw supplies the complete queued batch here; throwing blocks safe-output publication. const filename = process.env.GH_AW_AGENT_OUTPUT; if (!filename) { throw new Error('GH_AW_AGENT_OUTPUT is required; refusing to publish unvalidated triage output.'); } - validateTriageOutput(JSON.parse(fs.readFileSync(filename, 'utf8'))); + if (validateTriageOutputFile(filename)) { + console.log('Queued label changes after the triage summary.'); + } console.log('Triage output validated before safe-output publication.'); } -module.exports = { validateTriageOutput }; +module.exports = { normalizeTriageOutput, validateTriageOutput, validateTriageOutputFile }; diff --git a/.github/workflows/issue-triage.lock.yml b/.github/workflows/issue-triage.lock.yml index 614f0b1d2b..15ce7de50d 100644 --- a/.github/workflows/issue-triage.lock.yml +++ b/.github/workflows/issue-triage.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"0536852fc19f879532af2ce9eb87722351ee430b3f4043ee4dfc753103cd32ec","body_hash":"3b96e18888d3eed5db80c8f9ba8ff306377ed19256af3b2a6232f756c60e32ff","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","agent_model":"auto","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"62680070cc4bac9b2a1fcf147ff51a59c2269704af8184c319d528de70b80194","body_hash":"3b96e18888d3eed5db80c8f9ba8ff306377ed19256af3b2a6232f756c60e32ff","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","agent_model":"auto","engine_versions":{"copilot":"1.0.80"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"9271a1804551c0dc4fb0085a97979950aa2f8489","version":"v0.88.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12","digest":"sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12","digest":"sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12","digest":"sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.15","digest":"sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.15@sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e","pinned_image":"ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","issue_read","list_branches","list_commits","list_issue_types","list_issues","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_issues","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","add_labels","missing_data","missing_tool","noop","remove_labels"]}]} # This file was automatically generated by gh-aw (v0.88.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1837,14 +1837,16 @@ jobs: with: script: | const path = require('node:path'); - const fs = require('node:fs'); - const { validateTriageOutput } = require(path.join(process.env.GITHUB_WORKSPACE, + const { validateTriageOutputFile } = require(path.join(process.env.GITHUB_WORKSPACE, 'triage-validator', '.github', 'scripts', 'validate-triage-output.cjs')); const filename = process.env.GH_AW_AGENT_OUTPUT; if (!filename) { throw new Error('Missing agent output; refusing to publish unvalidated triage output.'); } - validateTriageOutput(JSON.parse(fs.readFileSync(filename, 'utf8'))); + // Publication reads this file in queue order; labels must follow the summary. + if (validateTriageOutputFile(filename)) { + core.info('Queued label changes after the triage summary.'); + } core.info('Triage output validated before safe-output publication.'); - name: Process Safe Outputs id: process_safe_outputs diff --git a/.github/workflows/issue-triage.md b/.github/workflows/issue-triage.md index 5682dbe674..8512defd80 100644 --- a/.github/workflows/issue-triage.md +++ b/.github/workflows/issue-triage.md @@ -71,14 +71,16 @@ safe-outputs: with: script: | const path = require('node:path'); - const fs = require('node:fs'); - const { validateTriageOutput } = require(path.join(process.env.GITHUB_WORKSPACE, + const { validateTriageOutputFile } = require(path.join(process.env.GITHUB_WORKSPACE, 'triage-validator', '.github', 'scripts', 'validate-triage-output.cjs')); const filename = process.env.GH_AW_AGENT_OUTPUT; if (!filename) { throw new Error('Missing agent output; refusing to publish unvalidated triage output.'); } - validateTriageOutput(JSON.parse(fs.readFileSync(filename, 'utf8'))); + // Publication reads this file in queue order; labels must follow the summary. + if (validateTriageOutputFile(filename)) { + core.info('Queued label changes after the triage summary.'); + } core.info('Triage output validated before safe-output publication.'); # One triage summary per run. `hide-older-comments` collapses previous # summaries so only the latest is visible.