Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions .github/scripts/tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,11 @@ They also reject partial safe-output batches with recorded errors while
allowing C# generic type syntax in completed summaries. Draft prefixes such as
`TODO:` and `TBD:` are rejected in required fields and list items; ordinary
mentions of those markers within completed prose remain valid.
Coverage also includes multiline template placeholders, label operations queued
before the summary, and contradictory no-op/incomplete results.
Coverage also includes multiline template placeholders and contradictory
no-op/incomplete results. Label operations queued before the summary are moved
after it, because safe outputs publish in queue order; the reordered batch must
still pass validation before it is written back to `GH_AW_AGENT_OUTPUT` for
publication, and the compiled workflow is checked for that wiring.
Comparison operators in completed prose remain valid; angle-bracket detection
targets known template fields and instructions rather than arbitrary bracketed text.
Generic arguments such as `Task<Result>` and `List<Item>`, including nested
Expand Down
150 changes: 149 additions & 1 deletion .github/scripts/tests/validate-triage-output.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,16 @@
// See the LICENSE file in the project root for more information.

const assert = require('node:assert/strict');
const { spawnSync } = require('node:child_process');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { test } = require('node:test');
const { validateTriageOutput } = require('../validate-triage-output.cjs');
const {
normalizeTriageOutput,
validateTriageOutput,
validateTriageOutputFile,
} = require('../validate-triage-output.cjs');

const summary = `## \u{1F50D} Triage Summary

Expand Down Expand Up @@ -61,6 +69,146 @@ test('rejects label operations before the summary', () => {
}
});

// The agent queued its label change first in the failed runs for #4855, #4879, #4904, #4940,
// #4953, #4965, and #4969, so the gate rejected otherwise complete summaries.
const labelFirst = (type) => {
const body = type === 'add_labels' ? summary :
summary.replace('Missing: SQL Server version', 'All required environment details provided');
return {
items: [{ type, labels: [waitingLabel] }, { type: 'add_comment', body }],
errors: [],
};
};

test('queues a label operation recorded before the summary after it', () => {
for (const type of ['add_labels', 'remove_labels']) {
// Arrange
const value = labelFirst(type);
const queued = structuredClone(value);

// Act
const normalized = normalizeTriageOutput(value);

// Assert
// The summary is published first, and the reordered batch passes the gate.
assert.deepEqual(normalized.items, [queued.items[1], queued.items[0]]);
assert.deepEqual(normalized.errors, []);
validateTriageOutput(normalized);
// The queued batch is not mutated.
assert.deepEqual(value, queued);
}
});

test('leaves batches without one summary or an early label operation unchanged', () => {
const label = { type: 'add_labels', labels: [waitingLabel] };
const comment = output(summary).items[0];
for (const value of [
null,
{},
{ items: null },
{ items: [null, comment] },
output(summary),
{ items: [{ type: 'noop', reason: 'No new information' }] },
{ items: [label] },
{ items: [label, comment, comment] },
]) {
// Act
const normalized = normalizeTriageOutput(value);

// Assert
assert.equal(normalized, value);
}
});

test('still rejects invalid batches after queueing labels after the summary', () => {
const label = { type: 'add_labels', labels: [waitingLabel] };
const comment = output(summary).items[0];
for (const [items, message] of [
[[label, label, comment], /exactly one/],
[[label, { type: 'add_comment', body: 'test simple body' }], /Triage Summary/],
[[{ type: 'remove_labels', labels: [waitingLabel] }, comment], /add_labels/],
[[{ ...label, labels: ['Unrelated label'] }, comment], /label/i],
]) {
// Act and Assert
assert.throws(() => validateTriageOutput(normalizeTriageOutput({ items })), message);
}
});

test('rewrites the queued batch file only after a reordered batch passes validation', () => {
// Arrange
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'triage-output-'));
const filename = path.join(directory, 'agent_output.json');
try {
for (const type of ['add_labels', 'remove_labels']) {
// Act
const value = labelFirst(type);
fs.writeFileSync(filename, JSON.stringify(value));
const reordered = validateTriageOutputFile(filename);

// Assert
assert.equal(reordered, true);
assert.deepEqual(JSON.parse(fs.readFileSync(filename, 'utf8')).items,
[value.items[1], value.items[0]]);
}

// An already ordered batch is validated without rewriting the file.
const ordered = JSON.stringify(output(summary), null, 2);
fs.writeFileSync(filename, ordered);
assert.equal(validateTriageOutputFile(filename), false);
assert.equal(fs.readFileSync(filename, 'utf8'), ordered);

// A rejected batch leaves the queued file unchanged.
const invalid = JSON.stringify({
items: [{ type: 'add_labels', labels: [waitingLabel] },
{ type: 'add_comment', body: 'test simple body' }],
});
fs.writeFileSync(filename, invalid);
assert.throws(() => validateTriageOutputFile(filename), /Triage Summary/);
assert.equal(fs.readFileSync(filename, 'utf8'), invalid);
} finally {
// Cleanup
fs.rmSync(directory, { recursive: true, force: true });
}
});

test('command line validation rewrites the batch that safe outputs publish', () => {
// Arrange
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'triage-output-'));
const filename = path.join(directory, 'agent_output.json');
const value = labelFirst('add_labels');
fs.writeFileSync(filename, JSON.stringify(value));
try {
// Act
const result = spawnSync(process.execPath,
[path.join(__dirname, '..', 'validate-triage-output.cjs')],
{ env: { ...process.env, GH_AW_AGENT_OUTPUT: filename }, encoding: 'utf8' });

// Assert
assert.equal(result.status, 0, result.stderr);
assert.match(result.stdout, /Queued label changes after the triage summary/);
assert.deepEqual(JSON.parse(fs.readFileSync(filename, 'utf8')).items,
[value.items[1], value.items[0]]);
} finally {
// Cleanup
fs.rmSync(directory, { recursive: true, force: true });
}
});

test('the compiled workflow validates and rewrites the batch before publishing it', () => {
// Arrange
const workflow = fs.readFileSync(
path.join(__dirname, '..', '..', 'workflows', 'issue-triage.lock.yml'), 'utf8');

// Act
const validate = workflow.indexOf('name: Validate triage output before publishing');
const rewrite = workflow.indexOf('validateTriageOutputFile(filename)', validate);
const publish = workflow.indexOf('name: Process Safe Outputs', validate);

// Assert
assert.ok(validate >= 0 && rewrite > validate && publish > rewrite);
assert.doesNotMatch(workflow.slice(validate, publish), /validateTriageOutput\(JSON\.parse/);
});

test('preserves explicit no-op and failure reporting without a comment', () => {
for (const type of ['noop', 'report_incomplete', 'missing_tool', 'missing_data']) {
validateTriageOutput({ items: [{ type, reason: 'Required context unavailable' }] });
Expand Down
55 changes: 51 additions & 4 deletions .github/scripts/validate-triage-output.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,13 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

// Pre-publication gate for issue-triage safe outputs: reject incomplete summaries
// and partial batches before any queued comment or label change reaches GitHub.
// Pre-publication gate for issue-triage safe outputs: queue label changes after the
// summary, then reject incomplete summaries and partial batches before any queued
// comment or label change reaches GitHub.
const fs = require('node:fs');

const LABEL_TYPES = ['add_labels', 'remove_labels'];

const TEMPLATE_PLACEHOLDER_PATTERN = /<([^<>]+)>/g;
const TEMPLATE_FIELD_PATTERN =
/^(?:list|component|(?:the\s+)?affected\s+component|missing\s+details)$/i;
Expand Down Expand Up @@ -51,6 +54,35 @@ function meaningful(value) {
!/^(?:todo|tbd|test(?: message| simple body)?(?: please ignore)?|placeholder)$/i.test(text);
}

// gh-aw publishes safe outputs in queue order, and the agent may queue its label change before
// the summary. Move label changes after a single summary so validation judges the batch that
// will be published; other batches are returned unchanged for validation to judge.
function normalizeTriageOutput(output) {
const items = output?.items;
if (!Array.isArray(items) || items.some(item => !item || typeof item.type !== 'string')) {
return output;
}
const comments = items.filter(item => item.type === 'add_comment');
if (comments.length !== 1) {
return output;
}
const commentIndex = items.indexOf(comments[0]);
const before = items.slice(0, commentIndex);
const labels = before.filter(item => LABEL_TYPES.includes(item.type));
if (labels.length === 0) {
return output;
}
return {
...output,
items: [
...before.filter(item => !LABEL_TYPES.includes(item.type)),
comments[0],
...labels,
...items.slice(commentIndex + 1),
],
};
}

function validateTriageOutput(output) {
if (!output || !Array.isArray(output.items) || output.items.length === 0 ||
output.items.some(item => !item || typeof item.type !== 'string')) {
Expand Down Expand Up @@ -148,14 +180,29 @@ function validateTriageOutput(output) {
}
}

// Validates the queued batch in place. Safe-output publication reads the same file, so a
// reordered batch is written back only after it passes validation. Returns true if reordered.
function validateTriageOutputFile(filename) {
const queued = JSON.parse(fs.readFileSync(filename, 'utf8'));
const output = normalizeTriageOutput(queued);
validateTriageOutput(output);
if (output === queued) {
return false;
}
fs.writeFileSync(filename, JSON.stringify(output));
return true;
}

if (require.main === module) {
// gh-aw supplies the complete queued batch here; throwing blocks safe-output publication.
const filename = process.env.GH_AW_AGENT_OUTPUT;
if (!filename) {
throw new Error('GH_AW_AGENT_OUTPUT is required; refusing to publish unvalidated triage output.');
}
validateTriageOutput(JSON.parse(fs.readFileSync(filename, 'utf8')));
if (validateTriageOutputFile(filename)) {
console.log('Queued label changes after the triage summary.');
}
console.log('Triage output validated before safe-output publication.');
}

module.exports = { validateTriageOutput };
module.exports = { normalizeTriageOutput, validateTriageOutput, validateTriageOutputFile };
10 changes: 6 additions & 4 deletions .github/workflows/issue-triage.lock.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 5 additions & 3 deletions .github/workflows/issue-triage.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,14 +71,16 @@ safe-outputs:
with:
script: |
const path = require('node:path');
const fs = require('node:fs');
const { validateTriageOutput } = require(path.join(process.env.GITHUB_WORKSPACE,
const { validateTriageOutputFile } = require(path.join(process.env.GITHUB_WORKSPACE,
'triage-validator', '.github', 'scripts', 'validate-triage-output.cjs'));
const filename = process.env.GH_AW_AGENT_OUTPUT;
if (!filename) {
throw new Error('Missing agent output; refusing to publish unvalidated triage output.');
}
validateTriageOutput(JSON.parse(fs.readFileSync(filename, 'utf8')));
// Publication reads this file in queue order; labels must follow the summary.
if (validateTriageOutputFile(filename)) {
core.info('Queued label changes after the triage summary.');
}
core.info('Triage output validated before safe-output publication.');
# One triage summary per run. `hide-older-comments` collapses previous
# summaries so only the latest is visible.
Expand Down
Loading