Skip to content

Commit f57988e

Browse files
Remove label management from issue triage (#5013)
Keep triage comment-only and use the latest summary for author follow-ups. Update validation, regressions, and the generated workflow. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
1 parent 92aa2a1 commit f57988e

5 files changed

Lines changed: 94 additions & 335 deletions

File tree

‎.github/scripts/tests/README.md‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,27 +39,27 @@ node --test .github/scripts/tests/validate-triage-output.test.cjs
3939
```
4040

4141
These tests also run in `verify-aw-lock.yml`. They cover the validation gate
42-
before triage comments and labels are published: complete summaries, explicit
42+
before triage comments are published: complete summaries, explicit
4343
no-op/failure results, placeholder comments, unfinished templates, quoted/fenced
4444
examples, duplicate comments, and mixed success/incomplete outcomes.
4545
They also reject partial safe-output batches with recorded errors while
4646
allowing C# generic type syntax in completed summaries. Draft prefixes such as
4747
`TODO:` and `TBD:` are rejected in required fields and list items; ordinary
4848
mentions of those markers within completed prose remain valid.
4949
Coverage also includes multiline template placeholders and contradictory
50-
no-op/incomplete results. Label operations queued before the summary are moved
51-
after it, because safe outputs publish in queue order; the reordered batch must
52-
still pass validation before it is written back to `GH_AW_AGENT_OUTPUT` for
53-
publication, and the compiled workflow is checked for that wiring.
50+
no-op/incomplete results. Label operations are rejected in every queue position,
51+
with or without a summary. Validation leaves `GH_AW_AGENT_OUTPUT` unchanged,
52+
and the compiled workflow is checked for that wiring and the absence of label tools.
5453
Comparison operators in completed prose remain valid; angle-bracket detection
5554
targets known template fields and instructions rather than arbitrary bracketed text.
5655
Generic arguments such as `Task<Result>` and `List<Item>`, including nested
5756
types, and XML documentation tags are covered by acceptance regressions.
5857
The summary heading must start at the beginning of the body; leading indentation
5958
cannot turn a code-block heading into an accepted Markdown heading.
60-
Initial and follow-up summaries must queue exactly one Waiting for Author label
61-
operation matching the Environment row; on-demand summaries must leave labels
62-
unchanged. Tests also reject diagnostic placeholders within required fields,
59+
Initial, follow-up, and on-demand summaries require no label operations,
60+
regardless of the Environment row. Author follow-ups use the latest workflow
61+
summary to determine whether environment details are still missing, not labels.
62+
Tests also reject diagnostic placeholders within required fields,
6363
duplicate required rows/sections, and unfinished content in additional prose.
6464

6565
## Shell script tests

‎.github/scripts/tests/validate-triage-output.test.cjs‎

Lines changed: 40 additions & 139 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ const os = require('node:os');
99
const path = require('node:path');
1010
const { test } = require('node:test');
1111
const {
12-
normalizeTriageOutput,
1312
validateTriageOutput,
1413
validateTriageOutputFile,
1514
} = require('../validate-triage-output.cjs');
@@ -33,15 +32,8 @@ Cancellation leaves the operation running. Investigate the async cancellation pa
3332
- Ask the author for their SQL Server version.
3433
`;
3534

36-
const waitingLabel = 'Auto-Triage: Waiting for Author';
3735
const output = (body) => ({
38-
items: [
39-
{ type: 'add_comment', body },
40-
...(!body.includes('Triage Summary (on-demand re-triage)') ? [{
41-
type: /\| Environment \| (?:Missing:|.*Partial:)/.test(body) ? 'add_labels' : 'remove_labels',
42-
labels: [waitingLabel],
43-
}] : []),
44-
],
36+
items: [{ type: 'add_comment', body }],
4537
errors: [],
4638
});
4739

@@ -52,115 +44,41 @@ test('accepts initial, follow-up, and on-demand summaries with multiline content
5244
validateTriageOutput(output(summary.replaceAll('\n', '\r\n')));
5345
});
5446

55-
test('accepts a summary followed by an allowed label operation', () => {
56-
for (const type of ['add_labels', 'remove_labels']) {
57-
const body = type === 'add_labels' ? summary :
58-
summary.replace('Missing: SQL Server version', 'All required environment details provided');
59-
const value = output(body);
60-
validateTriageOutput(value);
61-
}
62-
});
63-
64-
test('rejects label operations before the summary', () => {
65-
for (const type of ['add_labels', 'remove_labels']) {
66-
const value = output(summary);
67-
value.items.unshift({ type, labels: ['Auto-Triage: Waiting for Author'] });
68-
assert.throws(() => validateTriageOutput(value), /before/);
69-
}
70-
});
71-
72-
// The agent queued its label change first in the failed runs for #4855, #4879, #4904, #4940,
73-
// #4953, #4965, and #4969, so the gate rejected otherwise complete summaries.
74-
const labelFirst = (type) => {
75-
const body = type === 'add_labels' ? summary :
76-
summary.replace('Missing: SQL Server version', 'All required environment details provided');
77-
return {
78-
items: [{ type, labels: [waitingLabel] }, { type: 'add_comment', body }],
79-
errors: [],
80-
};
81-
};
82-
83-
test('queues a label operation recorded before the summary after it', () => {
84-
for (const type of ['add_labels', 'remove_labels']) {
85-
// Arrange
86-
const value = labelFirst(type);
87-
const queued = structuredClone(value);
88-
89-
// Act
90-
const normalized = normalizeTriageOutput(value);
91-
92-
// Assert
93-
// The summary is published first, and the reordered batch passes the gate.
94-
assert.deepEqual(normalized.items, [queued.items[1], queued.items[0]]);
95-
assert.deepEqual(normalized.errors, []);
96-
validateTriageOutput(normalized);
97-
// The queued batch is not mutated.
98-
assert.deepEqual(value, queued);
99-
}
100-
});
101-
102-
test('leaves batches without one summary or an early label operation unchanged', () => {
103-
const label = { type: 'add_labels', labels: [waitingLabel] };
104-
const comment = output(summary).items[0];
105-
for (const value of [
106-
null,
107-
{},
108-
{ items: null },
109-
{ items: [null, comment] },
110-
output(summary),
111-
{ items: [{ type: 'noop', reason: 'No new information' }] },
112-
{ items: [label] },
113-
{ items: [label, comment, comment] },
114-
]) {
115-
// Act
116-
const normalized = normalizeTriageOutput(value);
117-
118-
// Assert
119-
assert.equal(normalized, value);
120-
}
121-
});
122-
123-
test('still rejects invalid batches after queueing labels after the summary', () => {
124-
const label = { type: 'add_labels', labels: [waitingLabel] };
47+
test('rejects label operations before or after a summary and without one', () => {
48+
// Arrange
12549
const comment = output(summary).items[0];
126-
for (const [items, message] of [
127-
[[label, label, comment], /exactly one/],
128-
[[label, { type: 'add_comment', body: 'test simple body' }], /Triage Summary/],
129-
[[{ type: 'remove_labels', labels: [waitingLabel] }, comment], /add_labels/],
130-
[[{ ...label, labels: ['Unrelated label'] }, comment], /label/i],
131-
]) {
132-
// Act and Assert
133-
assert.throws(() => validateTriageOutput(normalizeTriageOutput({ items })), message);
50+
for (const type of ['add_labels', 'remove_labels']) {
51+
const label = { type, labels: ['Any label'] };
52+
for (const items of [
53+
[label, comment],
54+
[comment, label],
55+
[label],
56+
[{ type: 'noop', reason: 'No new information' }, label],
57+
[{ type: 'report_incomplete', reason: 'Context unavailable' }, label],
58+
]) {
59+
// Act and Assert
60+
assert.throws(() => validateTriageOutput({ items }), /cannot change issue labels/);
61+
}
13462
}
13563
});
13664

137-
test('rewrites the queued batch file only after a reordered batch passes validation', () => {
65+
test('validates the queued batch without rewriting its file', () => {
13866
// Arrange
13967
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'triage-output-'));
14068
const filename = path.join(directory, 'agent_output.json');
14169
try {
142-
for (const type of ['add_labels', 'remove_labels']) {
143-
// Act
144-
const value = labelFirst(type);
145-
fs.writeFileSync(filename, JSON.stringify(value));
146-
const reordered = validateTriageOutputFile(filename);
147-
148-
// Assert
149-
assert.equal(reordered, true);
150-
assert.deepEqual(JSON.parse(fs.readFileSync(filename, 'utf8')).items,
151-
[value.items[1], value.items[0]]);
152-
}
70+
const queued = JSON.stringify(output(summary), null, 2);
71+
fs.writeFileSync(filename, queued);
15372

154-
// An already ordered batch is validated without rewriting the file.
155-
const ordered = JSON.stringify(output(summary), null, 2);
156-
fs.writeFileSync(filename, ordered);
157-
assert.equal(validateTriageOutputFile(filename), false);
158-
assert.equal(fs.readFileSync(filename, 'utf8'), ordered);
73+
// Act
74+
validateTriageOutputFile(filename);
75+
76+
// Assert
77+
assert.equal(fs.readFileSync(filename, 'utf8'), queued);
15978

16079
// A rejected batch leaves the queued file unchanged.
16180
const invalid = JSON.stringify({
162-
items: [{ type: 'add_labels', labels: [waitingLabel] },
163-
{ type: 'add_comment', body: 'test simple body' }],
81+
items: [{ type: 'add_comment', body: 'test simple body' }],
16482
});
16583
fs.writeFileSync(filename, invalid);
16684
assert.throws(() => validateTriageOutputFile(filename), /Triage Summary/);
@@ -171,12 +89,12 @@ test('rewrites the queued batch file only after a reordered batch passes validat
17189
}
17290
});
17391

174-
test('command line validation rewrites the batch that safe outputs publish', () => {
92+
test('command line validation accepts a comment-only batch without rewriting it', () => {
17593
// Arrange
17694
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'triage-output-'));
17795
const filename = path.join(directory, 'agent_output.json');
178-
const value = labelFirst('add_labels');
179-
fs.writeFileSync(filename, JSON.stringify(value));
96+
const queued = JSON.stringify(output(summary), null, 2);
97+
fs.writeFileSync(filename, queued);
18098
try {
18199
// Act
182100
const result = spawnSync(process.execPath,
@@ -185,28 +103,28 @@ test('command line validation rewrites the batch that safe outputs publish', ()
185103

186104
// Assert
187105
assert.equal(result.status, 0, result.stderr);
188-
assert.match(result.stdout, /Queued label changes after the triage summary/);
189-
assert.deepEqual(JSON.parse(fs.readFileSync(filename, 'utf8')).items,
190-
[value.items[1], value.items[0]]);
106+
assert.match(result.stdout, /Triage output validated before safe-output publication/);
107+
assert.equal(fs.readFileSync(filename, 'utf8'), queued);
191108
} finally {
192109
// Cleanup
193110
fs.rmSync(directory, { recursive: true, force: true });
194111
}
195112
});
196113

197-
test('the compiled workflow validates and rewrites the batch before publishing it', () => {
114+
test('the compiled workflow validates the batch before publishing without label tools', () => {
198115
// Arrange
199116
const workflow = fs.readFileSync(
200117
path.join(__dirname, '..', '..', 'workflows', 'issue-triage.lock.yml'), 'utf8');
201118

202119
// Act
203120
const validate = workflow.indexOf('name: Validate triage output before publishing');
204-
const rewrite = workflow.indexOf('validateTriageOutputFile(filename)', validate);
121+
const validationCall = workflow.indexOf('validateTriageOutputFile(filename)', validate);
205122
const publish = workflow.indexOf('name: Process Safe Outputs', validate);
206123

207124
// Assert
208-
assert.ok(validate >= 0 && rewrite > validate && publish > rewrite);
125+
assert.ok(validate >= 0 && validationCall > validate && publish > validationCall);
209126
assert.doesNotMatch(workflow.slice(validate, publish), /validateTriageOutput\(JSON\.parse/);
127+
assert.doesNotMatch(workflow, /add_labels|remove_labels|Auto-Triage: Waiting for Author/);
210128
});
211129

212130
test('preserves explicit no-op and failure reporting without a comment', () => {
@@ -215,7 +133,7 @@ test('preserves explicit no-op and failure reporting without a comment', () => {
215133
}
216134
});
217135

218-
test('enforces label state for initial, follow-up, and on-demand summaries', () => {
136+
test('accepts all environment states without labels for each triage scenario', () => {
219137
for (const suffix of ['', ' (updated after author response)', ' (on-demand re-triage)']) {
220138
for (const environment of [
221139
'Missing: SQL Server version',
@@ -225,25 +143,9 @@ test('enforces label state for initial, follow-up, and on-demand summaries', ()
225143
]) {
226144
const body = summary.replace('Triage Summary', `Triage Summary${suffix}`)
227145
.replace('Missing: SQL Server version', environment);
228-
const onDemand = suffix.includes('on-demand');
229-
const expected = /Missing:|Partial:/.test(environment) ? 'add_labels' : 'remove_labels';
230-
const comment = { type: 'add_comment', body };
231-
const label = { type: expected, labels: [waitingLabel] };
232-
validateTriageOutput({ items: onDemand ? [comment] : [comment, label] });
233-
const invalid = onDemand ? [
234-
[comment, label],
235-
[comment, { ...label, type: expected === 'add_labels' ? 'remove_labels' : 'add_labels' }],
236-
] : [
237-
[comment],
238-
[comment, { ...label, type: expected === 'add_labels' ? 'remove_labels' : 'add_labels' }],
239-
[comment, label, label],
240-
[comment, { ...label, labels: [] }],
241-
[comment, { ...label, labels: ['Unrelated label'] }],
242-
[comment, { ...label, labels: [waitingLabel, 'Unrelated label'] }],
243-
];
244-
for (const items of invalid) {
245-
assert.throws(() => validateTriageOutput({ items }), /label/i);
246-
}
146+
147+
// Act and Assert
148+
assert.doesNotThrow(() => validateTriageOutput(output(body)));
247149
}
248150
}
249151
});
@@ -502,8 +404,7 @@ test('rejects multiple comments and mixed success/incomplete outcomes', () => {
502404

503405
test('rejects output errors even when a complete summary is queued', () => {
504406
const value = output(summary);
505-
value.items.push({ type: 'add_labels', labels: ['Auto-Triage: Waiting for Author'] });
506-
value.errors.push({ type: 'remove_labels', message: 'Label operation exceeded quota' });
407+
value.errors.push({ type: 'add_comment', message: 'Comment operation exceeded quota' });
507408
assert.throws(() => validateTriageOutput(value), /errors/);
508409
assert.throws(() => validateTriageOutput({
509410
items: [{ type: 'noop', reason: 'No new information' }],
@@ -517,6 +418,6 @@ test('rejects missing output, empty results, and label-only results', () => {
517418
assert.throws(() => validateTriageOutput(value), /output|result/);
518419
}
519420
assert.throws(() => validateTriageOutput({
520-
items: [{ type: 'add_labels', labels: ['Auto-Triage: Waiting for Author'] }],
521-
}), /summary/);
421+
items: [{ type: 'add_labels', labels: ['Any label'] }],
422+
}), /cannot change issue labels/);
522423
});

0 commit comments

Comments
 (0)