Add a npx hereby validate command to group all repo validations (#64161)

Author: weswigham

.github/copilot-instructions.md

@@ -30,18 +30,18 @@ IF THESE COMMANDS FAIL, CI WILL FAIL, AND YOUR PR WILL BE REJECTED OUT OF HAND.
 FIXING ERRORS FROM THESE COMMANDS IS YOUR HIGHEST PRIORITY.
 ENSURE YOU DO THE RIGHT THINGS TO MAKE THEM PASS.
 ```sh
-npx hereby build  # Build the project
-npx hereby test   # Run tests
-npx hereby lint   # Run linters
-npx hereby format # Format the code
+npx hereby validate # Build, test, lint, and format the project
 ```
 </critical>
 
 If you are writing or testing TS API features (eg, code in packages/typescript/src/api/async/api.ts), additionally, you need to run
 ```sh
-npx hereby test:api
+npx hereby validate --api # Also run the TypeScript API tests
 ```
-which is not run as part of the primary suite.
+instead. API tests are not run by `npx hereby validate` without `--api`.
+
+If as part of your change you edit tools, benchmarks, or other ancillary parts of the repository, you should instead run `npx hereby validate --all` to ensure *everything* is working correctly. This will
+most accurately approximate the results of a full CI run.
 
 ## Compiler Features, Fixes, and Tests
 
@@ -134,10 +134,8 @@ Were alternate fixes considered? Describe them briefly if so
 ## Copilot Checklist
 
 <!-- don't lie! -->
-I successfully ran these commands at the end of my session, and they completed without error:
- * [ ] npx hereby build
- * [ ] npx hereby test
- * [ ] npx hereby lint
- * [ ] npx hereby format
+I successfully ran the applicable command at the end of my session, and it completed without error:
+ * [ ] npx hereby validate
+ * [ ] npx hereby validate --api (for TypeScript API changes)
 
 ```

Herebyfile.mjs

@@ -103,6 +103,8 @@ const { values: rawOptions } = parseArgs({
     options: {
         tests: { type: "string", short: "t" },
         fix: { type: "boolean" },
+        api: { type: "boolean" },
+        all: { type: "boolean" },
         debug: { type: "boolean" },
         dirty: { type: "boolean" },
         release: { type: "boolean" },
@@ -967,13 +969,15 @@ export const generateAST = task({
     run: () => run("node", ["./tools/scripts/tsc/generate.ts"]),
 });
 
+async function runGenerateAPI() {
+    await run("go", ["-C", "./tools", "run", "./gen-proto", "../tsc/internal/api/proto.go", "../packages/typescript/src/api/proto.generated.ts"]);
+    await run("npx", ["dprint", "fmt", "packages/typescript/src/api/proto.generated.ts"]);
+}
+
 export const generateAPI = task({
     name: "generate:api",
     description: "Generates API files from internal/api/proto.go and internal/api/session.go.",
-    run: async () => {
-        await run("go", ["-C", "./tools", "run", "./gen-proto", "../tsc/internal/api/proto.go", "../packages/typescript/src/api/proto.generated.ts"]);
-        await run("npx", ["dprint", "fmt", "packages/typescript/src/api/proto.generated.ts"]);
-    },
+    run: runGenerateAPI,
 });
 
 // ── Vendored npm dependencies ───────────────────────────────────
@@ -1186,13 +1190,16 @@ export const testTsc = task({
     run: runTests,
 });
 
+export const testExtension = task({
+    name: "test:extension",
+    description: "Runs the VS Code extension tests.",
+    run: runTestExtension,
+});
+
 export const test = task({
     name: "test",
-    description: "Runs all tests. This is the most typical test task to need.",
-    run: async () => {
-        await runTests();
-        await runTestExtension();
-    },
+    description: "Alias for test:tsc.",
+    dependencies: [testTsc],
 });
 
 async function runTestBenchmarks() {
@@ -1223,12 +1230,6 @@ export const testTools = task({
     run: runTestTools,
 });
 
-export const testExtension = task({
-    name: "test:extension",
-    description: "Runs the VS Code extension tests.",
-    run: runTestExtension,
-});
-
 export const buildAPI = task({
     name: "build:api",
     description: "Builds @typescript/typescript JS API.",
@@ -1237,13 +1238,15 @@ export const buildAPI = task({
     },
 });
 
+async function runBuildAPITests() {
+    await run("npm", ["run", "-w", "@typescript/typescript", "build:test"]);
+}
+
 export const buildAPITests = task({
     name: "build:api:test",
     description: "Builds the @typescript/typescript JS API tests.",
     dependencies: [generateEnums, generateAPI],
-    run: async () => {
-        await run("npm", ["run", "-w", "@typescript/typescript", "build:test"]);
-    },
+    run: runBuildAPITests,
 });
 
 export const testAPI = task({
@@ -1364,6 +1367,61 @@ async function runFormat() {
     await run("dprint", ["fmt"]);
 }
 
+export const validate = task({
+    name: "validate",
+    description: "Builds, tests, lints, and formats the repo. Pass --api to include API tests, or --all to include all ancilliary repository tests.",
+    dependencies: [build],
+    run: async () => {
+        /** @type {{ name: string; error: unknown }[]} */
+        const failures = [];
+        /** @param {string} name @param {() => Promise<void>} action */
+        const runValidation = async (name, action) => {
+            try {
+                await action();
+            }
+            catch (error) {
+                failures.push({ name, error });
+                console.error(styleText("red", `${name} failed; continuing validation.`));
+            }
+        };
+
+        await runValidation("test:tsc", runTests);
+        await runValidation("test:extension", runTestExtension);
+        if (options.api || options.all) {
+            await runGenerateEnums(); // prereqs for test:api not included in `validate` deps
+            await runGenerateAPI();
+            await runBuildAPITests();
+            await runValidation("test:api", runTestAPI);
+        }
+        if (options.all) {
+            await runValidation("test:benchmarks", runTestBenchmarks);
+            await runValidation("test:tools", runTestTools);
+            await runValidation("test:smoke", runSmokeTest); // in CI this is run with `--race`
+        }
+        await runValidation("lint", runLint);
+        await runValidation("format", runFormat);
+
+        if (failures.length) {
+            throw new AggregateError(
+                failures.map(failure => failure.error),
+                `Validation failed: ${failures.map(failure => failure.name).join(", ")}`,
+            );
+        }
+    },
+});
+
+async function runSmokeTest() {
+    await run("./built/local/tsc", ["-p", "./tsc/testdata/fixtures/compiler", "--noEmit", "--singleThreaded"]);
+    await run("./built/local/tsc", ["-p", "./tsc/testdata/fixtures/compiler", "--noEmit"]);
+}
+
+export const smokeTest = task({
+    name: "test:smoke",
+    description: "Runs the smoke tests.",
+    dependencies: [build],
+    run: runSmokeTest,
+});
+
 export const checkFormat = task({
     name: "check:format",
     description: "Checks that the repo is formatted.",