[AIAA] Add deterministic data-collection scripts

.github/workflows/assessment-test.yml

@@ -0,0 +1,19 @@
+name: Assessment tests
+
+on:
+  pull_request:
+    paths:
+      - scripts/assessment/**
+      - .github/workflows/assessment-test.yml
+
+permissions:
+  contents: read
+
+jobs:
+  assessment-test:
+    name: ASSESSMENT tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/npm-ci-via-cached-nvmrc
+      - run: npm run test:assessment

.prettierignore

@@ -1 +1,5 @@
 /docs/localization/ja
+
+# Test fixtures are frozen inputs; formatting would change the bytes
+# the assessment collection tests hash.
+/scripts/assessment/test/fixtures/

package.json

@@ -37,6 +37,7 @@
     "precheck:links": "npm run build",
     "seq": "bash -c 'for cmd in \"$@\"; do npm run $cmd || exit 1; done' - ",
     "serve": "npm run docus:serve",
+    "test:assessment": "node --test scripts/assessment/test/*.test.mjs",
     "test:unit": "node --experimental-strip-types --test 'lib/**/*.test.mts'",
     "test": "npm run check && npm run test:unit",
     "typecheck": "tsc",

scripts/assessment/collect.mjs

@@ -0,0 +1,179 @@
+#!/usr/bin/env node
+// Deterministic data collection for documentation assessments.
+// Inventories one or more repository checkouts, extracts markdown
+// links, optionally fetches sites and checks external links, and
+// writes a content-hash manifest so every quantitative claim traces
+// to a committed, re-runnable step and post-collection edits are
+// detectable.
+//
+//   node scripts/assessment/collect.mjs --repo <path> [--repo <path>]
+//     [--site <url>] [--check-links] --out <dir>
+//   node scripts/assessment/collect.mjs verify --out <dir>
+
+import path from 'node:path';
+import process from 'node:process';
+import { buildInventory, markdownPaths } from './lib/inventory.mjs';
+import { buildLinkReport } from './lib/links.mjs';
+import { resolveHeadSha } from './lib/git.mjs';
+import { checkLinks, fetchSites } from './lib/sitefetch.mjs';
+import { verifyManifest, writeCollection } from './lib/manifest.mjs';
+import { stableStringify } from './lib/util.mjs';
+
+const USAGE =
+  'usage: collect.mjs [verify] --repo <path> [--site <url>] [--check-links] --out <dir>';
+
+function takeValue(argv, i, flag) {
+  const value = argv[i + 1];
+  if (value == null || value.startsWith('--')) {
+    throw new Error(`missing value for ${flag}`);
+  }
+  return value;
+}
+
+function parseArgs(argv) {
+  const args = {
+    verify: false,
+    repos: [],
+    sites: [],
+    checkLinks: false,
+    out: null,
+  };
+  let i = 0;
+  if (argv[0] === 'verify') {
+    args.verify = true;
+    i = 1;
+  }
+  while (i < argv.length) {
+    const flag = argv[i];
+    if (flag === '--repo') {
+      args.repos.push(takeValue(argv, i, flag));
+      i += 2;
+    } else if (flag === '--site') {
+      args.sites.push(takeValue(argv, i, flag));
+      i += 2;
+    } else if (flag === '--out') {
+      args.out = takeValue(argv, i, flag);
+      i += 2;
+    } else if (flag === '--check-links') {
+      args.checkLinks = true;
+      i += 1;
+    } else throw new Error(`unknown flag: ${flag}`);
+  }
+  if (!args.out) throw new Error('missing --out');
+  if (!args.verify && args.repos.length === 0 && args.sites.length === 0) {
+    throw new Error('nothing to collect: pass --repo or --site');
+  }
+  for (const repoPath of args.repos) {
+    if (outputRelPath(repoPath, args.out) === '') {
+      throw new Error('--out must not be a repository root');
+    }
+  }
+  return args;
+}
+
+// Relative posix path of the output directory inside a repository, so
+// a rerun never inventories its own previous outputs; null when the
+// output directory lives outside the repository.
+function outputRelPath(repoPath, out) {
+  const rel = path.relative(path.resolve(repoPath), path.resolve(out));
+  if (rel === '') return '';
+  if (rel.startsWith('..') || path.isAbsolute(rel)) return null;
+  return rel.split(path.sep).join('/');
+}
+
+// The manifest records the command without --out: the output location
+// is wherever the manifest lives, and omitting it keeps two runs into
+// different directories byte-identical.
+function commandLine(args) {
+  const parts = ['collect.mjs'];
+  for (const repo of args.repos) parts.push('--repo', repo);
+  for (const site of args.sites) parts.push('--site', site);
+  if (args.checkLinks) parts.push('--check-links');
+  return parts.join(' ');
+}
+
+async function collect(args) {
+  const outputs = {};
+  const repos = [];
+  const repoReports = [];
+  const linkReports = [];
+  for (const repoPath of args.repos) {
+    const excludeRel = outputRelPath(repoPath, args.out);
+    const inventory = buildInventory(repoPath, {
+      exclude: excludeRel ? [excludeRel] : [],
+    });
+    const report = buildLinkReport(repoPath, markdownPaths(inventory));
+    repos.push({ path: repoPath, sha: resolveHeadSha(repoPath) });
+    repoReports.push({ path: repoPath, ...inventory });
+    linkReports.push({ path: repoPath, ...report });
+  }
+  outputs['inventory.json'] = stableStringify({ repos: repoReports }) + '\n';
+  outputs['links.json'] = stableStringify({ repos: linkReports }) + '\n';
+
+  const sites = [];
+  if (args.sites.length > 0) {
+    const fetched = await fetchSites(args.sites);
+    fetched.forEach((entry, index) => {
+      const { body, ...meta } = entry;
+      if (body) {
+        // A body implies the fetch succeeded, so the URL parses; on
+        // failure the entry (including an unparseable URL) is written
+        // through as evidence untouched.
+        const host = new URL(entry.url).host;
+        const name = `sites/${String(index + 1).padStart(3, '0')}-${host}.body`;
+        outputs[name] = body;
+        meta.bodyFile = name;
+      }
+      sites.push(meta);
+    });
+    outputs['site-fetches.json'] = stableStringify({ sites }) + '\n';
+  }
+
+  if (args.checkLinks) {
+    const external = [
+      ...new Set(linkReports.flatMap((report) => report.external)),
+    ].sort();
+    outputs['link-status.json'] =
+      stableStringify({ checked: await checkLinks(external) }) + '\n';
+  }
+
+  writeCollection(args.out, {
+    commands: [commandLine(args)],
+    repos,
+    sites,
+    outputs,
+  });
+  process.stdout.write(`collected into ${args.out}\n`);
+}
+
+function verify(args) {
+  const result = verifyManifest(args.out);
+  if (result.ok) {
+    process.stdout.write('manifest ok\n');
+    return;
+  }
+  for (const name of result.mismatched) {
+    process.stderr.write(`mismatched: ${name}\n`);
+  }
+  for (const name of result.missing) {
+    process.stderr.write(`missing: ${name}\n`);
+  }
+  for (const name of result.unexpected) {
+    process.stderr.write(`unexpected: ${name}\n`);
+  }
+  process.exit(1);
+}
+
+async function main() {
+  let args;
+  try {
+    args = parseArgs(process.argv.slice(2));
+  } catch (err) {
+    process.stderr.write(`${err.message}\n${USAGE}\n`);
+    process.exit(2);
+  }
+  if (args.verify) verify(args);
+  else await collect(args);
+}
+
+await main();

scripts/assessment/lib/git.mjs

@@ -0,0 +1,17 @@
+import { execFileSync } from 'node:child_process';
+
+function defaultExec(cmd, args) {
+  return execFileSync(cmd, args, {
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+}
+
+export function resolveHeadSha(dir, execImpl = defaultExec) {
+  try {
+    const out = execImpl('git', ['-C', dir, 'rev-parse', 'HEAD']).trim();
+    return /^[0-9a-f]{40}$/i.test(out) ? out : null;
+  } catch {
+    return null;
+  }
+}

scripts/assessment/lib/inventory.mjs

@@ -0,0 +1,79 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { sha256Hex } from './util.mjs';
+
+const MARKDOWN_EXTENSIONS = new Set(['.md', '.mdx', '.markdown']);
+const SKIPPED_DIRECTORIES = new Set(['.git', 'node_modules']);
+
+function walk(root, rel, { files, symlinks, exclude }) {
+  const entries = fs.readdirSync(path.join(root, rel), {
+    withFileTypes: true,
+  });
+  for (const entry of entries) {
+    const relPath = rel ? `${rel}/${entry.name}` : entry.name;
+    if (exclude.has(relPath)) continue;
+    if (entry.isSymbolicLink()) {
+      // Recorded, never followed: the link target string is the
+      // content, so links pointing outside the repository cannot pull
+      // outside bytes into the inventory.
+      symlinks.push(relPath);
+    } else if (entry.isDirectory()) {
+      if (!SKIPPED_DIRECTORIES.has(entry.name)) {
+        walk(root, relPath, { files, symlinks, exclude });
+      }
+    } else if (entry.isFile()) {
+      files.push(relPath);
+    }
+  }
+}
+
+export function buildInventory(root, { exclude = [] } = {}) {
+  const filePaths = [];
+  const symlinkPaths = [];
+  walk(root, '', {
+    files: filePaths,
+    symlinks: symlinkPaths,
+    exclude: new Set(exclude),
+  });
+  filePaths.sort();
+  symlinkPaths.sort();
+  const files = filePaths.map((relPath) => {
+    const content = fs.readFileSync(path.join(root, relPath));
+    return { path: relPath, bytes: content.length, sha256: sha256Hex(content) };
+  });
+  const symlinks = symlinkPaths.map((relPath) => {
+    const target = fs.readlinkSync(path.join(root, relPath));
+    return {
+      path: relPath,
+      target,
+      bytes: Buffer.byteLength(target),
+      sha256: sha256Hex(target),
+    };
+  });
+  const byExtension = {};
+  let markdownCount = 0;
+  for (const file of files) {
+    const ext = path.posix.extname(file.path).toLowerCase();
+    if (ext) byExtension[ext] = (byExtension[ext] ?? 0) + 1;
+    if (MARKDOWN_EXTENSIONS.has(ext)) markdownCount += 1;
+  }
+  return {
+    files,
+    symlinks,
+    totals: {
+      fileCount: files.length,
+      symlinkCount: symlinks.length,
+      byteCount: files.reduce((n, f) => n + f.bytes, 0),
+      markdownCount,
+      byExtension,
+    },
+  };
+}
+
+export function markdownPaths(inventory) {
+  return inventory.files
+    .map((f) => f.path)
+    .filter((p) =>
+      MARKDOWN_EXTENSIONS.has(path.posix.extname(p).toLowerCase()),
+    );
+}

scripts/assessment/lib/links.mjs

@@ -0,0 +1,84 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+// Deliberate limitations, documented rather than hidden: destinations
+// with unescaped parentheses need the angle-bracket form to be seen,
+// and indented (non-fenced) code blocks are still scanned. A markdown
+// AST would lift both at the cost of a new dependency.
+const INLINE_LINK = /!?\[[^\]]*\]\(([^)\s<][^)\s]*)(?:\s+"[^"]*")?\)/g;
+const ANGLE_LINK = /!?\[[^\]]*\]\(<([^>]+)>(?:\s+"[^"]*")?\)/g;
+const REFERENCE_DEFINITION = /^\s*\[[^\]]+\]:\s*(\S+)/gm;
+const AUTOLINK = /<(https?:\/\/[^>\s]+)>/g;
+
+const FENCE = /^ {0,3}(`{3,}|~{3,})/;
+
+// Blanks out fenced code blocks and inline code spans so example
+// links inside them (common in documentation about documentation) do
+// not pollute the inventory.
+function stripCodeRegions(markdown) {
+  const out = [];
+  let fence = null;
+  for (const line of markdown.split('\n')) {
+    const opener = line.match(FENCE);
+    if (fence) {
+      if (
+        opener &&
+        opener[1][0] === fence[0] &&
+        opener[1].length >= fence.length
+      ) {
+        fence = null;
+      }
+      out.push('');
+    } else if (opener) {
+      fence = opener[1];
+      out.push('');
+    } else {
+      out.push(line.replace(/(`+)[^`]*\1/g, ' '));
+    }
+  }
+  return out.join('\n');
+}
+
+function classify(url) {
+  if (url.startsWith('#')) return 'anchor';
+  if (/^https?:\/\//i.test(url)) return 'external';
+  return 'internal';
+}
+
+export function extractLinks(markdown) {
+  const source = stripCodeRegions(markdown);
+  const urls = new Set();
+  for (const pattern of [
+    INLINE_LINK,
+    ANGLE_LINK,
+    REFERENCE_DEFINITION,
+    AUTOLINK,
+  ]) {
+    for (const match of source.matchAll(pattern)) {
+      urls.add(match[1]);
+    }
+  }
+  return [...urls].sort().map((url) => ({ url, kind: classify(url) }));
+}
+
+export function buildLinkReport(root, markdownRelPaths) {
+  const perFile = {};
+  const byKind = {
+    external: new Set(),
+    internal: new Set(),
+    anchor: new Set(),
+  };
+  for (const relPath of markdownRelPaths) {
+    const links = extractLinks(
+      fs.readFileSync(path.join(root, relPath), 'utf8'),
+    );
+    perFile[relPath] = links;
+    for (const link of links) byKind[link.kind].add(link.url);
+  }
+  return {
+    perFile,
+    external: [...byKind.external].sort(),
+    internal: [...byKind.internal].sort(),
+    anchors: [...byKind.anchor].sort(),
+  };
+}