From 75363fe67157271b47158eab4a7ee0f9500d362d Mon Sep 17 00:00:00 2001
From: DevelopmentCats <christofer@coder.com>
Date: Thu, 20 Aug 2026 12:38:31 -0500
Subject: [PATCH 1/2] scorecard: verify README image references to prevent
 false positives
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The LLM scoring Visual preview only saw markdown image references like
`![alt](../../.images/foo.gif)` but never received proof that the
referenced file exists (images are stored in a shared .images/ directory
outside the module folder).

This led to score variance — sometimes the LLM inferred the image exists,
sometimes it marked the reference as "not included in module content".
PR #1057 hit this: vscode-web dropped from 82 to 79 due to a 5→2 Visual
preview regression, despite the GIF existing and rendering correctly.

Fix: verifyReadmeImages() now resolves relative image paths and checks
existence with existsSync(). The context sent to the LLM includes a
verification section:

  === README IMAGE VERIFICATION ===
    ✓ ../../.images/vscode-web.gif — exists (5277.4 KB)
    → https://example.com/img.png — external URL

Or for broken references:

    ✗ ../.images/missing.png — NOT FOUND

This makes scoring deterministic and also catches real broken references
(e.g. jetbrains-gateway has a typo: ../.images/ instead of ../../.images/).

Fixes false positive from: https://github.com/coder/registry/pull/1057
---
 .github/scorecard/score-modules.ts | 70 ++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)

diff --git a/.github/scorecard/score-modules.ts b/.github/scorecard/score-modules.ts
index e12deb34f..0c46b61b4 100644
--- a/.github/scorecard/score-modules.ts
+++ b/.github/scorecard/score-modules.ts
@@ -65,6 +65,13 @@ interface Args {
   prReport?: string;
 }
 
+interface ImageVerification {
+  reference: string;
+  resolvedPath: string;
+  exists: boolean;
+  sizeBytes?: number;
+}
+
 function parseArgs(): Args {
   const args: Args = { dryRun: false };
   const argv = process.argv.slice(2);
@@ -96,6 +103,49 @@ async function readTruncated(filePath: string): Promise<string> {
   return content.slice(0, MAX_FILE_BYTES) + "\n... [truncated]";
 }
 
+async function verifyReadmeImages(
+  moduleName: string,
+): Promise<ImageVerification[]> {
+  const readmePath = path.join(MODULES_DIR, moduleName, "README.md");
+  const readme = await readFile(readmePath, "utf8");
+  const moduleDir = path.join(MODULES_DIR, moduleName);
+  const imageRegex = /!\[[^\]]*\]\(([^)]+)\)/g;
+  const results: ImageVerification[] = [];
+
+  let match;
+  while ((match = imageRegex.exec(readme)) !== null) {
+    const reference = match[1];
+
+    if (reference.startsWith("http://") || reference.startsWith("https://")) {
+      results.push({
+        reference,
+        resolvedPath: "(external)",
+        exists: true,
+      });
+      continue;
+    }
+
+    const resolvedPath = path.resolve(moduleDir, reference);
+    const verification: ImageVerification = {
+      reference,
+      resolvedPath,
+      exists: existsSync(resolvedPath),
+    };
+
+    if (verification.exists) {
+      try {
+        const { stat } = await import("node:fs/promises");
+        const stats = await stat(resolvedPath);
+        verification.sizeBytes = stats.size;
+      } catch {}
+    }
+
+    results.push(verification);
+  }
+
+  return results;
+}
+
 async function gatherModuleContext(moduleName: string): Promise<string> {
   const dir = path.join(MODULES_DIR, moduleName);
   const parts: string[] = [];
@@ -114,6 +164,26 @@ async function gatherModuleContext(moduleName: string): Promise<string> {
     const full = path.join(dir, f);
     parts.push(`=== FILE: ${f} ===\n${await readTruncated(full)}`);
   }
+
+  const imageVerifications = await verifyReadmeImages(moduleName);
+  if (imageVerifications.length > 0) {
+    const verificationLines = imageVerifications.map((v) => {
+      if (v.resolvedPath === "(external)") {
+        return `  → ${v.reference} — external URL`;
+      } else if (v.exists) {
+        const size = v.sizeBytes
+          ? ` (${(v.sizeBytes / 1024).toFixed(1)} KB)`
+          : "";
+        return `  ✓ ${v.reference} — exists${size}`;
+      } else {
+        return `  ✗ ${v.reference} — NOT FOUND`;
+      }
+    });
+    parts.push(
+      `=== README IMAGE VERIFICATION ===\n${verificationLines.join("\n")}`,
+    );
+  }
+
   return parts.join("\n\n");
 }
 

From afd7c1f68ea37aa64a6f18486224ef172d880b22 Mon Sep 17 00:00:00 2001
From: DevelopmentCats <christofer@coder.com>
Date: Thu, 20 Aug 2026 12:50:34 -0500
Subject: [PATCH 2/2] address review: add path bounds check and type annotation

- Type match as RegExpExecArray | null
- Skip absolute paths
- Validate resolved paths stay within REGISTRY_ROOT
---
 .github/scorecard/score-modules.ts | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/.github/scorecard/score-modules.ts b/.github/scorecard/score-modules.ts
index 0c46b61b4..97d1af43f 100644
--- a/.github/scorecard/score-modules.ts
+++ b/.github/scorecard/score-modules.ts
@@ -112,7 +112,7 @@ async function verifyReadmeImages(
   const imageRegex = /!\[[^\]]*\]\(([^)]+)\)/g;
   const results: ImageVerification[] = [];
 
-  let match;
+  let match: RegExpExecArray | null;
   while ((match = imageRegex.exec(readme)) !== null) {
     const reference = match[1];
 
@@ -125,7 +125,15 @@ async function verifyReadmeImages(
       continue;
     }
 
+    if (path.isAbsolute(reference)) {
+      continue;
+    }
+
     const resolvedPath = path.resolve(moduleDir, reference);
+    if (!resolvedPath.startsWith(REGISTRY_ROOT)) {
+      continue;
+    }
+
     const verification: ImageVerification = {
       reference,
       resolvedPath,