ci(python): require full runtime mutation matrix

.github/workflows/ci.yml

@@ -58,16 +58,38 @@ jobs:
             reports/hypothesis/seed.txt
           if-no-files-found: ignore
 
-  mutation:
-    name: Mutation Gate
+  mutation-plan:
+    name: Mutation inventory
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    outputs:
+      modules: ${{ steps.plan.outputs.modules }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - name: Inventory every handwritten runtime module
+        id: plan
+        run: |
+          modules=$(bash scripts/mutation.sh --matrix)
+          echo "modules=$modules" >> "$GITHUB_OUTPUT"
+
+  mutation-module:
+    name: Mutation module (${{ matrix.module }})
+    needs: mutation-plan
     runs-on: ubuntu-latest
     timeout-minutes: 40
     permissions:
       contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        module: ${{ fromJSON(needs.mutation-plan.outputs.modules) }}
     steps:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
         with:
-          fetch-depth: 0
           persist-credentials: false
       - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
@@ -76,18 +98,30 @@ jobs:
         with:
           version: "0.12.17"
       - run: uv sync --locked
-      - name: Mutate changed and critical runtime modules
+      - name: Mutate assigned runtime modules
         env:
-          MUTATION_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }}
+          MUTATION_MODULE: ${{ matrix.module }}
         run: uv run --locked poe mutation
       - name: Preserve mutation outcomes
         if: ${{ always() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: mutation-outcomes
+          name: mutation-outcomes-${{ matrix.module }}
           path: reports/mutation.json
           if-no-files-found: error
 
+  mutation:
+    name: Mutation Gate
+    if: ${{ always() }}
+    needs: [mutation-plan, mutation-module]
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Require every mutation module to succeed
+        env:
+          RESULTS: ${{ toJSON(needs.*.result) }}
+        run: jq -e 'length == 2 and all(. == "success")' <<< "$RESULTS"
+
   quality:
     name: Quality Gate
     if: ${{ always() }}

.github/workflows/mutation-audit.yml

@@ -25,7 +25,7 @@ jobs:
         with:
           version: "0.12.17"
       - run: uv sync --locked
-      - run: uv run --locked poe mutation-full
+      - run: uv run --locked poe mutation
       - name: Preserve mutation outcomes
         if: ${{ always() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

maintainers/mutation-testing.md

@@ -1,20 +1,25 @@
 # Mutation testing
 
-`uv run --locked poe quality` runs all native checks and mutates every changed
-handwritten runtime module plus the lock acquisition, guard, renewal, and worker
-modules. CI uses the same `checks` and `mutation` tasks in separate jobs, then
-requires both through `Quality Gate`. The weekly `poe mutation-full` task audits
-every handwritten runtime module without a debt baseline.
+`uv run --locked poe quality` runs all native checks and mutates every
+handwritten runtime module. CI runs the same `checks` and `mutation` tasks,
+assigning one handwritten module to each independent matrix job. Git's current
+handwritten module inventory determines the jobs, including newly added modules.
+`Mutation Gate` requires every module job, and `Quality Gate` requires it and the Python test matrix. The
+weekly audit runs the same full mutation task without a debt baseline.
 
 Mutmut's [native configuration](https://mutmut.readthedocs.io/en/latest/) lives
 in `pyproject.toml`; it excludes only the generated OpenAPI client. Mutmut can
-select modules by name but has no Git-changed-module option and returns success
-when mutants survive. `scripts/mutation.sh` selects module names from Git and
-`scripts/mutation_results.py` reads only those modules' native metadata. The
+select modules by name but returns success when mutants survive.
+`scripts/mutation.sh` selects the full handwritten inventory from Git and
+`scripts/mutation_results.py` reads each selected module's native metadata. The
 report at `reports/mutation.json` distinguishes killed, statically invalid,
 surviving, uncovered, timed-out, crashed, interrupted, and missing results.
+Mutmut creates mutants inside functions. Export-only modules remain in the
+inventory; the runner verifies that they define no functions and records them
+as unmutatable. Coverage and installed-package checks still include them.
 A pytest internal error is a harness crash, not a killed mutant.
-The pinned Pyrefly check rejects type-invalid realtime and auth mutants before pytest;
+The pinned Pyrefly check covers the handwritten runtime and rejects
+type-invalid mutants before pytest;
 the report counts these as `type_checked`, separately from test-killed mutants.
 Surviving, uncovered, timed-out, crashed, and incomplete mutants still fail.
 Mutmut passes pytest `-x` so a selected test's first assertion failure kills the
@@ -29,12 +34,13 @@ Mutmut uses its native forkserver isolation because forking from a process that
 has already run asyncio tests can crash a worker before its tests report a result.
 The mutation runner sets `NO_PROXY=*` for its hermetic transport tests because
 macOS system-proxy discovery can abort after a fork with active threads.
-The pinned pytest-order plugin runs bounded callback and presence assertions
-first when mutmut's unordered test selection could otherwise reach a blocked test.
+The pinned pytest-order plugin runs bounded callback, presence, and fetch-worker
+assertions first when mutmut's unordered test selection could otherwise reach a
+blocked test.
 
-The full audit runs mutmut once across the entire source tree. It reports any
-surviving mutants without treating them as an approved baseline. Equivalent
-mutants require a reviewed, exact exception before a gate can accept them.
+The local and weekly full runs execute the inventory in one Mutmut process.
+Equivalent mutants require a reviewed, exact exception before a gate can
+accept them.
 
 Ruff, Mypy, Basedpyright, pytest, and Tox tasks pass `pyproject.toml`
 explicitly. Their documented config-file precedence can otherwise select a

maintainers/quality-policy.lock.json

@@ -299,16 +299,9 @@
         "pyrefly",
         "check",
         "--output-format=json",
-        "src/volcano_sdk/realtime.py",
-        "src/volcano_sdk/auth.py",
-        "src/volcano_sdk/storage.py",
-        "src/volcano_sdk/functions.py",
-        "src/volcano_sdk/durable_authoring.py",
-        "src/volcano_sdk/database.py",
-        "src/volcano_sdk/_function_resolution.py",
-        "src/volcano_sdk/_transport.py",
-        "src/volcano_sdk/_session.py",
-        "src/volcano_sdk/_session_operations.py"
+        "--project-excludes",
+        "src/volcano_sdk/_generated/**",
+        "src/volcano_sdk"
       ]
     },
     "mypy": {
@@ -400,7 +393,6 @@
         "generated": "python -m scripts.check_openapi",
         "lint": "ruff check --config pyproject.toml .",
         "mutation": "bash scripts/mutation.sh",
-        "mutation-full": "MUTATION_FULL=1 bash scripts/mutation.sh",
         "mypy": "mypy --config-file pyproject.toml",
         "package-check": {
           "interpreter": "bash",

pyproject.toml

@@ -458,16 +458,8 @@ partial_branches = []
 source_paths = ["src/volcano_sdk"]
 type_check_command = [
   "pyrefly", "check", "--output-format=json",
-  "src/volcano_sdk/realtime.py",
-  "src/volcano_sdk/auth.py",
-  "src/volcano_sdk/storage.py",
-  "src/volcano_sdk/functions.py",
-  "src/volcano_sdk/durable_authoring.py",
-  "src/volcano_sdk/database.py",
-  "src/volcano_sdk/_function_resolution.py",
-  "src/volcano_sdk/_transport.py",
-  "src/volcano_sdk/_session.py",
-  "src/volcano_sdk/_session_operations.py",
+  "--project-excludes", "src/volcano_sdk/_generated/**",
+  "src/volcano_sdk",
 ]
 cache_invalidation_files = ["tests/**/*.py"]
 pytest_add_cli_args_test_selection = ["tests/unit"]
@@ -498,7 +490,6 @@ types = ["mypy", "basedpyright"]
 mypy = "mypy --config-file pyproject.toml"
 basedpyright = "basedpyright --project pyproject.toml"
 mutation = "bash scripts/mutation.sh"
-mutation-full = "MUTATION_FULL=1 bash scripts/mutation.sh"
 
 [tool.poe.tasks.test]
 interpreter = "bash"

scripts/check_quality_policy.py

@@ -18,7 +18,7 @@
     from collections.abc import Iterable
 
 GENERATED = "src/volcano_sdk/_generated"
-LOCK_SHA256 = "489b1a41632f67d528dbb10267bb8921c38c78726c430d990b4ee99ebdd9236a"
+LOCK_SHA256 = "3eea92085dce48f7454bc9e2b82854cc99a8787083da471609ae41da57d75560"
 TYPE_FIXTURES = {
     "tests/typing/contract_steps.py",
     "tests/typing/durable_callbacks.py",

scripts/mutation.sh

@@ -4,71 +4,87 @@ set -euo pipefail
 # Forked macOS workers must not query SystemConfiguration through urllib/httpx.
 export NO_PROXY='*' no_proxy='*'
 
-# Refresh mutmut's test-to-mutant map so newly added tests are selected.
-rm -rf -- mutants
+paths=()
+modules=()
+while IFS= read -r -d '' path; do
+  if [[ $path == src/volcano_sdk/*.py && $path != src/volcano_sdk/_generated/* && -f $path ]]; then
+    module=${path#src/}
+    module=${module%.py}
+    if [[ $module == */__init__ ]]; then
+      module=${module%/__init__}
+    fi
+    module=${module//\//.}
+    if [[ ! $module =~ ^[a-zA-Z_][a-zA-Z_0-9]*(\.[a-zA-Z_][a-zA-Z_0-9]*)*$ ]]; then
+      echo "Invalid Python runtime module: $path" >&2
+      exit 1
+    fi
+    for existing in "${modules[@]-}"; do
+      if [[ $module == "$existing" ]]; then
+        echo "Duplicate Python runtime module: $module" >&2
+        exit 1
+      fi
+    done
+    paths+=("$path")
+    modules+=("$module")
+  fi
+done < <(git ls-files --cached --others --exclude-standard -z -- src/volcano_sdk)
+
+if (( ${#paths[@]} == 0 )); then
+  echo 'No handwritten SDK runtime modules found' >&2
+  exit 1
+fi
+if [[ ${1:-} == --matrix && $# == 1 ]]; then
+  printf '['
+  for index in "${!modules[@]}"; do
+    ((index == 0)) || printf ','
+    printf '"%s"' "${modules[$index]}"
+  done
+  printf ']\n'
+  exit
+fi
+if (( $# != 0 )); then
+  echo 'Usage: scripts/mutation.sh [--matrix]' >&2
+  exit 2
+fi
 
 mkdir -p reports
 targets=reports/mutation-targets.bin
 failed=reports/mutation-failed.bin
 : > "$targets"
 : > "$failed"
+patterns=()
+for index in "${!paths[@]}"; do
+  if [[ -n ${MUTATION_MODULE:-} ]] && [[ ${modules[$index]} != "$MUTATION_MODULE" ]]; then
+    continue
+  fi
+  path=${paths[$index]}
+  selected_path=$path
+  printf '%s\0' "$path" >> "$targets"
+  patterns+=("${modules[$index]}.x*")
+done
 
-modules=()
-add_module() {
-  for known in "${modules[@]-}"; do
-    [[ $known == "$1" ]] && return
-  done
-  modules+=("$1")
-}
-if [[ ${MUTATION_FULL:-0} == 1 ]]; then
-  git ls-files -z 'src/volcano_sdk/*.py' > reports/mutation-source.bin
-  while IFS= read -r -d '' path; do
-    if [[ $path != src/volcano_sdk/_generated/* && -f $path ]]; then
-      add_module "$path"
-    fi
-  done < reports/mutation-source.bin
-else
-  for path in \
-    src/volcano_sdk/locks.py \
-    src/volcano_sdk/_lock_guard.py \
-    src/volcano_sdk/_lock_renewer.py \
-    src/volcano_sdk/_lock_worker.py; do
-    add_module "$path"
-  done
-
-  base=${MUTATION_BASE_SHA:-origin/main}
-  ancestor=$(git merge-base "$base" HEAD)
-  changed() {
-    git diff --name-only --diff-filter=ACMRT -z "$ancestor" HEAD
-    git diff --name-only --diff-filter=ACMRT -z HEAD
-    git ls-files -z --others --exclude-standard
-  }
-  changed > reports/mutation-changed.bin
-  while IFS= read -r -d '' path; do
-    if [[ $path == src/volcano_sdk/*.py && $path != src/volcano_sdk/_generated/* && -f $path ]]; then
-      add_module "$path"
-    fi
-  done < reports/mutation-changed.bin
+if (( ${#patterns[@]} == 0 )); then
+  echo "Unknown mutation module: ${MUTATION_MODULE:-}" >&2
+  exit 2
 fi
 
-for path in "${modules[@]}"; do
-  printf '%s\0' "$path" >> "$targets"
-done
+# A fresh run must not inherit stale test-to-mutant mappings or verdicts.
+rm -rf -- mutants
 
-if [[ ${MUTATION_FULL:-0} == 1 ]]; then
-  if ! mutmut run --max-children 1; then
-    printf '%s\0' "full mutation run" >> "$failed"
-  fi
-else
-  patterns=()
-  for path in "${modules[@]}"; do
-    module=${path#src/}
-    module=${module%.py}
-    patterns+=("${module//\//.}.x*")
-  done
-  if ! mutmut run --max-children 1 "${patterns[@]}"; then
-    printf '%s\0' "scoped mutation run" >> "$failed"
-  fi
+# Mutmut rejects an exact wildcard for a module with no functions. Record that
+# module explicitly instead of treating a native no-match assertion as a kill.
+if [[ -n ${MUTATION_MODULE:-} ]] && ! python -c '
+import sys
+from pathlib import Path
+from scripts.mutation_results import has_functions
+raise SystemExit(0 if has_functions(Path(sys.argv[1])) else 1)
+' "$selected_path"; then
+  python -m scripts.mutation_results "$targets" "$failed"
+  exit
+fi
+
+if ! mutmut run --max-children 1 "${patterns[@]}"; then
+  printf '%s\0' 'mutation run' >> "$failed"
 fi
 
 python -m scripts.mutation_results "$targets" "$failed"

scripts/mutation_results.py

@@ -134,7 +134,7 @@ def main(targets_path: Path, failed_path: Path) -> int:
         counts.update(module_counts)
         if empty:
             unmutatable.append(name)
-    if not counts:
+    if not counts and (not targets or len(unmutatable) != len(targets)):
         failures.append("No mutants were tested")
     failures.extend(
         f"{name}: {count} mutant(s)"

src/volcano_sdk/_realtime_fetch_worker.py

@@ -293,9 +293,11 @@ async def _fetch_and_deliver(
         if isinstance(result, BaseException):
             await self._deliver_failure(jobs, result)
             return
-        if len(result) != len(jobs):
-            raise RuntimeError(_INVALID_RESULT_COUNT)
-        for job, record in zip(jobs, result, strict=True):
+        try:
+            pairs = tuple(zip(jobs, result, strict=True))
+        except ValueError:
+            raise RuntimeError(_INVALID_RESULT_COUNT) from None
+        for job, record in pairs:
             await self._deliver(PostgresFetchOutcome(job=job, record=record))
 
     async def _deliver_failure(