ci: adopt Gremlins mutation testing (advisory)

.github/workflows/mutation.yml

@@ -0,0 +1,136 @@
+name: Mutation testing
+
+on:
+  pull_request:
+  schedule:
+    # Sunday 02:00 Costa Rica (UTC-6, no DST) = 08:00 UTC
+    - cron: '0 8 * * 0'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  changed-code:
+    name: changed code
+    if: ${{ github.event_name == 'pull_request' && github.repository == 'Backblaze/blazer' && github.actor != 'dependabot[bot]' && github.event.pull_request.user.login != 'dependabot[bot]' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6
+        with:
+          go-version: '1.27.3'
+          cache-dependency-path: '**/go.sum'
+      - name: Install Gremlins
+        run: |
+          go install github.com/go-gremlins/gremlins/cmd/gremlins@v0.6.0
+          echo "$(go env GOPATH)/bin" >>"$GITHUB_PATH"
+      - name: Mutate changed B2 and base code
+        continue-on-error: true
+        env:
+          B2_APPLICATION_KEY_ID: ''
+          B2_APPLICATION_KEY: ''
+          B2_ACCOUNT_ID: ''
+          B2_ACCOUNT_KEY: ''
+          BASE_REF: ${{ github.base_ref }}
+        shell: bash
+        run: |
+          set -o pipefail
+          {
+            status=0
+            for package in b2 base; do
+              if ! gremlins unleash --diff "origin/${BASE_REF}" --timeout-coefficient 20 --workers 2 --output "mutation-${package}.json" "./${package}"; then
+                status=1
+              fi
+            done
+            exit "$status"
+          } 2>&1 | tee mutation.log
+      - name: Write mutation summary
+        if: ${{ always() }}
+        shell: bash
+        run: |
+          {
+            echo '## Mutation testing: changed code'
+            echo
+            if [ -f mutation.log ]; then
+              tail -n 40 mutation.log
+            else
+              echo 'Mutation testing did not produce a report.'
+            fi
+          } >>"$GITHUB_STEP_SUMMARY"
+      - name: Upload mutation report
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: mutation-report
+          path: |
+            mutation-*.json
+            mutation.log
+          if-no-files-found: warn
+          retention-days: 30
+
+  full-suite:
+    name: full suite
+    if: ${{ github.event_name != 'pull_request' && github.repository == 'Backblaze/blazer' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 300
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6
+        with:
+          go-version: '1.27.3'
+          cache-dependency-path: '**/go.sum'
+      - name: Install Gremlins
+        run: |
+          go install github.com/go-gremlins/gremlins/cmd/gremlins@v0.6.0
+          echo "$(go env GOPATH)/bin" >>"$GITHUB_PATH"
+      - name: Mutate B2 and base code
+        continue-on-error: true
+        env:
+          B2_APPLICATION_KEY_ID: ''
+          B2_APPLICATION_KEY: ''
+          B2_ACCOUNT_ID: ''
+          B2_ACCOUNT_KEY: ''
+        shell: bash
+        run: |
+          set -o pipefail
+          {
+            status=0
+            for package in b2 base; do
+              if ! gremlins unleash --timeout-coefficient 20 --workers 2 --output "mutation-${package}.json" "./${package}"; then
+                status=1
+              fi
+            done
+            exit "$status"
+          } 2>&1 | tee mutation.log
+      - name: Write mutation summary
+        if: ${{ always() }}
+        shell: bash
+        run: |
+          {
+            echo '## Mutation testing: full suite'
+            echo
+            if [ -f mutation.log ]; then
+              tail -n 40 mutation.log
+            else
+              echo 'Mutation testing did not produce a report.'
+            fi
+          } >>"$GITHUB_STEP_SUMMARY"
+      - name: Upload mutation report
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: mutation-report
+          path: |
+            mutation-*.json
+            mutation.log
+          if-no-files-found: warn
+          retention-days: 30

.gremlins.yaml

@@ -0,0 +1,12 @@
+unleash:
+  # Only ./b2 and ./base are mutated; neither contains generated non-test files.
+  # internal/b2assets, examples and bin are outside this scope; Gremlins skips _test.go.
+  # Some B2 and base tests take longer under mutation; avoid false timeout kills.
+  timeout-coefficient: 20
+  # Keep local and CI mutation runs within the runner's memory and CPU budget.
+  workers: 2
+  threshold:
+    # Mutation testing reports trends for now; it must not gate merges or releases.
+    efficacy: 0
+    # Mutation coverage is advisory while live-only behavior remains unexercised offline.
+    mutant-coverage: 0

Makefile

@@ -2,10 +2,13 @@
 
 GOLANGCI_LINT_VERSION := v2.14.0
 CHANGIE_VERSION := v1.24.0
+GREMLINS_VERSION := v0.6.0
 CHANGIE := go run github.com/miniscruff/changie@$(CHANGIE_VERSION)
+GREMLINS ?= gremlins
+MUTATION_PACKAGES := ./b2 ./base
 BASE_REF ?= origin/main
 
-.PHONY: release-check ci build vet fmt fmt-check tidy-check verify lint test b2keys crosscompile changelog-check changelog-new scripts-test
+.PHONY: release-check ci build vet fmt fmt-check tidy-check verify lint test b2keys crosscompile changelog-check changelog-new scripts-test mutation mutation-diff
 
 ci: build vet fmt-check tidy-check verify lint test b2keys crosscompile changelog-check scripts-test
 
@@ -60,6 +63,17 @@ changelog-new:
 scripts-test:
 	for t in scripts/*_test.sh; do "$$t" || exit 1; done
 
+# Install the pinned CLI first: go install github.com/go-gremlins/gremlins/cmd/gremlins@$(GREMLINS_VERSION)
+mutation:
+	@set -e; for package in $(MUTATION_PACKAGES); do \
+		$(GREMLINS) unleash --timeout-coefficient 20 --workers 2 "$$package"; \
+	done
+
+mutation-diff:
+	@set -e; for package in $(MUTATION_PACKAGES); do \
+		$(GREMLINS) unleash --diff $(BASE_REF) --timeout-coefficient 20 --workers 2 "$$package"; \
+	done
+
 # Holds a release while solvable security or quality findings are open (see RELEASE.md).
 release-check:
 	scripts/release-check.sh

docs/testing.md

@@ -16,6 +16,81 @@ how to run it, and how to show that a test pins the behavior a PR claims.
 changelog check. Cross-compiling proves the code builds for a platform, not that it runs there; native
 Windows and macOS runs happen in CI.
 
+## Mutation testing
+
+[Gremlins](https://github.com/go-gremlins/gremlins) v0.6.0 mutates Go source and runs the package tests
+against each change. It is installed as a CLI, so it adds no `go.mod` dependency. Install the pinned
+version once, then run the whole baseline or only code changed from `origin/main`:
+
+```sh
+go install github.com/go-gremlins/gremlins/cmd/gremlins@v0.6.0
+make mutation
+make mutation-diff
+```
+
+The pull-request `changed-code` job in [mutation.yml](../.github/workflows/mutation.yml) runs the latter
+against changed `b2` and `base` code. The scheduled and manually dispatched `full-suite` job runs the
+whole `b2` and `base` mutation suite and uploads its JSON reports.
+
+Both jobs upload `mutation-*.json` and `mutation.log`, including reports from partial or failed runs.
+For discovery without executing mutants, run `gremlins unleash --dry-run ./base`, then
+`gremlins unleash --dry-run ./b2`. Version v0.6.0 accepts one package path per invocation.
+
+### Why Gremlins
+
+The [candidate comparison in #97](https://github.com/Backblaze/blazer/issues/97)
+evaluated these approaches:
+
+| Tool | Strengths | Tradeoffs for blazer |
+| --- | --- | --- |
+| Gremlins v0.6.0 | Coverage-guided CLI, diff mode, JSON reports and configurable workers, timeouts and thresholds. | Pre-1.0 flags can change, so pin the version. Timeouts need tuning. No per-line suppression was found in the reviewed documentation. |
+| go-mutesting (Avito fork) v2.3.1 | CLI with more mutation types, custom test commands, a blacklist and comment-based exclusions. | Its documentation warns about false positives. The #97 review found no coverage guidance, so more mutants and longer runs are expected. It requires Go 1.27. |
+| Ooze v0.2.0 | Runs from a Go test with configurable commands and prints living-mutant diffs. | Imported as a library, so it would add a dependency or require a separate module. It runs the whole test command per mutant and defaults to serial execution. |
+
+Gremlins is the primary tool because its CLI preserves the zero-dependency library, and coverage guidance
+and diff mode fit the cost of PR runs. The comparison above summarizes #97's repository and documentation
+review, not a measured Avito benchmark. Avito remains a candidate for a separate cross-check; Ooze is
+deferred because adopting a mutation library would require a separate module.
+
+Only `./b2` and `./base` are mutation targets. Neither contains generated non-test files;
+`internal/b2assets`, `examples` and `bin` are outside those packages, and Gremlins skips `_test.go` files.
+There is no additional `exclude-files` configuration to maintain for this scope.
+
+Gremlins reports **efficacy** as killed mutations divided by killed plus lived mutations: it describes how
+well the tests distinguish the mutations they exercise. **Mutator coverage** is killed plus lived divided
+by killed plus lived plus not covered: it describes how much source the unit test run exercised. `not
+covered` is expected in this repository: much B2 behavior is live-only and cannot run without credentials.
+Timeouts are reported separately.
+
+The refreshed main baseline, measured at `c3b00f1` with `--timeout-coefficient 20 --workers 2`, unit tests
+only and no `-race`, is:
+
+| Package | Killed | Lived | Not covered | Timed out | Efficacy | Mutator coverage | Previous efficacy / coverage |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- |
+| `base` | 208 | 17 | 61 | 0 | 92.44% | 78.67% | 71.88% / 20.25% |
+| `b2` | 754 | 27 | 27 | 17 | 96.54% | 96.66% | 64.05% / 47.96% |
+
+See [#290](https://github.com/Backblaze/blazer/issues/290) for the remaining surviving and uncovered gaps.
+
+### How to accept or suppress a surviving mutant
+
+Download the mutation report from either workflow job and inspect each `lived` mutant's file, line and
+operator. If it changes observable behavior, add a focused test with an independent expected value.
+Apply the mutation alone, show that the test fails for that behavior, then restore it and show a pass.
+
+If a mutant is equivalent, record the location, mutation and reason in the PR or a linked issue, and
+add a nearby comment when it explains a lasting invariant. Accept it explicitly rather than weakening
+an assertion to improve the score. Keep accepted mutants visible in the advisory report.
+
+Gremlins v0.6.0 has file-level `exclude-files` patterns, but no documented per-line suppression. Use a
+file exclusion only for generated or non-library code if the target scope expands. Do not exclude a
+library file just to suppress one accepted mutant.
+
+`not covered` means the credential-free test run did not reach the code. Add a hermetic test where
+possible, or record the live-only limitation. A `timed out` mutant is unresolved, not proof of a kill:
+investigate test stability and timeout tuning, and keep mutation runs free of `-race` to avoid false
+kills from the known `TestReuploadPart*` race. Thresholds remain zero during this advisory rollout.
+
 ## Showing a test pins the fix
 
 For every behavior a PR adds or fixes: