diff --git a/.github/scripts/compare-benchmarks.py b/.github/scripts/compare-benchmarks.py new file mode 100755 index 0000000..e513f1a --- /dev/null +++ b/.github/scripts/compare-benchmarks.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Compare two JMH JSON result files, print a Markdown table. + +Used by benchmark-lto.yml to summarize LTO's throughput impact per platform: +compare-benchmarks.py +""" +import json +import sys + + +def load(path): + with open(path) as f: + results = json.load(f) + return { + (r["benchmark"], tuple(sorted(r.get("params", {}).items()))): r + for r in results + } + + +def main(): + if len(sys.argv) != 3: + print("usage: compare-benchmarks.py ", file=sys.stderr) + return 1 + + baseline = load(sys.argv[1]) + variant = load(sys.argv[2]) + + print("| Benchmark | Params | Baseline | Variant | Delta |") + print("|---|---|---:|---:|---:|") + for key in sorted(baseline): + if key not in variant: + continue + b, v = baseline[key]["primaryMetric"], variant[key]["primaryMetric"] + # scoreError is "NaN" (a JSON string, not a number) when JMH can't + # compute a confidence interval - e.g. a single measurement iteration. + b_score, b_err, unit = b["score"], float(b["scoreError"]), b["scoreUnit"] + v_score, v_err = v["score"], float(v["scoreError"]) + delta = (v_score - b_score) / b_score * 100 if b_score else 0.0 + name = key[0].rsplit(".", 1)[-1] + params = ", ".join(f"{k}={pv}" for k, pv in key[1]) or "-" + print( + f"| {name} | {params} | {b_score:.3f} ± {b_err:.3f} {unit} " + f"| {v_score:.3f} ± {v_err:.3f} {unit} | {delta:+.1f}% |" + ) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/benchmark-lto.yml b/.github/workflows/benchmark-lto.yml new file mode 100644 index 0000000..3541a98 --- /dev/null +++ b/.github/workflows/benchmark-lto.yml @@ -0,0 +1,86 @@ +name: Benchmark LTO impact + +# Manual, exploratory: compares JMH throughput between main and an LTO +# variant branch (see scripts/build-zstd.sh - currently -flto is Linux-only, +# zig's Mach-O backend doesn't support it) on every runner OS the repo +# supports, so the real-vs-no-op split shows up as CI evidence per platform +# instead of a manual claim. Not part of ci.yml: this is throughput +# exploration, not a correctness gate, and JMH runs are too slow to run on +# every push/PR. + +on: + workflow_dispatch: + inputs: + lto_ref: + description: Branch/ref with the LTO change to compare against main. + required: false + default: experiment/lto-linux + +jobs: + benchmark: + name: ${{ matrix.classifier }} + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-latest, classifier: linux-x86_64 } + - { os: ubuntu-24.04-arm, classifier: linux-aarch64 } + - { os: macos-14, classifier: osx-aarch64 } + - { os: windows-latest, classifier: windows-x86_64 } + runs-on: ${{ matrix.os }} + steps: + - name: Checkout baseline (main, with zstd submodule) + uses: actions/checkout@v7 + with: + submodules: recursive + path: baseline + + - name: Checkout LTO variant (${{ inputs.lto_ref }}, with zstd submodule) + uses: actions/checkout@v7 + with: + submodules: recursive + ref: ${{ inputs.lto_ref }} + path: lto + + - name: Set up JDK 25 + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: '25' + cache: maven + + - name: Set up Zig + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2 + with: + version: 0.16.0 + + # `ompressBenchmark` matches both CompressBenchmark and + # DecompressBenchmark method FQNs (JMH's filter is a substring/regex + # search), but not GoldenCorpusBenchmark - keeps this to the quick + # synthetic-payload suite, not the slower real-corpus one. + - name: Build + run baseline benchmark + shell: bash + working-directory: baseline + run: | + ./mvnw -B -ntp -q -pl benchmark -am package -DskipTests + java -jar benchmark/target/benchmarks.jar ompressBenchmark \ + -f 1 -wi 2 -i 5 -p size=65536 -rf json -rff ../baseline-results.json + + - name: Build + run LTO-variant benchmark + shell: bash + working-directory: lto + run: | + ./mvnw -B -ntp -q -pl benchmark -am package -DskipTests + java -jar benchmark/target/benchmarks.jar ompressBenchmark \ + -f 1 -wi 2 -i 5 -p size=65536 -rf json -rff ../lto-results.json + + - name: Summarize + shell: bash + run: | + { + echo "### ${{ matrix.classifier }}" + echo + python3 baseline/.github/scripts/compare-benchmarks.py \ + baseline-results.json lto-results.json + echo + } >> "$GITHUB_STEP_SUMMARY"