From ced17caf5f85ce0951bf34aa56053ebf071ba5da Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Fri, 29 May 2026 12:16:24 +0000 Subject: [PATCH 1/4] [INFRA] Unify Coursier cache to a single key across all jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace 8 distinct per-job Coursier cache keys (`$matrix.java-$matrix.hadoop-coursier-`, `pyspark-coursier-`, `sparkr-coursier-`, `docs-coursier-` (×2), `tpcds-coursier-`, `docker-integration-coursier-`, `k8s-integration-coursier-`) with a single `coursier-` key written exclusively by the `precompile` job and restored read-only (`actions/cache/restore`) by all consumers. The near-duplicate per-job Coursier caches were consuming ~4.5 GB on master and ~5.2 GB on branch-4.x, leaving the 10 GB repo-wide cache budget almost entirely full. Old maintenance branches (4.0, 4.1, 4.2, 3.5) had their caches evicted before their next CI run and were always cold. With one writer per branch the footprint shrinks to ~1.4 GB, leaving room for all actively-maintained branches simultaneously. The `precompile` job already builds with every profile (`-Phadoop-3 -Pyarn -Pspark-ganglia-lgpl -Phadoop-cloud -Phive -Pkubernetes -Pjvm-profiler -Pkinesis-asl -Phive-thriftserver -Pdocker-integration-tests -Pvolcano`) so its `~/.cache/coursier` is a superset of every consumer job's dependency closure. Generated-by: Claude Code (claude-sonnet-4-6) --- .github/workflows/build_and_test.yml | 76 +++++++++++++--------------- 1 file changed, 34 insertions(+), 42 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 66c3f89ebbab..701bb14eb18a 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -378,15 +378,13 @@ jobs: key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} restore-keys: | build- - - name: Cache Coursier local repository - uses: actions/cache@v5 + - name: Restore Coursier local repository + uses: actions/cache/restore@v5 with: path: ~/.cache/coursier - key: ${{ matrix.java }}-${{ matrix.hadoop }}-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + key: coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | - ${{ matrix.java }}-${{ matrix.hadoop }}-coursier- - precompile-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} - precompile-coursier- + coursier- - name: Free up disk space run: | if [ -f ./dev/free_disk_space ]; then @@ -616,9 +614,9 @@ jobs: uses: actions/cache@v5 with: path: ~/.cache/coursier - key: precompile-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + key: coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | - precompile-coursier- + coursier- - name: Install Java ${{ inputs.java }} uses: actions/setup-java@v5 with: @@ -738,13 +736,13 @@ jobs: key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} restore-keys: | build- - - name: Cache Coursier local repository - uses: actions/cache@v5 + - name: Restore Coursier local repository + uses: actions/cache/restore@v5 with: path: ~/.cache/coursier - key: pyspark-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + key: coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | - pyspark-coursier- + coursier- - name: Free up disk space shell: 'script -q -e -c "bash {0}"' run: ./dev/free_disk_space_container @@ -888,13 +886,13 @@ jobs: key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} restore-keys: | build- - - name: Cache Coursier local repository - uses: actions/cache@v5 + - name: Restore Coursier local repository + uses: actions/cache/restore@v5 with: path: ~/.cache/coursier - key: sparkr-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + key: coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | - sparkr-coursier- + coursier- - name: Free up disk space run: ./dev/free_disk_space_container - name: Install Java ${{ inputs.java }} @@ -1039,13 +1037,13 @@ jobs: key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} restore-keys: | build- - - name: Cache Coursier local repository - uses: actions/cache@v5 + - name: Restore Coursier local repository + uses: actions/cache/restore@v5 with: path: ~/.cache/coursier - key: docs-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + key: coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | - docs-coursier- + coursier- - name: Cache Maven local repository uses: actions/cache@v5 with: @@ -1238,13 +1236,13 @@ jobs: key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} restore-keys: | build- - - name: Cache Coursier local repository - uses: actions/cache@v5 + - name: Restore Coursier local repository + uses: actions/cache/restore@v5 with: path: ~/.cache/coursier - key: docs-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + key: coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | - docs-coursier- + coursier- - name: Cache Maven local repository uses: actions/cache@v5 with: @@ -1433,15 +1431,13 @@ jobs: key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} restore-keys: | build- - - name: Cache Coursier local repository - uses: actions/cache@v5 + - name: Restore Coursier local repository + uses: actions/cache/restore@v5 with: path: ~/.cache/coursier - key: tpcds-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + key: coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | - tpcds-coursier- - precompile-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} - precompile-coursier- + coursier- - name: Install Java ${{ inputs.java }} uses: actions/setup-java@v5 with: @@ -1553,15 +1549,13 @@ jobs: key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} restore-keys: | build- - - name: Cache Coursier local repository - uses: actions/cache@v5 + - name: Restore Coursier local repository + uses: actions/cache/restore@v5 with: path: ~/.cache/coursier - key: docker-integration-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + key: coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | - docker-integration-coursier- - precompile-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} - precompile-coursier- + coursier- - name: Install Java ${{ inputs.java }} uses: actions/setup-java@v5 with: @@ -1641,15 +1635,13 @@ jobs: key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} restore-keys: | build- - - name: Cache Coursier local repository - uses: actions/cache@v5 + - name: Restore Coursier local repository + uses: actions/cache/restore@v5 with: path: ~/.cache/coursier - key: k8s-integration-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + key: coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | - k8s-integration-coursier- - precompile-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} - precompile-coursier- + coursier- - name: Free up disk space run: | if [ -f ./dev/free_disk_space ]; then From 90d40509d662834453ab9c632aff3e2a5583c6d6 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Fri, 29 May 2026 12:22:38 +0000 Subject: [PATCH 2/4] [INFRA] Also allow build job to save Coursier cache as fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the `build` (Scala test matrix) job use `actions/cache@v5` instead of `actions/cache/restore` for the Coursier step, so it can act as a fallback writer when `precompile` is absent or its cache save fails. When `precompile` succeeds, `build` gets an exact key hit on `coursier-` and GHA automatically skips saving (caches are immutable) — no duplicate entry. When `precompile` fails or is skipped (e.g. an infra-only PR where build=false for precompile's condition, or a transient precompile failure covered by continue-on-error), `build` runs the full SBT compilation, populates ~/.cache/coursier, and seeds the cache for subsequent runs. All other consumers (pyspark, sparkr, lint, docs, tpcds-1g, docker-integration-tests, k8s-integration-tests) remain read-only via `actions/cache/restore` since they use the precompile artifact and do not do full SBT dependency resolution from scratch. Generated-by: Claude Code (claude-sonnet-4-6) --- .github/workflows/build_and_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 701bb14eb18a..faaa34f4c417 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -378,8 +378,8 @@ jobs: key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} restore-keys: | build- - - name: Restore Coursier local repository - uses: actions/cache/restore@v5 + - name: Cache Coursier local repository + uses: actions/cache@v5 with: path: ~/.cache/coursier key: coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} From 00fe57bdfa7027795bcab03d522669d550c94fcc Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Fri, 29 May 2026 12:26:01 +0000 Subject: [PATCH 3/4] [INFRA] Unify Coursier cache key in python_hosted_runner_test.yml Rename `pyspark-coursier-` to `coursier-` to match the unified key introduced in build_and_test.yml, so this workflow benefits from the shared cache pool instead of maintaining a separate per-workflow copy. benchmark.yml (benchmark-coursier--) and build_python_connect*.yml (coursier-build-spark-connect-python-only-) are intentionally left unchanged: benchmark runs with a user-supplied JDK and has a hardcoded coursier path reference; build_python_connect is deliberately isolated to a connect-only subset build. Generated-by: Claude Code (claude-sonnet-4-6) --- .github/workflows/python_hosted_runner_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python_hosted_runner_test.yml b/.github/workflows/python_hosted_runner_test.yml index eb0430bfe6c2..18ef69af09f4 100644 --- a/.github/workflows/python_hosted_runner_test.yml +++ b/.github/workflows/python_hosted_runner_test.yml @@ -137,9 +137,9 @@ jobs: uses: actions/cache@v5 with: path: ~/.cache/coursier - key: pyspark-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + key: coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | - pyspark-coursier- + coursier- - name: Install Java ${{ matrix.java }} uses: actions/setup-java@v5 with: From 998fca614975eb3336d461a45e7e9b63a15f1705 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Fri, 29 May 2026 12:48:52 +0000 Subject: [PATCH 4/4] [INFRA] Restore OS/java/hadoop prefix on build job's Coursier cache key Give the build (Scala test matrix) job a specific key `$runner.os-$matrix.java-$matrix.hadoop-coursier-` so that different OS/JDK/Hadoop combinations each maintain a tailored cache entry, falling back to the precompile superset (`coursier-`) when cold. The precompile job keeps the plain `coursier-` key since it has no OS/matrix dimension and is the shared base-level writer. Generated-by: Claude Code (claude-sonnet-4-6) --- .github/workflows/build_and_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index faaa34f4c417..ebe713f3463f 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -382,8 +382,9 @@ jobs: uses: actions/cache@v5 with: path: ~/.cache/coursier - key: coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + key: ${{ runner.os }}-${{ matrix.java }}-${{ matrix.hadoop }}-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} restore-keys: | + ${{ runner.os }}-${{ matrix.java }}-${{ matrix.hadoop }}-coursier- coursier- - name: Free up disk space run: |