WIP: CI TESTING ENH: Two-level CastXML/igenerator build cache + Python CI workflow

.github/workflows/arm.yml

@@ -102,6 +102,7 @@ jobs:
           echo "CCACHE_SLOPPINESS=pch_defines,time_macros" >> "$GITHUB_ENV"
           echo "CCACHE_DIR=${{ runner.temp }}/ccache" >> "$GITHUB_ENV"
           echo "CCACHE_MAXSIZE=5G" >> "$GITHUB_ENV"
+          echo "ITK_WRAP_CACHE=${{ runner.temp }}/itk-castxml-cache" >> "$GITHUB_ENV"
           if [ "$RUNNER_OS" == "Linux" ]; then
             sudo apt-get update -qq && sudo apt-get install -y ccache locales
             sudo locale-gen de_DE.UTF-8
@@ -118,6 +119,16 @@ jobs:
           restore-keys: |
             ccache-v4-${{ runner.os }}-${{ matrix.name }}-
 
+      - name: Restore CastXML cache
+        if: matrix.python-version != ''
+        id: restore-castxml-cache
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ runner.temp }}/itk-castxml-cache
+          key: castxml-v1-${{ runner.os }}-arm-python-${{ github.sha }}
+          restore-keys: |
+            castxml-v1-${{ runner.os }}-arm-python-
+
       - name: Restore ExternalData object store
         id: restore-externaldata
         uses: actions/cache/restore@v5
@@ -202,6 +213,13 @@ jobs:
           path: ${{ runner.temp }}/ccache
           key: ccache-v4-${{ runner.os }}-${{ matrix.name }}-${{ github.sha }}
 
+      - name: Save CastXML cache
+        if: ${{ !cancelled() && matrix.python-version != '' }}
+        uses: actions/cache/save@v5
+        with:
+          path: ${{ runner.temp }}/itk-castxml-cache
+          key: castxml-v1-${{ runner.os }}-arm-python-${{ github.sha }}
+
       # ExternalData object store is populated by
       # .github/workflows/populate-externaldata-cache.yml — a dedicated
       # workflow whose only job is to prefetch every CID and write the

.github/workflows/python.yml

@@ -0,0 +1,179 @@
+name: ITK.Pixi.Python
+
+on:
+    push:
+        branches:
+            - main
+            - 'release*'
+        paths-ignore:
+            - '*.md'
+            - LICENSE
+            - NOTICE
+            - 'Documentation/**'
+            - 'Utilities/Debugger/**'
+            - 'Utilities/ITKv5Preparation/**'
+            - 'Utilities/Maintenance/**'
+            - 'Modules/Remote/*.remote.cmake'
+    pull_request:
+        paths-ignore:
+            - '*.md'
+            - LICENSE
+            - NOTICE
+            - 'Documentation/**'
+            - 'Utilities/Debugger/**'
+            - 'Utilities/ITKv5Preparation/**'
+            - 'Utilities/Maintenance/**'
+            - 'Modules/Remote/*.remote.cmake'
+
+concurrency:
+  group: '${{ github.workflow }}@${{ github.head_ref || github.ref }}'
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+    Pixi-Python:
+      runs-on: ${{ matrix.os }}
+      timeout-minutes: 360
+      strategy:
+        fail-fast: false
+        matrix:
+          # windows-2022 excluded: itk_end_wrap_module.cmake generates an
+          # igenerator command that exceeds cmd.exe's 8191-char batch-file
+          # line limit for large modules (e.g. ITKImageIntensity, 59
+          # submodules). Fix requires response-file support in cmake custom
+          # command, tracked separately.
+          os: [ubuntu-24.04, macos-15]
+      steps:
+        - name: Checkout
+          uses: actions/checkout@v5
+          with:
+            fetch-depth: 5
+            clean: true
+
+        - name: Configure ccache environment
+          shell: bash
+          run: |
+            echo "CCACHE_BASEDIR=${GITHUB_WORKSPACE}" >> "$GITHUB_ENV"
+            echo "CCACHE_COMPILERCHECK=content" >> "$GITHUB_ENV"
+            echo "CCACHE_NOHASHDIR=true" >> "$GITHUB_ENV"
+            echo "CCACHE_SLOPPINESS=pch_defines,time_macros" >> "$GITHUB_ENV"
+            echo "CCACHE_DIR=${{ runner.temp }}/ccache" >> "$GITHUB_ENV"
+            echo "CCACHE_MAXSIZE=5G" >> "$GITHUB_ENV"
+            if [ "$RUNNER_OS" == "Linux" ]; then
+              sudo apt-get update -qq && sudo apt-get install -y locales
+              sudo locale-gen de_DE.UTF-8
+            fi
+
+        - name: Configure CastXML cache environment
+          shell: bash
+          run: |
+            echo "ITK_WRAP_CACHE=${{ runner.temp }}/itk-castxml-cache" >> "$GITHUB_ENV"
+
+        - name: Restore compiler cache
+          id: restore-ccache
+          uses: actions/cache/restore@v5
+          with:
+            path: ${{ runner.temp }}/ccache
+            key: ccache-v4-${{ runner.os }}-pixi-python-${{ github.sha }}
+            restore-keys: |
+              ccache-v4-${{ runner.os }}-pixi-python-
+
+        - name: Restore CastXML cache
+          id: restore-castxml-cache
+          uses: actions/cache/restore@v5
+          with:
+            path: ${{ runner.temp }}/itk-castxml-cache
+            key: castxml-v1-${{ runner.os }}-pixi-python-${{ github.sha }}
+            restore-keys: |
+              castxml-v1-${{ runner.os }}-pixi-python-
+
+        - name: Restore ExternalData object store
+          id: restore-externaldata
+          uses: actions/cache/restore@v5
+          with:
+            path: ${{ runner.temp }}/ExternalData
+            key: externaldata-v1-${{ hashFiles('**/*.cid') }}
+            restore-keys: |
+              externaldata-v1-
+
+        - name: Disk space before build (Ubuntu)
+          if: matrix.os == 'ubuntu-24.04'
+          run: df -h /
+
+        - name: Free Disk Space (Ubuntu)
+          if: matrix.os == 'ubuntu-24.04'
+          uses: BRAINSia/free-disk-space@v2
+          with:
+            removalmode: "rmz"
+            swap-storage:   "true"
+            haskell:        "true"
+            dotnet:         "true"
+            docker-images:  "false"
+            tool-cache:     "true"
+            android:        "false"
+            large-packages: "true"
+            mandb:          "true"
+
+        - name: Export ExternalData_OBJECT_STORES
+          shell: bash
+          run: |
+            echo "ExternalData_OBJECT_STORES=${{ runner.temp }}/ExternalData" >> "$GITHUB_ENV"
+
+        - name: Set up Pixi
+          uses: prefix-dev/setup-pixi@v0.9.5
+
+        - name: Show ccache configuration, stats and maintenance
+          shell: bash
+          run: |
+            pixi run -e python ccache --zero-stats
+            pixi run -e python ccache --evict-older-than 7d
+            pixi run -e python ccache --show-config
+
+        - name: Configure
+          run: pixi run configure-python-ci
+
+        - name: Fetch ExternalData
+          shell: bash
+          run: pixi run -e python cmake --build build-python --target ITKData
+
+        - name: Build
+          run: |
+            df -h / || true
+            pixi run build-python-ci
+            df -h / || true
+
+        - name: Free disk space after build
+          shell: bash
+          run: |
+            find build-python -type f \( -path '*/CMakeFiles/*' -o -path '*.dir/*' \) \( -name "*.o" -o -name "*.obj" \) -delete
+            find build-python/lib -type f \( -name "*.a" -o -name "*.lib" \) -delete 2>/dev/null || true
+            pixi run -e python ccache --evict-older-than 1d 2>/dev/null || true
+            pixi run -e python ccache --cleanup 2>/dev/null || true
+            df -h / || true
+
+        - name: Test
+          run: pixi run test-python-ci
+
+        - name: Save compiler cache
+          if: ${{ !cancelled() }}
+          uses: actions/cache/save@v5
+          with:
+            path: ${{ runner.temp }}/ccache
+            key: ccache-v4-${{ runner.os }}-pixi-python-${{ github.sha }}
+
+        - name: Save CastXML cache
+          if: ${{ !cancelled() }}
+          uses: actions/cache/save@v5
+          with:
+            path: ${{ runner.temp }}/itk-castxml-cache
+            key: castxml-v1-${{ runner.os }}-pixi-python-${{ github.sha }}
+
+        - name: Save ExternalData object store
+          if: ${{ !cancelled() }}
+          uses: actions/cache/save@v5
+          with:
+            path: ${{ runner.temp }}/ExternalData
+            key: externaldata-v1-${{ hashFiles('**/*.cid') }}
+
+        - name: ccache stats
+          if: always()
+          run: pixi run -e python ccache --show-stats

CMake/itkWrapCastXMLCacheSupport.cmake

@@ -0,0 +1,60 @@
+###############################################################################
+# Content-addressed two-level cache for CastXML wrapping steps.
+#
+# When ITK_WRAP_CASTXML_CACHE is ON, a Python wrapper replaces
+# `ccache castxml` for every .xml generation step. The wrapper computes
+# a two-level SHA-256 key:
+#   L1 (fast, ~0.2s): direct inputs (cxx + castxml.inc + compiler flags)
+#   L2 (robust, ~1s): sha256(L1_key + `castxml -E` preprocessed output)
+#
+# On L2 hit: restores .xml from cache, saving the full CastXML run.
+# On miss:   runs castxml normally and populates the cache.
+#
+# Cache location: $ITK_WRAP_CACHE env var (default: ~/.cache/itk-wrap)
+
+set(
+  _ITK_WRAP_CASTXML_CACHE_SCRIPT_DEFAULT
+  "${ITK_SOURCE_DIR}/Wrapping/Generators/CastXML/itk-castxml-cache.py"
+)
+
+option(
+  ITK_WRAP_CASTXML_CACHE
+  "Use a content-addressed two-level cache for CastXML wrapping steps."
+  ON
+)
+mark_as_advanced(ITK_WRAP_CASTXML_CACHE)
+
+if(ITK_WRAP_CASTXML_CACHE)
+  set(
+    ITK_WRAP_CASTXML_CACHE_SCRIPT
+    "${_ITK_WRAP_CASTXML_CACHE_SCRIPT_DEFAULT}"
+    CACHE FILEPATH
+    "Path to the CastXML content-addressed cache wrapper script"
+  )
+  mark_as_advanced(ITK_WRAP_CASTXML_CACHE_SCRIPT)
+
+  if(NOT EXISTS "${ITK_WRAP_CASTXML_CACHE_SCRIPT}")
+    message(
+      FATAL_ERROR
+      "ITK_WRAP_CASTXML_CACHE is ON but the wrapper script was not found:\n"
+      "  ${ITK_WRAP_CASTXML_CACHE_SCRIPT}\n"
+      "Set ITK_WRAP_CASTXML_CACHE_SCRIPT to the correct path or turn off ITK_WRAP_CASTXML_CACHE."
+    )
+  endif()
+
+  if(NOT Python3_EXECUTABLE)
+    message(
+      FATAL_ERROR
+      "ITK_WRAP_CASTXML_CACHE requires Python3_EXECUTABLE to be set."
+    )
+  endif()
+
+  message(STATUS "CastXML content-addressed cache enabled")
+  message(STATUS "  Script: ${ITK_WRAP_CASTXML_CACHE_SCRIPT}")
+  message(
+    STATUS
+    "  Cache root: set ITK_WRAP_CACHE env var at build time (default: ~/.cache/itk-wrap)"
+  )
+endif()
+
+unset(_ITK_WRAP_CASTXML_CACHE_SCRIPT_DEFAULT)

Testing/ContinuousIntegration/AzurePipelinesLinux.yml

@@ -55,6 +55,7 @@ jobs:
         df -h /
       displayName: 'Free preinstalled software'
 
+
     - checkout: self
       clean: true
       fetchDepth: 5

Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml

@@ -36,6 +36,7 @@ variables:
   CCACHE_NOHASHDIR: 'true'
   CCACHE_SLOPPINESS: pch_defines,time_macros
   CCACHE_MAXSIZE: 8G
+  ITK_WRAP_CACHE: $(Pipeline.Workspace)/.castxml-cache
 jobs:
 - job: Linux
   timeoutInMinutes: 0
@@ -55,6 +56,7 @@ jobs:
         df -h /
       displayName: 'Free preinstalled software'
 
+
     - checkout: self
       clean: true
       fetchDepth: 5
@@ -102,6 +104,14 @@ jobs:
         path: $(CCACHE_DIR)
       displayName: 'Restore ccache'
 
+    - task: Cache@2
+      inputs:
+        key: '"castxml-v1" | "$(Agent.OS)" | "LinuxPython" | "$(Build.SourceVersion)"'
+        restoreKeys: |
+          "castxml-v1" | "$(Agent.OS)" | "LinuxPython"
+        path: $(ITK_WRAP_CACHE)
+      displayName: 'Restore CastXML cache'
+
     - bash: |
         set -x
         ccache --zero-stats

Testing/ContinuousIntegration/AzurePipelinesMacOSPython.yml

@@ -36,6 +36,7 @@ variables:
   CCACHE_NOHASHDIR: 'true'
   CCACHE_SLOPPINESS: pch_defines,time_macros
   CCACHE_MAXSIZE: 8G
+  ITK_WRAP_CACHE: $(Pipeline.Workspace)/.castxml-cache
 jobs:
 - job: macOS
   timeoutInMinutes: 0
@@ -106,6 +107,14 @@ jobs:
         path: $(CCACHE_DIR)
       displayName: 'Restore ccache'
 
+    - task: Cache@2
+      inputs:
+        key: '"castxml-v1" | "$(Agent.OS)" | "macOSPython" | "$(Build.SourceVersion)"'
+        restoreKeys: |
+          "castxml-v1" | "$(Agent.OS)" | "macOSPython"
+        path: $(ITK_WRAP_CACHE)
+      displayName: 'Restore CastXML cache'
+
     - bash: |
         set -x
         ccache --zero-stats

Testing/ContinuousIntegration/AzurePipelinesWindowsPython.yml

@@ -36,6 +36,7 @@ variables:
   CCACHE_NOHASHDIR: 'true'
   CCACHE_SLOPPINESS: pch_defines,time_macros
   CCACHE_MAXSIZE: 8G
+  ITK_WRAP_CACHE: $(Pipeline.Workspace)/.castxml-cache
 jobs:
 - job: Windows
   timeoutInMinutes: 0
@@ -84,6 +85,14 @@ jobs:
         path: $(CCACHE_DIR)
       displayName: 'Restore ccache'
 
+    - task: Cache@2
+      inputs:
+        key: '"castxml-v1" | "$(Agent.OS)" | "WindowsPython" | "$(Build.SourceVersion)"'
+        restoreKeys: |
+          "castxml-v1" | "$(Agent.OS)" | "WindowsPython"
+        path: $(ITK_WRAP_CACHE)
+      displayName: 'Restore CastXML cache'
+
     - bash: |
         set -x
         ccache --zero-stats

Wrapping/CMakeLists.txt

@@ -152,6 +152,10 @@ endif()
 
 include(ConfigureWrapping.cmake)
 
+if(ITK_WRAP_PYTHON)
+  include(itkWrapCastXMLCacheSupport)
+endif()
+
 ###############################################################################
 # Configure specific wrapper modules
 ###############################################################################