ENH: Cache ExternalData object store across CI runs

.github/workflows/arm.yml

@@ -30,7 +30,6 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:
-    ExternalDataVersion: 5.4.5
     CMakeVersion: "4.0.1"
 
 jobs:
@@ -119,6 +118,19 @@ jobs:
           restore-keys: |
             ccache-v4-${{ runner.os }}-${{ matrix.name }}-
 
+      - name: Restore ExternalData object store
+        id: restore-externaldata
+        uses: actions/cache/restore@v5
+        with:
+          # ExternalData blobs are platform-agnostic; share a single cache
+          # entry across every workflow/job/OS in the repo.  The SHA suffix
+          # produces an immutable save key per run; restore-keys falls
+          # through to the most recent prior cache under the same version.
+          path: ${{ runner.temp }}/ExternalData
+          key: externaldata-v1-${{ hashFiles('**/*.cid') }}
+          restore-keys: |
+            externaldata-v1-
+
       - name: Show ccache configuration, stats and maintenance
         shell: bash
         run: |
@@ -144,14 +156,15 @@ jobs:
           cmakeVersion: ~${{ env.CMakeVersion }}
           ninjaVersion: latest
 
-      - name: Download dashboard script and testing data
+      - name: Download dashboard script
         run: |
           set -x
           git clone -b dashboard --single-branch https://github.com/InsightSoftwareConsortium/ITK.git ITK-dashboard
 
-          curl -L https://github.com/InsightSoftwareConsortium/ITK/releases/download/v${{ env.ExternalDataVersion }}/InsightData-${{ env.ExternalDataVersion }}.tar.gz -O
-          cmake -E tar xfz InsightData-${{ env.ExternalDataVersion }}.tar.gz
-          cmake -E rename InsightToolkit-${{ env.ExternalDataVersion }}/.ExternalData/CID ${{ github.workspace }}/.ExternalData/CID
+      - name: Export ExternalData_OBJECT_STORES
+        shell: bash
+        run: |
+          echo "ExternalData_OBJECT_STORES=${{ runner.temp }}/ExternalData" >> "$GITHUB_ENV"
 
       - name: Configure CTest script
         run: |
@@ -189,6 +202,15 @@ jobs:
           path: ${{ runner.temp }}/ccache
           key: ccache-v4-${{ runner.os }}-${{ matrix.name }}-${{ github.sha }}
 
+      # ExternalData object store is populated by
+      # .github/workflows/populate-externaldata-cache.yml — a dedicated
+      # workflow whose only job is to prefetch every CID and write the
+      # shared cache entry.  Consumer workflows (this one, pixi.yml, the
+      # Azure pipelines) restore-only.  Do not reintroduce a Save step
+      # here: races across platforms overwrite the single shared key with
+      # a fraction of the expected blobs and poison every subsequent
+      # restore.
+
       - name: ccache stats
         if: always()
         run: ccache --show-stats

.github/workflows/pixi.yml

@@ -30,7 +30,6 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 env:
-    ExternalDataVersion: 5.4.5
 
 
 jobs:
@@ -74,6 +73,19 @@ jobs:
             restore-keys: |
               ccache-v4-${{ runner.os }}-pixi-cxx-
 
+        - name: Restore ExternalData object store
+          id: restore-externaldata
+          uses: actions/cache/restore@v5
+          with:
+            # ExternalData blobs are platform-agnostic; share a single cache
+            # entry across every workflow/job/OS in the repo.  The SHA suffix
+            # produces an immutable save key per run; restore-keys falls
+            # through to the most recent prior cache under the same version.
+            path: ${{ runner.temp }}/ExternalData
+            key: externaldata-v1-${{ hashFiles('**/*.cid') }}
+            restore-keys: |
+              externaldata-v1-
+
         - name: Show ccache configuration, stats and maintenance
           shell: bash
           run: |
@@ -116,11 +128,10 @@ jobs:
             # TIME_REPORT:         overall | 526        seconds | 31GiB        |
             # TIME_REPORT: == find ====================================================
 
-        - name: Download testing data
+        - name: Export ExternalData_OBJECT_STORES
+          shell: bash
           run: |
-            curl -L https://github.com/InsightSoftwareConsortium/ITK/releases/download/v${{ env.ExternalDataVersion }}/InsightData-${{ env.ExternalDataVersion }}.tar.gz -O
-            cmake -E tar xfz InsightData-${{ env.ExternalDataVersion }}.tar.gz
-            cmake -E rename InsightToolkit-${{ env.ExternalDataVersion }}/.ExternalData/CID ${{ github.workspace }}/.ExternalData/CID
+            echo "ExternalData_OBJECT_STORES=${{ runner.temp }}/ExternalData" >> "$GITHUB_ENV"
 
         - name: Set up Pixi
           uses: prefix-dev/setup-pixi@v0.8.1
@@ -142,10 +153,6 @@ jobs:
             # Remove object files and static libraries (not needed for testing)
             find build -type f -name "*.o" -delete
             find build -type f -name "*.a" -delete
-            # Remove downloaded data tarballs (already extracted)
-            rm -f InsightData-*.tar.gz
-            # Remove extracted source tarball directory
-            rm -rf InsightToolkit-${{ env.ExternalDataVersion }}
             # Trim ccache to stay within CCACHE_MAXSIZE and remove orphaned entries
             ccache --cleanup 2>/dev/null || true
             echo "****** df -h /  -- post cleanup"
@@ -169,6 +176,14 @@ jobs:
             path: ${{ runner.temp }}/ccache
             key: ccache-v4-${{ runner.os }}-pixi-cxx-${{ github.sha }}
 
+        # ExternalData object store is populated by
+        # .github/workflows/populate-externaldata-cache.yml — a dedicated
+        # workflow whose only job is to prefetch every CID and write the
+        # shared cache entry. Consumer workflows restore-only. Do not
+        # reintroduce a prefetch-and-save pair here: races across
+        # platforms overwrite the single shared key with a fraction of
+        # the expected blobs and poison every subsequent restore.
+
         - name: ccache stats
           if: always()
           run: ccache --show-stats

.github/workflows/populate-externaldata-cache.yml

@@ -0,0 +1,102 @@
+name: Populate ExternalData Cache
+
+# Single owner of the shared "externaldata-v1-<hashFiles>" GitHub Actions
+# cache entry. Every other workflow restores this entry but never saves
+# it — see the comments in arm.yml and pixi.yml for the race that a
+# multi-writer design caused.
+#
+# The job prefetches every .cid referenced in the source tree through
+# the same gateway list CMake/ITKExternalData.cmake uses, verifies that
+# all objects landed on disk, and only then saves the cache. If any
+# object is missing the save is skipped so a later run can try again
+# without poisoning the key.
+
+on:
+  # PRs that add or modify .cid references produce a new hashFiles
+  # digest, so the cache needs to be repopulated for that digest.
+  pull_request:
+    paths:
+      - '**/*.cid'
+  # Keep main and release branches' caches populated as new .cid files
+  # land.
+  push:
+    branches:
+      - main
+      - 'release*'
+    paths:
+      - '**/*.cid'
+  # Nightly safety net: if a populate run was skipped because some CIDs
+  # were unreachable on one day, the next night retries.
+  schedule:
+    - cron: '17 5 * * *'
+  workflow_dispatch:
+
+concurrency:
+  # Only one populate job per hashFiles digest makes sense, but we key
+  # the concurrency group on the branch ref since hashFiles requires a
+  # checkout. Mid-flight runs cancel; the final one wins.
+  group: 'externaldata-populate@${{ github.head_ref || github.ref }}'
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  actions: write  # needed to manage cache entries
+
+jobs:
+  populate:
+    name: Populate shared ExternalData cache
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 1
+
+      - name: Restore ExternalData object store
+        id: restore-externaldata
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ runner.temp }}/ExternalData
+          key: externaldata-v1-${{ hashFiles('**/*.cid') }}
+
+      - name: Skip if cache already complete
+        if: steps.restore-externaldata.outputs.cache-hit == 'true'
+        run: echo "Cache already present for this hashFiles digest - nothing to do."
+
+      - name: Prefetch every CID
+        if: steps.restore-externaldata.outputs.cache-hit != 'true'
+        shell: bash
+        env:
+          EXTERNALDATA_STORE: ${{ runner.temp }}/ExternalData
+        run: |
+          python3 Utilities/Maintenance/PrefetchCIDContentLinks.py \
+            --repo-root . \
+            --store "$EXTERNALDATA_STORE"
+
+      # Integrity gate: refuse to save unless every unique CID in the
+      # source tree has an object on disk. A partial save under the
+      # shared key would propagate holes to every consumer workflow.
+      - name: Verify completeness
+        if: steps.restore-externaldata.outputs.cache-hit != 'true'
+        shell: bash
+        env:
+          EXTERNALDATA_STORE: ${{ runner.temp }}/ExternalData
+        run: |
+          expected=$(find . -name '*.cid' -not -path './.git/*' -print0 \
+                       | xargs -0 -I{} cat {} \
+                       | sort -u | wc -l | tr -d ' ')
+          present=$(find "$EXTERNALDATA_STORE/cid" -type f 2>/dev/null | wc -l | tr -d ' ')
+          echo "expected unique CIDs: $expected"
+          echo "present on disk     : $present"
+          if [ "$present" -lt "$expected" ]; then
+            echo "::error::ExternalData prefetch produced $present/$expected objects; refusing to save a partial cache."
+            exit 1
+          fi
+
+      - name: Save ExternalData object store
+        if: steps.restore-externaldata.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v5
+        with:
+          path: ${{ runner.temp }}/ExternalData
+          key: externaldata-v1-${{ hashFiles('**/*.cid') }}

Testing/ContinuousIntegration/AzurePipelinesBatch.yml

@@ -11,7 +11,7 @@ trigger:
 pr: none
 
 variables:
-  ExternalDataVersion: 5.4.5
+  ExternalData_OBJECT_STORES: $(Pipeline.Workspace)/ExternalData
   CCACHE_DIR: $(Pipeline.Workspace)/.ccache
   CCACHE_BASEDIR: $(Build.SourcesDirectory)
   CCACHE_COMPILERCHECK: content
@@ -61,12 +61,16 @@ jobs:
 
       - script: |
           git clone -b dashboard --single-branch https://github.com/InsightSoftwareConsortium/ITK.git ITK-dashboard
-
-          curl -L https://github.com/InsightSoftwareConsortium/ITK/releases/download/v$(ExternalDataVersion)/InsightData-$(ExternalDataVersion).tar.gz -O
-          cmake -E tar xfz InsightData-$(ExternalDataVersion).tar.gz
-          cmake -E rename InsightToolkit-$(ExternalDataVersion)/.ExternalData/CID $(Build.SourcesDirectory)/.ExternalData/CID
         workingDirectory: $(Agent.BuildDirectory)
-        displayName: 'Download dashboard script and testing data'
+        displayName: 'Download dashboard script'
+
+      - task: Cache@2
+        inputs:
+          key: '"externaldata" | "v1" | **/*.cid'
+          restoreKeys: |
+            "externaldata" | "v1"
+          path: $(ExternalData_OBJECT_STORES)
+        displayName: 'Restore ExternalData object store'
 
       - task: Cache@2
         inputs:

Testing/ContinuousIntegration/AzurePipelinesLinux.yml

@@ -27,7 +27,7 @@ pr:
     - Utilities/Maintenance/*
     - Modules/Remote/*.remote.cmake
 variables:
-  ExternalDataVersion: 5.4.5
+  ExternalData_OBJECT_STORES: $(Pipeline.Workspace)/ExternalData
   CCACHE_DIR: $(Pipeline.Workspace)/.ccache
   CCACHE_BASEDIR: $(Build.SourcesDirectory)
   CCACHE_COMPILERCHECK: content
@@ -67,12 +67,16 @@ jobs:
     - bash: |
         set -x
         git clone -b dashboard --single-branch https://github.com/InsightSoftwareConsortium/ITK.git ITK-dashboard
-
-        curl -L https://github.com/InsightSoftwareConsortium/ITK/releases/download/v$(ExternalDataVersion)/InsightData-$(ExternalDataVersion).tar.gz -O
-        cmake -E tar xfz InsightData-$(ExternalDataVersion).tar.gz
-        cmake -E rename InsightToolkit-$(ExternalDataVersion)/.ExternalData/CID $(Build.SourcesDirectory)/.ExternalData/CID
       workingDirectory: $(Agent.BuildDirectory)
-      displayName: 'Download dashboard script and testing data'
+      displayName: 'Download dashboard script'
+
+    - task: Cache@2
+      inputs:
+        key: '"externaldata" | "v1" | **/*.cid'
+        restoreKeys: |
+          "externaldata" | "v1"
+        path: $(ExternalData_OBJECT_STORES)
+      displayName: 'Restore ExternalData object store'
 
     - task: Cache@2
       inputs:
@@ -168,12 +172,16 @@ jobs:
     - bash: |
         set -x
         git clone -b dashboard --single-branch https://github.com/InsightSoftwareConsortium/ITK.git ITK-dashboard
-
-        curl -L https://github.com/InsightSoftwareConsortium/ITK/releases/download/v$(ExternalDataVersion)/InsightData-$(ExternalDataVersion).tar.gz -O
-        cmake -E tar xfz InsightData-$(ExternalDataVersion).tar.gz
-        cmake -E rename InsightToolkit-$(ExternalDataVersion)/.ExternalData/CID $(Build.SourcesDirectory)/.ExternalData/CID
       workingDirectory: $(Agent.BuildDirectory)
-      displayName: 'Download dashboard script and testing data'
+      displayName: 'Download dashboard script'
+
+    - task: Cache@2
+      inputs:
+        key: '"externaldata" | "v1" | **/*.cid'
+        restoreKeys: |
+          "externaldata" | "v1"
+        path: $(ExternalData_OBJECT_STORES)
+      displayName: 'Restore ExternalData object store'
 
     - task: Cache@2
       inputs:
@@ -273,11 +281,17 @@ jobs:
     - bash: |
         set -x
         git clone -b dashboard --single-branch https://github.com/InsightSoftwareConsortium/ITK.git ITK-dashboard
-        curl -L https://github.com/InsightSoftwareConsortium/ITK/releases/download/v$(ExternalDataVersion)/InsightData-$(ExternalDataVersion).tar.gz -O
-        cmake -E tar xfz InsightData-$(ExternalDataVersion).tar.gz
-        cmake -E rename InsightToolkit-$(ExternalDataVersion)/.ExternalData/CID $(Build.SourcesDirectory)/.ExternalData/CID
       workingDirectory: $(Agent.BuildDirectory)
-      displayName: "Download dashboard script and testing data"
+      displayName: "Download dashboard script"
+
+    - task: Cache@2
+      inputs:
+        key: '"externaldata" | "v1" | **/*.cid'
+        restoreKeys: |
+          "externaldata" | "v1"
+        path: $(ExternalData_OBJECT_STORES)
+      displayName: 'Restore ExternalData object store'
+
     - task: Cache@2
       inputs:
         key: '"ccache-v4" | "$(Agent.OS)" | "LinuxCxx20" | "$(Build.SourceVersion)"'

Testing/ContinuousIntegration/AzurePipelinesLinuxPython.yml

@@ -27,7 +27,7 @@ pr:
     - Utilities/Maintenance/*
     - Modules/Remote/*.remote.cmake
 variables:
-  ExternalDataVersion: 5.4.5
+  ExternalData_OBJECT_STORES: $(Pipeline.Workspace)/ExternalData
   CCACHE_DIR: $(Pipeline.Workspace)/.ccache
   CCACHE_BASEDIR: $(Build.SourcesDirectory)
   CCACHE_COMPILERCHECK: content
@@ -67,14 +67,17 @@ jobs:
 
     - bash: |
         set -x
-
         git clone -b dashboard --single-branch https://github.com/InsightSoftwareConsortium/ITK.git ITK-dashboard
-
-        curl -L https://github.com/InsightSoftwareConsortium/ITK/releases/download/v$(ExternalDataVersion)/InsightData-$(ExternalDataVersion).tar.gz -O
-        cmake -E tar xfz InsightData-$(ExternalDataVersion).tar.gz
-        cmake -E rename InsightToolkit-$(ExternalDataVersion)/.ExternalData/CID $(Build.SourcesDirectory)/.ExternalData/CID
       workingDirectory: $(Agent.BuildDirectory)
-      displayName: 'Download dashboard script and testing data'
+      displayName: 'Download dashboard script'
+
+    - task: Cache@2
+      inputs:
+        key: '"externaldata" | "v1" | **/*.cid'
+        restoreKeys: |
+          "externaldata" | "v1"
+        path: $(ExternalData_OBJECT_STORES)
+      displayName: 'Restore ExternalData object store'
 
     - task: Cache@2
       inputs: