From 1fa903e27086590a0c68666176926d5de9ea697f Mon Sep 17 00:00:00 2001 From: David Calavera <1050+calavera@users.noreply.github.com> Date: Thu, 30 Jul 2026 13:51:29 -0700 Subject: [PATCH 1/7] Persist uv environments on runner volumes --- .github/workflows/build-reference.yml | 74 +++++++++++++++++++++++++-- README.md | 57 ++++++++++++++++----- 2 files changed, 116 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-reference.yml b/.github/workflows/build-reference.yml index 5f53302..d32623c 100644 --- a/.github/workflows/build-reference.yml +++ b/.github/workflows/build-reference.yml @@ -9,10 +9,16 @@ on: permissions: contents: read +concurrency: + group: build-reference-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + jobs: build: runs-on: [self-hosted, tensorlake, tensorlake-small] timeout-minutes: 30 + env: + REFERENCE_PYTHON_VERSION: "3.11" steps: - name: Check out repository @@ -36,10 +42,59 @@ jobs: cache-local-path: /mnt/tensorlake-cache/uv - name: Install Python - run: uv python install 3.11 + run: uv python install "${REFERENCE_PYTHON_VERSION}" + + - name: Configure persistent uv environment + id: uv-environment + shell: bash + run: | + python_executable="$(uv python find "${REFERENCE_PYTHON_VERSION}")" + python_identity="$("${python_executable}" -c 'import platform, sys; print(f"{sys.implementation.name}-{platform.python_version()}")')" + lock_hash="$(sha256sum uv.lock | cut -d ' ' -f 1)" + scope_hash="$(printf '%s\0%s' "${GITHUB_WORKFLOW}" "${GITHUB_REF}" | sha256sum | cut -c 1-16)" + environment_key="${RUNNER_OS}-${RUNNER_ARCH}-${python_identity}/${GITHUB_JOB}-${scope_hash}-${lock_hash}" + cache_root="${TENSORLAKE_CACHE_DIR:-/mnt/tensorlake-cache}" + environment_dir="${cache_root}/uv-environments/${environment_key}" + environment_marker="${environment_dir}.ready" + + if [[ -f "${environment_dir}/pyvenv.cfg" && -f "${environment_marker}" ]]; then + environment_state="warm" + else + environment_state="cold" + fi + + mkdir -p "$(dirname "${environment_dir}")" + { + echo "UV_PROJECT_ENVIRONMENT=${environment_dir}" + echo "TENSORLAKE_UV_ENVIRONMENT_KEY=${environment_key}" + echo "TENSORLAKE_UV_ENVIRONMENT_MARKER=${environment_marker}" + echo "TENSORLAKE_UV_ENVIRONMENT_STATE=${environment_state}" + } >> "${GITHUB_ENV}" + { + echo "key=${environment_key}" + echo "state=${environment_state}" + } >> "${GITHUB_OUTPUT}" - name: Install dependencies - run: uv sync --locked --all-extras + id: uv-sync + shell: bash + run: | + started_ns="$(date +%s%N)" + uv sync --locked --all-extras + finished_ns="$(date +%s%N)" + touch "${TENSORLAKE_UV_ENVIRONMENT_MARKER}" + duration_ms="$(( (finished_ns - started_ns) / 1000000 ))" + duration_seconds="$(awk -v ms="${duration_ms}" 'BEGIN { printf "%.3f", ms / 1000 }')" + + echo "duration-ms=${duration_ms}" >> "${GITHUB_OUTPUT}" + echo "UV_SYNC_TIMING duration_ms=${duration_ms} environment_state=${TENSORLAKE_UV_ENVIRONMENT_STATE} key=${TENSORLAKE_UV_ENVIRONMENT_KEY}" + { + echo "### Persistent uv environment" + echo + echo "- State before sync: \`${TENSORLAKE_UV_ENVIRONMENT_STATE}\`" + echo "- Sync duration: \`${duration_seconds}s\`" + echo "- Key: \`${TENSORLAKE_UV_ENVIRONMENT_KEY}\`" + } >> "${GITHUB_STEP_SUMMARY}" - name: Build Python package run: uv build @@ -54,7 +109,20 @@ jobs: run: bash -n scripts/*.sh - name: Run tests - run: uv run --no-sync pytest -q + id: tests + shell: bash + run: | + started_ns="$(date +%s%N)" + uv run --no-sync pytest -q + finished_ns="$(date +%s%N)" + duration_ms="$(( (finished_ns - started_ns) / 1000000 ))" + duration_seconds="$(awk -v ms="${duration_ms}" 'BEGIN { printf "%.3f", ms / 1000 }')" + + echo "duration-ms=${duration_ms}" >> "${GITHUB_OUTPUT}" + echo "TEST_TIMING duration_ms=${duration_ms} environment_state=${TENSORLAKE_UV_ENVIRONMENT_STATE}" + { + echo "- Test duration: \`${duration_seconds}s\`" + } >> "${GITHUB_STEP_SUMMARY}" - name: Verify Docker run: docker run --rm hello-world diff --git a/README.md b/README.md index 6452d75..40556f2 100644 --- a/README.md +++ b/README.md @@ -276,15 +276,38 @@ that verification. The runner deliberately exports only the generic `TENSORLAKE_CACHE_DIR`; it does not set tool-specific cache variables. Point `setup-uv` at a child directory with its `cache-local-path` input. Its default `enable-cache: auto` mode does not upload a GitHub Actions cache from a -self-hosted runner: +self-hosted runner. To avoid copying packages across the Cloud Volume and sandbox file systems, +put uv's project environment on the volume too: ```yaml +concurrency: + group: uv-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +env: + PYTHON_VERSION: "3.11" + - uses: actions/checkout@v6 - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: cache-local-path: /mnt/tensorlake-cache/uv +- run: uv python install "${PYTHON_VERSION}" + +- name: Configure persistent uv environment + shell: bash + run: | + python_path="$(uv python find "${PYTHON_VERSION}")" + python_identity="$("${python_path}" -c 'import platform, sys; print(f"{sys.implementation.name}-{platform.python_version()}")')" + lock_hash="$(sha256sum uv.lock | cut -d ' ' -f 1)" + scope_hash="$(printf '%s\0%s' "${GITHUB_WORKFLOW}" "${GITHUB_REF}" | sha256sum | cut -c 1-16)" + environment_key="${RUNNER_OS}-${RUNNER_ARCH}-${python_identity}/${GITHUB_JOB}-${scope_hash}-${lock_hash}" + cache_root="${TENSORLAKE_CACHE_DIR:-/mnt/tensorlake-cache}" + environment_dir="${cache_root}/uv-environments/${environment_key}" + mkdir -p "$(dirname "${environment_dir}")" + echo "UV_PROJECT_ENVIRONMENT=${environment_dir}" >> "${GITHUB_ENV}" + - run: uv sync --locked ``` @@ -292,9 +315,18 @@ self-hosted runner: the same path is created on the sandbox's ephemeral disk, so a workflow that omits the verification step still runs without persistence. -The cache and `.venv` are on different file systems, so uv may copy artifacts instead of -hard-linking them. The volume still avoids repeated downloads and source builds, but benchmark the -result for dependency sets dominated by small prebuilt wheels. +The environment key separates incompatible operating systems, CPU architectures, exact Python +versions, jobs, Git refs, and lockfiles. The concurrency group serializes workflows for the same +ref, so two sandboxes do not update that ref's environment at once. Different refs get separate +directories even when they use the same lockfile. This matters because Cloud Volumes reconcile +concurrent same-path writes with last-writer-wins semantics rather than distributed file locking. + +Because `UV_CACHE_DIR` and `UV_PROJECT_ENVIRONMENT` are on the same mounted file system, uv can link +cached packages into the environment instead of copying them to the sandbox disk. A later run with +the same key reuses the synchronized environment. Python then imports packages from the mounted +volume, so compare both dependency-sync and test execution times for the workload. Remove obsolete +directories under `${TENSORLAKE_CACHE_DIR}/uv-environments` when their refs or lockfiles are no +longer needed. ### Rust, Cargo, and sccache @@ -414,11 +446,12 @@ volume snapshots for disposable cache data. ## Self-Test Workflow -`.github/workflows/build-reference.yml` runs on a `tensorlake-small` runner. It points uv at the -persistent volume automatically when available and uses the sandbox-local uv cache otherwise. It -installs and lints the Python project, builds its distribution, validates the setup scripts, runs -the test suite, and runs Docker's `hello-world` image to verify the runner's Docker daemon. The -reusable runner sandbox image still builds through `scripts/build-runner-image.sh`, because -`tensorlake/ubuntu-systemd` is a Tensorlake registered base rather than a public Docker Hub image. -Once the organization webhook is configured, pushes to `main`, pull requests, and manual dispatches -exercise the runner implementation from its own repository. +`.github/workflows/build-reference.yml` runs on a `tensorlake-small` runner. It places both uv's +cache and a compatibility-keyed project environment on the persistent volume, and records the +environment's cold/warm state and `uv sync` duration in the job summary. It installs and lints the +Python project, builds its distribution, validates the setup scripts, runs the test suite, and runs +Docker's `hello-world` image to verify the runner's Docker daemon. The reusable runner sandbox image +still builds through `scripts/build-runner-image.sh`, because `tensorlake/ubuntu-systemd` is a +Tensorlake registered base rather than a public Docker Hub image. Once the organization webhook is +configured, pushes to `main`, pull requests, and manual dispatches exercise the runner implementation +from its own repository. From e415d446c896b3a8f7e3ab377228ef9197b8b76d Mon Sep 17 00:00:00 2001 From: David Calavera <1050+calavera@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:02:44 -0700 Subject: [PATCH 2/7] Persist copied uv environment files --- .github/workflows/build-reference.yml | 6 ++++-- README.md | 18 +++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-reference.yml b/.github/workflows/build-reference.yml index d32623c..659ce50 100644 --- a/.github/workflows/build-reference.yml +++ b/.github/workflows/build-reference.yml @@ -19,6 +19,7 @@ jobs: timeout-minutes: 30 env: REFERENCE_PYTHON_VERSION: "3.11" + UV_LINK_MODE: copy steps: - name: Check out repository @@ -54,7 +55,7 @@ jobs: scope_hash="$(printf '%s\0%s' "${GITHUB_WORKFLOW}" "${GITHUB_REF}" | sha256sum | cut -c 1-16)" environment_key="${RUNNER_OS}-${RUNNER_ARCH}-${python_identity}/${GITHUB_JOB}-${scope_hash}-${lock_hash}" cache_root="${TENSORLAKE_CACHE_DIR:-/mnt/tensorlake-cache}" - environment_dir="${cache_root}/uv-environments/${environment_key}" + environment_dir="${cache_root}/uv-environments-v2/${environment_key}" environment_marker="${environment_dir}.ready" if [[ -f "${environment_dir}/pyvenv.cfg" && -f "${environment_marker}" ]]; then @@ -82,7 +83,8 @@ jobs: started_ns="$(date +%s%N)" uv sync --locked --all-extras finished_ns="$(date +%s%N)" - touch "${TENSORLAKE_UV_ENVIRONMENT_MARKER}" + printf '%s\n' "${TENSORLAKE_UV_ENVIRONMENT_KEY}" \ + > "${TENSORLAKE_UV_ENVIRONMENT_MARKER}" duration_ms="$(( (finished_ns - started_ns) / 1000000 ))" duration_seconds="$(awk -v ms="${duration_ms}" 'BEGIN { printf "%.3f", ms / 1000 }')" diff --git a/README.md b/README.md index 40556f2..3dd026a 100644 --- a/README.md +++ b/README.md @@ -286,6 +286,7 @@ concurrency: env: PYTHON_VERSION: "3.11" + UV_LINK_MODE: copy - uses: actions/checkout@v6 @@ -304,7 +305,7 @@ env: scope_hash="$(printf '%s\0%s' "${GITHUB_WORKFLOW}" "${GITHUB_REF}" | sha256sum | cut -c 1-16)" environment_key="${RUNNER_OS}-${RUNNER_ARCH}-${python_identity}/${GITHUB_JOB}-${scope_hash}-${lock_hash}" cache_root="${TENSORLAKE_CACHE_DIR:-/mnt/tensorlake-cache}" - environment_dir="${cache_root}/uv-environments/${environment_key}" + environment_dir="${cache_root}/uv-environments-v2/${environment_key}" mkdir -p "$(dirname "${environment_dir}")" echo "UV_PROJECT_ENVIRONMENT=${environment_dir}" >> "${GITHUB_ENV}" @@ -321,12 +322,15 @@ ref, so two sandboxes do not update that ref's environment at once. Different re directories even when they use the same lockfile. This matters because Cloud Volumes reconcile concurrent same-path writes with last-writer-wins semantics rather than distributed file locking. -Because `UV_CACHE_DIR` and `UV_PROJECT_ENVIRONMENT` are on the same mounted file system, uv can link -cached packages into the environment instead of copying them to the sandbox disk. A later run with -the same key reuses the synchronized environment. Python then imports packages from the mounted -volume, so compare both dependency-sync and test execution times for the workload. Remove obsolete -directories under `${TENSORLAKE_CACHE_DIR}/uv-environments` when their refs or lockfiles are no -longer needed. +Set `UV_LINK_MODE=copy` when creating a durable environment on a Cloud Volume. Although hardlinks +work within one live mount, cache-to-environment hardlinks are not materialized as independent files +when a later sandbox mounts the volume. Copy mode makes the first sync write every environment file +into the durable timeline. It does not copy packages back to the sandbox disk, and a later run with +the same key reuses the synchronized environment. + +Python then imports packages from the mounted volume, so compare both dependency-sync and test +execution times for the workload. Remove obsolete directories under +`${TENSORLAKE_CACHE_DIR}/uv-environments-v2` when their refs or lockfiles are no longer needed. ### Rust, Cargo, and sccache From 1c245b0888df826c54f307d2a7d1bbf3c68d572a Mon Sep 17 00:00:00 2001 From: David Calavera <1050+calavera@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:14:59 -0700 Subject: [PATCH 3/7] Publish uv environment contents --- .github/workflows/build-reference.yml | 10 ++++++++-- README.md | 27 +++++++++++++++++++++------ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-reference.yml b/.github/workflows/build-reference.yml index 659ce50..0e9bbd9 100644 --- a/.github/workflows/build-reference.yml +++ b/.github/workflows/build-reference.yml @@ -55,16 +55,18 @@ jobs: scope_hash="$(printf '%s\0%s' "${GITHUB_WORKFLOW}" "${GITHUB_REF}" | sha256sum | cut -c 1-16)" environment_key="${RUNNER_OS}-${RUNNER_ARCH}-${python_identity}/${GITHUB_JOB}-${scope_hash}-${lock_hash}" cache_root="${TENSORLAKE_CACHE_DIR:-/mnt/tensorlake-cache}" - environment_dir="${cache_root}/uv-environments-v2/${environment_key}" + environment_dir="${cache_root}/uv-environments-v3/${environment_key}" environment_marker="${environment_dir}.ready" + mkdir -p "$(dirname "${environment_dir}")" if [[ -f "${environment_dir}/pyvenv.cfg" && -f "${environment_marker}" ]]; then environment_state="warm" else environment_state="cold" + uv venv --clear --python "${python_executable}" "${environment_dir}" fi - mkdir -p "$(dirname "${environment_dir}")" + rm -f "${environment_dir}/.gitignore" "${environment_dir}/CACHEDIR.TAG" { echo "UV_PROJECT_ENVIRONMENT=${environment_dir}" echo "TENSORLAKE_UV_ENVIRONMENT_KEY=${environment_key}" @@ -83,8 +85,12 @@ jobs: started_ns="$(date +%s%N)" uv sync --locked --all-extras finished_ns="$(date +%s%N)" + rm -f "${UV_PROJECT_ENVIRONMENT}/.gitignore" \ + "${UV_PROJECT_ENVIRONMENT}/CACHEDIR.TAG" printf '%s\n' "${TENSORLAKE_UV_ENVIRONMENT_KEY}" \ > "${TENSORLAKE_UV_ENVIRONMENT_MARKER}" + sync + sleep 6 duration_ms="$(( (finished_ns - started_ns) / 1000000 ))" duration_seconds="$(awk -v ms="${duration_ms}" 'BEGIN { printf "%.3f", ms / 1000 }')" diff --git a/README.md b/README.md index 3dd026a..b682579 100644 --- a/README.md +++ b/README.md @@ -305,11 +305,22 @@ env: scope_hash="$(printf '%s\0%s' "${GITHUB_WORKFLOW}" "${GITHUB_REF}" | sha256sum | cut -c 1-16)" environment_key="${RUNNER_OS}-${RUNNER_ARCH}-${python_identity}/${GITHUB_JOB}-${scope_hash}-${lock_hash}" cache_root="${TENSORLAKE_CACHE_DIR:-/mnt/tensorlake-cache}" - environment_dir="${cache_root}/uv-environments-v2/${environment_key}" + environment_dir="${cache_root}/uv-environments-v3/${environment_key}" mkdir -p "$(dirname "${environment_dir}")" + if [[ ! -f "${environment_dir}.ready" ]]; then + uv venv --clear --python "${python_path}" "${environment_dir}" + fi + rm -f "${environment_dir}/.gitignore" "${environment_dir}/CACHEDIR.TAG" echo "UV_PROJECT_ENVIRONMENT=${environment_dir}" >> "${GITHUB_ENV}" -- run: uv sync --locked +- name: Synchronize and publish the environment + run: | + uv sync --locked + rm -f "${UV_PROJECT_ENVIRONMENT}/.gitignore" \ + "${UV_PROJECT_ENVIRONMENT}/CACHEDIR.TAG" + printf 'ready\n' > "${UV_PROJECT_ENVIRONMENT}.ready" + sync + sleep 6 ``` `setup-uv` sets `UV_CACHE_DIR` for the job from this input. If cache provisioning is unavailable, @@ -324,13 +335,17 @@ concurrent same-path writes with last-writer-wins semantics rather than distribu Set `UV_LINK_MODE=copy` when creating a durable environment on a Cloud Volume. Although hardlinks work within one live mount, cache-to-environment hardlinks are not materialized as independent files -when a later sandbox mounts the volume. Copy mode makes the first sync write every environment file -into the durable timeline. It does not copy packages back to the sandbox disk, and a later run with -the same key reuses the synchronized environment. +when a later sandbox mounts the volume. Also remove the `.gitignore` and `CACHEDIR.TAG` files that uv +creates in a virtual environment: those correctly mark an ordinary local venv as disposable, but +they exclude its contents from the Cloud Volume's durable timeline. Copy mode then makes the first +sync write every environment file into that timeline. It does not copy packages back to the sandbox +disk, and a later run with the same key reuses the synchronized environment. The final `sync` and +short wait let the ready marker pass through the volume's bounded autosave interval before the job +can exit. Python then imports packages from the mounted volume, so compare both dependency-sync and test execution times for the workload. Remove obsolete directories under -`${TENSORLAKE_CACHE_DIR}/uv-environments-v2` when their refs or lockfiles are no longer needed. +`${TENSORLAKE_CACHE_DIR}/uv-environments-v3` when their refs or lockfiles are no longer needed. ### Rust, Cargo, and sccache From 2a5dd796fbc63c71dcbc284c00bbf2dc7fd78a61 Mon Sep 17 00:00:00 2001 From: David Calavera <1050+calavera@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:19:10 -0700 Subject: [PATCH 4/7] Create durable uv environments without ignore markers --- .github/workflows/build-reference.yml | 7 ++----- README.md | 23 ++++++++++------------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-reference.yml b/.github/workflows/build-reference.yml index 0e9bbd9..8e3429f 100644 --- a/.github/workflows/build-reference.yml +++ b/.github/workflows/build-reference.yml @@ -55,7 +55,7 @@ jobs: scope_hash="$(printf '%s\0%s' "${GITHUB_WORKFLOW}" "${GITHUB_REF}" | sha256sum | cut -c 1-16)" environment_key="${RUNNER_OS}-${RUNNER_ARCH}-${python_identity}/${GITHUB_JOB}-${scope_hash}-${lock_hash}" cache_root="${TENSORLAKE_CACHE_DIR:-/mnt/tensorlake-cache}" - environment_dir="${cache_root}/uv-environments-v3/${environment_key}" + environment_dir="${cache_root}/uv-environments-v4/${environment_key}" environment_marker="${environment_dir}.ready" mkdir -p "$(dirname "${environment_dir}")" @@ -63,10 +63,9 @@ jobs: environment_state="warm" else environment_state="cold" - uv venv --clear --python "${python_executable}" "${environment_dir}" + "${python_executable}" -m venv --clear --without-pip "${environment_dir}" fi - rm -f "${environment_dir}/.gitignore" "${environment_dir}/CACHEDIR.TAG" { echo "UV_PROJECT_ENVIRONMENT=${environment_dir}" echo "TENSORLAKE_UV_ENVIRONMENT_KEY=${environment_key}" @@ -85,8 +84,6 @@ jobs: started_ns="$(date +%s%N)" uv sync --locked --all-extras finished_ns="$(date +%s%N)" - rm -f "${UV_PROJECT_ENVIRONMENT}/.gitignore" \ - "${UV_PROJECT_ENVIRONMENT}/CACHEDIR.TAG" printf '%s\n' "${TENSORLAKE_UV_ENVIRONMENT_KEY}" \ > "${TENSORLAKE_UV_ENVIRONMENT_MARKER}" sync diff --git a/README.md b/README.md index b682579..6821671 100644 --- a/README.md +++ b/README.md @@ -305,19 +305,16 @@ env: scope_hash="$(printf '%s\0%s' "${GITHUB_WORKFLOW}" "${GITHUB_REF}" | sha256sum | cut -c 1-16)" environment_key="${RUNNER_OS}-${RUNNER_ARCH}-${python_identity}/${GITHUB_JOB}-${scope_hash}-${lock_hash}" cache_root="${TENSORLAKE_CACHE_DIR:-/mnt/tensorlake-cache}" - environment_dir="${cache_root}/uv-environments-v3/${environment_key}" + environment_dir="${cache_root}/uv-environments-v4/${environment_key}" mkdir -p "$(dirname "${environment_dir}")" if [[ ! -f "${environment_dir}.ready" ]]; then - uv venv --clear --python "${python_path}" "${environment_dir}" + "${python_path}" -m venv --clear --without-pip "${environment_dir}" fi - rm -f "${environment_dir}/.gitignore" "${environment_dir}/CACHEDIR.TAG" echo "UV_PROJECT_ENVIRONMENT=${environment_dir}" >> "${GITHUB_ENV}" - name: Synchronize and publish the environment run: | uv sync --locked - rm -f "${UV_PROJECT_ENVIRONMENT}/.gitignore" \ - "${UV_PROJECT_ENVIRONMENT}/CACHEDIR.TAG" printf 'ready\n' > "${UV_PROJECT_ENVIRONMENT}.ready" sync sleep 6 @@ -335,17 +332,17 @@ concurrent same-path writes with last-writer-wins semantics rather than distribu Set `UV_LINK_MODE=copy` when creating a durable environment on a Cloud Volume. Although hardlinks work within one live mount, cache-to-environment hardlinks are not materialized as independent files -when a later sandbox mounts the volume. Also remove the `.gitignore` and `CACHEDIR.TAG` files that uv -creates in a virtual environment: those correctly mark an ordinary local venv as disposable, but -they exclude its contents from the Cloud Volume's durable timeline. Copy mode then makes the first -sync write every environment file into that timeline. It does not copy packages back to the sandbox -disk, and a later run with the same key reuses the synchronized environment. The final `sync` and -short wait let the ready marker pass through the volume's bounded autosave interval before the job -can exit. +when a later sandbox mounts the volume. Create the environment with Python's standard-library +`venv --without-pip` rather than `uv venv`: uv marks its generated environments as disposable with +`.gitignore` and `CACHEDIR.TAG`, which excludes their contents from the Cloud Volume's durable +timeline. Copy mode then makes the first sync write every environment file into that timeline. It +does not copy packages back to the sandbox disk, and a later run with the same key reuses the +synchronized environment. The final `sync` and short wait let the ready marker pass through the +volume's bounded autosave interval before the job can exit. Python then imports packages from the mounted volume, so compare both dependency-sync and test execution times for the workload. Remove obsolete directories under -`${TENSORLAKE_CACHE_DIR}/uv-environments-v3` when their refs or lockfiles are no longer needed. +`${TENSORLAKE_CACHE_DIR}/uv-environments-v4` when their refs or lockfiles are no longer needed. ### Rust, Cargo, and sccache From c6517171c5a8e5e2dae6f709c806df932e7cb931 Mon Sep 17 00:00:00 2001 From: David Calavera <1050+calavera@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:39:25 -0700 Subject: [PATCH 5/7] Cache uv environments as durable archives --- .github/workflows/build-reference.yml | 45 ++++++++--- README.md | 109 +++++++++++++++----------- 2 files changed, 96 insertions(+), 58 deletions(-) diff --git a/.github/workflows/build-reference.yml b/.github/workflows/build-reference.yml index 8e3429f..8a53679 100644 --- a/.github/workflows/build-reference.yml +++ b/.github/workflows/build-reference.yml @@ -19,7 +19,6 @@ jobs: timeout-minutes: 30 env: REFERENCE_PYTHON_VERSION: "3.11" - UV_LINK_MODE: copy steps: - name: Check out repository @@ -40,7 +39,7 @@ jobs: - name: Set up uv uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: - cache-local-path: /mnt/tensorlake-cache/uv + enable-cache: false - name: Install Python run: uv python install "${REFERENCE_PYTHON_VERSION}" @@ -55,21 +54,30 @@ jobs: scope_hash="$(printf '%s\0%s' "${GITHUB_WORKFLOW}" "${GITHUB_REF}" | sha256sum | cut -c 1-16)" environment_key="${RUNNER_OS}-${RUNNER_ARCH}-${python_identity}/${GITHUB_JOB}-${scope_hash}-${lock_hash}" cache_root="${TENSORLAKE_CACHE_DIR:-/mnt/tensorlake-cache}" - environment_dir="${cache_root}/uv-environments-v4/${environment_key}" - environment_marker="${environment_dir}.ready" - mkdir -p "$(dirname "${environment_dir}")" + environment_dir="${RUNNER_TEMP}/uv-environments/${environment_key}" + environment_archive="${cache_root}/uv-environment-archives-v1/${environment_key}.tar.gz" + restore_duration_ms=0 + mkdir -p "$(dirname "${environment_dir}")" "$(dirname "${environment_archive}")" - if [[ -f "${environment_dir}/pyvenv.cfg" && -f "${environment_marker}" ]]; then + if [[ -f "${environment_archive}" ]]; then environment_state="warm" + restore_started_ns="$(date +%s%N)" + tar -xzf "${environment_archive}" -C "$(dirname "${environment_dir}")" + restore_finished_ns="$(date +%s%N)" + restore_duration_ms="$(( (restore_finished_ns - restore_started_ns) / 1000000 ))" + if [[ ! -f "${environment_dir}/pyvenv.cfg" ]]; then + echo "::error::The persistent uv environment archive is invalid." + exit 1 + fi else environment_state="cold" - "${python_executable}" -m venv --clear --without-pip "${environment_dir}" fi { echo "UV_PROJECT_ENVIRONMENT=${environment_dir}" + echo "TENSORLAKE_UV_ENVIRONMENT_ARCHIVE=${environment_archive}" echo "TENSORLAKE_UV_ENVIRONMENT_KEY=${environment_key}" - echo "TENSORLAKE_UV_ENVIRONMENT_MARKER=${environment_marker}" + echo "TENSORLAKE_UV_ENVIRONMENT_RESTORE_DURATION_MS=${restore_duration_ms}" echo "TENSORLAKE_UV_ENVIRONMENT_STATE=${environment_state}" } >> "${GITHUB_ENV}" { @@ -84,20 +92,33 @@ jobs: started_ns="$(date +%s%N)" uv sync --locked --all-extras finished_ns="$(date +%s%N)" - printf '%s\n' "${TENSORLAKE_UV_ENVIRONMENT_KEY}" \ - > "${TENSORLAKE_UV_ENVIRONMENT_MARKER}" - sync - sleep 6 duration_ms="$(( (finished_ns - started_ns) / 1000000 ))" duration_seconds="$(awk -v ms="${duration_ms}" 'BEGIN { printf "%.3f", ms / 1000 }')" + publish_duration_ms=0 + + if [[ "${TENSORLAKE_UV_ENVIRONMENT_STATE}" == "cold" ]]; then + publish_started_ns="$(date +%s%N)" + temporary_archive="${TENSORLAKE_UV_ENVIRONMENT_ARCHIVE}.tmp.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}" + tar -czf "${temporary_archive}" \ + -C "$(dirname "${UV_PROJECT_ENVIRONMENT}")" \ + "$(basename "${UV_PROJECT_ENVIRONMENT}")" + mv -f "${temporary_archive}" "${TENSORLAKE_UV_ENVIRONMENT_ARCHIVE}" + sync + sleep 6 + publish_finished_ns="$(date +%s%N)" + publish_duration_ms="$(( (publish_finished_ns - publish_started_ns) / 1000000 ))" + fi echo "duration-ms=${duration_ms}" >> "${GITHUB_OUTPUT}" echo "UV_SYNC_TIMING duration_ms=${duration_ms} environment_state=${TENSORLAKE_UV_ENVIRONMENT_STATE} key=${TENSORLAKE_UV_ENVIRONMENT_KEY}" + echo "UV_ARCHIVE_TIMING restore_duration_ms=${TENSORLAKE_UV_ENVIRONMENT_RESTORE_DURATION_MS} publish_duration_ms=${publish_duration_ms} environment_state=${TENSORLAKE_UV_ENVIRONMENT_STATE}" { echo "### Persistent uv environment" echo echo "- State before sync: \`${TENSORLAKE_UV_ENVIRONMENT_STATE}\`" + echo "- Archive restore: \`${TENSORLAKE_UV_ENVIRONMENT_RESTORE_DURATION_MS}ms\`" echo "- Sync duration: \`${duration_seconds}s\`" + echo "- Archive publish: \`${publish_duration_ms}ms\`" echo "- Key: \`${TENSORLAKE_UV_ENVIRONMENT_KEY}\`" } >> "${GITHUB_STEP_SUMMARY}" diff --git a/README.md b/README.md index 6821671..1e3bfba 100644 --- a/README.md +++ b/README.md @@ -276,8 +276,23 @@ that verification. The runner deliberately exports only the generic `TENSORLAKE_CACHE_DIR`; it does not set tool-specific cache variables. Point `setup-uv` at a child directory with its `cache-local-path` input. Its default `enable-cache: auto` mode does not upload a GitHub Actions cache from a -self-hosted runner. To avoid copying packages across the Cloud Volume and sandbox file systems, -put uv's project environment on the volume too: +self-hosted runner: + +```yaml +- uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + cache-local-path: /mnt/tensorlake-cache/uv + +- run: uv sync --locked +``` + +The cache and `.venv` are on different file systems, so uv may copy artifacts instead of +hard-linking them. The volume still avoids repeated downloads and source builds, but benchmark the +result for dependency sets dominated by small prebuilt wheels. + +For a dependency set where those cross-filesystem copies are expensive, keep the live environment +on the sandbox's local disk and store a compressed, compatibility-keyed environment archive on the +volume: ```yaml concurrency: @@ -286,17 +301,16 @@ concurrency: env: PYTHON_VERSION: "3.11" - UV_LINK_MODE: copy - uses: actions/checkout@v6 - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: - cache-local-path: /mnt/tensorlake-cache/uv + enable-cache: false - run: uv python install "${PYTHON_VERSION}" -- name: Configure persistent uv environment +- name: Restore persistent uv environment shell: bash run: | python_path="$(uv python find "${PYTHON_VERSION}")" @@ -305,44 +319,46 @@ env: scope_hash="$(printf '%s\0%s' "${GITHUB_WORKFLOW}" "${GITHUB_REF}" | sha256sum | cut -c 1-16)" environment_key="${RUNNER_OS}-${RUNNER_ARCH}-${python_identity}/${GITHUB_JOB}-${scope_hash}-${lock_hash}" cache_root="${TENSORLAKE_CACHE_DIR:-/mnt/tensorlake-cache}" - environment_dir="${cache_root}/uv-environments-v4/${environment_key}" - mkdir -p "$(dirname "${environment_dir}")" - if [[ ! -f "${environment_dir}.ready" ]]; then - "${python_path}" -m venv --clear --without-pip "${environment_dir}" + environment_dir="${RUNNER_TEMP}/uv-environments/${environment_key}" + environment_archive="${cache_root}/uv-environment-archives-v1/${environment_key}.tar.gz" + mkdir -p "$(dirname "${environment_dir}")" "$(dirname "${environment_archive}")" + if [[ -f "${environment_archive}" ]]; then + tar -xzf "${environment_archive}" -C "$(dirname "${environment_dir}")" + environment_state=warm + else + environment_state=cold fi - echo "UV_PROJECT_ENVIRONMENT=${environment_dir}" >> "${GITHUB_ENV}" + { + echo "UV_PROJECT_ENVIRONMENT=${environment_dir}" + echo "UV_ENVIRONMENT_ARCHIVE=${environment_archive}" + echo "UV_ENVIRONMENT_STATE=${environment_state}" + } >> "${GITHUB_ENV}" -- name: Synchronize and publish the environment +- name: Synchronize and cache the environment run: | uv sync --locked - printf 'ready\n' > "${UV_PROJECT_ENVIRONMENT}.ready" - sync - sleep 6 + if [[ "${UV_ENVIRONMENT_STATE}" == "cold" ]]; then + temporary_archive="${UV_ENVIRONMENT_ARCHIVE}.tmp.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}" + tar -czf "${temporary_archive}" \ + -C "$(dirname "${UV_PROJECT_ENVIRONMENT}")" \ + "$(basename "${UV_PROJECT_ENVIRONMENT}")" + mv -f "${temporary_archive}" "${UV_ENVIRONMENT_ARCHIVE}" + sync + sleep 6 + fi ``` -`setup-uv` sets `UV_CACHE_DIR` for the job from this input. If cache provisioning is unavailable, -the same path is created on the sandbox's ephemeral disk, so a workflow that omits the verification -step still runs without persistence. - -The environment key separates incompatible operating systems, CPU architectures, exact Python -versions, jobs, Git refs, and lockfiles. The concurrency group serializes workflows for the same -ref, so two sandboxes do not update that ref's environment at once. Different refs get separate -directories even when they use the same lockfile. This matters because Cloud Volumes reconcile -concurrent same-path writes with last-writer-wins semantics rather than distributed file locking. - -Set `UV_LINK_MODE=copy` when creating a durable environment on a Cloud Volume. Although hardlinks -work within one live mount, cache-to-environment hardlinks are not materialized as independent files -when a later sandbox mounts the volume. Create the environment with Python's standard-library -`venv --without-pip` rather than `uv venv`: uv marks its generated environments as disposable with -`.gitignore` and `CACHEDIR.TAG`, which excludes their contents from the Cloud Volume's durable -timeline. Copy mode then makes the first sync write every environment file into that timeline. It -does not copy packages back to the sandbox disk, and a later run with the same key reuses the -synchronized environment. The final `sync` and short wait let the ready marker pass through the -volume's bounded autosave interval before the job can exit. - -Python then imports packages from the mounted volume, so compare both dependency-sync and test -execution times for the workload. Remove obsolete directories under -`${TENSORLAKE_CACHE_DIR}/uv-environments-v4` when their refs or lockfiles are no longer needed. +The archive key separates incompatible operating systems, CPU architectures, exact Python versions, +jobs, Git refs, and lockfiles. The local restore path is stable across runner sandboxes so virtual +environment script paths remain valid. The concurrency group serializes workflows for the same ref, +and the archive is published with an atomic rename so another run never restores a partial file. +Different refs get separate archives even when they use the same lockfile. + +This layout avoids both cross-filesystem package installation and Python's small-file reads over +FUSE. It also presents the Cloud Volume autosave engine with one sequential archive instead of +thousands of independently changing environment files. Remove obsolete archives under +`${TENSORLAKE_CACHE_DIR}/uv-environment-archives-v1` when their refs or lockfiles are no longer +needed. ### Rust, Cargo, and sccache @@ -462,12 +478,13 @@ volume snapshots for disposable cache data. ## Self-Test Workflow -`.github/workflows/build-reference.yml` runs on a `tensorlake-small` runner. It places both uv's -cache and a compatibility-keyed project environment on the persistent volume, and records the -environment's cold/warm state and `uv sync` duration in the job summary. It installs and lints the -Python project, builds its distribution, validates the setup scripts, runs the test suite, and runs -Docker's `hello-world` image to verify the runner's Docker daemon. The reusable runner sandbox image -still builds through `scripts/build-runner-image.sh`, because `tensorlake/ubuntu-systemd` is a -Tensorlake registered base rather than a public Docker Hub image. Once the organization webhook is -configured, pushes to `main`, pull requests, and manual dispatches exercise the runner implementation -from its own repository. +`.github/workflows/build-reference.yml` runs on a `tensorlake-small` runner. It keeps uv's live +environment on local sandbox storage and caches a compatibility-keyed compressed environment on the +persistent volume. The job summary records cold/warm state plus archive restore, `uv sync`, archive +publish, and test durations. It installs and lints the Python project, builds its distribution, +validates the setup scripts, runs the test suite, and runs Docker's `hello-world` image to verify the +runner's Docker daemon. The reusable runner sandbox image still builds through +`scripts/build-runner-image.sh`, because `tensorlake/ubuntu-systemd` is a Tensorlake registered base +rather than a public Docker Hub image. Once the organization webhook is configured, pushes to +`main`, pull requests, and manual dispatches exercise the runner implementation from its own +repository. From 121b0770500a8d93de24015324442461219ae2b8 Mon Sep 17 00:00:00 2001 From: David Calavera <1050+calavera@users.noreply.github.com> Date: Thu, 30 Jul 2026 16:33:38 -0700 Subject: [PATCH 6/7] Add reusable uv and Rust cache actions --- .github/workflows/build-reference.yml | 103 +------------- README.md | 113 ++++++---------- actions/setup-rust-cache/action.yml | 162 +++++++++++++++++++++++ actions/setup-uv-cache/action.yml | 184 ++++++++++++++++++++++++++ tests/test_configuration.py | 3 +- 5 files changed, 390 insertions(+), 175 deletions(-) create mode 100644 actions/setup-rust-cache/action.yml create mode 100644 actions/setup-uv-cache/action.yml diff --git a/.github/workflows/build-reference.yml b/.github/workflows/build-reference.yml index 8a53679..3f8fa3a 100644 --- a/.github/workflows/build-reference.yml +++ b/.github/workflows/build-reference.yml @@ -17,110 +17,17 @@ jobs: build: runs-on: [self-hosted, tensorlake, tensorlake-small] timeout-minutes: 30 - env: - REFERENCE_PYTHON_VERSION: "3.11" steps: - name: Check out repository uses: actions/checkout@v6 - - name: Verify persistent cache mount - shell: bash - run: | - if [[ -z "${TENSORLAKE_CACHE_DIR:-}" ]]; then - echo "::error::The runner did not provide a persistent cache directory." - exit 1 - fi - if ! mountpoint -q "${TENSORLAKE_CACHE_DIR}"; then - echo "::error::${TENSORLAKE_CACHE_DIR} is not a mounted Cloud Volume." - exit 1 - fi - - - name: Set up uv - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 - with: - enable-cache: false - - - name: Install Python - run: uv python install "${REFERENCE_PYTHON_VERSION}" - - - name: Configure persistent uv environment + - name: Set up cached uv environment id: uv-environment - shell: bash - run: | - python_executable="$(uv python find "${REFERENCE_PYTHON_VERSION}")" - python_identity="$("${python_executable}" -c 'import platform, sys; print(f"{sys.implementation.name}-{platform.python_version()}")')" - lock_hash="$(sha256sum uv.lock | cut -d ' ' -f 1)" - scope_hash="$(printf '%s\0%s' "${GITHUB_WORKFLOW}" "${GITHUB_REF}" | sha256sum | cut -c 1-16)" - environment_key="${RUNNER_OS}-${RUNNER_ARCH}-${python_identity}/${GITHUB_JOB}-${scope_hash}-${lock_hash}" - cache_root="${TENSORLAKE_CACHE_DIR:-/mnt/tensorlake-cache}" - environment_dir="${RUNNER_TEMP}/uv-environments/${environment_key}" - environment_archive="${cache_root}/uv-environment-archives-v1/${environment_key}.tar.gz" - restore_duration_ms=0 - mkdir -p "$(dirname "${environment_dir}")" "$(dirname "${environment_archive}")" - - if [[ -f "${environment_archive}" ]]; then - environment_state="warm" - restore_started_ns="$(date +%s%N)" - tar -xzf "${environment_archive}" -C "$(dirname "${environment_dir}")" - restore_finished_ns="$(date +%s%N)" - restore_duration_ms="$(( (restore_finished_ns - restore_started_ns) / 1000000 ))" - if [[ ! -f "${environment_dir}/pyvenv.cfg" ]]; then - echo "::error::The persistent uv environment archive is invalid." - exit 1 - fi - else - environment_state="cold" - fi - - { - echo "UV_PROJECT_ENVIRONMENT=${environment_dir}" - echo "TENSORLAKE_UV_ENVIRONMENT_ARCHIVE=${environment_archive}" - echo "TENSORLAKE_UV_ENVIRONMENT_KEY=${environment_key}" - echo "TENSORLAKE_UV_ENVIRONMENT_RESTORE_DURATION_MS=${restore_duration_ms}" - echo "TENSORLAKE_UV_ENVIRONMENT_STATE=${environment_state}" - } >> "${GITHUB_ENV}" - { - echo "key=${environment_key}" - echo "state=${environment_state}" - } >> "${GITHUB_OUTPUT}" - - - name: Install dependencies - id: uv-sync - shell: bash - run: | - started_ns="$(date +%s%N)" - uv sync --locked --all-extras - finished_ns="$(date +%s%N)" - duration_ms="$(( (finished_ns - started_ns) / 1000000 ))" - duration_seconds="$(awk -v ms="${duration_ms}" 'BEGIN { printf "%.3f", ms / 1000 }')" - publish_duration_ms=0 - - if [[ "${TENSORLAKE_UV_ENVIRONMENT_STATE}" == "cold" ]]; then - publish_started_ns="$(date +%s%N)" - temporary_archive="${TENSORLAKE_UV_ENVIRONMENT_ARCHIVE}.tmp.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}" - tar -czf "${temporary_archive}" \ - -C "$(dirname "${UV_PROJECT_ENVIRONMENT}")" \ - "$(basename "${UV_PROJECT_ENVIRONMENT}")" - mv -f "${temporary_archive}" "${TENSORLAKE_UV_ENVIRONMENT_ARCHIVE}" - sync - sleep 6 - publish_finished_ns="$(date +%s%N)" - publish_duration_ms="$(( (publish_finished_ns - publish_started_ns) / 1000000 ))" - fi - - echo "duration-ms=${duration_ms}" >> "${GITHUB_OUTPUT}" - echo "UV_SYNC_TIMING duration_ms=${duration_ms} environment_state=${TENSORLAKE_UV_ENVIRONMENT_STATE} key=${TENSORLAKE_UV_ENVIRONMENT_KEY}" - echo "UV_ARCHIVE_TIMING restore_duration_ms=${TENSORLAKE_UV_ENVIRONMENT_RESTORE_DURATION_MS} publish_duration_ms=${publish_duration_ms} environment_state=${TENSORLAKE_UV_ENVIRONMENT_STATE}" - { - echo "### Persistent uv environment" - echo - echo "- State before sync: \`${TENSORLAKE_UV_ENVIRONMENT_STATE}\`" - echo "- Archive restore: \`${TENSORLAKE_UV_ENVIRONMENT_RESTORE_DURATION_MS}ms\`" - echo "- Sync duration: \`${duration_seconds}s\`" - echo "- Archive publish: \`${publish_duration_ms}ms\`" - echo "- Key: \`${TENSORLAKE_UV_ENVIRONMENT_KEY}\`" - } >> "${GITHUB_STEP_SUMMARY}" + uses: ./actions/setup-uv-cache + with: + python-version: "3.11" + sync-args: --locked --all-extras - name: Build Python package run: uv build diff --git a/README.md b/README.md index 1e3bfba..22e804d 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ keeps the GitHub contract and removes the AWS scheduling layer: - `github_runner_orchestrator/cache.py` - Per-repository cache volume naming and provisioning. - `github_runner_orchestrator/github.py` - GitHub App JWT, installation token, and JIT runner APIs. - `github_runner_orchestrator/webhook.py` - HMAC verification and `workflow_job` filtering. +- `actions/setup-uv-cache/action.yml` - Reusable uv environment archive cache action. +- `actions/setup-rust-cache/action.yml` - Reusable Cargo, sccache, and target cache action. - `sandbox-image/Dockerfile` - Reusable GitHub Actions runner sandbox image built from an OCI base. - `uv.lock` - Locked application and development dependency versions. @@ -292,67 +294,37 @@ result for dependency sets dominated by small prebuilt wheels. For a dependency set where those cross-filesystem copies are expensive, keep the live environment on the sandbox's local disk and store a compressed, compatibility-keyed environment archive on the -volume: +volume with the repository's composite action: ```yaml concurrency: group: uv-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false -env: - PYTHON_VERSION: "3.11" - - uses: actions/checkout@v6 -- uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 +- name: Set up cached uv environment + id: uv-cache + uses: tensorlakeai/tensorlake-github-runners/actions/setup-uv-cache@main with: - enable-cache: false - -- run: uv python install "${PYTHON_VERSION}" - -- name: Restore persistent uv environment - shell: bash - run: | - python_path="$(uv python find "${PYTHON_VERSION}")" - python_identity="$("${python_path}" -c 'import platform, sys; print(f"{sys.implementation.name}-{platform.python_version()}")')" - lock_hash="$(sha256sum uv.lock | cut -d ' ' -f 1)" - scope_hash="$(printf '%s\0%s' "${GITHUB_WORKFLOW}" "${GITHUB_REF}" | sha256sum | cut -c 1-16)" - environment_key="${RUNNER_OS}-${RUNNER_ARCH}-${python_identity}/${GITHUB_JOB}-${scope_hash}-${lock_hash}" - cache_root="${TENSORLAKE_CACHE_DIR:-/mnt/tensorlake-cache}" - environment_dir="${RUNNER_TEMP}/uv-environments/${environment_key}" - environment_archive="${cache_root}/uv-environment-archives-v1/${environment_key}.tar.gz" - mkdir -p "$(dirname "${environment_dir}")" "$(dirname "${environment_archive}")" - if [[ -f "${environment_archive}" ]]; then - tar -xzf "${environment_archive}" -C "$(dirname "${environment_dir}")" - environment_state=warm - else - environment_state=cold - fi - { - echo "UV_PROJECT_ENVIRONMENT=${environment_dir}" - echo "UV_ENVIRONMENT_ARCHIVE=${environment_archive}" - echo "UV_ENVIRONMENT_STATE=${environment_state}" - } >> "${GITHUB_ENV}" + python-version: "3.11" + sync-args: --locked --all-extras -- name: Synchronize and cache the environment - run: | - uv sync --locked - if [[ "${UV_ENVIRONMENT_STATE}" == "cold" ]]; then - temporary_archive="${UV_ENVIRONMENT_ARCHIVE}.tmp.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}" - tar -czf "${temporary_archive}" \ - -C "$(dirname "${UV_PROJECT_ENVIRONMENT}")" \ - "$(basename "${UV_PROJECT_ENVIRONMENT}")" - mv -f "${temporary_archive}" "${UV_ENVIRONMENT_ARCHIVE}" - sync - sleep 6 - fi +- run: uv run --no-sync pytest ``` +`setup-uv-cache` installs uv and the requested Python version, restores a warm archive when one +exists, runs `uv sync`, and atomically publishes the environment after a cold sync. It accepts +`working-directory`, `lockfile`, and `cache-version` inputs for monorepos and manual invalidation. +Its `state`, `key`, `restore-duration-ms`, `sync-duration-ms`, and `publish-duration-ms` outputs are +also written to the job summary. Pin the action to a release tag or full commit in production +instead of tracking `main`. + The archive key separates incompatible operating systems, CPU architectures, exact Python versions, -jobs, Git refs, and lockfiles. The local restore path is stable across runner sandboxes so virtual -environment script paths remain valid. The concurrency group serializes workflows for the same ref, -and the archive is published with an atomic rename so another run never restores a partial file. -Different refs get separate archives even when they use the same lockfile. +jobs, Git refs, working directories, and lockfiles. The local restore path is stable across runner +sandboxes so virtual environment script paths remain valid. The concurrency group serializes +workflows for the same ref, and the archive is published with an atomic rename so another run never +restores a partial file. Different refs get separate archives even when they use the same lockfile. This layout avoids both cross-filesystem package installation and Python's small-file reads over FUSE. It also presents the Cloud Volume autosave engine with one sequential archive instead of @@ -373,32 +345,26 @@ Give each compatible build family its own cache namespace: cache: false # Install sccache with your preferred pinned action, package, or runner image. -- name: Configure persistent Rust caches - shell: bash - env: - CACHE_NAMESPACE: rust-workspace-v1 - run: | - platform="${RUNNER_OS}-${RUNNER_ARCH}" - cargo_home="${TENSORLAKE_CACHE_DIR}/cargo-home/${platform}" - sccache_dir="${TENSORLAKE_CACHE_DIR}/sccache/${platform}/${CACHE_NAMESPACE}" - mkdir -p "${cargo_home}" "${sccache_dir}" - echo "CARGO_HOME=${cargo_home}" >> "${GITHUB_ENV}" - echo "SCCACHE_DIR=${sccache_dir}" >> "${GITHUB_ENV}" - echo "SCCACHE_CACHE_SIZE=50G" >> "${GITHUB_ENV}" - echo "SCCACHE_BASEDIRS=${GITHUB_WORKSPACE}" >> "${GITHUB_ENV}" - echo "RUSTC_WRAPPER=sccache" >> "${GITHUB_ENV}" + +- name: Set up persistent Rust caches + id: rust-cache + uses: tensorlakeai/tensorlake-github-runners/actions/setup-rust-cache@main + with: + cache-namespace: rust-workspace-v1 + disable-incremental: "true" - name: Build run: | - sccache --start-server cargo build --locked --workspace sccache --show-stats - -- name: Stop sccache - if: always() - run: sccache --stop-server || true ``` +`setup-rust-cache` configures a filesystem-safe cache tree for the requested namespace and exports +`CARGO_HOME`, `SCCACHE_DIR`, `SCCACHE_CACHE_SIZE`, `SCCACHE_BASEDIRS`, and `RUSTC_WRAPPER`. +Toolchain and sccache installation stay under workflow control. Set `cache-sccache: "false"` when +sccache is unavailable, and use `sccache-cache-size` to change the default `50G` limit. The action +does not need to start the sccache server explicitly; the first wrapped compilation starts it. + Persisting `CARGO_HOME` reuses registry downloads, Git dependencies, and installed Cargo tools. Authenticate private registries with environment variables; do not run `cargo login` against the persistent `CARGO_HOME`, because it writes credentials. Cloud Volumes reconcile concurrent @@ -410,15 +376,12 @@ Some release builds benefit from reusing the complete `target` directory. Give e build family an explicit versioned namespace: ```yaml -- name: Select Cargo target cache - shell: bash - env: +- name: Set up persistent Rust caches + uses: tensorlakeai/tensorlake-github-runners/actions/setup-rust-cache@main + with: # Include the runtime ABI, Rust target, profile, and a manual version bump. - CACHE_NAMESPACE: rust-release-glibc-2.35-x86_64-unknown-linux-gnu-v2 - run: | - target_dir="${TENSORLAKE_CACHE_DIR}/cargo-target/${CACHE_NAMESPACE}" - mkdir -p "${target_dir}" - echo "CARGO_TARGET_DIR=${target_dir}" >> "${GITHUB_ENV}" + cache-namespace: rust-release-glibc-2.35-x86_64-unknown-linux-gnu-v2 + cache-target: "true" - run: cargo build --locked --release --target x86_64-unknown-linux-gnu ``` diff --git a/actions/setup-rust-cache/action.yml b/actions/setup-rust-cache/action.yml new file mode 100644 index 0000000..e500fd3 --- /dev/null +++ b/actions/setup-rust-cache/action.yml @@ -0,0 +1,162 @@ +name: Set up Tensorlake Rust cache +description: Configure Cargo, sccache, and optional target caches on a Tensorlake Cloud Volume. + +inputs: + cache-namespace: + description: Compatibility namespace shared only by equivalent Rust builds. + required: true + cache-version: + description: Manual cache namespace version used for invalidation. + required: false + default: v1 + working-directory: + description: Rust workspace directory used to normalize sccache paths. + required: false + default: . + cache-cargo-home: + description: Persist CARGO_HOME for registry downloads, Git dependencies, and installed tools. + required: false + default: "true" + cache-sccache: + description: Configure SCCACHE_DIR and RUSTC_WRAPPER. sccache must already be installed. + required: false + default: "true" + sccache-cache-size: + description: Maximum size passed through SCCACHE_CACHE_SIZE. + required: false + default: 50G + cache-target: + description: Persist CARGO_TARGET_DIR. Serialize writers and use only with compatible builds. + required: false + default: "false" + disable-incremental: + description: Set CARGO_INCREMENTAL=0 so Rust compilations are eligible for sccache. + required: false + default: "false" + +outputs: + key: + description: Filesystem-safe key derived from the namespace, version, OS, and architecture. + value: ${{ steps.configure.outputs.key }} + cargo-home: + description: Persistent CARGO_HOME, or an empty string when disabled. + value: ${{ steps.configure.outputs.cargo_home }} + sccache-dir: + description: Persistent SCCACHE_DIR, or an empty string when disabled. + value: ${{ steps.configure.outputs.sccache_dir }} + target-dir: + description: Persistent CARGO_TARGET_DIR, or an empty string when disabled. + value: ${{ steps.configure.outputs.target_dir }} + +runs: + using: composite + steps: + - name: Configure persistent Rust caches + id: configure + shell: bash + env: + INPUT_CACHE_CARGO_HOME: ${{ inputs.cache-cargo-home }} + INPUT_CACHE_NAMESPACE: ${{ inputs.cache-namespace }} + INPUT_CACHE_SCCACHE: ${{ inputs.cache-sccache }} + INPUT_CACHE_TARGET: ${{ inputs.cache-target }} + INPUT_CACHE_VERSION: ${{ inputs.cache-version }} + INPUT_DISABLE_INCREMENTAL: ${{ inputs.disable-incremental }} + INPUT_SCCACHE_CACHE_SIZE: ${{ inputs.sccache-cache-size }} + INPUT_WORKING_DIRECTORY: ${{ inputs.working-directory }} + run: | + set -euo pipefail + + if [[ -z "${TENSORLAKE_CACHE_DIR:-}" ]]; then + echo "::error::The runner did not provide TENSORLAKE_CACHE_DIR." + exit 1 + fi + if ! mountpoint -q "${TENSORLAKE_CACHE_DIR}"; then + echo "::error::${TENSORLAKE_CACHE_DIR} is not a mounted Tensorlake Cloud Volume." + exit 1 + fi + if [[ ! "${INPUT_CACHE_VERSION}" =~ ^[A-Za-z0-9._-]+$ ]]; then + echo "::error::cache-version may contain only letters, numbers, dots, underscores, and hyphens." + exit 1 + fi + + for value in \ + "${INPUT_CACHE_CARGO_HOME}" \ + "${INPUT_CACHE_SCCACHE}" \ + "${INPUT_CACHE_TARGET}" \ + "${INPUT_DISABLE_INCREMENTAL}"; do + if [[ "${value}" != "true" && "${value}" != "false" ]]; then + echo "::error::Boolean inputs must be either true or false." + exit 1 + fi + done + + if [[ "${INPUT_WORKING_DIRECTORY}" = /* ]]; then + working_directory="${INPUT_WORKING_DIRECTORY}" + else + working_directory="${GITHUB_WORKSPACE}/${INPUT_WORKING_DIRECTORY}" + fi + if [[ ! -d "${working_directory}" ]]; then + echo "::error::Working directory does not exist: ${working_directory}" + exit 1 + fi + working_directory="$(cd "${working_directory}" && pwd -P)" + + namespace_slug="$(printf '%s' "${INPUT_CACHE_NAMESPACE}" \ + | tr '[:upper:]' '[:lower:]' \ + | sed -E 's/[^a-z0-9._-]+/-/g; s/^-+//; s/-+$//' \ + | cut -c 1-48)" + namespace_slug="${namespace_slug:-rust-cache}" + namespace_hash="$(printf '%s\0%s' "${INPUT_CACHE_VERSION}" "${INPUT_CACHE_NAMESPACE}" | sha256sum | cut -c 1-12)" + cache_key="${namespace_slug}-${namespace_hash}" + platform="${RUNNER_OS}-${RUNNER_ARCH}" + cache_root="${TENSORLAKE_CACHE_DIR}/rust-cache-${INPUT_CACHE_VERSION}/${platform}/${cache_key}" + cargo_home="" + sccache_dir="" + target_dir="" + + if [[ "${INPUT_CACHE_CARGO_HOME}" == "true" ]]; then + cargo_home="${cache_root}/cargo-home" + mkdir -p "${cargo_home}" + echo "CARGO_HOME=${cargo_home}" >> "${GITHUB_ENV}" + fi + + if [[ "${INPUT_CACHE_SCCACHE}" == "true" ]]; then + if ! command -v sccache >/dev/null 2>&1; then + echo "::error::cache-sccache is enabled but sccache is not installed." + exit 1 + fi + sccache_dir="${cache_root}/sccache" + mkdir -p "${sccache_dir}" + { + echo "SCCACHE_DIR=${sccache_dir}" + echo "SCCACHE_CACHE_SIZE=${INPUT_SCCACHE_CACHE_SIZE}" + echo "SCCACHE_BASEDIRS=${working_directory}" + echo "RUSTC_WRAPPER=sccache" + } >> "${GITHUB_ENV}" + fi + + if [[ "${INPUT_CACHE_TARGET}" == "true" ]]; then + target_dir="${cache_root}/target" + mkdir -p "${target_dir}" + echo "CARGO_TARGET_DIR=${target_dir}" >> "${GITHUB_ENV}" + fi + + if [[ "${INPUT_DISABLE_INCREMENTAL}" == "true" ]]; then + echo "CARGO_INCREMENTAL=0" >> "${GITHUB_ENV}" + fi + + { + echo "key=${cache_key}" + echo "cargo_home=${cargo_home}" + echo "sccache_dir=${sccache_dir}" + echo "target_dir=${target_dir}" + } >> "${GITHUB_OUTPUT}" + + { + echo "### Tensorlake Rust cache" + echo + echo "- Key: \`${cache_key}\`" + echo "- Cargo home: \`${cargo_home:-disabled}\`" + echo "- sccache: \`${sccache_dir:-disabled}\`" + echo "- Cargo target: \`${target_dir:-disabled}\`" + } >> "${GITHUB_STEP_SUMMARY}" diff --git a/actions/setup-uv-cache/action.yml b/actions/setup-uv-cache/action.yml new file mode 100644 index 0000000..2d60c6c --- /dev/null +++ b/actions/setup-uv-cache/action.yml @@ -0,0 +1,184 @@ +name: Set up Tensorlake uv cache +description: Restore, synchronize, and persist a uv environment with a Tensorlake Cloud Volume. + +inputs: + python-version: + description: Python version passed to uv python install and uv python find. + required: true + sync-args: + description: Whitespace-separated arguments passed to uv sync. + required: false + default: --locked + working-directory: + description: Project directory containing the uv lockfile. + required: false + default: . + lockfile: + description: Lockfile path relative to the working directory. + required: false + default: uv.lock + cache-version: + description: Manual cache namespace version used for invalidation. + required: false + default: v1 + +outputs: + state: + description: Whether the environment archive was cold or warm. + value: ${{ steps.cache.outputs.state }} + key: + description: Compatibility key for the environment archive. + value: ${{ steps.cache.outputs.key }} + restore-duration-ms: + description: Time spent restoring the environment archive. + value: ${{ steps.cache.outputs.restore_duration_ms }} + sync-duration-ms: + description: Time spent running uv sync. + value: ${{ steps.cache.outputs.sync_duration_ms }} + publish-duration-ms: + description: Time spent creating and settling a cold environment archive. + value: ${{ steps.cache.outputs.publish_duration_ms }} + +runs: + using: composite + steps: + - name: Verify persistent cache mount + shell: bash + run: | + set -euo pipefail + if [[ -z "${TENSORLAKE_CACHE_DIR:-}" ]]; then + echo "::error::The runner did not provide TENSORLAKE_CACHE_DIR." + exit 1 + fi + if ! mountpoint -q "${TENSORLAKE_CACHE_DIR}"; then + echo "::error::${TENSORLAKE_CACHE_DIR} is not a mounted Tensorlake Cloud Volume." + exit 1 + fi + + - name: Set up uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + enable-cache: false + + - name: Install Python + shell: bash + env: + INPUT_PYTHON_VERSION: ${{ inputs.python-version }} + run: | + set -euo pipefail + uv python install "${INPUT_PYTHON_VERSION}" + + - name: Restore, synchronize, and persist uv environment + id: cache + shell: bash + env: + INPUT_CACHE_VERSION: ${{ inputs.cache-version }} + INPUT_LOCKFILE: ${{ inputs.lockfile }} + INPUT_PYTHON_VERSION: ${{ inputs.python-version }} + INPUT_SYNC_ARGS: ${{ inputs.sync-args }} + INPUT_WORKING_DIRECTORY: ${{ inputs.working-directory }} + run: | + set -euo pipefail + + if [[ ! "${INPUT_CACHE_VERSION}" =~ ^[A-Za-z0-9._-]+$ ]]; then + echo "::error::cache-version may contain only letters, numbers, dots, underscores, and hyphens." + exit 1 + fi + + if [[ "${INPUT_WORKING_DIRECTORY}" = /* ]]; then + working_directory="${INPUT_WORKING_DIRECTORY}" + else + working_directory="${GITHUB_WORKSPACE}/${INPUT_WORKING_DIRECTORY}" + fi + if [[ ! -d "${working_directory}" ]]; then + echo "::error::Working directory does not exist: ${working_directory}" + exit 1 + fi + working_directory="$(cd "${working_directory}" && pwd -P)" + + if [[ "${INPUT_LOCKFILE}" = /* ]]; then + lockfile_path="${INPUT_LOCKFILE}" + else + lockfile_path="${working_directory}/${INPUT_LOCKFILE}" + fi + if [[ ! -f "${lockfile_path}" ]]; then + echo "::error::uv lockfile does not exist: ${lockfile_path}" + exit 1 + fi + + cd "${working_directory}" + python_executable="$(uv python find "${INPUT_PYTHON_VERSION}")" + python_identity="$("${python_executable}" -c 'import platform, sys; print(f"{sys.implementation.name}-{platform.python_version()}")')" + lock_hash="$(sha256sum "${lockfile_path}" | cut -d ' ' -f 1)" + scope_hash="$(printf '%s\0%s\0%s' "${GITHUB_WORKFLOW}" "${GITHUB_REF}" "${INPUT_WORKING_DIRECTORY}" | sha256sum | cut -c 1-16)" + environment_key="${RUNNER_OS}-${RUNNER_ARCH}-${python_identity}/${GITHUB_JOB}-${scope_hash}-${lock_hash}" + environment_dir="${RUNNER_TEMP}/tensorlake-uv-environments/${environment_key}" + environment_archive="${TENSORLAKE_CACHE_DIR}/uv-environment-archives-${INPUT_CACHE_VERSION}/${environment_key}.tar.gz" + restore_duration_ms=0 + publish_duration_ms=0 + mkdir -p "$(dirname "${environment_dir}")" "$(dirname "${environment_archive}")" + + if [[ -f "${environment_archive}" ]]; then + environment_state="warm" + restore_started_ns="$(date +%s%N)" + tar -xzf "${environment_archive}" -C "$(dirname "${environment_dir}")" + restore_finished_ns="$(date +%s%N)" + restore_duration_ms="$(( (restore_finished_ns - restore_started_ns) / 1000000 ))" + if [[ ! -f "${environment_dir}/pyvenv.cfg" ]]; then + echo "::error::The persistent uv environment archive is invalid." + exit 1 + fi + else + environment_state="cold" + fi + + { + echo "UV_PROJECT_ENVIRONMENT=${environment_dir}" + echo "TENSORLAKE_UV_ENVIRONMENT_KEY=${environment_key}" + echo "TENSORLAKE_UV_ENVIRONMENT_STATE=${environment_state}" + } >> "${GITHUB_ENV}" + export UV_PROJECT_ENVIRONMENT="${environment_dir}" + + uv_sync_args=() + if [[ -n "${INPUT_SYNC_ARGS}" ]]; then + read -r -a uv_sync_args <<< "${INPUT_SYNC_ARGS}" + fi + sync_started_ns="$(date +%s%N)" + uv sync "${uv_sync_args[@]}" + sync_finished_ns="$(date +%s%N)" + sync_duration_ms="$(( (sync_finished_ns - sync_started_ns) / 1000000 ))" + + if [[ "${environment_state}" == "cold" ]]; then + publish_started_ns="$(date +%s%N)" + temporary_archive="${environment_archive}.tmp.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}" + trap 'rm -f "${temporary_archive}"' EXIT + tar -czf "${temporary_archive}" \ + -C "$(dirname "${environment_dir}")" \ + "$(basename "${environment_dir}")" + mv -f "${temporary_archive}" "${environment_archive}" + trap - EXIT + sync + sleep 6 + publish_finished_ns="$(date +%s%N)" + publish_duration_ms="$(( (publish_finished_ns - publish_started_ns) / 1000000 ))" + fi + + { + echo "state=${environment_state}" + echo "key=${environment_key}" + echo "restore_duration_ms=${restore_duration_ms}" + echo "sync_duration_ms=${sync_duration_ms}" + echo "publish_duration_ms=${publish_duration_ms}" + } >> "${GITHUB_OUTPUT}" + + echo "UV_SYNC_TIMING duration_ms=${sync_duration_ms} environment_state=${environment_state} key=${environment_key}" + echo "UV_ARCHIVE_TIMING restore_duration_ms=${restore_duration_ms} publish_duration_ms=${publish_duration_ms} environment_state=${environment_state}" + { + echo "### Tensorlake uv environment cache" + echo + echo "- State: \`${environment_state}\`" + echo "- Archive restore: \`${restore_duration_ms}ms\`" + echo "- uv sync: \`${sync_duration_ms}ms\`" + echo "- Archive publish: \`${publish_duration_ms}ms\`" + echo "- Key: \`${environment_key}\`" + } >> "${GITHUB_STEP_SUMMARY}" diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 931f178..e570300 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -48,8 +48,7 @@ def test_readme_explains_the_webhook_and_deployment_order() -> None: def test_self_hosted_workflow_builds_on_tensorlake_runner() -> None: workflow = Path(".github/workflows/build-reference.yml").read_text() assert "runs-on: [self-hosted, tensorlake, tensorlake-small]" in workflow - assert "astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2" in workflow - assert "uv sync --locked --all-extras" in workflow + assert "uses: ./actions/setup-uv-cache" in workflow assert "uv build" in workflow assert "uv run --no-sync pytest -q" in workflow assert "docker run --rm hello-world" in workflow From 862ceacdfa31c6c02fb341351864813165653f8d Mon Sep 17 00:00:00 2001 From: David Calavera <1050+calavera@users.noreply.github.com> Date: Thu, 30 Jul 2026 16:35:09 -0700 Subject: [PATCH 7/7] Constrain Python package discovery --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index f09c2f2..cf5352c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,9 @@ dependencies = [ requires = ["setuptools>=69"] build-backend = "setuptools.build_meta" +[tool.setuptools.packages.find] +include = ["github_runner_orchestrator*"] + [project.optional-dependencies] dev = [ "pytest>=8.0.0",