diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8f349fa..47d03a2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,8 +50,8 @@ jobs: - name: Assert committed inlined yamls match templates + scripts run: | - make inlined-yaml-files + make inline if ! git diff --exit-code inlined/; then - echo "::error::inlined/*.yaml is stale -- run 'make inlined-yaml-files' and commit the result" + echo "::error::inlined/*.yaml is stale -- run 'make inline' and commit the result" exit 1 fi diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4065176..aa36afd 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -49,7 +49,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Regenerate inlined yamls - run: make inlined-yaml-files + run: make inline - name: Restore cached binaries uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/README.md b/README.md index 4d138d2..aa19df6 100644 --- a/README.md +++ b/README.md @@ -55,13 +55,6 @@ BREAD_NET=publish spread # force port-publishing (e.g. to test the macOS path BREAD_NET=bridge spread # force bridge IPs ``` -one more macOS gotcha: spread packs the project with the host `tar`, and macOS `tar` injects `._*` AppleDouble sidecar files (resource-fork metadata) into the archive. they unpack as real files in the container and can break tools that scan for `*.yaml` etc. export `COPYFILE_DISABLE=1` so macOS `tar` skips them: - -``` -export COPYFILE_DISABLE=1 -spread -``` - ## install spread prefer a precompiled spread CLI over `go install`? same release ships statically-linked binaries for linux amd64 / arm64 / s390x / ppc64le: @@ -137,7 +130,7 @@ make build-bread-chisel-releases ARCH=arm64 regenerate distribution yamls (already committed under `inlined/`, but regenerate after script edits): ``` -make inlined-yaml-files +make inline ``` run the demo (LTS systems only -- 24.04 + 26.04 x amd64 / arm64). builds the required `bread` images first if missing: diff --git a/demo/spread.yaml b/demo/spread.yaml index e610401..afd11c1 100644 --- a/demo/spread.yaml +++ b/demo/spread.yaml @@ -11,6 +11,19 @@ exclude: - .git - .github +# macOS tar packs an AppleDouble `._*` sidecar next to every file; they unpack as +# real files on the remote. `exclude:` can't drop them (tar synthesises them below +# its glob filter), so extract and repack with COPYFILE_DISABLE=1. +repack: | + if [ "$(uname -s)" = Darwin ]; then + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + tar -xf - -C "$tmp" <&3 + ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + else + cat <&3 >&4 + fi + backends: docker: type: adhoc diff --git a/hack/inline_scripts.rb b/hack/inline_scripts.rb index 7c4f7b8..0762110 100755 --- a/hack/inline_scripts.rb +++ b/hack/inline_scripts.rb @@ -1,13 +1,12 @@ #!/usr/bin/env ruby # Inline `source scripts/.sh` references inside spread yaml templates. # -# For each line matching `allocate: source scripts/.sh` or -# `discard: source scripts/.sh`, substitute the script's content as a -# yaml block scalar under the matching key. +# For each line matching `: source scripts/.sh`, substitute the +# script's content as a yaml block scalar under the matching key. require "fileutils" -LINE_RE = /\A(?\s*)(?allocate|discard):\s*source\s+(?scripts\/[\w.-]+\.sh)\s*\z/ +LINE_RE = /\A(?\s*)(?allocate|discard|repack):\s*source\s+(?scripts\/[\w.-]+\.sh)\s*\z/ def inline_scripts(content, yaml_indent: 2) out = [] diff --git a/inlined/bread-22.04.yaml b/inlined/bread-22.04.yaml index bbb2308..2671c1c 100644 --- a/inlined/bread-22.04.yaml +++ b/inlined/bread-22.04.yaml @@ -11,6 +11,18 @@ exclude: - .git - .github +repack: | + # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as + # real files on the remote. Repack to remove them. + if [ "$(uname -s)" = Darwin ]; then + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + tar -xf - -C "$tmp" <&3 + ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + else + cat <&3 >&4 + fi + backends: docker: type: adhoc diff --git a/inlined/bread-24.04.yaml b/inlined/bread-24.04.yaml index 81a2c69..810aec9 100644 --- a/inlined/bread-24.04.yaml +++ b/inlined/bread-24.04.yaml @@ -11,6 +11,18 @@ exclude: - .git - .github +repack: | + # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as + # real files on the remote. Repack to remove them. + if [ "$(uname -s)" = Darwin ]; then + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + tar -xf - -C "$tmp" <&3 + ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + else + cat <&3 >&4 + fi + backends: docker: type: adhoc diff --git a/inlined/bread-25.10.yaml b/inlined/bread-25.10.yaml index fdf60ec..98adee1 100644 --- a/inlined/bread-25.10.yaml +++ b/inlined/bread-25.10.yaml @@ -11,6 +11,18 @@ exclude: - .git - .github +repack: | + # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as + # real files on the remote. Repack to remove them. + if [ "$(uname -s)" = Darwin ]; then + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + tar -xf - -C "$tmp" <&3 + ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + else + cat <&3 >&4 + fi + backends: docker: type: adhoc diff --git a/inlined/bread-26.04.yaml b/inlined/bread-26.04.yaml index 4a71f83..58a873a 100644 --- a/inlined/bread-26.04.yaml +++ b/inlined/bread-26.04.yaml @@ -11,6 +11,18 @@ exclude: - .git - .github +repack: | + # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as + # real files on the remote. Repack to remove them. + if [ "$(uname -s)" = Darwin ]; then + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + tar -xf - -C "$tmp" <&3 + ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + else + cat <&3 >&4 + fi + backends: docker: type: adhoc diff --git a/inlined/bread-26.10.yaml b/inlined/bread-26.10.yaml index 5d74d30..490932c 100644 --- a/inlined/bread-26.10.yaml +++ b/inlined/bread-26.10.yaml @@ -11,6 +11,18 @@ exclude: - .git - .github +repack: | + # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as + # real files on the remote. Repack to remove them. + if [ "$(uname -s)" = Darwin ]; then + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + tar -xf - -C "$tmp" <&3 + ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + else + cat <&3 >&4 + fi + backends: docker: type: adhoc diff --git a/inlined/bread-chisel-releases-22.04.yaml b/inlined/bread-chisel-releases-22.04.yaml index 1f9610f..fa3b434 100644 --- a/inlined/bread-chisel-releases-22.04.yaml +++ b/inlined/bread-chisel-releases-22.04.yaml @@ -18,6 +18,18 @@ exclude: - work - rootfs +repack: | + # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as + # real files on the remote. Repack to remove them. + if [ "$(uname -s)" = Darwin ]; then + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + tar -xf - -C "$tmp" <&3 + ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + else + cat <&3 >&4 + fi + backends: docker: type: adhoc diff --git a/inlined/bread-chisel-releases-24.04.yaml b/inlined/bread-chisel-releases-24.04.yaml index 84f217f..f6578c9 100644 --- a/inlined/bread-chisel-releases-24.04.yaml +++ b/inlined/bread-chisel-releases-24.04.yaml @@ -18,6 +18,18 @@ exclude: - work - rootfs +repack: | + # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as + # real files on the remote. Repack to remove them. + if [ "$(uname -s)" = Darwin ]; then + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + tar -xf - -C "$tmp" <&3 + ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + else + cat <&3 >&4 + fi + backends: docker: type: adhoc diff --git a/inlined/bread-chisel-releases-25.10.yaml b/inlined/bread-chisel-releases-25.10.yaml index 3c6916e..8786ab9 100644 --- a/inlined/bread-chisel-releases-25.10.yaml +++ b/inlined/bread-chisel-releases-25.10.yaml @@ -18,6 +18,18 @@ exclude: - work - rootfs +repack: | + # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as + # real files on the remote. Repack to remove them. + if [ "$(uname -s)" = Darwin ]; then + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + tar -xf - -C "$tmp" <&3 + ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + else + cat <&3 >&4 + fi + backends: docker: type: adhoc diff --git a/inlined/bread-chisel-releases-26.04.yaml b/inlined/bread-chisel-releases-26.04.yaml index 01227d7..9f3e26f 100644 --- a/inlined/bread-chisel-releases-26.04.yaml +++ b/inlined/bread-chisel-releases-26.04.yaml @@ -18,6 +18,18 @@ exclude: - work - rootfs +repack: | + # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as + # real files on the remote. Repack to remove them. + if [ "$(uname -s)" = Darwin ]; then + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + tar -xf - -C "$tmp" <&3 + ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + else + cat <&3 >&4 + fi + backends: docker: type: adhoc diff --git a/inlined/bread-chisel-releases-26.10.yaml b/inlined/bread-chisel-releases-26.10.yaml index 3ce2cdb..16f319b 100644 --- a/inlined/bread-chisel-releases-26.10.yaml +++ b/inlined/bread-chisel-releases-26.10.yaml @@ -18,6 +18,18 @@ exclude: - work - rootfs +repack: | + # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as + # real files on the remote. Repack to remove them. + if [ "$(uname -s)" = Darwin ]; then + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + tar -xf - -C "$tmp" <&3 + ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + else + cat <&3 >&4 + fi + backends: docker: type: adhoc diff --git a/makefile b/makefile index 8449f82..a70c018 100644 --- a/makefile +++ b/makefile @@ -56,7 +56,7 @@ help: ## Show this help awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-40s\033[0m %s\n", $$1, $$2}' .PHONY: all -all: build-all inlined-yaml-files ## Build all images + generate inlined yamls +all: build-all inline ## Build all images + generate inlined yamls .PHONY: build-all build-all: build-bread build-bread-chisel-releases ## Build all images (narrow via VER=... ARCH=...) @@ -84,7 +84,7 @@ SPREAD_ARGS ?= TEST_STAMPS := .stamp/bread-test-26.04-$(SELECTED_ARCH) \ $(foreach v,$(VERSIONS),.stamp/bread-$(v)-$(SELECTED_ARCH) .stamp/bread-chisel-releases-$(v)-$(SELECTED_ARCH)) .PHONY: test -test: $(TEST_STAMPS) inlined-yaml-files ## Run the spread test suite (host arch; ARCH=... to override, SPREAD_ARGS=... for flags) +test: $(TEST_STAMPS) inline ## Run the spread test suite (host arch; ARCH=... to override, SPREAD_ARGS=... for flags) cd tests && spread $(SPREAD_ARGS) outer:ubuntu-26.04-$(SELECTED_ARCH) .PHONY: check-base @@ -102,8 +102,8 @@ DEMO_STAMPS := $(foreach a,$(NATIVE_ARCHES),.stamp/bread-24.04-$(a) .stamp/bread demo: $(DEMO_STAMPS) ## Run the spread demo on LTS systems (24.04 + 26.04, native arches) $(MAKE) -C demo run -.PHONY: inlined-yaml-files -inlined-yaml-files: $(INLINED) ## Generate inlined/*.yaml from templates/*.yaml.in +.PHONY: inline +inline: $(INLINED) ## Generate inlined/*.yaml from templates/*.yaml.in # Hash-stamp pattern: stamp file contents = hash of all inputs that affect # this image. FORCE-dep makes us recompute hash each run; stamp content only diff --git a/scripts/spread_repack_bread.sh b/scripts/spread_repack_bread.sh new file mode 100644 index 0000000..6389089 --- /dev/null +++ b/scripts/spread_repack_bread.sh @@ -0,0 +1,10 @@ +# macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as +# real files on the remote. Repack to remove them. +if [ "$(uname -s)" = Darwin ]; then + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + tar -xf - -C "$tmp" <&3 + ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 +else + cat <&3 >&4 +fi diff --git a/templates/bread-22.04.yaml.in b/templates/bread-22.04.yaml.in index bf8e8f5..d73e867 100644 --- a/templates/bread-22.04.yaml.in +++ b/templates/bread-22.04.yaml.in @@ -11,6 +11,8 @@ exclude: - .git - .github +repack: source scripts/spread_repack_bread.sh + backends: docker: type: adhoc diff --git a/templates/bread-24.04.yaml.in b/templates/bread-24.04.yaml.in index 9f6a798..bda04df 100644 --- a/templates/bread-24.04.yaml.in +++ b/templates/bread-24.04.yaml.in @@ -11,6 +11,8 @@ exclude: - .git - .github +repack: source scripts/spread_repack_bread.sh + backends: docker: type: adhoc diff --git a/templates/bread-25.10.yaml.in b/templates/bread-25.10.yaml.in index 8c5219d..bb6f415 100644 --- a/templates/bread-25.10.yaml.in +++ b/templates/bread-25.10.yaml.in @@ -11,6 +11,8 @@ exclude: - .git - .github +repack: source scripts/spread_repack_bread.sh + backends: docker: type: adhoc diff --git a/templates/bread-26.04.yaml.in b/templates/bread-26.04.yaml.in index 9aabed8..c2e2de7 100644 --- a/templates/bread-26.04.yaml.in +++ b/templates/bread-26.04.yaml.in @@ -11,6 +11,8 @@ exclude: - .git - .github +repack: source scripts/spread_repack_bread.sh + backends: docker: type: adhoc diff --git a/templates/bread-26.10.yaml.in b/templates/bread-26.10.yaml.in index b808a76..2710a04 100644 --- a/templates/bread-26.10.yaml.in +++ b/templates/bread-26.10.yaml.in @@ -11,6 +11,8 @@ exclude: - .git - .github +repack: source scripts/spread_repack_bread.sh + backends: docker: type: adhoc diff --git a/templates/bread-chisel-releases-22.04.yaml.in b/templates/bread-chisel-releases-22.04.yaml.in index 9f2f923..51fbb34 100644 --- a/templates/bread-chisel-releases-22.04.yaml.in +++ b/templates/bread-chisel-releases-22.04.yaml.in @@ -18,6 +18,8 @@ exclude: - work - rootfs +repack: source scripts/spread_repack_bread.sh + backends: docker: type: adhoc diff --git a/templates/bread-chisel-releases-24.04.yaml.in b/templates/bread-chisel-releases-24.04.yaml.in index 791d80b..080edfe 100644 --- a/templates/bread-chisel-releases-24.04.yaml.in +++ b/templates/bread-chisel-releases-24.04.yaml.in @@ -18,6 +18,8 @@ exclude: - work - rootfs +repack: source scripts/spread_repack_bread.sh + backends: docker: type: adhoc diff --git a/templates/bread-chisel-releases-25.10.yaml.in b/templates/bread-chisel-releases-25.10.yaml.in index a461dda..bccf42a 100644 --- a/templates/bread-chisel-releases-25.10.yaml.in +++ b/templates/bread-chisel-releases-25.10.yaml.in @@ -18,6 +18,8 @@ exclude: - work - rootfs +repack: source scripts/spread_repack_bread.sh + backends: docker: type: adhoc diff --git a/templates/bread-chisel-releases-26.04.yaml.in b/templates/bread-chisel-releases-26.04.yaml.in index 2e8940a..0174650 100644 --- a/templates/bread-chisel-releases-26.04.yaml.in +++ b/templates/bread-chisel-releases-26.04.yaml.in @@ -18,6 +18,8 @@ exclude: - work - rootfs +repack: source scripts/spread_repack_bread.sh + backends: docker: type: adhoc diff --git a/templates/bread-chisel-releases-26.10.yaml.in b/templates/bread-chisel-releases-26.10.yaml.in index b14aca6..53fc3dd 100644 --- a/templates/bread-chisel-releases-26.10.yaml.in +++ b/templates/bread-chisel-releases-26.10.yaml.in @@ -18,6 +18,8 @@ exclude: - work - rootfs +repack: source scripts/spread_repack_bread.sh + backends: docker: type: adhoc diff --git a/tests/spread.yaml b/tests/spread.yaml index a798942..45db283 100644 --- a/tests/spread.yaml +++ b/tests/spread.yaml @@ -12,6 +12,18 @@ exclude: - .git - .github +# macOS tar packs an AppleDouble `._*` sidecar next to every file; they unpack as +# real files on the remote. Repack to remove them. +repack: | + if [ "$(uname -s)" = Darwin ]; then + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + tar -xf - -C "$tmp" <&3 + ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + else + cat <&3 >&4 + fi + backends: outer: type: adhoc