diff --git a/README.md b/README.md index aa19df6..d50b10b 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,12 @@ BREAD_NET=publish spread # force port-publishing (e.g. to test the macOS path BREAD_NET=bridge spread # force bridge IPs ``` +### tar under emulation (amd64 on apple silicon) + +ubuntu 26.04's patched GNU tar (`1.35+dfsg-4ubuntu0.x`) resolves extraction paths through a syscall Docker Desktop's Rosetta emulation does not implement, so in an amd64 container on apple silicon every archive entry below the top level fails with `Function not implemented`. spread hits this when it unpacks the project, reports `cannot send project content`, and after three tries gives up with `Cannot allocate ... after too many retries`. 26.10 and everything at or below 25.10 are unaffected, as are the qemu-emulated arches. + +the images work around it: `/bin/tar` is a shim that probes GNU tar once per container and, if it is broken, routes *extraction* to `bsdtar` (see `hack/tar-shim.sh`). creation stays on GNU tar, which spread needs for `--sort=name` when it packs artifacts. on unaffected hosts the shim is inert. + ## install spread prefer a precompiled spread CLI over `go install`? same release ships statically-linked binaries for linux amd64 / arm64 / s390x / ppc64le: @@ -93,6 +99,7 @@ spread-bread/ hash_inputs.sh # per-image input hash (drives stamp invalidation) check_base.sh # detect upstream ubuntu base digest drift; rewrite @sha256 pins inline_scripts.rb # splice scripts/*.sh into yaml templates + tar-shim.sh # image /bin/tar; routes extraction to bsdtar where gnu tar is broken scripts/ # allocate / discard scripts, one pair per flavour images/ # one Dockerfile per (flavour, ubuntu version) templates/ # yaml templates with `source scripts/...` markers diff --git a/demo/spread.yaml b/demo/spread.yaml index afd11c1..ef3b491 100644 --- a/demo/spread.yaml +++ b/demo/spread.yaml @@ -11,15 +11,16 @@ exclude: - .git - .github -# macOS tar packs an AppleDouble `._*` sidecar next to every file; they unpack as -# real files on the remote. `exclude:` can't drop them (tar synthesises them below -# its glob filter), so extract and repack with COPYFILE_DISABLE=1. +# macOS tar packs an AppleDouble `._*` sidecar next to every file and stores +# xattrs as pax headers; the sidecars unpack as real files on the remote and GNU +# tar warns about every xattr header. `exclude:` can't drop them (tar synthesises +# them below its glob filter), so extract and repack without either. repack: | if [ "$(uname -s)" = Darwin ]; then tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT tar -xf - -C "$tmp" <&3 - ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + ( cd "$tmp" && COPYFILE_DISABLE=1 tar --no-mac-metadata --no-xattrs -cf - . ) >&4 else cat <&3 >&4 fi @@ -61,7 +62,9 @@ backends: # snippet thanks to @lengau # https://github.com/canonical/charmcraft/blob/120a00a50f7ed3d0ae2fc2bea69e2e43b68b1594/spread.yaml#L72-L79 sleep 0.$RANDOM # Minimize chances of a race condition - export counter_file=".spread-worker-num" + # Kept out of the project tree: whatever lives there is packed up + # and shipped to every container. + export counter_file="${TMPDIR:-/tmp}/spread-bread-worker-num" instance_num=$( flock -x $counter_file bash -c ' [ -s $counter_file ] || echo 0 > $counter_file @@ -84,10 +87,42 @@ backends: -e "usr=$SPREAD_SYSTEM_USERNAME" \ -e "pass=$SPREAD_SYSTEM_PASSWORD" \ $publish_flag \ + --label spread-bread \ --name "$container_name" \ -d "$image" - until docker exec "$container_name" pgrep sshd; do sleep 1; done + # sshd is up within a second or two. The bound and the liveness check are + # backstops: a container that dies on start would otherwise spin here + # until spread's own timeout, which reports nothing about the container. + sshd_up="" + for ((i = 0; i < 60; i++)); do + if docker exec "$container_name" pgrep -x sshd >/dev/null 2>&1; then + sshd_up=1 + break + fi + if ! docker inspect "$container_name" --format '{{.State.Status}}' 2>/dev/null | grep -qx running; then + echo "$container_name is not running; reproduce with: docker run --rm --platform linux/$arch $image" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + sleep 1 + done + if [ -z "$sshd_up" ]; then + echo "sshd did not come up in $container_name after 60s" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + + # A container whose gnu tar cannot extract falls back to bsdtar (see + # hack/tar-shim.sh); say so rather than swapping the tool silently. + tar_backend=$(docker exec "$container_name" /usr/local/bin/bread-tar-shim --bread-probe 2>/dev/null || echo unknown) + if [ "$tar_backend" = bsdtar ]; then + note="note: gnu tar cannot extract in $container_name (host emulation); using bsdtar" + # spread buffers allocate output and only prints it on failure, so the + # terminal (when there is one) is the only channel a user actually reads. + echo "$note" + { [ -w /dev/tty ] && echo "$note" > /dev/tty; } 2>/dev/null || true + fi if [ "$mode" = publish ]; then port=$(docker port "$container_name" 22 | head -n1 | cut -d: -f2) @@ -109,24 +144,18 @@ backends: container_name="" case "$SPREAD_SYSTEM_ADDRESS" in *:*) + # Matched on the ports column, e.g. "127.0.0.1:32768->22/tcp". target_port="${SPREAD_SYSTEM_ADDRESS##*:}" - for cid in $(docker ps -a --format '{{.ID}}'); do - if docker port "$cid" 22 2>/dev/null | grep -q ":${target_port}\$"; then - container_name=$(docker inspect "$cid" --format '{{.Name}}' | sed 's#^/##') - break - fi - done + container_name=$(docker ps -a --filter label=spread-bread --format '{{.Names}} {{.Ports}}' | + awk -v m=":${target_port}->22/" 'index($0, m) { print $1; exit }') not_found="No container found with published port: $target_port" ;; *) - for cid in $(docker ps -a --filter "network=bridge" --format '{{.ID}}'); do - cname=$(docker inspect "$cid" --format '{{.Name}}' | sed 's/^\/\(.*\)/\1/') - cip=$(docker inspect "$cid" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}' || echo "") - if [ "$cip" == "$SPREAD_SYSTEM_ADDRESS" ]; then - container_name="$cname" - break - fi - done + ids=$(docker ps -a --filter label=spread-bread --filter network=bridge --format '{{.ID}}') + if [ -n "$ids" ]; then + container_name=$(docker inspect $ids --format '{{.Name}} {{.NetworkSettings.Networks.bridge.IPAddress}}' | + awk -v ip="$SPREAD_SYSTEM_ADDRESS" '$2 == ip { print substr($1, 2); exit }') + fi not_found="No container found with IP address: $SPREAD_SYSTEM_ADDRESS" ;; esac diff --git a/hack/hash_inputs.sh b/hack/hash_inputs.sh index 573a261..ad3edd4 100755 --- a/hack/hash_inputs.sh +++ b/hack/hash_inputs.sh @@ -39,6 +39,7 @@ case "$flavour" in "images/Dockerfile.bread-$ver" "hack/bread-warning.sh" "hack/banner.txt" + "hack/tar-shim.sh" ) ;; bread-chisel-releases) diff --git a/hack/tar-shim.sh b/hack/tar-shim.sh new file mode 100644 index 0000000..e7815f2 --- /dev/null +++ b/hack/tar-shim.sh @@ -0,0 +1,79 @@ +#!/bin/sh +# tar-shim: routes extraction to bsdtar on hosts where GNU tar cannot do it. +# +# Installed as /usr/bin/tar, with the real binary diverted to +# /usr/bin/tar.distrib. It has to sit on that path rather than earlier in +# PATH because spread invokes /bin/tar by absolute path. +# +# Ubuntu's patched tar (26.04 ships 1.35+dfsg-4ubuntu0.x) resolves extraction +# paths through a syscall Docker Desktop's Rosetta emulation does not +# implement, so in an amd64 container on Apple Silicon every entry below the +# top level fails with ENOSYS -- which spread reports as "cannot send project +# content", then as a failure to allocate the system. bsdtar is unaffected. +# +# Only extraction is routed: bsdtar has no --sort=name, which spread passes +# when packing artifacts. The probe runs once per container, so on unaffected +# hosts every call is plain GNU tar. + +_real=/usr/bin/tar.distrib +_checked=/run/bread-tar-checked +_broken=/run/bread-tar-broken + +_extracting() { + # Bare mode letters are only valid as the first argument; elsewhere an x + # is just as likely to be an option's value. + case "$1" in + (x*) return 0 ;; + esac + for _a in "$@"; do + case "$_a" in + (--) return 1 ;; + (--extract|--get) return 0 ;; + (--*) ;; + (-*x*) return 0 ;; + esac + done + return 1 +} + +_gnu_tar_broken() { + [ -e "$_broken" ] && return 0 + [ -e "$_checked" ] && return 1 + + _d=$(mktemp -d) || return 1 + _bad=0 + mkdir -p "$_d/a/b" && : > "$_d/a/b/f" \ + && "$_real" -cf "$_d/t.tar" -C "$_d" a && rm -rf "$_d/a" \ + && "$_real" -xf "$_d/t.tar" -C "$_d" 2>/dev/null && [ -f "$_d/a/b/f" ] \ + || _bad=1 + rm -rf "$_d" + + # Markers are a cache; a read-only /run just means every call probes. + : > "$_checked" 2>/dev/null + [ "$_bad" = 0 ] && return 1 + : > "$_broken" 2>/dev/null + return 0 +} + +# Reports which backend extraction would use, and warms the probe cache. The +# allocate scripts call it so the fallback shows up in the spread log instead +# of happening silently. +if [ "$1" = --bread-probe ]; then + if command -v bsdtar >/dev/null 2>&1 && _gnu_tar_broken; then + echo bsdtar + else + echo gnu + fi + exit 0 +fi + +if _extracting "$@" && command -v bsdtar >/dev/null 2>&1 && _gnu_tar_broken; then + # bsdtar has no bare mode letters, and spread sends with `tar xz`. + case "$1" in + (-*) ;; + (*) _mode="-$1"; shift; set -- "$_mode" "$@" ;; + esac + exec bsdtar --no-xattrs --no-mac-metadata "$@" +fi + +exec "$_real" "$@" diff --git a/images/Dockerfile.bread-22.04 b/images/Dockerfile.bread-22.04 index 15cf957..5194868 100644 --- a/images/Dockerfile.bread-22.04 +++ b/images/Dockerfile.bread-22.04 @@ -8,7 +8,7 @@ FROM docker.io/library/ubuntu:22.04@sha256:0e0a0fc6d18feda9db1590da249ac93e8d5ab RUN printf 'path-exclude /usr/share/man/*\npath-exclude /usr/share/doc/*\npath-exclude /usr/share/info/*\n' \ > /etc/dpkg/dpkg.cfg.d/01-nodoc && \ apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server libarchive-tools && \ mkdir /var/run/sshd || true && \ echo 'root:bread' | chpasswd && \ echo "PermitRootLogin yes" >> /etc/ssh/sshd_config && \ @@ -33,5 +33,12 @@ COPY hack/bread-warning.sh /etc/profile.d/00-bread-warning.sh RUN chmod 0644 /etc/bread-banner.txt /etc/profile.d/00-bread-warning.sh && \ printf '\n# bread-warning\n[ -r /etc/profile.d/00-bread-warning.sh ] && . /etc/profile.d/00-bread-warning.sh\n' >> /root/.bashrc +# Extraction shim over /bin/tar, inert unless the host's emulation breaks GNU +# tar. See hack/tar-shim.sh. +COPY hack/tar-shim.sh /usr/local/bin/bread-tar-shim +RUN chmod 0755 /usr/local/bin/bread-tar-shim && \ + dpkg-divert --local --divert /usr/bin/tar.distrib --rename /usr/bin/tar && \ + ln -s /usr/local/bin/bread-tar-shim /usr/bin/tar + EXPOSE 22 CMD ["/usr/sbin/sshd", "-D"] diff --git a/images/Dockerfile.bread-24.04 b/images/Dockerfile.bread-24.04 index b9b1804..0ef5c2b 100644 --- a/images/Dockerfile.bread-24.04 +++ b/images/Dockerfile.bread-24.04 @@ -8,7 +8,7 @@ FROM docker.io/library/ubuntu:24.04@sha256:4fbb8e6a8395de5a7550b33509421a2bafbc0 RUN printf 'path-exclude /usr/share/man/*\npath-exclude /usr/share/doc/*\npath-exclude /usr/share/info/*\n' \ > /etc/dpkg/dpkg.cfg.d/01-nodoc && \ apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server libarchive-tools && \ mkdir /var/run/sshd || true && \ echo 'root:bread' | chpasswd && \ echo "PermitRootLogin yes" >> /etc/ssh/sshd_config && \ @@ -33,5 +33,12 @@ COPY hack/bread-warning.sh /etc/profile.d/00-bread-warning.sh RUN chmod 0644 /etc/bread-banner.txt /etc/profile.d/00-bread-warning.sh && \ printf '\n# bread-warning\n[ -r /etc/profile.d/00-bread-warning.sh ] && . /etc/profile.d/00-bread-warning.sh\n' >> /root/.bashrc +# Extraction shim over /bin/tar, inert unless the host's emulation breaks GNU +# tar. See hack/tar-shim.sh. +COPY hack/tar-shim.sh /usr/local/bin/bread-tar-shim +RUN chmod 0755 /usr/local/bin/bread-tar-shim && \ + dpkg-divert --local --divert /usr/bin/tar.distrib --rename /usr/bin/tar && \ + ln -s /usr/local/bin/bread-tar-shim /usr/bin/tar + EXPOSE 22 CMD ["/usr/sbin/sshd", "-D"] diff --git a/images/Dockerfile.bread-25.10 b/images/Dockerfile.bread-25.10 index 767eb82..8d5df43 100644 --- a/images/Dockerfile.bread-25.10 +++ b/images/Dockerfile.bread-25.10 @@ -8,7 +8,7 @@ FROM docker.io/library/ubuntu:25.10@sha256:7cc5e35f6567ee8c66d2abb4aab0fd866669e RUN printf 'path-exclude /usr/share/man/*\npath-exclude /usr/share/doc/*\npath-exclude /usr/share/info/*\n' \ > /etc/dpkg/dpkg.cfg.d/01-nodoc && \ apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server libarchive-tools && \ mkdir /var/run/sshd || true && \ echo 'root:bread' | chpasswd && \ echo "PermitRootLogin yes" >> /etc/ssh/sshd_config && \ @@ -33,5 +33,12 @@ COPY hack/bread-warning.sh /etc/profile.d/00-bread-warning.sh RUN chmod 0644 /etc/bread-banner.txt /etc/profile.d/00-bread-warning.sh && \ printf '\n# bread-warning\n[ -r /etc/profile.d/00-bread-warning.sh ] && . /etc/profile.d/00-bread-warning.sh\n' >> /root/.bashrc +# Extraction shim over /bin/tar, inert unless the host's emulation breaks GNU +# tar. See hack/tar-shim.sh. +COPY hack/tar-shim.sh /usr/local/bin/bread-tar-shim +RUN chmod 0755 /usr/local/bin/bread-tar-shim && \ + dpkg-divert --local --divert /usr/bin/tar.distrib --rename /usr/bin/tar && \ + ln -s /usr/local/bin/bread-tar-shim /usr/bin/tar + EXPOSE 22 CMD ["/usr/sbin/sshd", "-D"] diff --git a/images/Dockerfile.bread-26.04 b/images/Dockerfile.bread-26.04 index 307eee7..59cab2b 100644 --- a/images/Dockerfile.bread-26.04 +++ b/images/Dockerfile.bread-26.04 @@ -8,7 +8,7 @@ FROM docker.io/library/ubuntu:26.04@sha256:3131b4cc82a783df6c9df078f86e01819a135 RUN printf 'path-exclude /usr/share/man/*\npath-exclude /usr/share/doc/*\npath-exclude /usr/share/info/*\n' \ > /etc/dpkg/dpkg.cfg.d/01-nodoc && \ apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server libarchive-tools && \ mkdir /var/run/sshd || true && \ echo 'root:bread' | chpasswd && \ echo "PermitRootLogin yes" >> /etc/ssh/sshd_config && \ @@ -33,5 +33,12 @@ COPY hack/bread-warning.sh /etc/profile.d/00-bread-warning.sh RUN chmod 0644 /etc/bread-banner.txt /etc/profile.d/00-bread-warning.sh && \ printf '\n# bread-warning\n[ -r /etc/profile.d/00-bread-warning.sh ] && . /etc/profile.d/00-bread-warning.sh\n' >> /root/.bashrc +# Extraction shim over /bin/tar, inert unless the host's emulation breaks GNU +# tar. See hack/tar-shim.sh. +COPY hack/tar-shim.sh /usr/local/bin/bread-tar-shim +RUN chmod 0755 /usr/local/bin/bread-tar-shim && \ + dpkg-divert --local --divert /usr/bin/tar.distrib --rename /usr/bin/tar && \ + ln -s /usr/local/bin/bread-tar-shim /usr/bin/tar + EXPOSE 22 CMD ["/usr/sbin/sshd", "-D"] diff --git a/images/Dockerfile.bread-26.10 b/images/Dockerfile.bread-26.10 index 84f170a..2faf951 100644 --- a/images/Dockerfile.bread-26.10 +++ b/images/Dockerfile.bread-26.10 @@ -8,7 +8,7 @@ FROM docker.io/library/ubuntu:26.10@sha256:694b773ee7e0d0b55ca74c095ac3309055589 RUN printf 'path-exclude /usr/share/man/*\npath-exclude /usr/share/doc/*\npath-exclude /usr/share/info/*\n' \ > /etc/dpkg/dpkg.cfg.d/01-nodoc && \ apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends openssh-server libarchive-tools && \ mkdir /var/run/sshd || true && \ echo 'root:bread' | chpasswd && \ echo "PermitRootLogin yes" >> /etc/ssh/sshd_config && \ @@ -33,5 +33,12 @@ COPY hack/bread-warning.sh /etc/profile.d/00-bread-warning.sh RUN chmod 0644 /etc/bread-banner.txt /etc/profile.d/00-bread-warning.sh && \ printf '\n# bread-warning\n[ -r /etc/profile.d/00-bread-warning.sh ] && . /etc/profile.d/00-bread-warning.sh\n' >> /root/.bashrc +# Extraction shim over /bin/tar, inert unless the host's emulation breaks GNU +# tar. See hack/tar-shim.sh. +COPY hack/tar-shim.sh /usr/local/bin/bread-tar-shim +RUN chmod 0755 /usr/local/bin/bread-tar-shim && \ + dpkg-divert --local --divert /usr/bin/tar.distrib --rename /usr/bin/tar && \ + ln -s /usr/local/bin/bread-tar-shim /usr/bin/tar + EXPOSE 22 CMD ["/usr/sbin/sshd", "-D"] diff --git a/inlined/bread-22.04.yaml b/inlined/bread-22.04.yaml index 2671c1c..3708741 100644 --- a/inlined/bread-22.04.yaml +++ b/inlined/bread-22.04.yaml @@ -12,13 +12,14 @@ exclude: - .github repack: | - # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as - # real files on the remote. Repack to remove them. + # macOS tar packs an AppleDouble ._* sidecar next to every file and stores + # xattrs as pax headers; the sidecars unpack as real files on the remote and + # GNU tar warns about every xattr header. Repack to remove both. if [ "$(uname -s)" = Darwin ]; then tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT tar -xf - -C "$tmp" <&3 - ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + ( cd "$tmp" && COPYFILE_DISABLE=1 tar --no-mac-metadata --no-xattrs -cf - . ) >&4 else cat <&3 >&4 fi @@ -63,7 +64,9 @@ backends: # snippet thanks to @lengau # https://github.com/canonical/charmcraft/blob/120a00a50f7ed3d0ae2fc2bea69e2e43b68b1594/spread.yaml#L72-L79 sleep 0.$RANDOM # Minimize chances of a race condition - export counter_file=".spread-worker-num" + # Kept out of the project tree: whatever lives there is packed up and + # shipped to every container. + export counter_file="${TMPDIR:-/tmp}/spread-bread-worker-num" instance_num=$( flock -x $counter_file bash -c ' [ -s $counter_file ] || echo 0 > $counter_file @@ -87,10 +90,42 @@ backends: -e "usr=$SPREAD_SYSTEM_USERNAME" \ -e "pass=$SPREAD_SYSTEM_PASSWORD" \ $publish_flag \ + --label spread-bread \ --name "$container_name" \ -d "$image" - until docker exec "$container_name" pgrep sshd; do sleep 1; done + # sshd is up within a second or two. The bound and the liveness check are + # backstops: a container that dies on start would otherwise spin here until + # spread's own timeout, which reports nothing about the container. + sshd_up="" + for ((i = 0; i < 60; i++)); do + if docker exec "$container_name" pgrep -x sshd >/dev/null 2>&1; then + sshd_up=1 + break + fi + if ! docker inspect "$container_name" --format '{{.State.Status}}' 2>/dev/null | grep -qx running; then + echo "$container_name is not running; reproduce with: docker run --rm --platform linux/$arch $image" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + sleep 1 + done + if [ -z "$sshd_up" ]; then + echo "sshd did not come up in $container_name after 60s" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + + # A container whose gnu tar cannot extract falls back to bsdtar (see + # hack/tar-shim.sh); say so rather than swapping the tool silently. + tar_backend=$(docker exec "$container_name" /usr/local/bin/bread-tar-shim --bread-probe 2>/dev/null || echo unknown) + if [ "$tar_backend" = bsdtar ]; then + note="note: gnu tar cannot extract in $container_name (host emulation); using bsdtar" + # spread buffers allocate output and only prints it on failure, so the + # terminal (when there is one) is the only channel a user actually reads. + echo "$note" + { [ -w /dev/tty ] && echo "$note" > /dev/tty; } 2>/dev/null || true + fi if [ "$mode" = publish ]; then # The ephemeral host port docker mapped to the container's sshd. @@ -114,26 +149,20 @@ backends: container_name="" case "$SPREAD_SYSTEM_ADDRESS" in *:*) - # publish mode: match the container by its published host port. + # publish mode: match the container by its published host port, as it + # shows up in the ports column, e.g. "127.0.0.1:32768->22/tcp". target_port="${SPREAD_SYSTEM_ADDRESS##*:}" - for cid in $(docker ps -a --format '{{.ID}}'); do - if docker port "$cid" 22 2>/dev/null | grep -q ":${target_port}\$"; then - container_name=$(docker inspect "$cid" --format '{{.Name}}' | sed 's#^/##') - break - fi - done + container_name=$(docker ps -a --filter label=spread-bread --format '{{.Names}} {{.Ports}}' | + awk -v m=":${target_port}->22/" 'index($0, m) { print $1; exit }') not_found="No container found with published port: $target_port" ;; *) # bridge mode: match the container by its bridge IP address. - for cid in $(docker ps -a --filter "network=bridge" --format '{{.ID}}'); do - cname=$(docker inspect "$cid" --format '{{.Name}}' | sed 's/^\/\(.*\)/\1/') - cip=$(docker inspect "$cid" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}' || echo "") - if [ "$cip" == "$SPREAD_SYSTEM_ADDRESS" ]; then - container_name="$cname" - break - fi - done + ids=$(docker ps -a --filter label=spread-bread --filter network=bridge --format '{{.ID}}') + if [ -n "$ids" ]; then + container_name=$(docker inspect $ids --format '{{.Name}} {{.NetworkSettings.Networks.bridge.IPAddress}}' | + awk -v ip="$SPREAD_SYSTEM_ADDRESS" '$2 == ip { print substr($1, 2); exit }') + fi not_found="No container found with IP address: $SPREAD_SYSTEM_ADDRESS" ;; esac diff --git a/inlined/bread-24.04.yaml b/inlined/bread-24.04.yaml index 810aec9..89955f2 100644 --- a/inlined/bread-24.04.yaml +++ b/inlined/bread-24.04.yaml @@ -12,13 +12,14 @@ exclude: - .github repack: | - # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as - # real files on the remote. Repack to remove them. + # macOS tar packs an AppleDouble ._* sidecar next to every file and stores + # xattrs as pax headers; the sidecars unpack as real files on the remote and + # GNU tar warns about every xattr header. Repack to remove both. if [ "$(uname -s)" = Darwin ]; then tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT tar -xf - -C "$tmp" <&3 - ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + ( cd "$tmp" && COPYFILE_DISABLE=1 tar --no-mac-metadata --no-xattrs -cf - . ) >&4 else cat <&3 >&4 fi @@ -63,7 +64,9 @@ backends: # snippet thanks to @lengau # https://github.com/canonical/charmcraft/blob/120a00a50f7ed3d0ae2fc2bea69e2e43b68b1594/spread.yaml#L72-L79 sleep 0.$RANDOM # Minimize chances of a race condition - export counter_file=".spread-worker-num" + # Kept out of the project tree: whatever lives there is packed up and + # shipped to every container. + export counter_file="${TMPDIR:-/tmp}/spread-bread-worker-num" instance_num=$( flock -x $counter_file bash -c ' [ -s $counter_file ] || echo 0 > $counter_file @@ -87,10 +90,42 @@ backends: -e "usr=$SPREAD_SYSTEM_USERNAME" \ -e "pass=$SPREAD_SYSTEM_PASSWORD" \ $publish_flag \ + --label spread-bread \ --name "$container_name" \ -d "$image" - until docker exec "$container_name" pgrep sshd; do sleep 1; done + # sshd is up within a second or two. The bound and the liveness check are + # backstops: a container that dies on start would otherwise spin here until + # spread's own timeout, which reports nothing about the container. + sshd_up="" + for ((i = 0; i < 60; i++)); do + if docker exec "$container_name" pgrep -x sshd >/dev/null 2>&1; then + sshd_up=1 + break + fi + if ! docker inspect "$container_name" --format '{{.State.Status}}' 2>/dev/null | grep -qx running; then + echo "$container_name is not running; reproduce with: docker run --rm --platform linux/$arch $image" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + sleep 1 + done + if [ -z "$sshd_up" ]; then + echo "sshd did not come up in $container_name after 60s" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + + # A container whose gnu tar cannot extract falls back to bsdtar (see + # hack/tar-shim.sh); say so rather than swapping the tool silently. + tar_backend=$(docker exec "$container_name" /usr/local/bin/bread-tar-shim --bread-probe 2>/dev/null || echo unknown) + if [ "$tar_backend" = bsdtar ]; then + note="note: gnu tar cannot extract in $container_name (host emulation); using bsdtar" + # spread buffers allocate output and only prints it on failure, so the + # terminal (when there is one) is the only channel a user actually reads. + echo "$note" + { [ -w /dev/tty ] && echo "$note" > /dev/tty; } 2>/dev/null || true + fi if [ "$mode" = publish ]; then # The ephemeral host port docker mapped to the container's sshd. @@ -114,26 +149,20 @@ backends: container_name="" case "$SPREAD_SYSTEM_ADDRESS" in *:*) - # publish mode: match the container by its published host port. + # publish mode: match the container by its published host port, as it + # shows up in the ports column, e.g. "127.0.0.1:32768->22/tcp". target_port="${SPREAD_SYSTEM_ADDRESS##*:}" - for cid in $(docker ps -a --format '{{.ID}}'); do - if docker port "$cid" 22 2>/dev/null | grep -q ":${target_port}\$"; then - container_name=$(docker inspect "$cid" --format '{{.Name}}' | sed 's#^/##') - break - fi - done + container_name=$(docker ps -a --filter label=spread-bread --format '{{.Names}} {{.Ports}}' | + awk -v m=":${target_port}->22/" 'index($0, m) { print $1; exit }') not_found="No container found with published port: $target_port" ;; *) # bridge mode: match the container by its bridge IP address. - for cid in $(docker ps -a --filter "network=bridge" --format '{{.ID}}'); do - cname=$(docker inspect "$cid" --format '{{.Name}}' | sed 's/^\/\(.*\)/\1/') - cip=$(docker inspect "$cid" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}' || echo "") - if [ "$cip" == "$SPREAD_SYSTEM_ADDRESS" ]; then - container_name="$cname" - break - fi - done + ids=$(docker ps -a --filter label=spread-bread --filter network=bridge --format '{{.ID}}') + if [ -n "$ids" ]; then + container_name=$(docker inspect $ids --format '{{.Name}} {{.NetworkSettings.Networks.bridge.IPAddress}}' | + awk -v ip="$SPREAD_SYSTEM_ADDRESS" '$2 == ip { print substr($1, 2); exit }') + fi not_found="No container found with IP address: $SPREAD_SYSTEM_ADDRESS" ;; esac diff --git a/inlined/bread-25.10.yaml b/inlined/bread-25.10.yaml index 98adee1..dfcff8e 100644 --- a/inlined/bread-25.10.yaml +++ b/inlined/bread-25.10.yaml @@ -12,13 +12,14 @@ exclude: - .github repack: | - # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as - # real files on the remote. Repack to remove them. + # macOS tar packs an AppleDouble ._* sidecar next to every file and stores + # xattrs as pax headers; the sidecars unpack as real files on the remote and + # GNU tar warns about every xattr header. Repack to remove both. if [ "$(uname -s)" = Darwin ]; then tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT tar -xf - -C "$tmp" <&3 - ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + ( cd "$tmp" && COPYFILE_DISABLE=1 tar --no-mac-metadata --no-xattrs -cf - . ) >&4 else cat <&3 >&4 fi @@ -63,7 +64,9 @@ backends: # snippet thanks to @lengau # https://github.com/canonical/charmcraft/blob/120a00a50f7ed3d0ae2fc2bea69e2e43b68b1594/spread.yaml#L72-L79 sleep 0.$RANDOM # Minimize chances of a race condition - export counter_file=".spread-worker-num" + # Kept out of the project tree: whatever lives there is packed up and + # shipped to every container. + export counter_file="${TMPDIR:-/tmp}/spread-bread-worker-num" instance_num=$( flock -x $counter_file bash -c ' [ -s $counter_file ] || echo 0 > $counter_file @@ -87,10 +90,42 @@ backends: -e "usr=$SPREAD_SYSTEM_USERNAME" \ -e "pass=$SPREAD_SYSTEM_PASSWORD" \ $publish_flag \ + --label spread-bread \ --name "$container_name" \ -d "$image" - until docker exec "$container_name" pgrep sshd; do sleep 1; done + # sshd is up within a second or two. The bound and the liveness check are + # backstops: a container that dies on start would otherwise spin here until + # spread's own timeout, which reports nothing about the container. + sshd_up="" + for ((i = 0; i < 60; i++)); do + if docker exec "$container_name" pgrep -x sshd >/dev/null 2>&1; then + sshd_up=1 + break + fi + if ! docker inspect "$container_name" --format '{{.State.Status}}' 2>/dev/null | grep -qx running; then + echo "$container_name is not running; reproduce with: docker run --rm --platform linux/$arch $image" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + sleep 1 + done + if [ -z "$sshd_up" ]; then + echo "sshd did not come up in $container_name after 60s" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + + # A container whose gnu tar cannot extract falls back to bsdtar (see + # hack/tar-shim.sh); say so rather than swapping the tool silently. + tar_backend=$(docker exec "$container_name" /usr/local/bin/bread-tar-shim --bread-probe 2>/dev/null || echo unknown) + if [ "$tar_backend" = bsdtar ]; then + note="note: gnu tar cannot extract in $container_name (host emulation); using bsdtar" + # spread buffers allocate output and only prints it on failure, so the + # terminal (when there is one) is the only channel a user actually reads. + echo "$note" + { [ -w /dev/tty ] && echo "$note" > /dev/tty; } 2>/dev/null || true + fi if [ "$mode" = publish ]; then # The ephemeral host port docker mapped to the container's sshd. @@ -114,26 +149,20 @@ backends: container_name="" case "$SPREAD_SYSTEM_ADDRESS" in *:*) - # publish mode: match the container by its published host port. + # publish mode: match the container by its published host port, as it + # shows up in the ports column, e.g. "127.0.0.1:32768->22/tcp". target_port="${SPREAD_SYSTEM_ADDRESS##*:}" - for cid in $(docker ps -a --format '{{.ID}}'); do - if docker port "$cid" 22 2>/dev/null | grep -q ":${target_port}\$"; then - container_name=$(docker inspect "$cid" --format '{{.Name}}' | sed 's#^/##') - break - fi - done + container_name=$(docker ps -a --filter label=spread-bread --format '{{.Names}} {{.Ports}}' | + awk -v m=":${target_port}->22/" 'index($0, m) { print $1; exit }') not_found="No container found with published port: $target_port" ;; *) # bridge mode: match the container by its bridge IP address. - for cid in $(docker ps -a --filter "network=bridge" --format '{{.ID}}'); do - cname=$(docker inspect "$cid" --format '{{.Name}}' | sed 's/^\/\(.*\)/\1/') - cip=$(docker inspect "$cid" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}' || echo "") - if [ "$cip" == "$SPREAD_SYSTEM_ADDRESS" ]; then - container_name="$cname" - break - fi - done + ids=$(docker ps -a --filter label=spread-bread --filter network=bridge --format '{{.ID}}') + if [ -n "$ids" ]; then + container_name=$(docker inspect $ids --format '{{.Name}} {{.NetworkSettings.Networks.bridge.IPAddress}}' | + awk -v ip="$SPREAD_SYSTEM_ADDRESS" '$2 == ip { print substr($1, 2); exit }') + fi not_found="No container found with IP address: $SPREAD_SYSTEM_ADDRESS" ;; esac diff --git a/inlined/bread-26.04.yaml b/inlined/bread-26.04.yaml index 58a873a..715bc9c 100644 --- a/inlined/bread-26.04.yaml +++ b/inlined/bread-26.04.yaml @@ -12,13 +12,14 @@ exclude: - .github repack: | - # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as - # real files on the remote. Repack to remove them. + # macOS tar packs an AppleDouble ._* sidecar next to every file and stores + # xattrs as pax headers; the sidecars unpack as real files on the remote and + # GNU tar warns about every xattr header. Repack to remove both. if [ "$(uname -s)" = Darwin ]; then tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT tar -xf - -C "$tmp" <&3 - ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + ( cd "$tmp" && COPYFILE_DISABLE=1 tar --no-mac-metadata --no-xattrs -cf - . ) >&4 else cat <&3 >&4 fi @@ -63,7 +64,9 @@ backends: # snippet thanks to @lengau # https://github.com/canonical/charmcraft/blob/120a00a50f7ed3d0ae2fc2bea69e2e43b68b1594/spread.yaml#L72-L79 sleep 0.$RANDOM # Minimize chances of a race condition - export counter_file=".spread-worker-num" + # Kept out of the project tree: whatever lives there is packed up and + # shipped to every container. + export counter_file="${TMPDIR:-/tmp}/spread-bread-worker-num" instance_num=$( flock -x $counter_file bash -c ' [ -s $counter_file ] || echo 0 > $counter_file @@ -87,10 +90,42 @@ backends: -e "usr=$SPREAD_SYSTEM_USERNAME" \ -e "pass=$SPREAD_SYSTEM_PASSWORD" \ $publish_flag \ + --label spread-bread \ --name "$container_name" \ -d "$image" - until docker exec "$container_name" pgrep sshd; do sleep 1; done + # sshd is up within a second or two. The bound and the liveness check are + # backstops: a container that dies on start would otherwise spin here until + # spread's own timeout, which reports nothing about the container. + sshd_up="" + for ((i = 0; i < 60; i++)); do + if docker exec "$container_name" pgrep -x sshd >/dev/null 2>&1; then + sshd_up=1 + break + fi + if ! docker inspect "$container_name" --format '{{.State.Status}}' 2>/dev/null | grep -qx running; then + echo "$container_name is not running; reproduce with: docker run --rm --platform linux/$arch $image" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + sleep 1 + done + if [ -z "$sshd_up" ]; then + echo "sshd did not come up in $container_name after 60s" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + + # A container whose gnu tar cannot extract falls back to bsdtar (see + # hack/tar-shim.sh); say so rather than swapping the tool silently. + tar_backend=$(docker exec "$container_name" /usr/local/bin/bread-tar-shim --bread-probe 2>/dev/null || echo unknown) + if [ "$tar_backend" = bsdtar ]; then + note="note: gnu tar cannot extract in $container_name (host emulation); using bsdtar" + # spread buffers allocate output and only prints it on failure, so the + # terminal (when there is one) is the only channel a user actually reads. + echo "$note" + { [ -w /dev/tty ] && echo "$note" > /dev/tty; } 2>/dev/null || true + fi if [ "$mode" = publish ]; then # The ephemeral host port docker mapped to the container's sshd. @@ -114,26 +149,20 @@ backends: container_name="" case "$SPREAD_SYSTEM_ADDRESS" in *:*) - # publish mode: match the container by its published host port. + # publish mode: match the container by its published host port, as it + # shows up in the ports column, e.g. "127.0.0.1:32768->22/tcp". target_port="${SPREAD_SYSTEM_ADDRESS##*:}" - for cid in $(docker ps -a --format '{{.ID}}'); do - if docker port "$cid" 22 2>/dev/null | grep -q ":${target_port}\$"; then - container_name=$(docker inspect "$cid" --format '{{.Name}}' | sed 's#^/##') - break - fi - done + container_name=$(docker ps -a --filter label=spread-bread --format '{{.Names}} {{.Ports}}' | + awk -v m=":${target_port}->22/" 'index($0, m) { print $1; exit }') not_found="No container found with published port: $target_port" ;; *) # bridge mode: match the container by its bridge IP address. - for cid in $(docker ps -a --filter "network=bridge" --format '{{.ID}}'); do - cname=$(docker inspect "$cid" --format '{{.Name}}' | sed 's/^\/\(.*\)/\1/') - cip=$(docker inspect "$cid" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}' || echo "") - if [ "$cip" == "$SPREAD_SYSTEM_ADDRESS" ]; then - container_name="$cname" - break - fi - done + ids=$(docker ps -a --filter label=spread-bread --filter network=bridge --format '{{.ID}}') + if [ -n "$ids" ]; then + container_name=$(docker inspect $ids --format '{{.Name}} {{.NetworkSettings.Networks.bridge.IPAddress}}' | + awk -v ip="$SPREAD_SYSTEM_ADDRESS" '$2 == ip { print substr($1, 2); exit }') + fi not_found="No container found with IP address: $SPREAD_SYSTEM_ADDRESS" ;; esac diff --git a/inlined/bread-26.10.yaml b/inlined/bread-26.10.yaml index 490932c..058a0cb 100644 --- a/inlined/bread-26.10.yaml +++ b/inlined/bread-26.10.yaml @@ -12,13 +12,14 @@ exclude: - .github repack: | - # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as - # real files on the remote. Repack to remove them. + # macOS tar packs an AppleDouble ._* sidecar next to every file and stores + # xattrs as pax headers; the sidecars unpack as real files on the remote and + # GNU tar warns about every xattr header. Repack to remove both. if [ "$(uname -s)" = Darwin ]; then tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT tar -xf - -C "$tmp" <&3 - ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + ( cd "$tmp" && COPYFILE_DISABLE=1 tar --no-mac-metadata --no-xattrs -cf - . ) >&4 else cat <&3 >&4 fi @@ -63,7 +64,9 @@ backends: # snippet thanks to @lengau # https://github.com/canonical/charmcraft/blob/120a00a50f7ed3d0ae2fc2bea69e2e43b68b1594/spread.yaml#L72-L79 sleep 0.$RANDOM # Minimize chances of a race condition - export counter_file=".spread-worker-num" + # Kept out of the project tree: whatever lives there is packed up and + # shipped to every container. + export counter_file="${TMPDIR:-/tmp}/spread-bread-worker-num" instance_num=$( flock -x $counter_file bash -c ' [ -s $counter_file ] || echo 0 > $counter_file @@ -87,10 +90,42 @@ backends: -e "usr=$SPREAD_SYSTEM_USERNAME" \ -e "pass=$SPREAD_SYSTEM_PASSWORD" \ $publish_flag \ + --label spread-bread \ --name "$container_name" \ -d "$image" - until docker exec "$container_name" pgrep sshd; do sleep 1; done + # sshd is up within a second or two. The bound and the liveness check are + # backstops: a container that dies on start would otherwise spin here until + # spread's own timeout, which reports nothing about the container. + sshd_up="" + for ((i = 0; i < 60; i++)); do + if docker exec "$container_name" pgrep -x sshd >/dev/null 2>&1; then + sshd_up=1 + break + fi + if ! docker inspect "$container_name" --format '{{.State.Status}}' 2>/dev/null | grep -qx running; then + echo "$container_name is not running; reproduce with: docker run --rm --platform linux/$arch $image" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + sleep 1 + done + if [ -z "$sshd_up" ]; then + echo "sshd did not come up in $container_name after 60s" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + + # A container whose gnu tar cannot extract falls back to bsdtar (see + # hack/tar-shim.sh); say so rather than swapping the tool silently. + tar_backend=$(docker exec "$container_name" /usr/local/bin/bread-tar-shim --bread-probe 2>/dev/null || echo unknown) + if [ "$tar_backend" = bsdtar ]; then + note="note: gnu tar cannot extract in $container_name (host emulation); using bsdtar" + # spread buffers allocate output and only prints it on failure, so the + # terminal (when there is one) is the only channel a user actually reads. + echo "$note" + { [ -w /dev/tty ] && echo "$note" > /dev/tty; } 2>/dev/null || true + fi if [ "$mode" = publish ]; then # The ephemeral host port docker mapped to the container's sshd. @@ -114,26 +149,20 @@ backends: container_name="" case "$SPREAD_SYSTEM_ADDRESS" in *:*) - # publish mode: match the container by its published host port. + # publish mode: match the container by its published host port, as it + # shows up in the ports column, e.g. "127.0.0.1:32768->22/tcp". target_port="${SPREAD_SYSTEM_ADDRESS##*:}" - for cid in $(docker ps -a --format '{{.ID}}'); do - if docker port "$cid" 22 2>/dev/null | grep -q ":${target_port}\$"; then - container_name=$(docker inspect "$cid" --format '{{.Name}}' | sed 's#^/##') - break - fi - done + container_name=$(docker ps -a --filter label=spread-bread --format '{{.Names}} {{.Ports}}' | + awk -v m=":${target_port}->22/" 'index($0, m) { print $1; exit }') not_found="No container found with published port: $target_port" ;; *) # bridge mode: match the container by its bridge IP address. - for cid in $(docker ps -a --filter "network=bridge" --format '{{.ID}}'); do - cname=$(docker inspect "$cid" --format '{{.Name}}' | sed 's/^\/\(.*\)/\1/') - cip=$(docker inspect "$cid" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}' || echo "") - if [ "$cip" == "$SPREAD_SYSTEM_ADDRESS" ]; then - container_name="$cname" - break - fi - done + ids=$(docker ps -a --filter label=spread-bread --filter network=bridge --format '{{.ID}}') + if [ -n "$ids" ]; then + container_name=$(docker inspect $ids --format '{{.Name}} {{.NetworkSettings.Networks.bridge.IPAddress}}' | + awk -v ip="$SPREAD_SYSTEM_ADDRESS" '$2 == ip { print substr($1, 2); exit }') + fi not_found="No container found with IP address: $SPREAD_SYSTEM_ADDRESS" ;; esac diff --git a/inlined/bread-chisel-releases-22.04.yaml b/inlined/bread-chisel-releases-22.04.yaml index fa3b434..5839216 100644 --- a/inlined/bread-chisel-releases-22.04.yaml +++ b/inlined/bread-chisel-releases-22.04.yaml @@ -19,13 +19,14 @@ exclude: - rootfs repack: | - # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as - # real files on the remote. Repack to remove them. + # macOS tar packs an AppleDouble ._* sidecar next to every file and stores + # xattrs as pax headers; the sidecars unpack as real files on the remote and + # GNU tar warns about every xattr header. Repack to remove both. if [ "$(uname -s)" = Darwin ]; then tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT tar -xf - -C "$tmp" <&3 - ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + ( cd "$tmp" && COPYFILE_DISABLE=1 tar --no-mac-metadata --no-xattrs -cf - . ) >&4 else cat <&3 >&4 fi @@ -67,7 +68,9 @@ backends: if [ "$mode" = bridge ]; then sleep 0.$RANDOM # Minimize chances of a race condition - export counter_file=".spread-worker-num" + # Kept out of the project tree: whatever lives there is packed up and + # shipped to every container. + export counter_file="${TMPDIR:-/tmp}/spread-bread-worker-num" instance_num=$( flock -x $counter_file bash -c ' [ -s $counter_file ] || echo 0 > $counter_file @@ -93,10 +96,42 @@ backends: -e "usr=$SPREAD_SYSTEM_USERNAME" \ -e "pass=$SPREAD_SYSTEM_PASSWORD" \ $publish_flag \ + --label spread-bread \ --name "$container_name" \ -d "$image" - until docker exec "$container_name" pgrep sshd; do sleep 1; done + # sshd is up within a second or two. The bound and the liveness check are + # backstops: a container that dies on start would otherwise spin here until + # spread's own timeout, which reports nothing about the container. + sshd_up="" + for ((i = 0; i < 60; i++)); do + if docker exec "$container_name" pgrep -x sshd >/dev/null 2>&1; then + sshd_up=1 + break + fi + if ! docker inspect "$container_name" --format '{{.State.Status}}' 2>/dev/null | grep -qx running; then + echo "$container_name is not running; reproduce with: docker run --rm --platform linux/$arch $image" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + sleep 1 + done + if [ -z "$sshd_up" ]; then + echo "sshd did not come up in $container_name after 60s" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + + # A container whose gnu tar cannot extract falls back to bsdtar (see + # hack/tar-shim.sh); say so rather than swapping the tool silently. + tar_backend=$(docker exec "$container_name" /usr/local/bin/bread-tar-shim --bread-probe 2>/dev/null || echo unknown) + if [ "$tar_backend" = bsdtar ]; then + note="note: gnu tar cannot extract in $container_name (host emulation); using bsdtar" + # spread buffers allocate output and only prints it on failure, so the + # terminal (when there is one) is the only channel a user actually reads. + echo "$note" + { [ -w /dev/tty ] && echo "$note" > /dev/tty; } 2>/dev/null || true + fi if [ "$mode" = publish ]; then # The ephemeral host port docker mapped to the container's sshd. @@ -120,26 +155,20 @@ backends: container_name="" case "$SPREAD_SYSTEM_ADDRESS" in *:*) - # publish mode: match the container by its published host port. + # publish mode: match the container by its published host port, as it + # shows up in the ports column, e.g. "127.0.0.1:32768->22/tcp". target_port="${SPREAD_SYSTEM_ADDRESS##*:}" - for cid in $(docker ps -a --format '{{.ID}}'); do - if docker port "$cid" 22 2>/dev/null | grep -q ":${target_port}\$"; then - container_name=$(docker inspect "$cid" --format '{{.Name}}' | sed 's#^/##') - break - fi - done + container_name=$(docker ps -a --filter label=spread-bread --format '{{.Names}} {{.Ports}}' | + awk -v m=":${target_port}->22/" 'index($0, m) { print $1; exit }') not_found="No container found with published port: $target_port" ;; *) # bridge mode: match the container by its bridge IP address. - for cid in $(docker ps -a --filter "network=bridge" --format '{{.ID}}'); do - cname=$(docker inspect "$cid" --format '{{.Name}}' | sed 's/^\/\(.*\)/\1/') - cip=$(docker inspect "$cid" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}' || echo "") - if [ "$cip" == "$SPREAD_SYSTEM_ADDRESS" ]; then - container_name="$cname" - break - fi - done + ids=$(docker ps -a --filter label=spread-bread --filter network=bridge --format '{{.ID}}') + if [ -n "$ids" ]; then + container_name=$(docker inspect $ids --format '{{.Name}} {{.NetworkSettings.Networks.bridge.IPAddress}}' | + awk -v ip="$SPREAD_SYSTEM_ADDRESS" '$2 == ip { print substr($1, 2); exit }') + fi not_found="No container found with IP address: $SPREAD_SYSTEM_ADDRESS" ;; esac diff --git a/inlined/bread-chisel-releases-24.04.yaml b/inlined/bread-chisel-releases-24.04.yaml index f6578c9..50093da 100644 --- a/inlined/bread-chisel-releases-24.04.yaml +++ b/inlined/bread-chisel-releases-24.04.yaml @@ -19,13 +19,14 @@ exclude: - rootfs repack: | - # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as - # real files on the remote. Repack to remove them. + # macOS tar packs an AppleDouble ._* sidecar next to every file and stores + # xattrs as pax headers; the sidecars unpack as real files on the remote and + # GNU tar warns about every xattr header. Repack to remove both. if [ "$(uname -s)" = Darwin ]; then tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT tar -xf - -C "$tmp" <&3 - ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + ( cd "$tmp" && COPYFILE_DISABLE=1 tar --no-mac-metadata --no-xattrs -cf - . ) >&4 else cat <&3 >&4 fi @@ -67,7 +68,9 @@ backends: if [ "$mode" = bridge ]; then sleep 0.$RANDOM # Minimize chances of a race condition - export counter_file=".spread-worker-num" + # Kept out of the project tree: whatever lives there is packed up and + # shipped to every container. + export counter_file="${TMPDIR:-/tmp}/spread-bread-worker-num" instance_num=$( flock -x $counter_file bash -c ' [ -s $counter_file ] || echo 0 > $counter_file @@ -93,10 +96,42 @@ backends: -e "usr=$SPREAD_SYSTEM_USERNAME" \ -e "pass=$SPREAD_SYSTEM_PASSWORD" \ $publish_flag \ + --label spread-bread \ --name "$container_name" \ -d "$image" - until docker exec "$container_name" pgrep sshd; do sleep 1; done + # sshd is up within a second or two. The bound and the liveness check are + # backstops: a container that dies on start would otherwise spin here until + # spread's own timeout, which reports nothing about the container. + sshd_up="" + for ((i = 0; i < 60; i++)); do + if docker exec "$container_name" pgrep -x sshd >/dev/null 2>&1; then + sshd_up=1 + break + fi + if ! docker inspect "$container_name" --format '{{.State.Status}}' 2>/dev/null | grep -qx running; then + echo "$container_name is not running; reproduce with: docker run --rm --platform linux/$arch $image" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + sleep 1 + done + if [ -z "$sshd_up" ]; then + echo "sshd did not come up in $container_name after 60s" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + + # A container whose gnu tar cannot extract falls back to bsdtar (see + # hack/tar-shim.sh); say so rather than swapping the tool silently. + tar_backend=$(docker exec "$container_name" /usr/local/bin/bread-tar-shim --bread-probe 2>/dev/null || echo unknown) + if [ "$tar_backend" = bsdtar ]; then + note="note: gnu tar cannot extract in $container_name (host emulation); using bsdtar" + # spread buffers allocate output and only prints it on failure, so the + # terminal (when there is one) is the only channel a user actually reads. + echo "$note" + { [ -w /dev/tty ] && echo "$note" > /dev/tty; } 2>/dev/null || true + fi if [ "$mode" = publish ]; then # The ephemeral host port docker mapped to the container's sshd. @@ -120,26 +155,20 @@ backends: container_name="" case "$SPREAD_SYSTEM_ADDRESS" in *:*) - # publish mode: match the container by its published host port. + # publish mode: match the container by its published host port, as it + # shows up in the ports column, e.g. "127.0.0.1:32768->22/tcp". target_port="${SPREAD_SYSTEM_ADDRESS##*:}" - for cid in $(docker ps -a --format '{{.ID}}'); do - if docker port "$cid" 22 2>/dev/null | grep -q ":${target_port}\$"; then - container_name=$(docker inspect "$cid" --format '{{.Name}}' | sed 's#^/##') - break - fi - done + container_name=$(docker ps -a --filter label=spread-bread --format '{{.Names}} {{.Ports}}' | + awk -v m=":${target_port}->22/" 'index($0, m) { print $1; exit }') not_found="No container found with published port: $target_port" ;; *) # bridge mode: match the container by its bridge IP address. - for cid in $(docker ps -a --filter "network=bridge" --format '{{.ID}}'); do - cname=$(docker inspect "$cid" --format '{{.Name}}' | sed 's/^\/\(.*\)/\1/') - cip=$(docker inspect "$cid" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}' || echo "") - if [ "$cip" == "$SPREAD_SYSTEM_ADDRESS" ]; then - container_name="$cname" - break - fi - done + ids=$(docker ps -a --filter label=spread-bread --filter network=bridge --format '{{.ID}}') + if [ -n "$ids" ]; then + container_name=$(docker inspect $ids --format '{{.Name}} {{.NetworkSettings.Networks.bridge.IPAddress}}' | + awk -v ip="$SPREAD_SYSTEM_ADDRESS" '$2 == ip { print substr($1, 2); exit }') + fi not_found="No container found with IP address: $SPREAD_SYSTEM_ADDRESS" ;; esac diff --git a/inlined/bread-chisel-releases-25.10.yaml b/inlined/bread-chisel-releases-25.10.yaml index 8786ab9..bf0d250 100644 --- a/inlined/bread-chisel-releases-25.10.yaml +++ b/inlined/bread-chisel-releases-25.10.yaml @@ -19,13 +19,14 @@ exclude: - rootfs repack: | - # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as - # real files on the remote. Repack to remove them. + # macOS tar packs an AppleDouble ._* sidecar next to every file and stores + # xattrs as pax headers; the sidecars unpack as real files on the remote and + # GNU tar warns about every xattr header. Repack to remove both. if [ "$(uname -s)" = Darwin ]; then tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT tar -xf - -C "$tmp" <&3 - ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + ( cd "$tmp" && COPYFILE_DISABLE=1 tar --no-mac-metadata --no-xattrs -cf - . ) >&4 else cat <&3 >&4 fi @@ -67,7 +68,9 @@ backends: if [ "$mode" = bridge ]; then sleep 0.$RANDOM # Minimize chances of a race condition - export counter_file=".spread-worker-num" + # Kept out of the project tree: whatever lives there is packed up and + # shipped to every container. + export counter_file="${TMPDIR:-/tmp}/spread-bread-worker-num" instance_num=$( flock -x $counter_file bash -c ' [ -s $counter_file ] || echo 0 > $counter_file @@ -93,10 +96,42 @@ backends: -e "usr=$SPREAD_SYSTEM_USERNAME" \ -e "pass=$SPREAD_SYSTEM_PASSWORD" \ $publish_flag \ + --label spread-bread \ --name "$container_name" \ -d "$image" - until docker exec "$container_name" pgrep sshd; do sleep 1; done + # sshd is up within a second or two. The bound and the liveness check are + # backstops: a container that dies on start would otherwise spin here until + # spread's own timeout, which reports nothing about the container. + sshd_up="" + for ((i = 0; i < 60; i++)); do + if docker exec "$container_name" pgrep -x sshd >/dev/null 2>&1; then + sshd_up=1 + break + fi + if ! docker inspect "$container_name" --format '{{.State.Status}}' 2>/dev/null | grep -qx running; then + echo "$container_name is not running; reproduce with: docker run --rm --platform linux/$arch $image" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + sleep 1 + done + if [ -z "$sshd_up" ]; then + echo "sshd did not come up in $container_name after 60s" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + + # A container whose gnu tar cannot extract falls back to bsdtar (see + # hack/tar-shim.sh); say so rather than swapping the tool silently. + tar_backend=$(docker exec "$container_name" /usr/local/bin/bread-tar-shim --bread-probe 2>/dev/null || echo unknown) + if [ "$tar_backend" = bsdtar ]; then + note="note: gnu tar cannot extract in $container_name (host emulation); using bsdtar" + # spread buffers allocate output and only prints it on failure, so the + # terminal (when there is one) is the only channel a user actually reads. + echo "$note" + { [ -w /dev/tty ] && echo "$note" > /dev/tty; } 2>/dev/null || true + fi if [ "$mode" = publish ]; then # The ephemeral host port docker mapped to the container's sshd. @@ -120,26 +155,20 @@ backends: container_name="" case "$SPREAD_SYSTEM_ADDRESS" in *:*) - # publish mode: match the container by its published host port. + # publish mode: match the container by its published host port, as it + # shows up in the ports column, e.g. "127.0.0.1:32768->22/tcp". target_port="${SPREAD_SYSTEM_ADDRESS##*:}" - for cid in $(docker ps -a --format '{{.ID}}'); do - if docker port "$cid" 22 2>/dev/null | grep -q ":${target_port}\$"; then - container_name=$(docker inspect "$cid" --format '{{.Name}}' | sed 's#^/##') - break - fi - done + container_name=$(docker ps -a --filter label=spread-bread --format '{{.Names}} {{.Ports}}' | + awk -v m=":${target_port}->22/" 'index($0, m) { print $1; exit }') not_found="No container found with published port: $target_port" ;; *) # bridge mode: match the container by its bridge IP address. - for cid in $(docker ps -a --filter "network=bridge" --format '{{.ID}}'); do - cname=$(docker inspect "$cid" --format '{{.Name}}' | sed 's/^\/\(.*\)/\1/') - cip=$(docker inspect "$cid" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}' || echo "") - if [ "$cip" == "$SPREAD_SYSTEM_ADDRESS" ]; then - container_name="$cname" - break - fi - done + ids=$(docker ps -a --filter label=spread-bread --filter network=bridge --format '{{.ID}}') + if [ -n "$ids" ]; then + container_name=$(docker inspect $ids --format '{{.Name}} {{.NetworkSettings.Networks.bridge.IPAddress}}' | + awk -v ip="$SPREAD_SYSTEM_ADDRESS" '$2 == ip { print substr($1, 2); exit }') + fi not_found="No container found with IP address: $SPREAD_SYSTEM_ADDRESS" ;; esac diff --git a/inlined/bread-chisel-releases-26.04.yaml b/inlined/bread-chisel-releases-26.04.yaml index 9f3e26f..d95124e 100644 --- a/inlined/bread-chisel-releases-26.04.yaml +++ b/inlined/bread-chisel-releases-26.04.yaml @@ -19,13 +19,14 @@ exclude: - rootfs repack: | - # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as - # real files on the remote. Repack to remove them. + # macOS tar packs an AppleDouble ._* sidecar next to every file and stores + # xattrs as pax headers; the sidecars unpack as real files on the remote and + # GNU tar warns about every xattr header. Repack to remove both. if [ "$(uname -s)" = Darwin ]; then tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT tar -xf - -C "$tmp" <&3 - ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + ( cd "$tmp" && COPYFILE_DISABLE=1 tar --no-mac-metadata --no-xattrs -cf - . ) >&4 else cat <&3 >&4 fi @@ -67,7 +68,9 @@ backends: if [ "$mode" = bridge ]; then sleep 0.$RANDOM # Minimize chances of a race condition - export counter_file=".spread-worker-num" + # Kept out of the project tree: whatever lives there is packed up and + # shipped to every container. + export counter_file="${TMPDIR:-/tmp}/spread-bread-worker-num" instance_num=$( flock -x $counter_file bash -c ' [ -s $counter_file ] || echo 0 > $counter_file @@ -93,10 +96,42 @@ backends: -e "usr=$SPREAD_SYSTEM_USERNAME" \ -e "pass=$SPREAD_SYSTEM_PASSWORD" \ $publish_flag \ + --label spread-bread \ --name "$container_name" \ -d "$image" - until docker exec "$container_name" pgrep sshd; do sleep 1; done + # sshd is up within a second or two. The bound and the liveness check are + # backstops: a container that dies on start would otherwise spin here until + # spread's own timeout, which reports nothing about the container. + sshd_up="" + for ((i = 0; i < 60; i++)); do + if docker exec "$container_name" pgrep -x sshd >/dev/null 2>&1; then + sshd_up=1 + break + fi + if ! docker inspect "$container_name" --format '{{.State.Status}}' 2>/dev/null | grep -qx running; then + echo "$container_name is not running; reproduce with: docker run --rm --platform linux/$arch $image" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + sleep 1 + done + if [ -z "$sshd_up" ]; then + echo "sshd did not come up in $container_name after 60s" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + + # A container whose gnu tar cannot extract falls back to bsdtar (see + # hack/tar-shim.sh); say so rather than swapping the tool silently. + tar_backend=$(docker exec "$container_name" /usr/local/bin/bread-tar-shim --bread-probe 2>/dev/null || echo unknown) + if [ "$tar_backend" = bsdtar ]; then + note="note: gnu tar cannot extract in $container_name (host emulation); using bsdtar" + # spread buffers allocate output and only prints it on failure, so the + # terminal (when there is one) is the only channel a user actually reads. + echo "$note" + { [ -w /dev/tty ] && echo "$note" > /dev/tty; } 2>/dev/null || true + fi if [ "$mode" = publish ]; then # The ephemeral host port docker mapped to the container's sshd. @@ -120,26 +155,20 @@ backends: container_name="" case "$SPREAD_SYSTEM_ADDRESS" in *:*) - # publish mode: match the container by its published host port. + # publish mode: match the container by its published host port, as it + # shows up in the ports column, e.g. "127.0.0.1:32768->22/tcp". target_port="${SPREAD_SYSTEM_ADDRESS##*:}" - for cid in $(docker ps -a --format '{{.ID}}'); do - if docker port "$cid" 22 2>/dev/null | grep -q ":${target_port}\$"; then - container_name=$(docker inspect "$cid" --format '{{.Name}}' | sed 's#^/##') - break - fi - done + container_name=$(docker ps -a --filter label=spread-bread --format '{{.Names}} {{.Ports}}' | + awk -v m=":${target_port}->22/" 'index($0, m) { print $1; exit }') not_found="No container found with published port: $target_port" ;; *) # bridge mode: match the container by its bridge IP address. - for cid in $(docker ps -a --filter "network=bridge" --format '{{.ID}}'); do - cname=$(docker inspect "$cid" --format '{{.Name}}' | sed 's/^\/\(.*\)/\1/') - cip=$(docker inspect "$cid" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}' || echo "") - if [ "$cip" == "$SPREAD_SYSTEM_ADDRESS" ]; then - container_name="$cname" - break - fi - done + ids=$(docker ps -a --filter label=spread-bread --filter network=bridge --format '{{.ID}}') + if [ -n "$ids" ]; then + container_name=$(docker inspect $ids --format '{{.Name}} {{.NetworkSettings.Networks.bridge.IPAddress}}' | + awk -v ip="$SPREAD_SYSTEM_ADDRESS" '$2 == ip { print substr($1, 2); exit }') + fi not_found="No container found with IP address: $SPREAD_SYSTEM_ADDRESS" ;; esac diff --git a/inlined/bread-chisel-releases-26.10.yaml b/inlined/bread-chisel-releases-26.10.yaml index 16f319b..89f64a5 100644 --- a/inlined/bread-chisel-releases-26.10.yaml +++ b/inlined/bread-chisel-releases-26.10.yaml @@ -19,13 +19,14 @@ exclude: - rootfs repack: | - # macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as - # real files on the remote. Repack to remove them. + # macOS tar packs an AppleDouble ._* sidecar next to every file and stores + # xattrs as pax headers; the sidecars unpack as real files on the remote and + # GNU tar warns about every xattr header. Repack to remove both. if [ "$(uname -s)" = Darwin ]; then tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT tar -xf - -C "$tmp" <&3 - ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + ( cd "$tmp" && COPYFILE_DISABLE=1 tar --no-mac-metadata --no-xattrs -cf - . ) >&4 else cat <&3 >&4 fi @@ -67,7 +68,9 @@ backends: if [ "$mode" = bridge ]; then sleep 0.$RANDOM # Minimize chances of a race condition - export counter_file=".spread-worker-num" + # Kept out of the project tree: whatever lives there is packed up and + # shipped to every container. + export counter_file="${TMPDIR:-/tmp}/spread-bread-worker-num" instance_num=$( flock -x $counter_file bash -c ' [ -s $counter_file ] || echo 0 > $counter_file @@ -93,10 +96,42 @@ backends: -e "usr=$SPREAD_SYSTEM_USERNAME" \ -e "pass=$SPREAD_SYSTEM_PASSWORD" \ $publish_flag \ + --label spread-bread \ --name "$container_name" \ -d "$image" - until docker exec "$container_name" pgrep sshd; do sleep 1; done + # sshd is up within a second or two. The bound and the liveness check are + # backstops: a container that dies on start would otherwise spin here until + # spread's own timeout, which reports nothing about the container. + sshd_up="" + for ((i = 0; i < 60; i++)); do + if docker exec "$container_name" pgrep -x sshd >/dev/null 2>&1; then + sshd_up=1 + break + fi + if ! docker inspect "$container_name" --format '{{.State.Status}}' 2>/dev/null | grep -qx running; then + echo "$container_name is not running; reproduce with: docker run --rm --platform linux/$arch $image" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + sleep 1 + done + if [ -z "$sshd_up" ]; then + echo "sshd did not come up in $container_name after 60s" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + + # A container whose gnu tar cannot extract falls back to bsdtar (see + # hack/tar-shim.sh); say so rather than swapping the tool silently. + tar_backend=$(docker exec "$container_name" /usr/local/bin/bread-tar-shim --bread-probe 2>/dev/null || echo unknown) + if [ "$tar_backend" = bsdtar ]; then + note="note: gnu tar cannot extract in $container_name (host emulation); using bsdtar" + # spread buffers allocate output and only prints it on failure, so the + # terminal (when there is one) is the only channel a user actually reads. + echo "$note" + { [ -w /dev/tty ] && echo "$note" > /dev/tty; } 2>/dev/null || true + fi if [ "$mode" = publish ]; then # The ephemeral host port docker mapped to the container's sshd. @@ -120,26 +155,20 @@ backends: container_name="" case "$SPREAD_SYSTEM_ADDRESS" in *:*) - # publish mode: match the container by its published host port. + # publish mode: match the container by its published host port, as it + # shows up in the ports column, e.g. "127.0.0.1:32768->22/tcp". target_port="${SPREAD_SYSTEM_ADDRESS##*:}" - for cid in $(docker ps -a --format '{{.ID}}'); do - if docker port "$cid" 22 2>/dev/null | grep -q ":${target_port}\$"; then - container_name=$(docker inspect "$cid" --format '{{.Name}}' | sed 's#^/##') - break - fi - done + container_name=$(docker ps -a --filter label=spread-bread --format '{{.Names}} {{.Ports}}' | + awk -v m=":${target_port}->22/" 'index($0, m) { print $1; exit }') not_found="No container found with published port: $target_port" ;; *) # bridge mode: match the container by its bridge IP address. - for cid in $(docker ps -a --filter "network=bridge" --format '{{.ID}}'); do - cname=$(docker inspect "$cid" --format '{{.Name}}' | sed 's/^\/\(.*\)/\1/') - cip=$(docker inspect "$cid" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}' || echo "") - if [ "$cip" == "$SPREAD_SYSTEM_ADDRESS" ]; then - container_name="$cname" - break - fi - done + ids=$(docker ps -a --filter label=spread-bread --filter network=bridge --format '{{.ID}}') + if [ -n "$ids" ]; then + container_name=$(docker inspect $ids --format '{{.Name}} {{.NetworkSettings.Networks.bridge.IPAddress}}' | + awk -v ip="$SPREAD_SYSTEM_ADDRESS" '$2 == ip { print substr($1, 2); exit }') + fi not_found="No container found with IP address: $SPREAD_SYSTEM_ADDRESS" ;; esac diff --git a/makefile b/makefile index a70c018..4570a58 100644 --- a/makefile +++ b/makefile @@ -156,7 +156,8 @@ shell: .stamp/bread-26.04-$(SELECTED_ARCH) ## Drop into a bread:26.04 shell (ho .PHONY: nuke-spread nuke-spread: ## Kill stray spread processes + force-remove bread containers -pkill spread - -$(DOCKER) ps | grep bread | cut -d' ' -f1 | xargs -r $(DOCKER) rm --force + -$(DOCKER) ps -aq --filter label=spread-bread | xargs -r $(DOCKER) rm --force + -$(DOCKER) ps -aq --filter name='^bread-' | xargs -r $(DOCKER) rm --force .PHONY: clean clean: ## Remove built images, stamps, generated inlined yamls, cached binaries diff --git a/scripts/spread_allocate_bread-chisel-releases.sh b/scripts/spread_allocate_bread-chisel-releases.sh index 1a95d2a..80d4999 100755 --- a/scripts/spread_allocate_bread-chisel-releases.sh +++ b/scripts/spread_allocate_bread-chisel-releases.sh @@ -31,7 +31,9 @@ echo "net mode: $mode" if [ "$mode" = bridge ]; then sleep 0.$RANDOM # Minimize chances of a race condition - export counter_file=".spread-worker-num" + # Kept out of the project tree: whatever lives there is packed up and + # shipped to every container. + export counter_file="${TMPDIR:-/tmp}/spread-bread-worker-num" instance_num=$( flock -x $counter_file bash -c ' [ -s $counter_file ] || echo 0 > $counter_file @@ -57,10 +59,42 @@ docker run \ -e "usr=$SPREAD_SYSTEM_USERNAME" \ -e "pass=$SPREAD_SYSTEM_PASSWORD" \ $publish_flag \ + --label spread-bread \ --name "$container_name" \ -d "$image" -until docker exec "$container_name" pgrep sshd; do sleep 1; done +# sshd is up within a second or two. The bound and the liveness check are +# backstops: a container that dies on start would otherwise spin here until +# spread's own timeout, which reports nothing about the container. +sshd_up="" +for ((i = 0; i < 60; i++)); do + if docker exec "$container_name" pgrep -x sshd >/dev/null 2>&1; then + sshd_up=1 + break + fi + if ! docker inspect "$container_name" --format '{{.State.Status}}' 2>/dev/null | grep -qx running; then + echo "$container_name is not running; reproduce with: docker run --rm --platform linux/$arch $image" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + sleep 1 +done +if [ -z "$sshd_up" ]; then + echo "sshd did not come up in $container_name after 60s" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 +fi + +# A container whose gnu tar cannot extract falls back to bsdtar (see +# hack/tar-shim.sh); say so rather than swapping the tool silently. +tar_backend=$(docker exec "$container_name" /usr/local/bin/bread-tar-shim --bread-probe 2>/dev/null || echo unknown) +if [ "$tar_backend" = bsdtar ]; then + note="note: gnu tar cannot extract in $container_name (host emulation); using bsdtar" + # spread buffers allocate output and only prints it on failure, so the + # terminal (when there is one) is the only channel a user actually reads. + echo "$note" + { [ -w /dev/tty ] && echo "$note" > /dev/tty; } 2>/dev/null || true +fi if [ "$mode" = publish ]; then # The ephemeral host port docker mapped to the container's sshd. diff --git a/scripts/spread_allocate_bread.sh b/scripts/spread_allocate_bread.sh index 7318ed6..306590f 100755 --- a/scripts/spread_allocate_bread.sh +++ b/scripts/spread_allocate_bread.sh @@ -34,7 +34,9 @@ if [ "$mode" = bridge ]; then # snippet thanks to @lengau # https://github.com/canonical/charmcraft/blob/120a00a50f7ed3d0ae2fc2bea69e2e43b68b1594/spread.yaml#L72-L79 sleep 0.$RANDOM # Minimize chances of a race condition - export counter_file=".spread-worker-num" + # Kept out of the project tree: whatever lives there is packed up and + # shipped to every container. + export counter_file="${TMPDIR:-/tmp}/spread-bread-worker-num" instance_num=$( flock -x $counter_file bash -c ' [ -s $counter_file ] || echo 0 > $counter_file @@ -58,10 +60,42 @@ docker run \ -e "usr=$SPREAD_SYSTEM_USERNAME" \ -e "pass=$SPREAD_SYSTEM_PASSWORD" \ $publish_flag \ + --label spread-bread \ --name "$container_name" \ -d "$image" -until docker exec "$container_name" pgrep sshd; do sleep 1; done +# sshd is up within a second or two. The bound and the liveness check are +# backstops: a container that dies on start would otherwise spin here until +# spread's own timeout, which reports nothing about the container. +sshd_up="" +for ((i = 0; i < 60; i++)); do + if docker exec "$container_name" pgrep -x sshd >/dev/null 2>&1; then + sshd_up=1 + break + fi + if ! docker inspect "$container_name" --format '{{.State.Status}}' 2>/dev/null | grep -qx running; then + echo "$container_name is not running; reproduce with: docker run --rm --platform linux/$arch $image" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + sleep 1 +done +if [ -z "$sshd_up" ]; then + echo "sshd did not come up in $container_name after 60s" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 +fi + +# A container whose gnu tar cannot extract falls back to bsdtar (see +# hack/tar-shim.sh); say so rather than swapping the tool silently. +tar_backend=$(docker exec "$container_name" /usr/local/bin/bread-tar-shim --bread-probe 2>/dev/null || echo unknown) +if [ "$tar_backend" = bsdtar ]; then + note="note: gnu tar cannot extract in $container_name (host emulation); using bsdtar" + # spread buffers allocate output and only prints it on failure, so the + # terminal (when there is one) is the only channel a user actually reads. + echo "$note" + { [ -w /dev/tty ] && echo "$note" > /dev/tty; } 2>/dev/null || true +fi if [ "$mode" = publish ]; then # The ephemeral host port docker mapped to the container's sshd. diff --git a/scripts/spread_discard_bread-chisel-releases.sh b/scripts/spread_discard_bread-chisel-releases.sh index e95e01f..6d3918d 100755 --- a/scripts/spread_discard_bread-chisel-releases.sh +++ b/scripts/spread_discard_bread-chisel-releases.sh @@ -11,26 +11,20 @@ echo "Discarding container for system: $SPREAD_SYSTEM_ADDRESS" container_name="" case "$SPREAD_SYSTEM_ADDRESS" in *:*) - # publish mode: match the container by its published host port. + # publish mode: match the container by its published host port, as it + # shows up in the ports column, e.g. "127.0.0.1:32768->22/tcp". target_port="${SPREAD_SYSTEM_ADDRESS##*:}" - for cid in $(docker ps -a --format '{{.ID}}'); do - if docker port "$cid" 22 2>/dev/null | grep -q ":${target_port}\$"; then - container_name=$(docker inspect "$cid" --format '{{.Name}}' | sed 's#^/##') - break - fi - done + container_name=$(docker ps -a --filter label=spread-bread --format '{{.Names}} {{.Ports}}' | + awk -v m=":${target_port}->22/" 'index($0, m) { print $1; exit }') not_found="No container found with published port: $target_port" ;; *) # bridge mode: match the container by its bridge IP address. - for cid in $(docker ps -a --filter "network=bridge" --format '{{.ID}}'); do - cname=$(docker inspect "$cid" --format '{{.Name}}' | sed 's/^\/\(.*\)/\1/') - cip=$(docker inspect "$cid" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}' || echo "") - if [ "$cip" == "$SPREAD_SYSTEM_ADDRESS" ]; then - container_name="$cname" - break - fi - done + ids=$(docker ps -a --filter label=spread-bread --filter network=bridge --format '{{.ID}}') + if [ -n "$ids" ]; then + container_name=$(docker inspect $ids --format '{{.Name}} {{.NetworkSettings.Networks.bridge.IPAddress}}' | + awk -v ip="$SPREAD_SYSTEM_ADDRESS" '$2 == ip { print substr($1, 2); exit }') + fi not_found="No container found with IP address: $SPREAD_SYSTEM_ADDRESS" ;; esac diff --git a/scripts/spread_discard_bread.sh b/scripts/spread_discard_bread.sh index e95e01f..6d3918d 100755 --- a/scripts/spread_discard_bread.sh +++ b/scripts/spread_discard_bread.sh @@ -11,26 +11,20 @@ echo "Discarding container for system: $SPREAD_SYSTEM_ADDRESS" container_name="" case "$SPREAD_SYSTEM_ADDRESS" in *:*) - # publish mode: match the container by its published host port. + # publish mode: match the container by its published host port, as it + # shows up in the ports column, e.g. "127.0.0.1:32768->22/tcp". target_port="${SPREAD_SYSTEM_ADDRESS##*:}" - for cid in $(docker ps -a --format '{{.ID}}'); do - if docker port "$cid" 22 2>/dev/null | grep -q ":${target_port}\$"; then - container_name=$(docker inspect "$cid" --format '{{.Name}}' | sed 's#^/##') - break - fi - done + container_name=$(docker ps -a --filter label=spread-bread --format '{{.Names}} {{.Ports}}' | + awk -v m=":${target_port}->22/" 'index($0, m) { print $1; exit }') not_found="No container found with published port: $target_port" ;; *) # bridge mode: match the container by its bridge IP address. - for cid in $(docker ps -a --filter "network=bridge" --format '{{.ID}}'); do - cname=$(docker inspect "$cid" --format '{{.Name}}' | sed 's/^\/\(.*\)/\1/') - cip=$(docker inspect "$cid" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}' || echo "") - if [ "$cip" == "$SPREAD_SYSTEM_ADDRESS" ]; then - container_name="$cname" - break - fi - done + ids=$(docker ps -a --filter label=spread-bread --filter network=bridge --format '{{.ID}}') + if [ -n "$ids" ]; then + container_name=$(docker inspect $ids --format '{{.Name}} {{.NetworkSettings.Networks.bridge.IPAddress}}' | + awk -v ip="$SPREAD_SYSTEM_ADDRESS" '$2 == ip { print substr($1, 2); exit }') + fi not_found="No container found with IP address: $SPREAD_SYSTEM_ADDRESS" ;; esac diff --git a/scripts/spread_repack_bread.sh b/scripts/spread_repack_bread.sh index 6389089..c08c598 100644 --- a/scripts/spread_repack_bread.sh +++ b/scripts/spread_repack_bread.sh @@ -1,10 +1,11 @@ -# macOS tar packs an AppleDouble ._* sidecar next to every file; they unpack as -# real files on the remote. Repack to remove them. +# macOS tar packs an AppleDouble ._* sidecar next to every file and stores +# xattrs as pax headers; the sidecars unpack as real files on the remote and +# GNU tar warns about every xattr header. Repack to remove both. if [ "$(uname -s)" = Darwin ]; then tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT tar -xf - -C "$tmp" <&3 - ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + ( cd "$tmp" && COPYFILE_DISABLE=1 tar --no-mac-metadata --no-xattrs -cf - . ) >&4 else cat <&3 >&4 fi diff --git a/tests/_inner-bread-chisel-releases/tar-shim/task.yaml b/tests/_inner-bread-chisel-releases/tar-shim/task.yaml new file mode 100644 index 0000000..8c40255 --- /dev/null +++ b/tests/_inner-bread-chisel-releases/tar-shim/task.yaml @@ -0,0 +1,55 @@ +summary: tar shim contract (gnu tar diverted, every spread tar invocation works) + +execute: | + set -e + + test -x /usr/bin/tar.distrib || { + echo "gnu tar not diverted to /usr/bin/tar.distrib" + exit 1 + } + test -L /usr/bin/tar || { + echo "/usr/bin/tar is not the shim symlink" + exit 1 + } + bsdtar --version >/dev/null || { + echo "bsdtar missing" + exit 1 + } + /bin/tar --version | grep -q "GNU tar" || { + echo "tar --version not passed through to gnu tar" + exit 1 + } + + work=$(mktemp -d) + trap "rm -rf $work" EXIT + mkdir -p "$work/src/a/b" + : > "$work/src/a/b/f" + + # Nested entries are what breaks under a host whose emulation the patched + # gnu tar trips over. + /bin/tar -cf "$work/t.tar" -C "$work/src" a + mkdir "$work/out" + /bin/tar -xf "$work/t.tar" -C "$work/out" + test -f "$work/out/a/b/f" || { + echo "nested extraction through /bin/tar failed" + exit 1 + } + + # spread unpacks the project with bare mode letters (`tar xz`), which bsdtar + # does not accept unprefixed. + /bin/tar -C "$work/src" -czf "$work/t.tgz" a + mkdir "$work/out2" + ( cd "$work/out2" && /bin/tar xz < "$work/t.tgz" ) + test -f "$work/out2/a/b/f" || { + echo "bare-mode extraction failed" + exit 1 + } + + # ... and packs artifacts with flags only gnu tar has, so creation must never + # be routed to bsdtar. + /bin/tar -C "$work/src" -cz --sort=name --ignore-failed-read -- a >/dev/null || { + echo "gnu-only create flags rejected; shim misrouted creation" + exit 1 + } + + echo "ok: $SPREAD_SYSTEM ($(/usr/local/bin/bread-tar-shim --bread-probe))" diff --git a/tests/_inner-bread/tar-shim/task.yaml b/tests/_inner-bread/tar-shim/task.yaml new file mode 100644 index 0000000..8c40255 --- /dev/null +++ b/tests/_inner-bread/tar-shim/task.yaml @@ -0,0 +1,55 @@ +summary: tar shim contract (gnu tar diverted, every spread tar invocation works) + +execute: | + set -e + + test -x /usr/bin/tar.distrib || { + echo "gnu tar not diverted to /usr/bin/tar.distrib" + exit 1 + } + test -L /usr/bin/tar || { + echo "/usr/bin/tar is not the shim symlink" + exit 1 + } + bsdtar --version >/dev/null || { + echo "bsdtar missing" + exit 1 + } + /bin/tar --version | grep -q "GNU tar" || { + echo "tar --version not passed through to gnu tar" + exit 1 + } + + work=$(mktemp -d) + trap "rm -rf $work" EXIT + mkdir -p "$work/src/a/b" + : > "$work/src/a/b/f" + + # Nested entries are what breaks under a host whose emulation the patched + # gnu tar trips over. + /bin/tar -cf "$work/t.tar" -C "$work/src" a + mkdir "$work/out" + /bin/tar -xf "$work/t.tar" -C "$work/out" + test -f "$work/out/a/b/f" || { + echo "nested extraction through /bin/tar failed" + exit 1 + } + + # spread unpacks the project with bare mode letters (`tar xz`), which bsdtar + # does not accept unprefixed. + /bin/tar -C "$work/src" -czf "$work/t.tgz" a + mkdir "$work/out2" + ( cd "$work/out2" && /bin/tar xz < "$work/t.tgz" ) + test -f "$work/out2/a/b/f" || { + echo "bare-mode extraction failed" + exit 1 + } + + # ... and packs artifacts with flags only gnu tar has, so creation must never + # be routed to bsdtar. + /bin/tar -C "$work/src" -cz --sort=name --ignore-failed-read -- a >/dev/null || { + echo "gnu-only create flags rejected; shim misrouted creation" + exit 1 + } + + echo "ok: $SPREAD_SYSTEM ($(/usr/local/bin/bread-tar-shim --bread-probe))" diff --git a/tests/spread.yaml b/tests/spread.yaml index 45db283..3b7448b 100644 --- a/tests/spread.yaml +++ b/tests/spread.yaml @@ -12,14 +12,15 @@ exclude: - .git - .github -# macOS tar packs an AppleDouble `._*` sidecar next to every file; they unpack as -# real files on the remote. Repack to remove them. +# macOS tar packs an AppleDouble `._*` sidecar next to every file and stores +# xattrs as pax headers; the sidecars unpack as real files on the remote and GNU +# tar warns about every xattr header. Repack to remove both. repack: | if [ "$(uname -s)" = Darwin ]; then tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT tar -xf - -C "$tmp" <&3 - ( cd "$tmp" && shopt -s dotglob && COPYFILE_DISABLE=1 tar -cf - -- * ) >&4 + ( cd "$tmp" && COPYFILE_DISABLE=1 tar --no-mac-metadata --no-xattrs -cf - . ) >&4 else cat <&3 >&4 fi @@ -45,7 +46,9 @@ backends: # Unique container name per worker. sleep 0.$RANDOM - export counter_file=".spread-worker-num" + # Kept out of the project tree: whatever lives there is packed up and + # shipped to every container. + export counter_file="${TMPDIR:-/tmp}/spread-bread-worker-num" instance_num=$( flock -x $counter_file bash -c ' [ -s $counter_file ] || echo 0 > $counter_file @@ -67,10 +70,31 @@ backends: -e DEBIAN_FRONTEND=noninteractive \ -e "usr=$SPREAD_SYSTEM_USERNAME" \ -e "pass=$SPREAD_SYSTEM_PASSWORD" \ + --label spread-bread \ --name "$container_name" \ -d "$image" - until docker exec "$container_name" pgrep sshd; do sleep 1; done + # sshd is up within a second or two. The bound and the liveness check are + # backstops: a container that dies on start would otherwise spin here + # until spread's own timeout, which reports nothing about the container. + sshd_up="" + for ((i = 0; i < 60; i++)); do + if docker exec "$container_name" pgrep -x sshd >/dev/null 2>&1; then + sshd_up=1 + break + fi + if ! docker inspect "$container_name" --format '{{.State.Status}}' 2>/dev/null | grep -qx running; then + echo "$container_name is not running; reproduce with: docker run --rm --platform linux/$arch $image" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi + sleep 1 + done + if [ -z "$sshd_up" ]; then + echo "sshd did not come up in $container_name after 60s" >&2 + docker logs "$container_name" >&2 2>&1 || true + exit 1 + fi ADDRESS "$(docker inspect "$container_name" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}')" @@ -78,14 +102,11 @@ backends: set -e container_name="" - for cid in $(docker ps -a --filter "network=bridge" --format '{{.ID}}'); do - cname=$(docker inspect "$cid" --format '{{.Name}}' | sed 's/^\/\(.*\)/\1/') - cip=$(docker inspect "$cid" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}' || echo "") - if [ "$cip" == "$SPREAD_SYSTEM_ADDRESS" ]; then - container_name="$cname" - break - fi - done + ids=$(docker ps -a --filter label=spread-bread --filter network=bridge --format '{{.ID}}') + if [ -n "$ids" ]; then + container_name=$(docker inspect $ids --format '{{.Name}} {{.NetworkSettings.Networks.bridge.IPAddress}}' | + awk -v ip="$SPREAD_SYSTEM_ADDRESS" '$2 == ip { print substr($1, 2); exit }') + fi if [ -n "$container_name" ]; then echo "Removing outer container: $container_name"