From b07c1319e3170853cd7d156f0ecb6764b3455295 Mon Sep 17 00:00:00 2001 From: JY Tan Date: Thu, 19 Mar 2026 00:01:16 -0700 Subject: [PATCH 1/6] Add repro --- .github/workflows/main.yml | 16 +++++ scripts/repro_minimal_devices_ci.sh | 94 +++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 scripts/repro_minimal_devices_ci.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e5d232e..fe50cf1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -117,6 +117,22 @@ jobs: - name: Run smoke tests run: FENCE_TEST_NETWORK=1 ./scripts/smoke_test.sh ./fence + repro-minimal-devices: + name: Repro minimal /dev (${{ matrix.runner }}) + runs-on: ${{ matrix.runner }} + continue-on-error: true + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + runner: [ubuntu-22.04, ubuntu-24.04] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run nested Docker repro + run: bash ./scripts/repro_minimal_devices_ci.sh + test-macos: name: Test (macOS) runs-on: macos-latest diff --git a/scripts/repro_minimal_devices_ci.sh b/scripts/repro_minimal_devices_ci.sh new file mode 100644 index 0000000..d7a0b0d --- /dev/null +++ b/scripts/repro_minimal_devices_ci.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +set -euo pipefail + +IMAGE="${FENCE_REPRO_IMAGE:-golang:1.25-bookworm}" + +echo "=== Host diagnostics ===" +echo "pwd: $PWD" +echo "kernel: $(uname -r)" +docker version --format 'docker server={{.Server.Version}} client={{.Client.Version}}' +docker info --format 'docker os={{.OperatingSystem}} kernel={{.KernelVersion}} cgroup={{.CgroupVersion}}' + +docker run --rm \ + --cap-add SYS_ADMIN \ + --security-opt seccomp=unconfined \ + --security-opt apparmor=unconfined \ + -v "$PWD":/src \ + -w /src \ + "$IMAGE" \ + bash -c ' + set -euo pipefail + + echo "=== Container diagnostics ===" + echo "kernel: $(uname -r)" + echo "user: $(id)" + + apt-get update + apt-get install -y bubblewrap socat python3 python3-pip + + go build -o /usr/local/bin/fence ./cmd/fence + python3 -m pip install -q grpcio + + cat >/tmp/fence.json <<'"'"'EOF'"'"' +{"devices":{"mode":"minimal"}} +EOF + + echo "=== Device probe under fence ===" + device_status=0 + /usr/local/bin/fence --settings /tmp/fence.json -- python3 - <<'"'"'PY'"'"' || device_status=$? +import os +import stat +import sys + +failures = 0 + +print(f"uid={os.getuid()} euid={os.geteuid()} cwd={os.getcwd()}") +print("/dev entries:", ", ".join(sorted(os.listdir("/dev")))) + +for path in ("/dev/null", "/dev/random", "/dev/urandom"): + st = os.stat(path) + kind = "char" if stat.S_ISCHR(st.st_mode) else "other" + device_id = "-" + if stat.S_ISCHR(st.st_mode): + device_id = f"{os.major(st.st_rdev)}:{os.minor(st.st_rdev)}" + print(f"{path} mode={oct(stat.S_IMODE(st.st_mode))} kind={kind} rdev={device_id}") + try: + fd = os.open(path, os.O_RDONLY) + os.close(fd) + print(path, "ok") + except OSError as exc: + print(path, f"errno={exc.errno} strerror={exc.strerror}") + failures += 1 + +print("getrandom:", len(os.getrandom(1))) +sys.exit(1 if failures else 0) +PY + echo "device_status=${device_status}" + + echo "=== gRPC startup probe under fence ===" + grpc_status=0 + /usr/local/bin/fence --settings /tmp/fence.json -- python3 - <<'"'"'PY'"'"' || grpc_status=$? +from concurrent import futures + +import grpc + +print("before grpc.server()", flush=True) +server = grpc.server(futures.ThreadPoolExecutor(max_workers=1)) +print("after grpc.server()", flush=True) +port = server.add_insecure_port("[::]:50051") +print(f"add_insecure_port={port}", flush=True) +server.start() +print("after server.start()", flush=True) +server.stop(0) +print("after server.stop()", flush=True) +PY + echo "grpc_status=${grpc_status}" + + if [ "${device_status}" -ne 0 ] || [ "${grpc_status}" -ne 0 ]; then + echo "=== Reproducer detected a failure ===" + exit 1 + fi + + echo "=== Reproducer did not detect the issue ===" + ' From e9368ca900572e8dcbdcb0f3bf941a36caf3e67a Mon Sep 17 00:00:00 2001 From: JY Tan Date: Thu, 19 Mar 2026 00:03:23 -0700 Subject: [PATCH 2/6] buildvcs=false --- scripts/repro_minimal_devices_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/repro_minimal_devices_ci.sh b/scripts/repro_minimal_devices_ci.sh index d7a0b0d..9e35ae1 100644 --- a/scripts/repro_minimal_devices_ci.sh +++ b/scripts/repro_minimal_devices_ci.sh @@ -27,7 +27,7 @@ docker run --rm \ apt-get update apt-get install -y bubblewrap socat python3 python3-pip - go build -o /usr/local/bin/fence ./cmd/fence + go build -buildvcs=false -o /usr/local/bin/fence ./cmd/fence python3 -m pip install -q grpcio cat >/tmp/fence.json <<'"'"'EOF'"'"' From 97b078a2d9b112202cc2b5a2dc8e79640b28d599 Mon Sep 17 00:00:00 2001 From: JY Tan Date: Thu, 19 Mar 2026 00:05:24 -0700 Subject: [PATCH 3/6] python env --- scripts/repro_minimal_devices_ci.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/repro_minimal_devices_ci.sh b/scripts/repro_minimal_devices_ci.sh index 9e35ae1..3e92814 100644 --- a/scripts/repro_minimal_devices_ci.sh +++ b/scripts/repro_minimal_devices_ci.sh @@ -25,10 +25,13 @@ docker run --rm \ echo "user: $(id)" apt-get update - apt-get install -y bubblewrap socat python3 python3-pip + apt-get install -y bubblewrap socat python3 python3-venv go build -buildvcs=false -o /usr/local/bin/fence ./cmd/fence - python3 -m pip install -q grpcio + VENV_DIR=/tmp/fence-repro-venv + python3 -m venv "${VENV_DIR}" + "${VENV_DIR}/bin/python" -m pip install -q grpcio + PYTHON_BIN="${VENV_DIR}/bin/python" cat >/tmp/fence.json <<'"'"'EOF'"'"' {"devices":{"mode":"minimal"}} @@ -36,7 +39,7 @@ EOF echo "=== Device probe under fence ===" device_status=0 - /usr/local/bin/fence --settings /tmp/fence.json -- python3 - <<'"'"'PY'"'"' || device_status=$? + /usr/local/bin/fence --settings /tmp/fence.json -- "${PYTHON_BIN}" - <<'"'"'PY'"'"' || device_status=$? import os import stat import sys @@ -68,7 +71,7 @@ PY echo "=== gRPC startup probe under fence ===" grpc_status=0 - /usr/local/bin/fence --settings /tmp/fence.json -- python3 - <<'"'"'PY'"'"' || grpc_status=$? + /usr/local/bin/fence --settings /tmp/fence.json -- "${PYTHON_BIN}" - <<'"'"'PY'"'"' || grpc_status=$? from concurrent import futures import grpc From 9512ef58c9a69095e087a426190704a7d6ee83b5 Mon Sep 17 00:00:00 2001 From: JY Tan Date: Thu, 19 Mar 2026 00:07:58 -0700 Subject: [PATCH 4/6] Use fresh tmp --- scripts/repro_minimal_devices_ci.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/repro_minimal_devices_ci.sh b/scripts/repro_minimal_devices_ci.sh index 3e92814..f89c21e 100644 --- a/scripts/repro_minimal_devices_ci.sh +++ b/scripts/repro_minimal_devices_ci.sh @@ -28,7 +28,8 @@ docker run --rm \ apt-get install -y bubblewrap socat python3 python3-venv go build -buildvcs=false -o /usr/local/bin/fence ./cmd/fence - VENV_DIR=/tmp/fence-repro-venv + VENV_DIR="$(mktemp -d /src/.fence-repro-venv.XXXXXX)" + trap "rm -rf \"${VENV_DIR}\"" EXIT python3 -m venv "${VENV_DIR}" "${VENV_DIR}/bin/python" -m pip install -q grpcio PYTHON_BIN="${VENV_DIR}/bin/python" From 3fd132ace75a89e0741a7236d7cd324183d3bff7 Mon Sep 17 00:00:00 2001 From: JY Tan Date: Thu, 19 Mar 2026 00:16:33 -0700 Subject: [PATCH 5/6] Add more stuff --- .github/workflows/main.yml | 27 +++++++- scripts/repro_minimal_devices_ci.sh | 99 ++++++++++++++++++++++++++--- 2 files changed, 115 insertions(+), 11 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fe50cf1..646fd22 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -118,19 +118,42 @@ jobs: run: FENCE_TEST_NETWORK=1 ./scripts/smoke_test.sh ./fence repro-minimal-devices: - name: Repro minimal /dev (${{ matrix.runner }}) + name: Repro minimal /dev (${{ matrix.runner }}, ${{ matrix.image_name }}) runs-on: ${{ matrix.runner }} continue-on-error: true timeout-minutes: 20 strategy: fail-fast: false matrix: - runner: [ubuntu-22.04, ubuntu-24.04] + include: + - runner: ubuntu-22.04 + image_name: golang-bookworm + image: golang:1.25-bookworm + - runner: ubuntu-22.04 + image_name: python39-slim + image: python:3.9-slim + - runner: ubuntu-24.04 + image_name: golang-bookworm + image: golang:1.25-bookworm + - runner: ubuntu-24.04 + image_name: python39-slim + image: python:3.9-slim steps: - name: Checkout uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Build fence binary for nested repro + run: go build -buildvcs=false -o ./fence-repro-bin ./cmd/fence + - name: Run nested Docker repro + env: + FENCE_REPRO_IMAGE: ${{ matrix.image }} run: bash ./scripts/repro_minimal_devices_ci.sh test-macos: diff --git a/scripts/repro_minimal_devices_ci.sh b/scripts/repro_minimal_devices_ci.sh index f89c21e..1aff1ea 100644 --- a/scripts/repro_minimal_devices_ci.sh +++ b/scripts/repro_minimal_devices_ci.sh @@ -17,17 +17,29 @@ docker run --rm \ -v "$PWD":/src \ -w /src \ "$IMAGE" \ - bash -c ' - set -euo pipefail + sh -c ' + set -eu echo "=== Container diagnostics ===" echo "kernel: $(uname -r)" echo "user: $(id)" apt-get update - apt-get install -y bubblewrap socat python3 python3-venv + apt-get install -y bubblewrap socat + if ! command -v python3 >/dev/null 2>&1; then + apt-get install -y python3 python3-venv + elif ! python3 -m venv --help >/dev/null 2>&1; then + apt-get install -y python3-venv + fi - go build -buildvcs=false -o /usr/local/bin/fence ./cmd/fence + if [ -x /src/fence-repro-bin ] && /src/fence-repro-bin --version >/dev/null 2>&1; then + install -m 0755 /src/fence-repro-bin /usr/local/bin/fence + elif command -v go >/dev/null 2>&1; then + go build -buildvcs=false -o /usr/local/bin/fence ./cmd/fence + else + echo "No usable Linux fence binary found at /src/fence-repro-bin and Go is unavailable in the container" >&2 + exit 1 + fi VENV_DIR="$(mktemp -d /src/.fence-repro-venv.XXXXXX)" trap "rm -rf \"${VENV_DIR}\"" EXIT python3 -m venv "${VENV_DIR}" @@ -38,7 +50,22 @@ docker run --rm \ {"devices":{"mode":"minimal"}} EOF - echo "=== Device probe under fence ===" + cat >/tmp/fence-replay-like.json <<'"'"'EOF'"'"' +{ + "devices": { "mode": "minimal" }, + "network": { + "allowedDomains": ["localhost", "127.0.0.1"], + "allowLocalBinding": true, + "allowLocalOutbound": false, + "allowAllUnixSockets": true + }, + "filesystem": { + "allowWrite": ["/src", "/tmp"] + } +} +EOF + + echo "=== Device probe under fence (baseline) ===" device_status=0 /usr/local/bin/fence --settings /tmp/fence.json -- "${PYTHON_BIN}" - <<'"'"'PY'"'"' || device_status=$? import os @@ -68,9 +95,41 @@ for path in ("/dev/null", "/dev/random", "/dev/urandom"): print("getrandom:", len(os.getrandom(1))) sys.exit(1 if failures else 0) PY - echo "device_status=${device_status}" + echo "device_status(baseline)=${device_status}" + + replay_device_status=0 + echo "=== Device probe under fence (replay-like) ===" + /usr/local/bin/fence -p 8000 --settings /tmp/fence-replay-like.json -- "${PYTHON_BIN}" - <<'"'"'PY'"'"' || replay_device_status=$? +import os +import stat +import sys + +failures = 0 + +print(f"uid={os.getuid()} euid={os.geteuid()} cwd={os.getcwd()}") +print("/dev entries:", ", ".join(sorted(os.listdir("/dev")))) + +for path in ("/dev/null", "/dev/random", "/dev/urandom"): + st = os.stat(path) + kind = "char" if stat.S_ISCHR(st.st_mode) else "other" + device_id = "-" + if stat.S_ISCHR(st.st_mode): + device_id = f"{os.major(st.st_rdev)}:{os.minor(st.st_rdev)}" + print(f"{path} mode={oct(stat.S_IMODE(st.st_mode))} kind={kind} rdev={device_id}") + try: + fd = os.open(path, os.O_RDONLY) + os.close(fd) + print(path, "ok") + except OSError as exc: + print(path, f"errno={exc.errno} strerror={exc.strerror}") + failures += 1 + +print("getrandom:", len(os.getrandom(1))) +sys.exit(1 if failures else 0) +PY + echo "device_status(replay-like)=${replay_device_status}" - echo "=== gRPC startup probe under fence ===" + echo "=== gRPC startup probe under fence (baseline) ===" grpc_status=0 /usr/local/bin/fence --settings /tmp/fence.json -- "${PYTHON_BIN}" - <<'"'"'PY'"'"' || grpc_status=$? from concurrent import futures @@ -87,9 +146,31 @@ print("after server.start()", flush=True) server.stop(0) print("after server.stop()", flush=True) PY - echo "grpc_status=${grpc_status}" + echo "grpc_status(baseline)=${grpc_status}" + + replay_grpc_status=0 + echo "=== gRPC startup probe under fence (replay-like) ===" + /usr/local/bin/fence -p 8000 --settings /tmp/fence-replay-like.json -- "${PYTHON_BIN}" - <<'"'"'PY'"'"' || replay_grpc_status=$? +from concurrent import futures + +import grpc + +print("before grpc.server()", flush=True) +server = grpc.server(futures.ThreadPoolExecutor(max_workers=1)) +print("after grpc.server()", flush=True) +port = server.add_insecure_port("[::]:50051") +print(f"add_insecure_port={port}", flush=True) +server.start() +print("after server.start()", flush=True) +server.stop(0) +print("after server.stop()", flush=True) +PY + echo "grpc_status(replay-like)=${replay_grpc_status}" - if [ "${device_status}" -ne 0 ] || [ "${grpc_status}" -ne 0 ]; then + if [ "${device_status}" -ne 0 ] || \ + [ "${replay_device_status}" -ne 0 ] || \ + [ "${grpc_status}" -ne 0 ] || \ + [ "${replay_grpc_status}" -ne 0 ]; then echo "=== Reproducer detected a failure ===" exit 1 fi From 2f4fb183976af15b2de9de47e625c53a56b230af Mon Sep 17 00:00:00 2001 From: JY Tan Date: Thu, 19 Mar 2026 00:21:08 -0700 Subject: [PATCH 6/6] Fix --- scripts/repro_minimal_devices_ci.sh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/scripts/repro_minimal_devices_ci.sh b/scripts/repro_minimal_devices_ci.sh index 1aff1ea..bbfdf88 100644 --- a/scripts/repro_minimal_devices_ci.sh +++ b/scripts/repro_minimal_devices_ci.sh @@ -27,11 +27,24 @@ docker run --rm \ apt-get update apt-get install -y bubblewrap socat if ! command -v python3 >/dev/null 2>&1; then - apt-get install -y python3 python3-venv - elif ! python3 -m venv --help >/dev/null 2>&1; then - apt-get install -y python3-venv + apt-get install -y python3 fi + VENV_PROBE_DIR="$(mktemp -d /tmp/fence-venv-probe.XXXXXX)" + if ! python3 -m venv "${VENV_PROBE_DIR}" >/tmp/fence-venv-probe.log 2>&1; then + PYTHON_VENV_PACKAGE="$(python3 - <<'"'"'PY'"'"' +import sys +print(f"python{sys.version_info.major}.{sys.version_info.minor}-venv") +PY +)" + if ! apt-get install -y "${PYTHON_VENV_PACKAGE}" python3-venv; then + apt-get install -y "${PYTHON_VENV_PACKAGE}" || apt-get install -y python3-venv + fi + rm -rf "${VENV_PROBE_DIR}" + python3 -m venv "${VENV_PROBE_DIR}" + fi + rm -rf "${VENV_PROBE_DIR}" /tmp/fence-venv-probe.log + if [ -x /src/fence-repro-bin ] && /src/fence-repro-bin --version >/dev/null 2>&1; then install -m 0755 /src/fence-repro-bin /usr/local/bin/fence elif command -v go >/dev/null 2>&1; then