From dd3e7628bff4ff2cad9aa8752dcd7cce5e40f047 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:14:49 +0200 Subject: [PATCH 1/6] ci: add fresh-process Bun test batching --- scripts/ci/run-bun-test-batches.sh | 156 +++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 scripts/ci/run-bun-test-batches.sh diff --git a/scripts/ci/run-bun-test-batches.sh b/scripts/ci/run-bun-test-batches.sh new file mode 100644 index 000000000..c74c56c45 --- /dev/null +++ b/scripts/ci/run-bun-test-batches.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly SHARD_SPEC="${1:-}" +readonly BATCH_SIZE="${BUN_TEST_BATCH_SIZE:-12}" + +usage() { + echo "usage: $0 " >&2 + exit 64 +} + +if [[ ! "$SHARD_SPEC" =~ ^([1-9][0-9]*)/([1-9][0-9]*)$ ]]; then + usage +fi + +readonly SHARD_INDEX="${BASH_REMATCH[1]}" +readonly SHARD_COUNT="${BASH_REMATCH[2]}" + +if (( SHARD_INDEX > SHARD_COUNT )); then + usage +fi +if [[ ! "$BATCH_SIZE" =~ ^[1-9][0-9]*$ ]]; then + echo "BUN_TEST_BATCH_SIZE must be a positive integer, got: $BATCH_SIZE" >&2 + exit 64 +fi + +is_general_test_file() { + local path="$1" + + case "$path" in + tests/api-storage-policy*.test.ts|tests/api-storage.test.ts|tests/api-usage.test.ts) + return 1 + ;; + esac + + case "$path" in + *.test.js|*.test.jsx|*.test.ts|*.test.tsx|*_test.js|*_test.jsx|*_test.ts|*_test.tsx|*.spec.js|*.spec.jsx|*.spec.ts|*.spec.tsx|*_spec.js|*_spec.jsx|*_spec.ts|*_spec.tsx) + return 0 + ;; + *) + return 1 + ;; + esac +} + +is_bun_runtime_crash() { + local status="$1" + local log_file="$2" + + case "$status" in + 132|133|134|135|136|137|139) + return 0 + ;; + esac + + grep -Eqi \ + 'oh no: Bun has crashed|Segmentation fault at address|Illegal instruction|Bus error|Aborted \(core dumped\)' \ + "$log_file" +} + +LAST_FAILURE_KIND="" + +run_batch_once() { + local batch_number="$1" + local attempt="$2" + shift 2 + local -a files=("$@") + local log_file + local status + + log_file="$(mktemp -t ocx-bun-test-batch.XXXXXX)" + + echo "::group::shard ${SHARD_SPEC} batch ${batch_number}/${TOTAL_BATCHES} attempt ${attempt} (${#files[@]} files)" + printf ' %s\n' "${files[@]}" + + set +e + bun test --isolate "${files[@]}" 2>&1 | tee "$log_file" + status="${PIPESTATUS[0]}" + set -e + + echo "::endgroup::" + + if (( status == 0 )); then + LAST_FAILURE_KIND="" + rm -f -- "$log_file" + return 0 + fi + + if is_bun_runtime_crash "$status" "$log_file"; then + LAST_FAILURE_KIND="runtime" + echo "::warning::Bun runtime crash in shard ${SHARD_SPEC} batch ${batch_number} (exit ${status}, attempt ${attempt})." + rm -f -- "$log_file" + return "$status" + fi + + LAST_FAILURE_KIND="test" + echo "::error::Test failure in shard ${SHARD_SPEC} batch ${batch_number} (exit ${status}); not retrying assertion/test failures." + rm -f -- "$log_file" + return "$status" +} + +mapfile -d '' -t ALL_TEST_FILES < <( + find tests -type f -print0 \ + | LC_ALL=C sort -z +) + +SELECTED_FILES=() +general_index=0 +for path in "${ALL_TEST_FILES[@]}"; do + if ! is_general_test_file "$path"; then + continue + fi + + if (( general_index % SHARD_COUNT == SHARD_INDEX - 1 )); then + SELECTED_FILES+=("$path") + fi + ((general_index += 1)) +done + +if (( ${#SELECTED_FILES[@]} == 0 )); then + echo "No tests selected for shard ${SHARD_SPEC}." >&2 + exit 1 +fi + +readonly TOTAL_BATCHES=$(( (${#SELECTED_FILES[@]} + BATCH_SIZE - 1) / BATCH_SIZE )) +echo "Shard ${SHARD_SPEC}: ${#SELECTED_FILES[@]} files in ${TOTAL_BATCHES} fresh Bun processes (batch size <= ${BATCH_SIZE})." + +for ((batch_index = 0; batch_index < TOTAL_BATCHES; batch_index += 1)); do + start=$(( batch_index * BATCH_SIZE )) + batch=("${SELECTED_FILES[@]:start:BATCH_SIZE}") + batch_number=$(( batch_index + 1 )) + + if run_batch_once "$batch_number" 1 "${batch[@]}"; then + continue + else + status=$? + fi + + if [[ "$LAST_FAILURE_KIND" != "runtime" ]]; then + exit "$status" + fi + + echo "Retrying shard ${SHARD_SPEC} batch ${batch_number} once in a fresh Bun process..." + if run_batch_once "$batch_number" 2 "${batch[@]}"; then + echo "::warning::Shard ${SHARD_SPEC} batch ${batch_number} passed on the single runtime-crash retry." + continue + else + status=$? + fi + + if [[ "$LAST_FAILURE_KIND" == "runtime" ]]; then + echo "::error::Bun runtime crash repeated in shard ${SHARD_SPEC} batch ${batch_number}; failing after one retry." + fi + + exit "$status" +done From 1aceb4b8541ce69e88b1c9d67d99d9362e5993a7 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:16:50 +0200 Subject: [PATCH 2/6] ci: run Linux test shards in fresh-process batches --- .github/workflows/ci.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01ceb64ee..19e00db97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -233,8 +233,10 @@ jobs: # The suite, split by file across four Linux runners. # - # `bun test --shard=i/N` sorts test files by path and deals them round-robin, - # so the split is deterministic for the files that remain in this lane. + # `scripts/ci/run-bun-test-batches.sh` mirrors Bun's sorted round-robin shard + # assignment, then runs each shard in small batches so every batch gets a fresh + # Bun process. The helper prints the exact files before each batch and retries + # only a Bun runtime crash once; ordinary test failures are never retried. # Storage-policy API tests and api-usage are deliberately excluded here and run # in dedicated jobs below. Bun 1.3.14 can corrupt the Linux isolate/epoll state # around those Worker-heavy harnesses; keeping them out of the general shards @@ -293,8 +295,10 @@ jobs: cd gui bun run build - - name: Test - run: bun test --isolate tests --path-ignore-patterns 'tests/api-storage-policy*.test.ts' --path-ignore-patterns 'tests/api-storage.test.ts' --path-ignore-patterns 'tests/api-usage.test.ts' --shard=${{ matrix.shard }}/4 + - name: Test in fresh-process batches + env: + TEST_SHARD: ${{ matrix.shard }}/4 + run: bash scripts/ci/run-bun-test-batches.sh "$TEST_SHARD" # Bun 1.3.14 has shown a Linux isolate/epoll race around the storage-policy # harness. Keep the entire six-file family in one fresh process so a runtime From 4f35818f943c90a51e188c9874fa20ecae6e6245 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:21:52 +0200 Subject: [PATCH 3/6] test: preserve api-usage isolation contract --- tests/zz-ci-api-usage-isolation.test.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/zz-ci-api-usage-isolation.test.ts b/tests/zz-ci-api-usage-isolation.test.ts index d3dee352a..732fca358 100644 --- a/tests/zz-ci-api-usage-isolation.test.ts +++ b/tests/zz-ci-api-usage-isolation.test.ts @@ -20,10 +20,15 @@ test("Linux shards isolate api-usage into its own gated job", async () => { jobs?: Record; }; - const shardRun = workflow.jobs?.test?.steps?.find(step => step.name === "Test")?.run ?? ""; - expect(shardRun).toContain( - "--path-ignore-patterns 'tests/api-usage.test.ts'", - ); + const shardRun = workflow.jobs?.test?.steps?.find( + step => step.name === "Test in fresh-process batches", + )?.run ?? ""; + expect(shardRun).toContain("scripts/ci/run-bun-test-batches.sh"); + + const batchHelper = await Bun.file( + new URL("../scripts/ci/run-bun-test-batches.sh", import.meta.url), + ).text(); + expect(batchHelper).toContain("tests/api-usage.test.ts)"); const apiUsageJob = workflow.jobs?.["api-usage"]; expect(apiUsageJob?.["runs-on"]).toBe("ubuntu-latest"); From bb7da80793a1372a5a19ff7419bfd3a167438ef4 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:23:00 +0200 Subject: [PATCH 4/6] test: preserve storage-policy isolation contract --- tests/zz-ci-storage-policy-isolation.test.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/zz-ci-storage-policy-isolation.test.ts b/tests/zz-ci-storage-policy-isolation.test.ts index 802554bac..5e2876cf5 100644 --- a/tests/zz-ci-storage-policy-isolation.test.ts +++ b/tests/zz-ci-storage-policy-isolation.test.ts @@ -20,13 +20,16 @@ test("Linux shards isolate the storage API runtime family into its own gated job jobs?: Record; }; - const shardRun = workflow.jobs?.test?.steps?.find(step => step.name === "Test")?.run ?? ""; - expect(shardRun).toContain( - "--path-ignore-patterns 'tests/api-storage-policy*.test.ts'", - ); - expect(shardRun).toContain( - "--path-ignore-patterns 'tests/api-storage.test.ts'", - ); + const shardRun = workflow.jobs?.test?.steps?.find( + step => step.name === "Test in fresh-process batches", + )?.run ?? ""; + expect(shardRun).toContain("scripts/ci/run-bun-test-batches.sh"); + + const batchHelper = await Bun.file( + new URL("../scripts/ci/run-bun-test-batches.sh", import.meta.url), + ).text(); + expect(batchHelper).toContain("tests/api-storage-policy*.test.ts"); + expect(batchHelper).toContain("tests/api-storage.test.ts"); const storageJob = workflow.jobs?.["storage-policy"]; expect(storageJob?.["runs-on"]).toBe("ubuntu-latest"); From 4fb67f8807d905c7495642df5f96f39865167200 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:40:27 +0200 Subject: [PATCH 5/6] ci: bound Bun test batches with timeouts --- scripts/ci/run-bun-test-batches.sh | 38 +++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/scripts/ci/run-bun-test-batches.sh b/scripts/ci/run-bun-test-batches.sh index c74c56c45..3a1215a32 100644 --- a/scripts/ci/run-bun-test-batches.sh +++ b/scripts/ci/run-bun-test-batches.sh @@ -3,6 +3,8 @@ set -euo pipefail readonly SHARD_SPEC="${1:-}" readonly BATCH_SIZE="${BUN_TEST_BATCH_SIZE:-12}" +readonly BATCH_TIMEOUT_SECONDS="${BUN_TEST_BATCH_TIMEOUT_SECONDS:-120}" +readonly BATCH_KILL_GRACE_SECONDS="${BUN_TEST_BATCH_KILL_GRACE_SECONDS:-15}" usage() { echo "usage: $0 " >&2 @@ -23,6 +25,18 @@ if [[ ! "$BATCH_SIZE" =~ ^[1-9][0-9]*$ ]]; then echo "BUN_TEST_BATCH_SIZE must be a positive integer, got: $BATCH_SIZE" >&2 exit 64 fi +if [[ ! "$BATCH_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]]; then + echo "BUN_TEST_BATCH_TIMEOUT_SECONDS must be a positive integer, got: $BATCH_TIMEOUT_SECONDS" >&2 + exit 64 +fi +if [[ ! "$BATCH_KILL_GRACE_SECONDS" =~ ^[1-9][0-9]*$ ]]; then + echo "BUN_TEST_BATCH_KILL_GRACE_SECONDS must be a positive integer, got: $BATCH_KILL_GRACE_SECONDS" >&2 + exit 64 +fi +if ! command -v timeout >/dev/null 2>&1; then + echo "GNU timeout is required to bound Bun test batches." >&2 + exit 69 +fi is_general_test_file() { local path="$1" @@ -74,7 +88,9 @@ run_batch_once() { printf ' %s\n' "${files[@]}" set +e - bun test --isolate "${files[@]}" 2>&1 | tee "$log_file" + timeout --signal=TERM --kill-after="${BATCH_KILL_GRACE_SECONDS}s" \ + "${BATCH_TIMEOUT_SECONDS}s" \ + bun test --isolate "${files[@]}" 2>&1 | tee "$log_file" status="${PIPESTATUS[0]}" set -e @@ -86,6 +102,13 @@ run_batch_once() { return 0 fi + if (( status == 124 )); then + LAST_FAILURE_KIND="timeout" + echo "::warning::Bun batch timed out after ${BATCH_TIMEOUT_SECONDS}s in shard ${SHARD_SPEC} batch ${batch_number} (attempt ${attempt})." + rm -f -- "$log_file" + return "$status" + fi + if is_bun_runtime_crash "$status" "$log_file"; then LAST_FAILURE_KIND="runtime" echo "::warning::Bun runtime crash in shard ${SHARD_SPEC} batch ${batch_number} (exit ${status}, attempt ${attempt})." @@ -123,7 +146,7 @@ if (( ${#SELECTED_FILES[@]} == 0 )); then fi readonly TOTAL_BATCHES=$(( (${#SELECTED_FILES[@]} + BATCH_SIZE - 1) / BATCH_SIZE )) -echo "Shard ${SHARD_SPEC}: ${#SELECTED_FILES[@]} files in ${TOTAL_BATCHES} fresh Bun processes (batch size <= ${BATCH_SIZE})." +echo "Shard ${SHARD_SPEC}: ${#SELECTED_FILES[@]} files in ${TOTAL_BATCHES} fresh Bun processes (batch size <= ${BATCH_SIZE}, timeout ${BATCH_TIMEOUT_SECONDS}s)." for ((batch_index = 0; batch_index < TOTAL_BATCHES; batch_index += 1)); do start=$(( batch_index * BATCH_SIZE )) @@ -136,19 +159,22 @@ for ((batch_index = 0; batch_index < TOTAL_BATCHES; batch_index += 1)); do status=$? fi - if [[ "$LAST_FAILURE_KIND" != "runtime" ]]; then + if [[ "$LAST_FAILURE_KIND" != "runtime" && "$LAST_FAILURE_KIND" != "timeout" ]]; then exit "$status" fi - echo "Retrying shard ${SHARD_SPEC} batch ${batch_number} once in a fresh Bun process..." + retry_kind="$LAST_FAILURE_KIND" + echo "Retrying shard ${SHARD_SPEC} batch ${batch_number} once in a fresh Bun process after ${retry_kind} failure..." if run_batch_once "$batch_number" 2 "${batch[@]}"; then - echo "::warning::Shard ${SHARD_SPEC} batch ${batch_number} passed on the single runtime-crash retry." + echo "::warning::Shard ${SHARD_SPEC} batch ${batch_number} passed on the single ${retry_kind} retry." continue else status=$? fi - if [[ "$LAST_FAILURE_KIND" == "runtime" ]]; then + if [[ "$LAST_FAILURE_KIND" == "timeout" ]]; then + echo "::error::Bun batch timed out twice in shard ${SHARD_SPEC} batch ${batch_number}; failing after one retry." + elif [[ "$LAST_FAILURE_KIND" == "runtime" ]]; then echo "::error::Bun runtime crash repeated in shard ${SHARD_SPEC} batch ${batch_number}; failing after one retry." fi From fb38f4b2ea85eeedcb2895d46c863ba649c26213 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:52:05 +0200 Subject: [PATCH 6/6] ci: isolate unstable Bun batches per file --- scripts/ci/run-bun-test-batches.sh | 89 ++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 22 deletions(-) diff --git a/scripts/ci/run-bun-test-batches.sh b/scripts/ci/run-bun-test-batches.sh index 3a1215a32..6031deb91 100644 --- a/scripts/ci/run-bun-test-batches.sh +++ b/scripts/ci/run-bun-test-batches.sh @@ -74,17 +74,23 @@ is_bun_runtime_crash() { LAST_FAILURE_KIND="" -run_batch_once() { +run_test_once() { local batch_number="$1" - local attempt="$2" - shift 2 + local phase="$2" + local attempt="$3" + shift 3 local -a files=("$@") local log_file local status + local label="shard ${SHARD_SPEC} batch ${batch_number}/${TOTAL_BATCHES}" + + if [[ -n "$phase" ]]; then + label+=" ${phase}" + fi log_file="$(mktemp -t ocx-bun-test-batch.XXXXXX)" - echo "::group::shard ${SHARD_SPEC} batch ${batch_number}/${TOTAL_BATCHES} attempt ${attempt} (${#files[@]} files)" + echo "::group::${label} attempt ${attempt} (${#files[@]} files)" printf ' %s\n' "${files[@]}" set +e @@ -104,24 +110,72 @@ run_batch_once() { if (( status == 124 )); then LAST_FAILURE_KIND="timeout" - echo "::warning::Bun batch timed out after ${BATCH_TIMEOUT_SECONDS}s in shard ${SHARD_SPEC} batch ${batch_number} (attempt ${attempt})." + echo "::warning::Bun test process timed out after ${BATCH_TIMEOUT_SECONDS}s in ${label} (attempt ${attempt})." rm -f -- "$log_file" return "$status" fi if is_bun_runtime_crash "$status" "$log_file"; then LAST_FAILURE_KIND="runtime" - echo "::warning::Bun runtime crash in shard ${SHARD_SPEC} batch ${batch_number} (exit ${status}, attempt ${attempt})." + echo "::warning::Bun runtime crash in ${label} (exit ${status}, attempt ${attempt})." rm -f -- "$log_file" return "$status" fi LAST_FAILURE_KIND="test" - echo "::error::Test failure in shard ${SHARD_SPEC} batch ${batch_number} (exit ${status}); not retrying assertion/test failures." + echo "::error::Test failure in ${label} (exit ${status}); not retrying assertion/test failures." rm -f -- "$log_file" return "$status" } +recover_batch_file_by_file() { + local batch_number="$1" + local batch_failure_kind="$2" + shift 2 + local -a files=("$@") + local file + local file_index=0 + local status + local retry_kind + + echo "::warning::Shard ${SHARD_SPEC} batch ${batch_number} hit a ${batch_failure_kind}; rerunning its ${#files[@]} files one at a time in fresh Bun processes." + + for file in "${files[@]}"; do + ((file_index += 1)) + if run_test_once "$batch_number" "singleton ${file_index}/${#files[@]}" 1 "$file"; then + continue + else + status=$? + fi + + if [[ "$LAST_FAILURE_KIND" != "runtime" && "$LAST_FAILURE_KIND" != "timeout" ]]; then + echo "::error::Singleton isolation identified ${file} as a failing test file." + return "$status" + fi + + retry_kind="$LAST_FAILURE_KIND" + echo "Retrying ${file} once in another fresh Bun process after ${retry_kind} failure..." + if run_test_once "$batch_number" "singleton ${file_index}/${#files[@]}" 2 "$file"; then + echo "::warning::${file} passed on its single ${retry_kind} retry." + continue + else + status=$? + fi + + if [[ "$LAST_FAILURE_KIND" == "timeout" ]]; then + echo "::error::${file} timed out twice under singleton isolation; failing after one retry." + elif [[ "$LAST_FAILURE_KIND" == "runtime" ]]; then + echo "::error::Bun runtime crash repeated for ${file} under singleton isolation; failing after one retry." + else + echo "::error::${file} failed during singleton retry." + fi + return "$status" + done + + echo "::warning::Shard ${SHARD_SPEC} batch ${batch_number} passed under singleton isolation after the original ${batch_failure_kind}; continuing." + return 0 +} + mapfile -d '' -t ALL_TEST_FILES < <( find tests -type f -print0 \ | LC_ALL=C sort -z @@ -146,14 +200,15 @@ if (( ${#SELECTED_FILES[@]} == 0 )); then fi readonly TOTAL_BATCHES=$(( (${#SELECTED_FILES[@]} + BATCH_SIZE - 1) / BATCH_SIZE )) -echo "Shard ${SHARD_SPEC}: ${#SELECTED_FILES[@]} files in ${TOTAL_BATCHES} fresh Bun processes (batch size <= ${BATCH_SIZE}, timeout ${BATCH_TIMEOUT_SECONDS}s)." +echo "Shard ${SHARD_SPEC}: ${#SELECTED_FILES[@]} files in ${TOTAL_BATCHES} primary Bun processes (batch size <= ${BATCH_SIZE}, timeout ${BATCH_TIMEOUT_SECONDS}s)." +echo "Runtime crashes and timeouts fall back to one-file-per-process isolation; assertion/test failures do not retry." for ((batch_index = 0; batch_index < TOTAL_BATCHES; batch_index += 1)); do start=$(( batch_index * BATCH_SIZE )) batch=("${SELECTED_FILES[@]:start:BATCH_SIZE}") batch_number=$(( batch_index + 1 )) - if run_batch_once "$batch_number" 1 "${batch[@]}"; then + if run_test_once "$batch_number" "" 1 "${batch[@]}"; then continue else status=$? @@ -163,20 +218,10 @@ for ((batch_index = 0; batch_index < TOTAL_BATCHES; batch_index += 1)); do exit "$status" fi - retry_kind="$LAST_FAILURE_KIND" - echo "Retrying shard ${SHARD_SPEC} batch ${batch_number} once in a fresh Bun process after ${retry_kind} failure..." - if run_batch_once "$batch_number" 2 "${batch[@]}"; then - echo "::warning::Shard ${SHARD_SPEC} batch ${batch_number} passed on the single ${retry_kind} retry." + failure_kind="$LAST_FAILURE_KIND" + if recover_batch_file_by_file "$batch_number" "$failure_kind" "${batch[@]}"; then continue else - status=$? + exit $? fi - - if [[ "$LAST_FAILURE_KIND" == "timeout" ]]; then - echo "::error::Bun batch timed out twice in shard ${SHARD_SPEC} batch ${batch_number}; failing after one retry." - elif [[ "$LAST_FAILURE_KIND" == "runtime" ]]; then - echo "::error::Bun runtime crash repeated in shard ${SHARD_SPEC} batch ${batch_number}; failing after one retry." - fi - - exit "$status" done