diff --git a/.github/workflows/bindings.yml b/.github/workflows/bindings.yml index 357142e..0cad49f 100644 --- a/.github/workflows/bindings.yml +++ b/.github/workflows/bindings.yml @@ -10,6 +10,11 @@ on: - 'crates/adapters/el-engine-candle/**' - '.cargo/config.toml' - 'Makefile' + - 'scripts/install-cargo-tool.sh' + - 'scripts/retry-command.sh' + - 'tests/check_ci_workflows.py' + - 'tests/test-install-cargo-tool.sh' + - 'tests/test-retry-command.sh' - '.github/workflows/bindings.yml' pull_request: paths: @@ -21,6 +26,11 @@ on: - 'Cargo.lock' - 'Cargo.toml' - 'Makefile' + - 'scripts/install-cargo-tool.sh' + - 'scripts/retry-command.sh' + - 'tests/check_ci_workflows.py' + - 'tests/test-install-cargo-tool.sh' + - 'tests/test-retry-command.sh' - '.github/workflows/bindings.yml' jobs: @@ -34,6 +44,12 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Test el-ffi (host target) run: cargo test -p el-ffi + - name: Test cargo tool installer + run: bash tests/test-install-cargo-tool.sh + - name: Test command retry wrapper + run: bash tests/test-retry-command.sh + - name: Check CI workflow tool dependencies + run: python3 tests/check_ci_workflows.py # ── Android: aarch64-linux-android cross-compile ────────────────────────── android: @@ -77,7 +93,7 @@ jobs: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar RANLIB_aarch64_linux_android: >- ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ranlib - run: make codegen-rn + run: bash scripts/retry-command.sh make codegen-rn - uses: actions/upload-artifact@v4 with: name: rn-bindings @@ -113,7 +129,7 @@ jobs: set -euo pipefail tmp="$(mktemp -d)" archive="wasm-pack-${WASM_PACK_VERSION}-x86_64-unknown-linux-musl.tar.gz" - curl -fsSL \ + bash scripts/retry-command.sh curl -fsSL \ "https://github.com/wasm-bindgen/wasm-pack/releases/download/${WASM_PACK_VERSION}/${archive}" \ -o "${tmp}/${archive}" tar -xzf "${tmp}/${archive}" -C "${tmp}" @@ -132,13 +148,18 @@ jobs: flutter: name: Flutter/Dart codegen runs-on: ubuntu-latest + env: + FRB_CODEGEN_VERSION: 2.12.0 + CARGO_EXPAND_VERSION: 1.0.123 steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - uses: dart-lang/setup-dart@v1 - - name: Install flutter_rust_bridge_codegen - run: cargo install flutter_rust_bridge_codegen --version 2.12.0 --locked + - name: Install Flutter codegen cargo tools + run: | + bash scripts/install-cargo-tool.sh flutter_rust_bridge_codegen flutter_rust_bridge_codegen "$FRB_CODEGEN_VERSION" + bash scripts/install-cargo-tool.sh cargo-expand cargo-expand "$CARGO_EXPAND_VERSION" - name: Codegen — Flutter/Dart run: make codegen-flutter - uses: actions/upload-artifact@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 663ac40..60a2070 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -76,6 +76,15 @@ jobs: - name: cargo test --locked --workspace run: cargo test --locked --workspace + - name: Test cargo tool installer + run: bash tests/test-install-cargo-tool.sh + + - name: Test command retry wrapper + run: bash tests/test-retry-command.sh + + - name: Check CI workflow tool dependencies + run: python3 tests/check_ci_workflows.py + - name: cargo fmt --check run: cargo fmt --check @@ -122,7 +131,7 @@ jobs: ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar RANLIB_aarch64_linux_android: >- ${{ steps.setup-ndk.outputs.ndk-path }}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ranlib - run: make codegen-rn + run: bash scripts/retry-command.sh make codegen-rn - name: Upload RN bindings (TypeScript + JSI) uses: actions/upload-artifact@v4 with: @@ -168,7 +177,7 @@ jobs: set -euo pipefail tmp="$(mktemp -d)" archive="wasm-pack-${WASM_PACK_VERSION}-x86_64-unknown-linux-musl.tar.gz" - curl -fsSL \ + bash scripts/retry-command.sh curl -fsSL \ "https://github.com/wasm-bindgen/wasm-pack/releases/download/${WASM_PACK_VERSION}/${archive}" \ -o "${tmp}/${archive}" tar -xzf "${tmp}/${archive}" -C "${tmp}" @@ -190,13 +199,17 @@ jobs: runs-on: ubuntu-latest env: VERSION: ${{ needs.verify.outputs.version }} + FRB_CODEGEN_VERSION: 2.12.0 + CARGO_EXPAND_VERSION: 1.0.123 steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - uses: dart-lang/setup-dart@v1 - - name: Install flutter_rust_bridge_codegen - run: cargo install flutter_rust_bridge_codegen --version 2.12.0 --locked + - name: Install Flutter codegen cargo tools + run: | + bash scripts/install-cargo-tool.sh flutter_rust_bridge_codegen flutter_rust_bridge_codegen "$FRB_CODEGEN_VERSION" + bash scripts/install-cargo-tool.sh cargo-expand cargo-expand "$CARGO_EXPAND_VERSION" - name: Codegen — Flutter/Dart run: make codegen-flutter - name: Stamp pub.dev version from tag diff --git a/Cargo.lock b/Cargo.lock index 9f3edfa..0d95d38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -793,7 +793,7 @@ checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "el-bench" -version = "0.3.8" +version = "0.3.9" dependencies = [ "el-core", "el-engine-candle", @@ -803,7 +803,7 @@ dependencies = [ [[package]] name = "el-chat" -version = "0.3.8" +version = "0.3.9" dependencies = [ "el-core", "el-engine-candle", @@ -811,7 +811,7 @@ dependencies = [ [[package]] name = "el-cloud" -version = "0.3.8" +version = "0.3.9" dependencies = [ "el-core", "reqwest", @@ -822,11 +822,11 @@ dependencies = [ [[package]] name = "el-core" -version = "0.3.8" +version = "0.3.9" [[package]] name = "el-engine-candle" -version = "0.3.8" +version = "0.3.9" dependencies = [ "candle-core", "candle-transformers", @@ -838,7 +838,7 @@ dependencies = [ [[package]] name = "el-ffi" -version = "0.3.8" +version = "0.3.9" dependencies = [ "el-cloud", "el-core", @@ -852,7 +852,7 @@ dependencies = [ [[package]] name = "el-grammar" -version = "0.3.8" +version = "0.3.9" dependencies = [ "el-core", "el-provenance", @@ -861,21 +861,21 @@ dependencies = [ [[package]] name = "el-memory" -version = "0.3.8" +version = "0.3.9" dependencies = [ "el-core", ] [[package]] name = "el-provenance" -version = "0.3.8" +version = "0.3.9" dependencies = [ "el-core", ] [[package]] name = "el-provenance-ed25519" -version = "0.3.8" +version = "0.3.9" dependencies = [ "ed25519-dalek", "el-core", @@ -884,7 +884,7 @@ dependencies = [ [[package]] name = "el-runtime" -version = "0.3.8" +version = "0.3.9" dependencies = [ "el-core", "el-memory", @@ -894,14 +894,14 @@ dependencies = [ [[package]] name = "el-safety" -version = "0.3.8" +version = "0.3.9" dependencies = [ "el-core", ] [[package]] name = "el-telemetry" -version = "0.3.8" +version = "0.3.9" dependencies = [ "el-core", ] diff --git a/Cargo.toml b/Cargo.toml index db512f7..2a2d296 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ exclude = [ [workspace.package] edition = "2021" -version = "0.3.8" +version = "0.3.9" license = "Apache-2.0" rust-version = "1.96" repository = "Edge-Native LLM SDK" @@ -48,16 +48,16 @@ repository = "Edge-Native LLM SDK" # This is the single source of truth for internal dep version requirements; # member crates reference them with `{ workspace = true }` and add no inline # version fields. `cargo set-version --workspace ` updates every entry. -el-core = { path = "crates/el-core", version = "0.3.8" } -el-memory = { path = "crates/el-memory", version = "0.3.8" } -el-telemetry = { path = "crates/el-telemetry", version = "0.3.8" } -el-provenance = { path = "crates/el-provenance", version = "0.3.8" } -el-safety = { path = "crates/el-safety", version = "0.3.8" } -el-runtime = { path = "crates/el-runtime", version = "0.3.8" } -el-grammar = { path = "crates/el-grammar", version = "0.3.8" } -el-provenance-ed25519 = { path = "crates/adapters/el-provenance-ed25519", version = "0.3.8" } -el-engine-candle = { path = "crates/adapters/el-engine-candle", version = "0.3.8" } -el-cloud = { path = "crates/adapters/el-cloud", version = "0.3.8" } +el-core = { path = "crates/el-core", version = "0.3.9" } +el-memory = { path = "crates/el-memory", version = "0.3.9" } +el-telemetry = { path = "crates/el-telemetry", version = "0.3.9" } +el-provenance = { path = "crates/el-provenance", version = "0.3.9" } +el-safety = { path = "crates/el-safety", version = "0.3.9" } +el-runtime = { path = "crates/el-runtime", version = "0.3.9" } +el-grammar = { path = "crates/el-grammar", version = "0.3.9" } +el-provenance-ed25519 = { path = "crates/adapters/el-provenance-ed25519", version = "0.3.9" } +el-engine-candle = { path = "crates/adapters/el-engine-candle", version = "0.3.9" } +el-cloud = { path = "crates/adapters/el-cloud", version = "0.3.9" } [profile.release] opt-level = 3 diff --git a/scripts/install-cargo-tool.sh b/scripts/install-cargo-tool.sh new file mode 100644 index 0000000..3412d0c --- /dev/null +++ b/scripts/install-cargo-tool.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then + echo "Usage: $0 [version]" >&2 + exit 64 +fi + +crate="$1" +binary="$2" +version="${3:-}" +attempts="${CARGO_INSTALL_ATTEMPTS:-3}" +retry_seconds="${CARGO_INSTALL_RETRY_SECONDS:-15}" + +installed_version_matches() { + command -v "$binary" >/dev/null 2>&1 || return 1 + + if [ -z "$version" ]; then + return 0 + fi + + "$binary" --version 2>/dev/null | grep -F -- "$version" >/dev/null +} + +if installed_version_matches; then + "$binary" --version + exit 0 +fi + +for attempt in $(seq 1 "$attempts"); do + echo "Installing ${crate}${version:+ ${version}} (attempt ${attempt}/${attempts})..." + + args=(install "$crate" --locked --force) + if [ -n "$version" ]; then + args=(install "$crate" --version "$version" --locked --force) + fi + + if cargo "${args[@]}"; then + "$binary" --version + exit 0 + fi + + if [ "$attempt" -lt "$attempts" ]; then + echo "Retrying ${crate} install in ${retry_seconds}s..." + sleep "$retry_seconds" + fi +done + +echo "ERROR: failed to install ${crate}${version:+ ${version}} after ${attempts} attempts" >&2 +exit 1 diff --git a/scripts/retry-command.sh b/scripts/retry-command.sh new file mode 100644 index 0000000..29b0375 --- /dev/null +++ b/scripts/retry-command.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -lt 1 ]; then + echo "Usage: $0 [args...]" >&2 + exit 64 +fi + +attempts="${COMMAND_ATTEMPTS:-3}" +retry_seconds="${COMMAND_RETRY_SECONDS:-15}" +status=0 + +for attempt in $(seq 1 "$attempts"); do + echo "Running command (attempt ${attempt}/${attempts}): $*" + + set +e + "$@" + status="$?" + set -e + + if [ "$status" -eq 0 ]; then + exit 0 + fi + + if [ "$attempt" -lt "$attempts" ]; then + echo "Command failed with status ${status}; retrying in ${retry_seconds}s..." + sleep "$retry_seconds" + fi +done + +echo "ERROR: command failed after ${attempts} attempts: $*" >&2 +exit "$status" diff --git a/tests/check_ci_workflows.py b/tests/check_ci_workflows.py new file mode 100644 index 0000000..374ea28 --- /dev/null +++ b/tests/check_ci_workflows.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Validate CI dependencies that are easy to accidentally leave implicit.""" + +from pathlib import Path +import sys + + +WORKFLOWS = [ + Path(".github/workflows/release.yml"), + Path(".github/workflows/bindings.yml"), +] + +RN_RETRY_COMMAND = "bash scripts/retry-command.sh make codegen-rn" +WASM_RETRY_COMMAND = "bash scripts/retry-command.sh curl -fsSL" + +REQUIRED_INSTALLS = [ + ( + "flutter_rust_bridge_codegen", + "scripts/install-cargo-tool.sh flutter_rust_bridge_codegen flutter_rust_bridge_codegen", + ), + ( + "cargo-expand", + "scripts/install-cargo-tool.sh cargo-expand cargo-expand", + ), +] + + +def check_workflow(path: Path) -> list[str]: + text = path.read_text(encoding="utf-8") + errors: list[str] = [] + codegen_pos = text.find("make codegen-flutter") + rn_codegen_pos = text.find("make codegen-rn") + + if codegen_pos < 0: + return [f"{path}: missing make codegen-flutter step"] + + for label, needle in REQUIRED_INSTALLS: + install_pos = text.find(needle) + if install_pos < 0: + errors.append(f"{path}: missing explicit {label} install before Flutter codegen") + elif install_pos > codegen_pos: + errors.append(f"{path}: installs {label} after Flutter codegen") + + if rn_codegen_pos >= 0 and RN_RETRY_COMMAND not in text: + errors.append(f"{path}: React Native codegen must run through retry wrapper") + + if "Install wasm-pack" in text and WASM_RETRY_COMMAND not in text: + errors.append(f"{path}: wasm-pack download must run through retry wrapper") + + return errors + + +def main() -> int: + errors: list[str] = [] + for workflow in WORKFLOWS: + errors.extend(check_workflow(workflow)) + + if errors: + print("\n".join(errors), file=sys.stderr) + return 1 + + print("OK: CI workflows guard Flutter tools and retry external codegen downloads") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test-install-cargo-tool.sh b/tests/test-install-cargo-tool.sh new file mode 100644 index 0000000..900dd05 --- /dev/null +++ b/tests/test-install-cargo-tool.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +mkdir -p "$TMP/bin" + +cat > "$TMP/bin/cargo" <<'SH' +#!/usr/bin/env bash +set -euo pipefail + +if [ "$1" != "install" ]; then + echo "unexpected cargo command: $*" >&2 + exit 64 +fi + +attempts_file="$FAKE_STATE/attempts" +attempts=0 +[ -f "$attempts_file" ] && attempts="$(cat "$attempts_file")" +attempts=$((attempts + 1)) +echo "$attempts" > "$attempts_file" +printf '%s\n' "$*" >> "$FAKE_STATE/cargo-args" + +if [ "$attempts" -eq 1 ]; then + echo "transient registry failure" >&2 + exit 101 +fi + +cat > "$FAKE_BIN/cargo-expand" <<'BIN' +#!/usr/bin/env bash +printf 'cargo-expand 1.2.3\n' +BIN +chmod +x "$FAKE_BIN/cargo-expand" +SH +chmod +x "$TMP/bin/cargo" + +OUTPUT="$TMP/output" +PATH="$TMP/bin:$PATH" \ +FAKE_BIN="$TMP/bin" \ +FAKE_STATE="$TMP" \ +CARGO_INSTALL_ATTEMPTS=2 \ +CARGO_INSTALL_RETRY_SECONDS=0 \ +bash "$ROOT/scripts/install-cargo-tool.sh" cargo-expand cargo-expand 1.2.3 > "$OUTPUT" + +test "$(cat "$TMP/attempts")" = "2" +grep -q -- "install cargo-expand --version 1.2.3 --locked --force" "$TMP/cargo-args" +grep -q -- "cargo-expand 1.2.3" "$OUTPUT" diff --git a/tests/test-retry-command.sh b/tests/test-retry-command.sh new file mode 100644 index 0000000..5e6513f --- /dev/null +++ b/tests/test-retry-command.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +mkdir -p "$TMP/bin" + +cat > "$TMP/bin/flaky-command" <<'SH' +#!/usr/bin/env bash +set -euo pipefail + +attempts_file="$FAKE_STATE/attempts" +attempts=0 +[ -f "$attempts_file" ] && attempts="$(cat "$attempts_file")" +attempts=$((attempts + 1)) +echo "$attempts" > "$attempts_file" +printf '%s\n' "$*" >> "$FAKE_STATE/args" + +if [ "$attempts" -eq 1 ]; then + echo "transient network failure" >&2 + exit 56 +fi + +printf 'ok\n' +SH +chmod +x "$TMP/bin/flaky-command" + +OUTPUT="$TMP/output" +PATH="$TMP/bin:$PATH" \ +FAKE_STATE="$TMP" \ +COMMAND_ATTEMPTS=2 \ +COMMAND_RETRY_SECONDS=0 \ +bash "$ROOT/scripts/retry-command.sh" flaky-command alpha beta > "$OUTPUT" + +test "$(cat "$TMP/attempts")" = "2" +grep -q -- "alpha beta" "$TMP/args" +grep -q -- "ok" "$OUTPUT"