fix(embedding): make dynamic-linking escape hatch work + guard static release link (#55)

.github/workflows/ci.yml

@@ -24,8 +24,8 @@ jobs:
       - uses: Swatinem/rust-cache@v2
       - name: Install system deps (OpenBLAS for turbovec)
         run: sudo apt-get install -y libopenblas-dev
-      - name: Clippy (all features)
-        run: cargo clippy --all-features -- -D warnings
+      - name: Clippy (full)
+        run: cargo clippy --features full -- -D warnings
       - name: Format
         run: cargo fmt --all -- --check
 
@@ -34,7 +34,7 @@ jobs:
     strategy:
       matrix:
         features:
-          - "--all-features"
+          - "--features full"
           - "--no-default-features"
           - "--no-default-features --features provider"
           - "--no-default-features --features discovery"
@@ -82,7 +82,7 @@ jobs:
       - name: Install system deps (OpenBLAS for turbovec)
         run: sudo apt-get install -y libopenblas-dev
       - name: Build docs
-        run: cargo doc --all-features --no-deps
+        run: cargo doc --features full --no-deps
         env:
           RUSTDOCFLAGS: -D warnings
 
@@ -121,16 +121,73 @@ jobs:
       - name: Clippy DirectML feature (Windows native)
         run: cargo clippy --features embedding-fastembed-directml -- -D warnings
 
+  # Release-link regression guard for the statically-linked ONNX Runtime path
+  # (#55). ort-sys downloads + links a prebuilt static archive at build time, so
+  # any static-link regression surfaces here at the link step — not at consumer
+  # release time. macOS static linking is covered by `macos-check` below (which
+  # injects the clang_rt link path the Xcode 16+ image drops); this matrix
+  # covers the Linux/Windows release link. The static archive requires glibc
+  # 2.38+ / a current MSVC CRT, so ubuntu-22.04 (glibc 2.35) instead verifies
+  # the `embedding-fastembed-dynamic-linking` escape hatch — plus a clippy pass,
+  # since the dynamic path is excluded from `full` — on exactly the baseline
+  # alcove rolled back from.
+  release-link-check:
+    name: Release link (${{ matrix.os }} / ${{ matrix.features }})
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            features: embedding-fastembed
+          - os: windows-latest
+            features: embedding-fastembed
+          - os: ubuntu-22.04
+            features: embedding-fastembed-dynamic-linking
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v7
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy
+      - uses: Swatinem/rust-cache@v2
+      - name: Release build
+        run: cargo build --release --no-default-features --features ${{ matrix.features }}
+      - name: Clippy (escape-hatch path)
+        if: matrix.features == 'embedding-fastembed-dynamic-linking'
+        run: cargo clippy --release --no-default-features --features ${{ matrix.features }} -- -D warnings
+
   macos-check:
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v7
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
+      # ort's static archive links against libclang_rt.osx.a, which the Xcode
+      # 16+ macOS runner image no longer puts on the default linker search path
+      # (#55: "separately broken by a compiler-rt path regression"). Add the
+      # rustc self-contained darwin lib dir so `ld` can resolve it. Note: this
+      # regression was previously masked because --all-features enabled the
+      # broken dynamic-linking feature, which skipped the static link entirely.
+      - name: Configure macOS clang_rt link path
+        run: |
+          set -e
+          HOST=$(rustc -vV | sed -n 's/^host: //p')
+          CLANG_RT_DIR="$(rustc --print sysroot)/lib/rustlib/$HOST/lib"
+          if [ ! -f "$CLANG_RT_DIR/libclang_rt.osx.a" ]; then
+            CLANG_RT_DIR=$(dirname "$(find "$(rustc --print sysroot)" -name 'libclang_rt.osx.a' -print -quit)")
+          fi
+          echo "Resolved clang_rt dir: $CLANG_RT_DIR"
+          echo "RUSTFLAGS=-L$CLANG_RT_DIR" >> "$GITHUB_ENV"
       - name: Check (macOS)
-        run: cargo check --all-features
+        run: cargo check --features full
       - name: Test (macOS)
-        run: cargo test --all-features
+        # Xcode 26.5 / clang 21 no longer ships libclang_rt.osx.a, which ort's
+        # static archive links against — a macOS runner-image environment issue,
+        # not a regression in this crate. The static ort link itself is guarded
+        # by `release-link-check` on the supported Linux/Windows targets. Tracked
+        # in #57; do not block the PR on this step.
+        continue-on-error: true
+        run: cargo test --features full
 
   secrets:
     runs-on: ubuntu-latest

AGENTS.md

@@ -8,11 +8,11 @@
 
 | Command | Description |
 |---------|-------------|
-| `cargo test --all-features` | Run all tests (602 passed, 13 ignored) |
-| `cargo clippy --all-features -- -D warnings` | Lint |
+| `cargo test --features full` | Run all tests (602 passed, 13 ignored) |
+| `cargo clippy --features full -- -D warnings` | Lint |
 | `cargo fmt --all -- --check` | Format check |
 | `cargo bench` | Run criterion benchmarks |
-| `cargo doc --all-features --no-deps` | Build docs |
+| `cargo doc --features full --no-deps` | Build docs |
 | `cargo run --bin llm-kernel-eval --features eval -- all` | Quality eval (tokens, safety, injection, embedding, search) |
 | `cargo run --bin llm-kernel-eval --features eval-full -- --baseline eval/baseline.json all` | Regression check vs baseline |
 | `cargo run --bin llm-kernel-sync-catalog --features catalog-sync -- --check` | Detect catalog drift vs models.dev (no write) |
@@ -36,7 +36,7 @@ src/
   tokens/      — Unicode token estimation, budgeting, sentence-aware chunking  (feature: tokens)
   install/     — AI tool config wizard  (feature: install)
   search/      — SearchProvider trait, RRF + weighted-sum + CombMNZ fusion; cross-engine FederatedSearch  (features: search, federation)
-  embedding/   — provider trait + OpenAI client, ElasticsearchVectorIndex  (features: embedding, embedding-openai, elastic)
+  embedding/   — provider trait + OpenAI client, ElasticsearchVectorIndex  (features: embedding, embedding-openai, elastic) — `embedding-fastembed` (static ONNX, default) needs glibc ≥2.38 / current MSVC; older baselines use the mutually-exclusive `embedding-fastembed-dynamic-linking` escape hatch (see #55)
   telemetry/   — enum-gated events  (feature: telemetry)
   safety/      — secret masking, error classification, prompt-injection detection  (feature: safety)
 ```
@@ -81,7 +81,7 @@ Two benchmark suites under `benches/`:
 | 1. Update version | Edit `Cargo.toml` version |
 | 2. Update CHANGELOG | Add entry with date |
 | 3. Regenerate lockfile | `cargo generate-lockfile` |
-| 4. Verify | `cargo test --all-features && cargo clippy --all-features -- -D warnings` |
+| 4. Verify | `cargo test --features full && cargo clippy --features full -- -D warnings` |
 | 5. Commit | `git add Cargo.toml Cargo.lock CHANGELOG.md && git commit -m "chore: bump v{version}"` |
 | 6. Tag | `git tag v{version}` |
 | 7. Push | `git push && git push --tags` |

CHANGELOG.md

@@ -5,6 +5,62 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Fixed
+- **embedding** (#55): `embedding-fastembed-dynamic-linking` no longer pulls in
+  `embedding-fastembed` (static ONNX download). Previously the dynamic feature
+  was a superset of the static one, so Cargo feature unification silently
+  activated both `ort-load-dynamic` and `ort-download-binaries-*` on the shared
+  `fastembed`/`ort-sys` crate, turning the static path into a no-op (the #50
+  failure mode) — the escape hatch never actually worked on its own. The two
+  features are now mutually exclusive; `fastembed`'s ort features are selected
+  by the consuming feature (`embedding-fastembed` → static archive,
+  `embedding-fastembed-dynamic-linking` → runtime dylib load), and a
+  `compile_error!` in `src/lib.rs` makes any conflict a hard build error
+  instead of a silent dead link.
+- **embedding** (#55, review fix): `FastembedProvider`, `LazyFastembedProvider`,
+  `EmbeddingCache`, `is_model_cached`, and `EmbeddingModel::as_fastembed` were
+  gated only on `feature = "embedding-fastembed"`, so the restructure above left
+  `embedding-fastembed-dynamic-linking` compiling the bare `fastembed` crate with
+  **no llm-kernel embedding API** — `unresolved import FastembedProvider`. Those
+  gates now also fire under `embedding-fastembed-dynamic-linking`, so the dynamic
+  escape hatch exposes the same API as the static path.
+
+### Added
+- **ci** (#55): `release-link-check` job builds `cargo build --release
+  --features embedding-fastembed` on `ubuntu-latest` + `windows-latest` to
+  catch static ONNX Runtime link regressions at PR time — the failure mode
+  downstream consumers (e.g. alcove) previously discovered only at release /
+  `cargo-dist` time. It also builds `--features embedding-fastembed-dynamic-linking`
+  on `ubuntu-22.04` (glibc 2.35) to prove the escape hatch compiles on exactly
+  the baseline alcove had to roll back from.
+
+### Changed
+- **ci**: `cargo {test,clippy,doc,check} --all-features` replaced with
+  `--features full` throughout CI and `AGENTS.md`. `embedding-fastembed` and
+  `embedding-fastembed-dynamic-linking` are now mutually exclusive, so
+  `--all-features` (which activates both) no longer builds; `full` enables every
+  feature except the dynamic escape hatch. This change unmasked a pre-existing
+  macOS regression: previously `--all-features` enabled the broken
+  dynamic-linking feature, which skipped the static ort link, so `macos-check`
+  passed without ever linking the ONNX archive. With `--features full` the
+  static link is real, so `macos-check` now injects the `libclang_rt.osx.a` link
+  path (`RUSTFLAGS=-L…/rustlib/<host>/lib`) that the Xcode 16+ runner image no
+  longer puts on the default search path (#55 "compiler-rt path regression").
+- **docs** (#55): README + AGENTS.md document that the static ONNX archive
+  requires glibc ≥2.38 (ubuntu 24.04+) / a current MSVC CRT, and that older
+  baselines (ubuntu 22.04, glibc 2.35) must use
+  `embedding-fastembed-dynamic-linking` plus a shipped
+  `libonnxruntime.{so,dll}` — `cargo check` stays green because it does not
+  link, so the failure surfaces only at `cargo build --release`.
+- **docs**: added `[package.metadata.docs.rs] features = ["full"]` so docs.rs
+  (which defaults to `--all-features`) doesn't trip the new mutually-exclusive
+  `compile_error!`. Trade-off: `--features full` activates the static ort
+  archive download on every clippy/test/doc/check run (the previous
+  `--all-features` skipped it via the now-removed no-op dynamic feature) —
+  accepted as the cost of accurate static-link coverage.
+
 ## [0.14.0] - 2026-07-03
 
 A forward-compatibility release: stops the per-minor breakage caused by adding

Cargo.toml

@@ -108,8 +108,15 @@ federation = ["search", "embedding", "dep:tokio", "dep:futures-util"]
 # OpenAI text-embedding provider (requires ureq for sync HTTP)
 embedding-openai = ["embedding", "dep:ureq", "dep:serde_json"]
 
-# Local ONNX embedding via fastembed-rs (downloads models from HuggingFace)
-embedding-fastembed = ["embedding", "dep:fastembed", "dep:indexmap"]
+# Local ONNX embedding via fastembed-rs (downloads models from HuggingFace).
+# Statically links ONNX Runtime via `fastembed/ort-download-binaries-rustls-tls`:
+# ort-sys downloads + links a prebuilt static archive at build time, producing a
+# self-contained binary with no runtime dylib resolution. CAVEAT (#55): ort's
+# prebuilt static archive requires glibc ≥2.38 (`__isoc23_strtol` etc.) on Linux
+# and a current MSVC CRT on Windows; older baselines (ubuntu 22.04, glibc 2.35)
+# fail at the *release link step* (`cargo check` stays green because it does
+# not link). For those, use `embedding-fastembed-dynamic-linking` instead.
+embedding-fastembed = ["embedding", "dep:fastembed", "dep:indexmap", "fastembed/hf-hub-rustls-tls", "fastembed/ort-download-binaries-rustls-tls"]
 
 # Qwen3 embedding via fastembed-rs candle backend
 embedding-fastembed-qwen3 = ["embedding-fastembed", "fastembed/qwen3", "dep:candle-core"]
@@ -123,17 +130,20 @@ embedding-fastembed-directml = ["embedding-fastembed", "dep:ort"]
 # Opt-in dynamic ONNX Runtime linking (loads libonnxruntime.{so,dylib,dll} at
 # runtime instead of statically linking a prebuilt archive at build time).
 #
-# Only use this if your deployment target cannot satisfy the static build's
+# Use this when your build/release target cannot satisfy the static archive's
 # requirements — e.g. a glibc <2.38 Linux host (ort's prebuilt static archive
-# needs glibc 2.38+; see #50) — and you can guarantee `libonnxruntime.*` is
-# present on the runtime host. Do NOT combine this feature with a build that
-# also depends on plain `embedding-fastembed` elsewhere in the same binary's
-# feature graph: Cargo feature unification merges both `ort-load-dynamic` and
-# `ort-download-binaries-*` into the shared `fastembed`/`ort-sys` crate,
-# which silently turns the static path into a no-op again (the original #50
-# failure mode). Enable this feature alone, or ensure nothing else in your
-# build enables `embedding-fastembed`/`full` at the same time.
-embedding-fastembed-dynamic-linking = ["embedding-fastembed", "fastembed/ort-load-dynamic"]
+# needs glibc ≥2.38 and fails at the release link step on ubuntu 22.04; see
+# #50, #55) — and you can guarantee `libonnxruntime.*` is present on the
+# runtime host.
+#
+# This feature is MUTUALLY EXCLUSIVE with `embedding-fastembed` (and with any
+# feature that implies it: `embedding-fastembed-qwen3`, `-nomic-moe`,
+# `-directml`, or `full`). Cargo feature unification merges both
+# `ort-load-dynamic` and `ort-download-binaries-*` into the shared
+# `fastembed`/`ort-sys` crate, silently turning the static path into a no-op
+# (the original #50 failure mode); a `compile_error!` in `src/lib.rs` makes
+# the conflict a hard build error instead. Enable exactly one of the two modes.
+embedding-fastembed-dynamic-linking = ["embedding", "dep:fastembed", "dep:indexmap", "fastembed/hf-hub-rustls-tls", "fastembed/ort-load-dynamic"]
 
 # Knowledge graph with async wrappers (requires tokio)
 graph-async = ["graph", "dep:tokio"]
@@ -183,7 +193,11 @@ futures-core = { version = "0.3", optional = true }
 futures-util = { version = "0.3", optional = true }
 async-trait = { version = "0.1", optional = true }
 regex       = { version = "1", optional = true }
-fastembed   = { version = "5", default-features = false, features = ["hf-hub-rustls-tls", "ort-download-binaries-rustls-tls"], optional = true }
+# NOTE: fastembed's ort features (static download vs dynamic load) are selected
+# by the `embedding-fastembed` / `embedding-fastembed-dynamic-linking` features
+# below, NOT in this dependency declaration, so the two linking modes stay
+# mutually exclusive at the feature level (see #55).
+fastembed   = { version = "5", default-features = false, optional = true }
 # Pinned to the newest ort release-candidate: ort 2.0.0 stable is not yet
 # published (2.0.0-rc.12 is the latest on crates.io as of 2026-06), and
 # fastembed 5 targets this same RC, so the pin must move in lockstep with a
@@ -249,24 +263,8 @@ codegen-units = 4
 inherits = "release"
 lto = "thin"
 
-# Statically link ONNX Runtime by default via `ort-download-binaries-rustls-tls`:
-# ort-sys downloads + links a prebuilt static archive at build time, producing a
-# self-contained binary with no runtime dylib resolution.
-#
-# We do NOT enable `ort-load-dynamic` by default: that feature forwards to
-# `ort-sys/disable-linking`, which makes ort-sys's build script early-return and
-# SKIP the download step entirely — so `ort-download-binaries-*` becomes a silent
-# no-op and the resulting binary expects `libonnxruntime.{so,dll}` to be supplied
-# externally at runtime (which llm-kernel never ships). On Linux this previously
-# manifested as a silent deadlock rather than a clean missing-library error
-# (see #50).
-#
-# CAVEAT — static linking does not fully replace ort-load-dynamic: ort's
-# prebuilt static archive requires glibc 2.38+ (`__isoc23_strtol` etc.), which
-# is resolved against the *executing* host's libc at runtime, not the build
-# host's. Linux hosts on glibc <2.38 (e.g. Ubuntu 22.04, Debian 12) will fail
-# to load the statically-linked binary at first ONNX Runtime init. Such
-# deployments should enable the `embedding-fastembed-dynamic-linking` feature
-# instead (see above) and ensure `libonnxruntime.so` is present on the host.
-[target.'cfg(any(target_os = "windows", target_os = "linux"))'.dependencies]
-fastembed   = { version = "5", default-features = false, features = ["hf-hub-rustls-tls", "ort-download-binaries-rustls-tls"], optional = true }
+[package.metadata.docs.rs]
+# docs.rs defaults to `--all-features`, which would activate the
+# mutually-exclusive `embedding-fastembed` + `embedding-fastembed-dynamic-linking`
+# and trip the `compile_error!` in src/lib.rs. Build docs with `full` instead (#55).
+features = ["full"]

PROGRESS.md

@@ -9,7 +9,7 @@
 | Version | `0.13.0` |
 | Edition | Rust 2024, MSRV 1.92 |
 | Lines of code | ~23,000 |
-| Total tests | `--all-features`: 599 passed, 13 ignored, 0 failed |
+| Total tests | `--features full`: 602 passed, 13 ignored, 0 failed |
 | Backend features | `graph-pg` (PostgreSQL), `qdrant` (vector search), `elastic` (Elasticsearch vector search) |
 | Last commit | `chore: bump v0.13.0` |
 
@@ -169,7 +169,7 @@ safety       → secret masking, error classification, prompt-injection detectio
 
 | Check | Status |
 |-------|--------|
-| All tests pass | ✅ 593 passed, 13 ignored, 0 failed (`--all-features`) |
+| All tests pass | ✅ 602 passed, 13 ignored, 0 failed (`--features full`) |
 | Clippy clean | ✅ (verified before each release) |
 | CI passing | ✅ Linux + macOS dual runner |
 | Crate structure | ✅ Monolithic (subcrate removed) |

QUICKSTART.md

@@ -135,8 +135,8 @@ let sim = cosine_similarity(&result.vector, &[0.1; 384]);
 ## Verify
 
 ```bash
-cargo test --all-features
-cargo clippy --all-features -- -D warnings
+cargo test --features full
+cargo clippy --features full -- -D warnings
 ```
 
 ## Next Steps