Create runner and job API

.github/workflows/deploy.yml

@@ -73,9 +73,6 @@ jobs:
       - name: Deploy Local API to Fly.io
         working-directory: ./services/api
         run: flyctl deploy --local-only --config fly/fly.dev.toml --wait-timeout 300
-      - name: Scale up Dev Machines
-        working-directory: ./services/api
-        run: flyctl scale vm performance-8x --app bencher-api-dev
       - name: Run Smoke Test
         id: smoke-test
         continue-on-error: true
@@ -89,10 +86,22 @@ jobs:
           --api-url https://bencher-api-dev.fly.dev \
           --username eustace.bagge@nowhere.com \
           --password "${{ secrets.TEST_BENCHER_API_TOKEN_ADMIN }}"
-      - name: Scale down Dev Machines
+      - name: OCI Conformance Test Summary
         if: always()
-        working-directory: ./services/api
-        run: flyctl scale vm shared-cpu-1x --app bencher-api-dev
+        run: |
+          echo "## OCI Conformance Test Results" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          if [ -f ./oci-conformance-results/test-output.log ]; then
+            echo "<details>" >> $GITHUB_STEP_SUMMARY
+            echo "<summary>Test Output</summary>" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo '```' >> $GITHUB_STEP_SUMMARY
+            cat ./oci-conformance-results/test-output.log >> $GITHUB_STEP_SUMMARY
+            echo '```' >> $GITHUB_STEP_SUMMARY
+            echo "</details>" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "No test output found." >> $GITHUB_STEP_SUMMARY
+          fi
       - name: Check Smoke Test Result
         if: steps.smoke-test.outcome == 'failure'
         run: exit 1

.github/workflows/test.yml

@@ -29,15 +29,16 @@ jobs:
     env:
       TEST_BILLING_KEY: ${{ secrets.TEST_BILLING_KEY }}
     steps:
+      - uses: jlumbroso/free-disk-space@main
+        with:
+          large-packages: false
       - uses: actions/checkout@v6
       - uses: ./.github/actions/setup-rust
         with:
           cache-key: all-features
           mold-version: ${{ inputs.mold-version }}
-      - name: cargo nextest run
-        run: RUST_BACKTRACE=1 cargo nextest run --all-features --no-capture --profile ci
-      - name: cargo test --doc
-        run: RUST_BACKTRACE=1 cargo test --doc --all-features
+      - name: Run Tests
+        run: ./scripts/test.sh
       - name: Upload Perf JPEG
         uses: actions/upload-artifact@v4
         with:
@@ -50,6 +51,9 @@ jobs:
     if: ${{ inputs.is-fork }}
     runs-on: ubuntu-22.04
     steps:
+      - uses: jlumbroso/free-disk-space@main
+        with:
+          large-packages: false
       - uses: actions/checkout@v6
       - uses: rui314/setup-mold@v1
         with:

CLAUDE.md

@@ -60,7 +60,7 @@ Running the seed tests:
 
 ```bash
 cargo fmt                      # Format Rust code
-cargo clippy --no-deps --all-features -- -Dwarnings  # Lint Rust code
+cargo clippy --no-deps --all-targets --all-features -- -Dwarnings  # Lint Rust code
 cargo check --no-default-features  # Verify build without Plus features
 
 cd services/console
@@ -111,7 +111,9 @@ lib/
   bencher_valid/     # Input validation (compiled to WASM for frontend)
 plus/             # Bencher Plus (commercial) features
   api_oci/           # OCI Distribution Spec registry endpoints
+  api_runners/       # Runner management and agent endpoints (Plus)
   bencher_oci_storage/ # OCI blob/manifest storage (S3-backed)
+  bencher_otel/      # OpenTelemetry instrumentation (Plus)
 tasks/            # Build tasks invoked via cargo aliases
 xtask/            # Administrative tasks (local only, not CI)
 ```
@@ -170,8 +172,12 @@ The API server includes an OCI Distribution Spec compliant container registry, r
   - `cargo check --no-default-features`
   - `cargo gen-types` (if the API changed at all)
 - Use idiomatic, strong types instead of `String` and `serde_json::Value` where possible
+- Database model fields should use strong validated types (e.g., `ProjectId`, `ProjectUuid`, `ProjectName`, `DateTime`, `VersionNumber`) with Diesel `ToSql`/`FromSql` impls rather than raw primitives (`i32`, `i64`, `String`). All conversion happens inside the Diesel impls, not in the model layer.
 - Avoid `select!` macros - use `futures_concurrency::stream::Merge::merge`
 - All time-based tests should be deterministic and use time manipulation not real wall-clock time
+- Use `bencher_json::Clock::Custom` (behind the `test-clock` feature) to inject a fake clock in tests instead of calling `DateTime::now()` directly. `Clock` is available on `ApiContext`.
+- Most wire type definitions are in the `bencher_valid` or `bencher_json` crate
+- Always pass strong types (`MyTypeId`, `MyTypeUuid`, etc) into a function instead of its stringly typed equivalent, even in tests
 
 ### Frontend (TypeScript)
 
@@ -211,13 +217,33 @@ When adding a new crate, update both Dockerfiles:
 - `services/api/Dockerfile`
 - `services/console/Dockerfile`
 
+## Runner System (Plus Feature)
+
+Bare metal benchmark runners that claim and execute jobs from the API. Server-scoped (runners serve all projects).
+
+**Key concepts:**
+- **Runner**: A registered agent authenticated via `bencher_runner_`-prefixed tokens (SHA-256 hashed in DB)
+- **Job**: Linked to a report, follows state machine: `pending → claimed → running → completed/failed/canceled`
+- **Claim endpoint**: Long-poll POST that atomically claims pending jobs (priority DESC, FIFO within tier)
+- **WebSocket channel**: Persistent connection for heartbeat and status updates during job execution
+
+**Key files:**
+- `plus/api_runners/` - Runner CRUD, token rotation, job claiming, job updates, WebSocket channel
+- `lib/bencher_json/src/runner/` - Shared JSON types (`JsonRunner`, `JsonJob`, `JobStatus`)
+- `lib/bencher_schema/src/model/runner/` - Database models and queries
+- `lib/api_projects/src/jobs.rs` - Project-scoped job listing (public API)
+- `lib/bencher_schema/migrations/2026-02-02-120000_runner/` - Migration for `runner` and `job` tables
+
+**Runner authentication** is separate from user auth — runner tokens use `Authorization: Bearer bencher_runner_<token>` and are validated by hashing and looking up `token_hash` in the `runner` table.
+
 ## Key Coordination Points
 
 Changes in these areas often require updates across multiple files:
 - **API endpoints** → run `cargo gen-types` → commit updated `openapi.json` + `bencher.ts`
 - **Rust types with `#[typeshare]`** → run `npm run typeshare` in console
 - **New crate** → update workspace `Cargo.toml` + both Dockerfiles
 - **Validation types** → may need WASM rebuild (`npm run wasm` in console)
+- **Runner endpoints** → behind `plus` feature flag; runner model in `bencher_schema` also gated on `plus`
 
 ## Documentation
 

Cargo.toml

@@ -50,6 +50,8 @@ api_users = { path = "lib/api_users" }
 # plus API endpoints
 api_oci = { path = "plus/api_oci" }
 # plus
+api_runners = { path = "plus/api_runners" }
+api_specs = { path = "plus/api_specs" }
 bencher_billing = { path = "plus/bencher_billing" }
 bencher_bing_index = { path = "plus/bencher_bing_index" }
 bencher_github_client = { path = "plus/bencher_github_client" }
@@ -87,6 +89,7 @@ dotenvy = "0.15"
 email_address = "0.2"
 fs-err = "3.2"
 futures = "0.3"
+futures-concurrency = "7"
 futures-util = "0.3"
 gix = { version = "0.75", default-features = false }
 gix-hash = "0.20"
@@ -148,6 +151,7 @@ tempfile = "3"
 thiserror = "2.0"
 tokio = "1.48"
 tokio-rustls = {version =  "0.26", default-features = false, features = ["logging", "tls12", "ring"]}
+tokio-tungstenite = "0.26"
 tokio-util = { version = "0.7", features = ["io"] }
 typeshare = "1.0"
 url = "2.5"

lib/api_auth/src/oauth/mod.rs

@@ -90,7 +90,7 @@ async fn handle_oauth_user(
         } else if let Some(organization_uuid) = oauth_state.claim() {
             let query_organization =
                 QueryOrganization::from_uuid(auth_conn!(context), organization_uuid)?;
-            query_organization.claim(context, &query_user).await?;
+            query_organization.claim(log, context, &query_user).await?;
         }
         (query_user, AuthAction::Login)
     } else {