From 58222b240621c0d9afd33895f9afee6f2938c4f1 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Wed, 29 Jul 2026 02:15:40 -0500 Subject: [PATCH 01/20] measure true idle from worker completion --- .../scenarios/artifact/runner/measurements.rs | 10 +- .../scenarios/artifact/runner/process.rs | 23 +- .../scenarios/artifact/runner/tests.rs | 29 +- .../src/embedding_qualification/worker.rs | 6 +- .../worker/operations/measure.rs | 364 ++++++++++++++++-- ...er-user-embedding-server-constant-set.json | 22 +- 6 files changed, 383 insertions(+), 71 deletions(-) diff --git a/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/measurements.rs b/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/measurements.rs index 579082006..c61e542fe 100644 --- a/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/measurements.rs +++ b/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/measurements.rs @@ -9,7 +9,7 @@ use super::analysis::{ }; use super::process::{ busy_retry_marker_timeout, busy_retry_worker_timeout, measurement_worker_timeout, - query_parameters, require_worker_success, + query_parameters, }; use super::{RunningWorker, ScenarioRunner, WorkerOutput, push_metric}; use crate::qualification::request::REQUIRED_METRICS; @@ -294,14 +294,6 @@ impl<'a> ScenarioRunner<'a> { )?; } - let idle_worker = self.spawn_worker("query", query_parameters(1), None)?; - let idle_output = self.finish_worker(idle_worker, measurement_worker_timeout("query"))?; - require_worker_success(&idle_output, "true_idle_owner")?; - let idle_owner = - self.record_worker_snapshot("measurement_true_idle_owner", &idle_output)?; - if !snapshot_has_resident_generation(&idle_owner) { - bail!("embedding_qualification_true_idle_owner_not_resident"); - } let measured = self.run_measure_worker( "measure_true_idle", "true_idle_exit", diff --git a/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/process.rs b/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/process.rs index d03f47e40..666504c6d 100644 --- a/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/process.rs +++ b/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/process.rs @@ -1,6 +1,4 @@ -use super::super::{ - CONTROL_TIMEOUT, ControlEvent, IDLE_EXIT_GRACE, POLL, QUEUE_SETUP_TIMEOUT, SNAPSHOT_TIMEOUT, -}; +use super::super::{CONTROL_TIMEOUT, ControlEvent, POLL, QUEUE_SETUP_TIMEOUT, SNAPSHOT_TIMEOUT}; use super::analysis::elapsed; use super::{ EMBEDDING_QUALIFICATION_WORKER_SCHEMA_VERSION, ProcessInvocation, RunningWorker, WorkerOutput, @@ -17,6 +15,8 @@ use std::path::{Path, PathBuf}; use std::process::{Child, ExitStatus}; use std::time::Duration; +pub(super) const MEASUREMENT_OWNER_ABSENCE_GRACE: Duration = Duration::from_secs(30); + pub(super) fn existing_control_events(directory: &Path) -> Result> { published_control_events(&directory.join(format!("{}.events.jsonl", qualification_nonce()?))) } @@ -284,12 +284,17 @@ pub(super) fn stall_worker_timeout() -> Duration { pub(super) fn measurement_worker_timeout(operation: &str) -> Duration { let budgets = EmbeddingClientBudgets::current(); if operation == "measure_true_idle" { - // The idle worker first proves the resident owner quiescent (bounded - // by the snapshot allowance), then waits out the server's own idle - // deadline plus the exit grace before the absence observation. - return Duration::from_millis(PER_USER_EMBEDDING_SERVER_IDLE_TIMEOUT_MS) - .saturating_add(IDLE_EXIT_GRACE) - .saturating_add(SNAPSHOT_TIMEOUT) + // The idle worker runs the product request that starts the measured + // idle epoch itself, then waits out the server's idle deadline plus + // the exit grace before the absence observation. + return budgets + .connect + .saturating_add(budgets.spawn) + .saturating_add(budgets.query_request) + .saturating_add(Duration::from_millis( + PER_USER_EMBEDDING_SERVER_IDLE_TIMEOUT_MS, + )) + .saturating_add(MEASUREMENT_OWNER_ABSENCE_GRACE) .saturating_add(SNAPSHOT_TIMEOUT) .saturating_add(CONTROL_TIMEOUT); } diff --git a/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/tests.rs b/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/tests.rs index 04664b8c1..bcc41d72c 100644 --- a/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/tests.rs +++ b/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/tests.rs @@ -7,9 +7,9 @@ use super::measurements::{ declared_phase_boundaries, declared_workload_id, measurement_span_interval, }; use super::process::{ - LOAD_ESTABLISHMENT_WAITS, busy_retry_worker_timeout, dead_client_setup_timeout, - load_establishment_budget, load_establishment_timeout, measurement_worker_timeout, - published_control_events, + LOAD_ESTABLISHMENT_WAITS, MEASUREMENT_OWNER_ABSENCE_GRACE, busy_retry_worker_timeout, + dead_client_setup_timeout, load_establishment_budget, load_establishment_timeout, + measurement_worker_timeout, published_control_events, }; use super::{ScenarioEvidence, WorkerOutput, opaque_measurement_sample_id}; use crate::qualification::request::{QualificationContracts, REQUIRED_METRICS, REQUIRED_SCENARIOS}; @@ -75,15 +75,26 @@ fn measurement_worker_budgets_dominate_the_deadlines_workers_honor() { measurement_worker_timeout("measure_resident_identity"), "the scenario residency probe must carry the same bulk-deadline budget as the residency measurement" ); - // The true-idle measurement worker waits out the server's own idle - // deadline before the absence observation; its watchdog must dominate - // that self-enforced wait plus its quiescence and absence-grace waits. + // The true-idle measurement worker now performs the product request that + // starts the idle epoch itself; its watchdog must cover that whole client + // chain before the server idle deadline and absence-grace wait. + assert_eq!( + MEASUREMENT_OWNER_ABSENCE_GRACE, + Duration::from_secs(30), + "the coordinator must mirror the measurement worker's owner-absence grace" + ); assert!( measurement_worker_timeout("measure_true_idle") - >= Duration::from_millis(PER_USER_EMBEDDING_SERVER_IDLE_TIMEOUT_MS) - .saturating_add(Duration::from_secs(30)) + >= budgets + .connect + .saturating_add(budgets.spawn) + .saturating_add(budgets.query_request) + .saturating_add(Duration::from_millis( + PER_USER_EMBEDDING_SERVER_IDLE_TIMEOUT_MS, + )) + .saturating_add(MEASUREMENT_OWNER_ABSENCE_GRACE) .saturating_add(SNAPSHOT_TIMEOUT), - "true-idle measurement budget must dominate the server idle deadline plus the worker's own waits" + "true-idle measurement budget must dominate its product request, server idle deadline, and worker waits" ); // The busy-retry worker seeds the held queues (queue-setup phase), then // after release drains queries bounded by its own 120s per-request diff --git a/crates/codestory-cli/src/embedding_qualification/worker.rs b/crates/codestory-cli/src/embedding_qualification/worker.rs index ec538f9a3..9acc431e9 100644 --- a/crates/codestory-cli/src/embedding_qualification/worker.rs +++ b/crates/codestory-cli/src/embedding_qualification/worker.rs @@ -228,7 +228,11 @@ fn run_measure_operation( &request.parameters, ), "measure_resident_identity" => run_measure_resident_identity(runtime, clock.as_ref()), - "measure_true_idle" => run_measure_true_idle(runtime, clock.as_ref()), + "measure_true_idle" => run_measure_true_idle( + &PerUserEmbeddingClient::for_runtime(runtime)?, + clock.as_ref(), + request.parameters.input_bytes, + ), "measure_busy_retry" => { let marker = request .retry_marker diff --git a/crates/codestory-cli/src/embedding_qualification/worker/operations/measure.rs b/crates/codestory-cli/src/embedding_qualification/worker/operations/measure.rs index 2433aeb43..2fea96180 100644 --- a/crates/codestory-cli/src/embedding_qualification/worker/operations/measure.rs +++ b/crates/codestory-cli/src/embedding_qualification/worker/operations/measure.rs @@ -19,7 +19,7 @@ use super::super::protocol::{ run_raw_protocol_exchange_with_input, validated_hello, write_protocol_frame, }; use super::ANTI_IDLE_PROTOCOL_DEADLINE_MS; -use super::owner_exit::{observe_owner_exit, wait_for_owner_exit}; +use super::owner_exit::{OwnerExitObservation, observe_owner_exit, wait_for_owner_exit}; use super::queue::{QueueOperation, run_queue_operation}; use anyhow::{Context, Result, bail}; use codestory_retrieval::{ @@ -350,30 +350,96 @@ pub(in crate::embedding_qualification::worker) fn run_measure_resident_identity( }) } -/// `true_idle_exit`: span start at the observation proving the resident owner -/// carries zero queued, active, or leased work -/// (`last_queued_active_or_leased_work_ended`), span end at the observation -/// that returned no owner (`engine_and_server_absent`). +fn true_idle_scheduler_is_drained( + active_request_count: u64, + query_depth: u64, + bulk_depth: u64, + lease_count: u64, +) -> bool { + active_request_count == 0 && query_depth == 0 && bulk_depth == 0 && lease_count == 0 +} + +fn snapshot_is_true_idle_boundary(snapshot: &EmbeddingServerSnapshot) -> bool { + snapshot.lifecycle == "resident" + && resident_engine_generation(snapshot) + && true_idle_scheduler_is_drained( + snapshot.scheduler.active_request_count, + snapshot.scheduler.query_depth, + snapshot.scheduler.bulk_depth, + snapshot.scheduler.lease_count, + ) +} + +fn same_true_idle_owner( + expected_server_instance_id: &str, + observed_server_instance_id: &str, +) -> bool { + expected_server_instance_id == observed_server_instance_id +} + +fn validate_true_idle_boundary( + expected_server_instance_id: &str, + snapshot: &EmbeddingServerSnapshot, +) -> Result<()> { + if !same_true_idle_owner( + expected_server_instance_id, + &snapshot.process.server_instance_id, + ) { + bail!("embedding_qualification_true_idle_owner_changed"); + } + if !snapshot_is_true_idle_boundary(snapshot) { + bail!("embedding_qualification_true_idle_not_quiescent"); + } + Ok(()) +} + +/// The product and observation operations used by the true-idle measurement. +/// +/// The worker implements this interface with the real per-user embedding +/// client. Tests use a scripted implementation to execute this same +/// `run_measure_true_idle` algorithm rather than a helper that production can +/// bypass. +pub(in crate::embedding_qualification::worker) trait TrueIdleMeasurementClient { + fn observe_snapshot(&self) -> Result>; + fn complete_product_query(&self, input: &str) -> Result<()>; + fn observe_owner_exit(&self) -> Result; +} + +impl TrueIdleMeasurementClient for PerUserEmbeddingClient { + fn observe_snapshot(&self) -> Result> { + self.observe() + } + + fn complete_product_query(&self, input: &str) -> Result<()> { + let _ = self.embed_query(input)?; + Ok(()) + } + + fn observe_owner_exit(&self) -> Result { + observe_owner_exit(self) + } +} + +/// `true_idle_exit`: this worker completes the last product request itself, +/// stamps its return as `last_queued_active_or_leased_work_ended`, proves the +/// resident scheduler is drained, then ends at the observation that returned +/// no owner (`engine_and_server_absent`). pub(in crate::embedding_qualification::worker) fn run_measure_true_idle( - runtime: &SidecarRuntimeConfig, + client: &dyn TrueIdleMeasurementClient, clock: &dyn AwakeMonotonicClock, + input_bytes: u32, ) -> Result { - let client = PerUserEmbeddingClient::for_runtime(runtime)?; - let idle_owner = wait_for_observed_snapshot( - &client, - clock, - SNAPSHOT_TIMEOUT, - "embedding_qualification_true_idle_not_quiescent", - |snapshot| { - snapshot.lifecycle == "resident" - && resident_engine_generation(snapshot) - && snapshot.scheduler.active_request_count == 0 - && snapshot.scheduler.query_depth == 0 - && snapshot.scheduler.bulk_depth == 0 - && snapshot.scheduler.lease_count == 0 - }, - )?; + let expected_owner = client + .observe_snapshot()? + .filter(resident_engine_generation) + .ok_or_else(|| anyhow::anyhow!("embedding_qualification_true_idle_owner_missing"))?; + let input = "q".repeat(input_bytes.max(1) as usize); + client.complete_product_query(&input)?; let start = begin_span(clock)?; + let idle_owner = client + .observe_snapshot()? + .ok_or_else(|| anyhow::anyhow!("embedding_qualification_true_idle_owner_missing"))?; + validate_true_idle_boundary(&expected_owner.process.server_instance_id, &idle_owner)?; let timeout = Duration::from_millis(PER_USER_EMBEDDING_SERVER_IDLE_TIMEOUT_MS) .saturating_add(OWNER_ABSENCE_GRACE); let wait_started = clock.now_ns(); @@ -382,7 +448,7 @@ pub(in crate::embedding_qualification::worker) fn run_measure_true_idle( wait_started, timeout, &idle_owner.process.server_instance_id, - || observe_owner_exit(&client), + || client.observe_owner_exit(), )?; let span = finish_span(clock, start)?; Ok(measurement(span, idle_owner)) @@ -652,7 +718,176 @@ fn validate_vector_response( #[cfg(test)] mod tests { - use super::workload_input; + use super::super::owner_exit::OwnerExitObservation; + use super::{ + TrueIdleMeasurementClient, run_measure_true_idle, true_idle_scheduler_is_drained, + validate_true_idle_boundary, workload_input, + }; + use anyhow::Result; + use codestory_retrieval::{ + AwakeMonotonicClock, EmbeddingServerAuthoritySnapshot, EmbeddingServerClockSnapshot, + EmbeddingServerEngineSnapshot, EmbeddingServerProcessSnapshot, + EmbeddingServerProtocolSnapshot, EmbeddingServerSchedulerSnapshot, EmbeddingServerSnapshot, + PER_USER_EMBEDDING_SERVER_SNAPSHOT_SCHEMA_VERSION, + }; + use std::collections::VecDeque; + use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + + struct ScriptClock { + now_ns: AtomicU64, + events: Arc>>, + } + + impl ScriptClock { + fn new(events: Arc>>) -> Self { + Self { + now_ns: AtomicU64::new(0), + events, + } + } + + fn set(&self, now_ns: u64) { + self.now_ns.store(now_ns, Ordering::Release); + } + } + + impl AwakeMonotonicClock for ScriptClock { + fn now_ns(&self) -> u64 { + let now_ns = self.now_ns.load(Ordering::Acquire); + if now_ns == 100 { + self.events + .lock() + .expect("lock events") + .push("span_started"); + } + now_ns + } + + fn sleep(&self, duration: Duration) { + self.now_ns.fetch_add( + u64::try_from(duration.as_nanos()).expect("test duration fits u64"), + Ordering::AcqRel, + ); + } + + fn snapshot(&self) -> EmbeddingServerClockSnapshot { + EmbeddingServerClockSnapshot { + domain: "awake_monotonic".into(), + api: "script_clock".into(), + boot_id: "test-boot".into(), + resolution_ns: 1, + } + } + } + + struct ScriptedTrueIdleClient { + clock: Arc, + events: Arc>>, + snapshots: Mutex>, + exit_observations: Mutex>, + product_calls: AtomicUsize, + } + + impl TrueIdleMeasurementClient for ScriptedTrueIdleClient { + fn observe_snapshot(&self) -> Result> { + let (snapshot, remaining) = { + let mut snapshots = self.snapshots.lock().expect("lock snapshots"); + let snapshot = snapshots + .pop_front() + .expect("the algorithm observed past its snapshot script"); + (snapshot, snapshots.len()) + }; + let event = if remaining == 1 { + "expected_owner_observed" + } else { + self.clock.set(145); + "same_owner_drained_observed" + }; + self.events.lock().expect("lock events").push(event); + Ok(Some(snapshot)) + } + + fn complete_product_query(&self, input: &str) -> Result<()> { + assert_eq!(input, "q".repeat(256)); + self.product_calls.fetch_add(1, Ordering::AcqRel); + self.clock.set(100); + self.events + .lock() + .expect("lock events") + .push("product_completed"); + Ok(()) + } + + fn observe_owner_exit(&self) -> Result { + let observation = self + .exit_observations + .lock() + .expect("lock exit observations") + .pop_front() + .expect("the algorithm observed past its exit script"); + let event = match &observation { + OwnerExitObservation::Present(_) => "same_owner_still_present", + OwnerExitObservation::Lost => "owner_connection_lost", + OwnerExitObservation::Absent => "owner_absence_observed", + }; + self.events.lock().expect("lock events").push(event); + Ok(observation) + } + } + + fn true_idle_snapshot( + owner: &str, + active_request_count: u64, + query_depth: u64, + bulk_depth: u64, + lease_count: u64, + ) -> EmbeddingServerSnapshot { + EmbeddingServerSnapshot { + schema_version: PER_USER_EMBEDDING_SERVER_SNAPSHOT_SCHEMA_VERSION, + event_sequence: 1, + lifecycle: "resident".into(), + clock: EmbeddingServerClockSnapshot { + domain: "awake_monotonic".into(), + api: "script_clock".into(), + boot_id: "test-boot".into(), + resolution_ns: 1, + }, + protocol: EmbeddingServerProtocolSnapshot::current(), + authority: EmbeddingServerAuthoritySnapshot { + endpoint_namespace_id: "endpoint".into(), + lifetime_authority_id: "authority".into(), + listener_id: "listener".into(), + peer_verified: true, + }, + process: EmbeddingServerProcessSnapshot { + server_instance_id: owner.into(), + pid: 42, + process_start_id: "server-start".into(), + executable_sha256: "a".repeat(64), + executable_version: "0.16.3".into(), + }, + scheduler: EmbeddingServerSchedulerSnapshot { + query_capacity: 64, + query_depth, + bulk_capacity: 64, + bulk_depth, + connection_count: 1, + active_request_count, + lease_count, + active_request: None, + }, + engine: Some(EmbeddingServerEngineSnapshot { + engine_owner_id: owner.into(), + native_worker_id: "native-worker".into(), + load_generation: 1, + model_load_count: 1, + successful_encode_count: 1, + }), + failure: None, + } + } #[test] fn workload_inputs_are_deterministic_ascii_and_distinct_per_ordinal() { @@ -666,4 +901,87 @@ mod tests { assert_ne!(first, other_repeat); assert_ne!(first, other_ordinal); } + + #[test] + fn true_idle_start_is_stamped_at_product_completion_before_observation_lag() { + let events = Arc::new(Mutex::new(Vec::new())); + let clock = Arc::new(ScriptClock::new(Arc::clone(&events))); + let client = ScriptedTrueIdleClient { + clock: Arc::clone(&clock), + events: Arc::clone(&events), + snapshots: Mutex::new(VecDeque::from([ + true_idle_snapshot("measured-owner", 0, 0, 0, 0), + true_idle_snapshot("measured-owner", 0, 0, 0, 0), + ])), + exit_observations: Mutex::new(VecDeque::from([ + OwnerExitObservation::Present("measured-owner".into()), + OwnerExitObservation::Absent, + ])), + product_calls: AtomicUsize::new(0), + }; + + let measurement = + run_measure_true_idle(&client, clock.as_ref(), 256).expect("measure true idle"); + + assert_eq!(client.product_calls.load(Ordering::Acquire), 1); + assert_eq!(measurement.span.awake_started_ns, 100); + assert!(measurement.span.awake_finished_ns >= 145); + assert_eq!( + measurement.snapshot.process.server_instance_id, + "measured-owner" + ); + assert_eq!( + *events.lock().expect("lock events"), + [ + "expected_owner_observed", + "product_completed", + "span_started", + "same_owner_drained_observed", + "same_owner_still_present", + "owner_absence_observed", + ] + ); + } + + #[test] + fn true_idle_boundary_rejects_each_queued_active_or_leased_shape() { + let occupied = [ + ("active", (1, 0, 0, 0)), + ("query", (0, 1, 0, 0)), + ("bulk", (0, 0, 1, 0)), + ("lease", (0, 0, 0, 1)), + ]; + assert!(true_idle_scheduler_is_drained(0, 0, 0, 0)); + for (label, state) in occupied { + assert!( + !true_idle_scheduler_is_drained(state.0, state.1, state.2, state.3), + "{label} work must keep the true-idle boundary open" + ); + let result = validate_true_idle_boundary( + "measured-owner", + &true_idle_snapshot("measured-owner", state.0, state.1, state.2, state.3), + ); + assert_eq!( + result + .expect_err("occupied scheduler must reject the sample") + .to_string(), + "embedding_qualification_true_idle_not_quiescent", + "{label} work must fail closed rather than shift the boundary" + ); + } + } + + #[test] + fn true_idle_boundary_rejects_a_different_owner_after_product_completion() { + let result = validate_true_idle_boundary( + "measured-owner", + &true_idle_snapshot("replacement-owner", 0, 0, 0, 0), + ); + assert_eq!( + result + .expect_err("replacement owner must reject the sample") + .to_string(), + "embedding_qualification_true_idle_owner_changed" + ); + } } diff --git a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json index 4aa4b19bb..dfd170e76 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json @@ -42,25 +42,7 @@ "pure_embedding_rpc_replay_limit": 1, "query_queue_capacity": 64 }, - "freeze_record": { - "calibration_bundle_sha256": "bdaf0d80a19af41cb1acdcc4d0ee7dbca4b90481d4e5616f72416b65923f5b0e", - "calibration_freeze_digest": "67e0af9456ddc1afc7ba2298b546024f36b1f92034b9b9e7363936ffbedae0b1", - "input_constant_set_sha256": "fb0326d180cc5e2e6cae46d70e5a7706d5ed423c30fb95b4b983ed61dcdc2e2c", - "measurement_protocol_sha256": "6d29a6a92dc6f9552d004806f0bb0785c040e57a9de25e3877d26bb76e78f04b", - "protocol_sha256": "f4a3fa4afb4d5bcd8e707a5e21b687cdd023dc3398b28ff6891a2318e89c5ec7", - "run_artifact_sha256s": [ - "184d5aa079369f8057acb71576a6a1e4dfd2f4e2f56ec20d17a9aab147c9f8dc", - "2fd6767d1195f9852ca56dfffb8511eb3c001e7f65450aed33c77e27a5b60d44", - "5ef180010acdcb6f518fefc9dac00a9da8277b4c4d397c0006efd9e1424d2c42", - "6e5e26f19b6028d87bd19ffca8d4dbec5d3ebaa0fd2037003723f7eef9d9f557", - "943449b5c7c187c397f2530011d08d5f56b9f41509bd182fbd7b03f970678116", - "fcad9b4c59792c1493ccc4ae96c9d3377efced89460b561cd1c4a2c7017377d7" - ], - "selected_at": "github-actions-run:30329118684:1", - "selection_rule": "all_preregistered_clean_runs_no_outlier_removal+slow_host_floors_v1", - "selection_source_commit": "6b0f4b2edb7d99fe43b63cfb0bd3a85ca9d0032c", - "selection_source_tree": "c759c770b6de9b0b5051a145d5862444052d5cdd" - }, + "freeze_record": null, "qualification_thresholds": { "backend_observed_accelerator_residency": 1, "bulk_documents_per_second": 2, @@ -78,5 +60,5 @@ }, "schema_version": 1, "selection_protocol": "codestory-per-user-embedding-server-v1", - "status": "frozen" + "status": "unfrozen" } From e5d029520d04c0ccd5af68cefc9e79294f9ffcb2 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Wed, 29 Jul 2026 10:35:11 -0500 Subject: [PATCH 02/20] use a fresh calibration live query --- .../runtime_bootstrap_continuity.py | 14 +++- .../self_test_runtime_bootstrap_scope.py | 82 +++++++++++++++---- 2 files changed, 81 insertions(+), 15 deletions(-) diff --git a/.github/scripts/packaged_agent_proof/runtime_bootstrap_continuity.py b/.github/scripts/packaged_agent_proof/runtime_bootstrap_continuity.py index 25087c94c..d9f5f0d28 100644 --- a/.github/scripts/packaged_agent_proof/runtime_bootstrap_continuity.py +++ b/.github/scripts/packaged_agent_proof/runtime_bootstrap_continuity.py @@ -22,6 +22,17 @@ from .server_identity import server_snapshot from .subprocess_control import McpProcess +_CALIBRATION_LIVE_QUERY_SUFFIX = " calibration live encode verification" + + +def _live_project_b_query(args: argparse.Namespace, setup: RuntimeSetup) -> str: + if args.proof_tier == "calibration": + # The cold phase already queried setup.query_b. Reusing it here can hit + # both the retrieval-result and embedding caches without exercising the + # resident native encoder whose counter this phase verifies. + return f"{setup.query_b}{_CALIBRATION_LIVE_QUERY_SUFFIX}" + return setup.query_b + def _managed_runtime( args: argparse.Namespace, @@ -108,8 +119,9 @@ def _live_retrieval( }, "packet-a", ) + project_b_query = _live_project_b_query(args, setup) live_tasks["search-b-live"] = lambda: hosts.host_b.search_until_ready( - {"project": str(setup.project_b), "query": setup.query_b, "why": True}, + {"project": str(setup.project_b), "query": project_b_query, "why": True}, "search-b-live", ) run_parallel(live_tasks) diff --git a/.github/scripts/packaged_agent_proof/self_test_runtime_bootstrap_scope.py b/.github/scripts/packaged_agent_proof/self_test_runtime_bootstrap_scope.py index fdc72d7fe..1ab3a4ac8 100644 --- a/.github/scripts/packaged_agent_proof/self_test_runtime_bootstrap_scope.py +++ b/.github/scripts/packaged_agent_proof/self_test_runtime_bootstrap_scope.py @@ -16,6 +16,9 @@ _QUESTION_A = "How does the large project activate?" _QUERY_A = "large_project_probe" _QUERY_B = "small_project_probe" +_CALIBRATION_LIVE_QUERY = ( + "small_project_probe calibration live encode verification" +) _MANIFEST = {"asset_target": "linux-x64"} @@ -56,39 +59,64 @@ def _run_live_retrieval_case( proof_tier: str, *, trap_project_a: bool = False, -) -> Mock: +) -> tuple[Mock, dict]: setup = _setup() cold = _cold() args = _args(proof_tier) host_a = Mock() host_b = Mock() + cache_state = { + "successful_encode_count": cold.snapshot_a["engine"][ + "successful_encode_count" + ], + "seen_queries": {_QUERY_B}, + } + + def advance_for_packet(*_arguments: object) -> tuple[dict, int]: + cache_state["successful_encode_count"] += 1 + return {"self_test": "packet"}, 1 + if trap_project_a: host_a.tool_until_ready.side_effect = ProofFailure( "calibration scheduled a second broad project-A request" ) else: - host_a.tool_until_ready.return_value = ({"self_test": "packet"}, 1) + host_a.tool_until_ready.side_effect = advance_for_packet host_a.search_until_ready.side_effect = ProofFailure( f"{proof_tier} moved its broad project-A request from packet to search" ) - host_b.search_until_ready.return_value = ({"self_test": "search"}, 1) - host_b.engine_diagnostics.return_value = {"self_test": "diagnostics"} + + def search_project_b(arguments: dict, _label: str) -> tuple[dict, int]: + query = arguments["query"] + if query not in cache_state["seen_queries"]: + cache_state["seen_queries"].add(query) + cache_state["successful_encode_count"] += 1 + return {"self_test": "search"}, 1 + + def engine_diagnostics(*_arguments: object) -> dict: + return { + "engine": { + "successful_encode_count": cache_state[ + "successful_encode_count" + ] + }, + "process": {"server_instance_id": "server-self-test"}, + } + + host_b.search_until_ready.side_effect = search_project_b + host_b.engine_diagnostics.side_effect = engine_diagnostics hosts = SimpleNamespace( host_a=host_a, host_b=host_b, start_a="host-a-start", start_b="host-b-start", ) - after = { - "engine": {"successful_encode_count": 8}, - "process": {"server_instance_id": "server-self-test"}, - } memory = {"self_test": "five-process-memory"} with ( patch.object( runtime_bootstrap_continuity, "server_snapshot", - return_value=after, + side_effect=lambda diagnostics, _manifest, *, require_resident: diagnostics, ) as snapshot_check, patch.object( runtime_bootstrap_continuity, @@ -107,13 +135,23 @@ def _run_live_retrieval_case( observed == memory, f"{proof_tier} live retrieval omitted five-process memory evidence", ) + expected_query = ( + _CALIBRATION_LIVE_QUERY + if proof_tier == "calibration" + else _QUERY_B + ) + if proof_tier == "calibration": + require( + expected_query.strip().lower() != _QUERY_B.strip().lower(), + "calibration live query reused the cold-phase embedding cache key", + ) require( host_b.method_calls == [ call.search_until_ready( { "project": str(_PROJECT_B), - "query": _QUERY_B, + "query": expected_query, "why": True, }, "search-b-live", @@ -122,8 +160,13 @@ def _run_live_retrieval_case( ], f"{proof_tier} live retrieval changed the bounded project-B path", ) + after = memory_check.call_args.kwargs["snapshot"] + require( + after["engine"]["successful_encode_count"] == 8, + f"{proof_tier} live retrieval did not model one fresh native encode", + ) snapshot_check.assert_called_once_with( - host_b.engine_diagnostics.return_value, + after, _MANIFEST, require_resident=True, ) @@ -140,17 +183,24 @@ def _run_live_retrieval_case( manifest=_MANIFEST, expected_backend="CPU", ) - return host_a + return host_a, cache_state def _live_retrieval_scope_tests() -> None: - host_a = _run_live_retrieval_case("calibration", trap_project_a=True) + host_a, cache_state = _run_live_retrieval_case( + "calibration", + trap_project_a=True, + ) require( host_a.method_calls == [], "calibration scheduled a second broad operation on project A", ) + require( + cache_state["seen_queries"] == {_QUERY_B, _CALIBRATION_LIVE_QUERY}, + "calibration did not prove a cache-distinct project-B query", + ) for proof_tier in ("hosted_package", "protected_hardware", "installed_runtime"): - host_a = _run_live_retrieval_case(proof_tier) + host_a, cache_state = _run_live_retrieval_case(proof_tier) require( host_a.method_calls == [ @@ -166,6 +216,10 @@ def _live_retrieval_scope_tests() -> None: ], f"{proof_tier} no longer requires the project-A packet path", ) + require( + cache_state["seen_queries"] == {_QUERY_B}, + f"{proof_tier} changed the established project-B query", + ) def _run_continuity_case(proof_tier: str, expected_project: Path) -> None: From 5cd73eb57e76ab26813a4a35782f026f8d39f2c5 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Wed, 29 Jul 2026 14:33:45 -0500 Subject: [PATCH 03/20] accept native gpu backend families --- .../scenarios/artifact/runner/measurements.rs | 66 +++++++++++++++++-- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/measurements.rs b/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/measurements.rs index 2da99ecad..06f6dc2d8 100644 --- a/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/measurements.rs +++ b/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/measurements.rs @@ -841,7 +841,7 @@ fn validate_constant_engine_evidence( || identity.load_error.is_some() || identity.policy != "accelerated" || identity.backend.eq_ignore_ascii_case("cpu") - || !identity.backend.eq_ignore_ascii_case(expected_backend) + || !constant_backend_matches_expected(&identity.backend, expected_backend) || identity.model_digest != expected_model_sha256 || identity.materialized_model_sha256 != expected_model_sha256 || !identity.embedded_model @@ -852,6 +852,29 @@ fn validate_constant_engine_evidence( Ok(()) } +fn constant_backend_matches_expected(observed: &str, expected: &str) -> bool { + let Some(expected_family) = constant_backend_family(expected) else { + return false; + }; + constant_backend_family(observed) == Some(expected_family) +} + +fn constant_backend_family(value: &str) -> Option<&'static str> { + let normalized = value.trim().to_ascii_lowercase(); + match normalized.as_str() { + "metal" | "mtl" => Some("metal"), + "vulkan" => Some("vulkan"), + value + if value.strip_prefix("vulkan").is_some_and(|suffix| { + !suffix.is_empty() && suffix.bytes().all(|byte| byte.is_ascii_digit()) + }) => + { + Some("vulkan") + } + _ => None, + } +} + fn expected_materialization_reuse(run_index: u32) -> Result { match run_index { 1 => Ok(false), @@ -1088,11 +1111,42 @@ mod constant_calibration_tests { fn engine_precondition_binds_accelerated_backend_model_and_reuse_only() { let server = server_identity("server-a"); let metal = engine_identity("server-a", "Metal", false); - validate_constant_engine_evidence(&metal, &server, "metal", &"a".repeat(64), false) - .expect("Metal identity"); - let vulkan = engine_identity("server-a", "Vulkan", false); - validate_constant_engine_evidence(&vulkan, &server, "vulkan", &"a".repeat(64), false) - .expect("Vulkan identity"); + for (observed, expected) in [ + ("Metal", "metal"), + ("MTL", "metal"), + ("Vulkan", "vulkan"), + ("Vulkan0", "vulkan"), + ] { + let identity = engine_identity("server-a", observed, false); + validate_constant_engine_evidence(&identity, &server, expected, &"a".repeat(64), false) + .unwrap_or_else(|error| { + panic!("{observed} must satisfy expected GPU family {expected}: {error}") + }); + } + for (observed, expected) in [ + ("CPU", "metal"), + ("cpu_explicit", "metal"), + ("", "metal"), + ("unknown", "metal"), + ("metal-cpu", "metal"), + ("mtl0", "metal"), + ("MTL", "vulkan"), + ("Vulkan", "metal"), + ("vulkan-cpu", "vulkan"), + ] { + let identity = engine_identity("server-a", observed, false); + assert!( + validate_constant_engine_evidence( + &identity, + &server, + expected, + &"a".repeat(64), + false, + ) + .is_err(), + "{observed} must not satisfy expected GPU family {expected}" + ); + } let mut invalid = metal.clone(); invalid.policy = "cpu_explicit".into(); From 7c245279a3356bd5ed1841c1d29b453f2b02f99f Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Wed, 29 Jul 2026 15:01:40 -0500 Subject: [PATCH 04/20] freeze embedding server constants --- ...er-user-embedding-server-constant-set.json | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json index 70b89b877..7d8ac092d 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json @@ -1,15 +1,15 @@ { "calibration_required_values": { "capacity_retry_policy": { - "retry_after_ms": 40, + "retry_after_ms": 59, "retry_class": "after_capacity_change", "retry_condition_source": "named_condition_from_typed_capacity_response" }, "connect_timeout_ms": 2000, "election_backoff_policy": { - "initial_backoff_ms": 7, + "initial_backoff_ms": 8, "jitter": "sha256(process_start_id||attempt) modulo inclusive [initial_backoff_ms,maximum_backoff_ms]", - "maximum_backoff_ms": 102 + "maximum_backoff_ms": 104 }, "hard_native_no_progress_ms": 385431, "request_deadlines_ms": { @@ -43,7 +43,22 @@ "query_queue_capacity": 64, "true_idle_observation_grace_ms": 2500 }, - "freeze_record": null, + "freeze_record": { + "calibration_bundle_sha256": "aeb9b86a0129a5d5dae870ccbc3e75959b92cc96e058b8c48975c21c3846aae5", + "calibration_freeze_digest": "7f227a1a30dca7c6ae279ea5bd1982d96a2f39cf11ca441463199047318b92b3", + "input_constant_set_sha256": "c67187b461e07bc09352cbb2b3eaaf1760780e32b284e6d127de9d49adb3c51b", + "measurement_protocol_sha256": "ce4bc4911c13d3e52438638e3abb5383e23cd9b400893805f589cdf6350e16e1", + "protocol_sha256": "f4a3fa4afb4d5bcd8e707a5e21b687cdd023dc3398b28ff6891a2318e89c5ec7", + "run_artifact_sha256s": [ + "427c0db697cb9ff82f8291055701fb4060369699c48c45957b24cb9957ca60af", + "93cc3597879a1e0a207738eab8185867354799ddab2563fc5a62dedb2129451a", + "cff89ed425a94d7e2fe31b72706e3ee91df4882c584730fda98e39f0c0264789" + ], + "selected_at": "github-actions-run:30485668299:1", + "selection_rule": "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2", + "selection_source_commit": "5cd73eb57e76ab26813a4a35782f026f8d39f2c5", + "selection_source_tree": "6f608347ebc721a1407902947c4563bf1b1e764d" + }, "qualification_thresholds": { "backend_observed_accelerator_residency": 1, "bulk_documents_per_second": 2, @@ -61,5 +76,5 @@ }, "schema_version": 1, "selection_protocol": "codestory-per-user-embedding-server-v1", - "status": "unfrozen" + "status": "frozen" } From 4f425deebae2071f71caa898c2020c68a387d1cd Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Wed, 29 Jul 2026 16:33:05 -0500 Subject: [PATCH 05/20] unfreeze embedding constants for repaired candidate --- ...er-user-embedding-server-constant-set.json | 25 ++++--------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json index 7d8ac092d..70b89b877 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json @@ -1,15 +1,15 @@ { "calibration_required_values": { "capacity_retry_policy": { - "retry_after_ms": 59, + "retry_after_ms": 40, "retry_class": "after_capacity_change", "retry_condition_source": "named_condition_from_typed_capacity_response" }, "connect_timeout_ms": 2000, "election_backoff_policy": { - "initial_backoff_ms": 8, + "initial_backoff_ms": 7, "jitter": "sha256(process_start_id||attempt) modulo inclusive [initial_backoff_ms,maximum_backoff_ms]", - "maximum_backoff_ms": 104 + "maximum_backoff_ms": 102 }, "hard_native_no_progress_ms": 385431, "request_deadlines_ms": { @@ -43,22 +43,7 @@ "query_queue_capacity": 64, "true_idle_observation_grace_ms": 2500 }, - "freeze_record": { - "calibration_bundle_sha256": "aeb9b86a0129a5d5dae870ccbc3e75959b92cc96e058b8c48975c21c3846aae5", - "calibration_freeze_digest": "7f227a1a30dca7c6ae279ea5bd1982d96a2f39cf11ca441463199047318b92b3", - "input_constant_set_sha256": "c67187b461e07bc09352cbb2b3eaaf1760780e32b284e6d127de9d49adb3c51b", - "measurement_protocol_sha256": "ce4bc4911c13d3e52438638e3abb5383e23cd9b400893805f589cdf6350e16e1", - "protocol_sha256": "f4a3fa4afb4d5bcd8e707a5e21b687cdd023dc3398b28ff6891a2318e89c5ec7", - "run_artifact_sha256s": [ - "427c0db697cb9ff82f8291055701fb4060369699c48c45957b24cb9957ca60af", - "93cc3597879a1e0a207738eab8185867354799ddab2563fc5a62dedb2129451a", - "cff89ed425a94d7e2fe31b72706e3ee91df4882c584730fda98e39f0c0264789" - ], - "selected_at": "github-actions-run:30485668299:1", - "selection_rule": "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2", - "selection_source_commit": "5cd73eb57e76ab26813a4a35782f026f8d39f2c5", - "selection_source_tree": "6f608347ebc721a1407902947c4563bf1b1e764d" - }, + "freeze_record": null, "qualification_thresholds": { "backend_observed_accelerator_residency": 1, "bulk_documents_per_second": 2, @@ -76,5 +61,5 @@ }, "schema_version": 1, "selection_protocol": "codestory-per-user-embedding-server-v1", - "status": "frozen" + "status": "unfrozen" } From f511689e0a7164ee11f3a453c736276cdf89ee5c Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Wed, 29 Jul 2026 16:59:52 -0500 Subject: [PATCH 06/20] freeze embedding server constants --- ...er-user-embedding-server-constant-set.json | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json index 70b89b877..26777e0b8 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json @@ -1,13 +1,13 @@ { "calibration_required_values": { "capacity_retry_policy": { - "retry_after_ms": 40, + "retry_after_ms": 48, "retry_class": "after_capacity_change", "retry_condition_source": "named_condition_from_typed_capacity_response" }, "connect_timeout_ms": 2000, "election_backoff_policy": { - "initial_backoff_ms": 7, + "initial_backoff_ms": 8, "jitter": "sha256(process_start_id||attempt) modulo inclusive [initial_backoff_ms,maximum_backoff_ms]", "maximum_backoff_ms": 102 }, @@ -43,7 +43,22 @@ "query_queue_capacity": 64, "true_idle_observation_grace_ms": 2500 }, - "freeze_record": null, + "freeze_record": { + "calibration_bundle_sha256": "6c8a2d449adeb763851e4b54024ce43149a8fc4875d0439edd5e9b0199179a4d", + "calibration_freeze_digest": "afe12bc64d9ec09d3085c8622198d4692b2b1f86c9d514448c0d02319e2f75db", + "input_constant_set_sha256": "c67187b461e07bc09352cbb2b3eaaf1760780e32b284e6d127de9d49adb3c51b", + "measurement_protocol_sha256": "ce4bc4911c13d3e52438638e3abb5383e23cd9b400893805f589cdf6350e16e1", + "protocol_sha256": "f4a3fa4afb4d5bcd8e707a5e21b687cdd023dc3398b28ff6891a2318e89c5ec7", + "run_artifact_sha256s": [ + "18a61ecbbbecb689dc32c6e50ef2930be710776e16c994fa7445128f3f0fa78f", + "b1603c475f1ae54cbe1207d2ce3842638c3ee4f9c598234fbf4cfe5f032cac21", + "cdeed297e9f098f6c542cec901e4a5c3292edf0b0654f841520e31e555d6712c" + ], + "selected_at": "github-actions-run:30493877118:1", + "selection_rule": "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2", + "selection_source_commit": "4f425deebae2071f71caa898c2020c68a387d1cd", + "selection_source_tree": "0a003c8b13b1ff2947f88a3f088285f29dbd944c" + }, "qualification_thresholds": { "backend_observed_accelerator_residency": 1, "bulk_documents_per_second": 2, @@ -61,5 +76,5 @@ }, "schema_version": 1, "selection_protocol": "codestory-per-user-embedding-server-v1", - "status": "unfrozen" + "status": "frozen" } From 08451fa2cfc6665821248bebc60bd8b5636c1258 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Wed, 29 Jul 2026 17:56:22 -0500 Subject: [PATCH 07/20] unfreeze constants after workflow repair --- ...er-user-embedding-server-constant-set.json | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json index 26777e0b8..70b89b877 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json @@ -1,13 +1,13 @@ { "calibration_required_values": { "capacity_retry_policy": { - "retry_after_ms": 48, + "retry_after_ms": 40, "retry_class": "after_capacity_change", "retry_condition_source": "named_condition_from_typed_capacity_response" }, "connect_timeout_ms": 2000, "election_backoff_policy": { - "initial_backoff_ms": 8, + "initial_backoff_ms": 7, "jitter": "sha256(process_start_id||attempt) modulo inclusive [initial_backoff_ms,maximum_backoff_ms]", "maximum_backoff_ms": 102 }, @@ -43,22 +43,7 @@ "query_queue_capacity": 64, "true_idle_observation_grace_ms": 2500 }, - "freeze_record": { - "calibration_bundle_sha256": "6c8a2d449adeb763851e4b54024ce43149a8fc4875d0439edd5e9b0199179a4d", - "calibration_freeze_digest": "afe12bc64d9ec09d3085c8622198d4692b2b1f86c9d514448c0d02319e2f75db", - "input_constant_set_sha256": "c67187b461e07bc09352cbb2b3eaaf1760780e32b284e6d127de9d49adb3c51b", - "measurement_protocol_sha256": "ce4bc4911c13d3e52438638e3abb5383e23cd9b400893805f589cdf6350e16e1", - "protocol_sha256": "f4a3fa4afb4d5bcd8e707a5e21b687cdd023dc3398b28ff6891a2318e89c5ec7", - "run_artifact_sha256s": [ - "18a61ecbbbecb689dc32c6e50ef2930be710776e16c994fa7445128f3f0fa78f", - "b1603c475f1ae54cbe1207d2ce3842638c3ee4f9c598234fbf4cfe5f032cac21", - "cdeed297e9f098f6c542cec901e4a5c3292edf0b0654f841520e31e555d6712c" - ], - "selected_at": "github-actions-run:30493877118:1", - "selection_rule": "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2", - "selection_source_commit": "4f425deebae2071f71caa898c2020c68a387d1cd", - "selection_source_tree": "0a003c8b13b1ff2947f88a3f088285f29dbd944c" - }, + "freeze_record": null, "qualification_thresholds": { "backend_observed_accelerator_residency": 1, "bulk_documents_per_second": 2, @@ -76,5 +61,5 @@ }, "schema_version": 1, "selection_protocol": "codestory-per-user-embedding-server-v1", - "status": "frozen" + "status": "unfrozen" } From e38fbe5ba15b7b284d99665bc3bff005e765ac9a Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Wed, 29 Jul 2026 18:17:57 -0500 Subject: [PATCH 08/20] freeze constants from repaired calibration --- ...er-user-embedding-server-constant-set.json | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json index 70b89b877..7d2d23f20 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json @@ -1,13 +1,13 @@ { "calibration_required_values": { "capacity_retry_policy": { - "retry_after_ms": 40, + "retry_after_ms": 57, "retry_class": "after_capacity_change", "retry_condition_source": "named_condition_from_typed_capacity_response" }, "connect_timeout_ms": 2000, "election_backoff_policy": { - "initial_backoff_ms": 7, + "initial_backoff_ms": 8, "jitter": "sha256(process_start_id||attempt) modulo inclusive [initial_backoff_ms,maximum_backoff_ms]", "maximum_backoff_ms": 102 }, @@ -43,7 +43,22 @@ "query_queue_capacity": 64, "true_idle_observation_grace_ms": 2500 }, - "freeze_record": null, + "freeze_record": { + "calibration_bundle_sha256": "650c558b108c6473f0f244aaeb8680938f45cb51ad30cba3cc42ccaf330de004", + "calibration_freeze_digest": "14f02f80a8bf0f68d02e820b5d16d8b171839cf60ecbf643ddc6191c9f9bb40e", + "input_constant_set_sha256": "c67187b461e07bc09352cbb2b3eaaf1760780e32b284e6d127de9d49adb3c51b", + "measurement_protocol_sha256": "ce4bc4911c13d3e52438638e3abb5383e23cd9b400893805f589cdf6350e16e1", + "protocol_sha256": "f4a3fa4afb4d5bcd8e707a5e21b687cdd023dc3398b28ff6891a2318e89c5ec7", + "run_artifact_sha256s": [ + "65789b6090a68a45ab770513fc2d20ed3a228a371f1dbdd76aec6ce568dfd8e8", + "b74a95d375f6e3e37b182d91a120b2d05071f2016bf7d49dc3355bb8a2b42dec", + "bdd9340b0020be257fa2af59cb7231eed5270ee2bf3509b04bdc30cc4861da11" + ], + "selected_at": "github-actions-run:30498515242:1", + "selection_rule": "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2", + "selection_source_commit": "08451fa2cfc6665821248bebc60bd8b5636c1258", + "selection_source_tree": "2215e54a1ef49f1b1bb8f0b2dab5bcdaaec19706" + }, "qualification_thresholds": { "backend_observed_accelerator_residency": 1, "bulk_documents_per_second": 2, @@ -61,5 +76,5 @@ }, "schema_version": 1, "selection_protocol": "codestory-per-user-embedding-server-v1", - "status": "unfrozen" + "status": "frozen" } From 34154f213a411bb0c89b2fe103ea641f454ffb79 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Wed, 29 Jul 2026 20:47:32 -0500 Subject: [PATCH 09/20] unfreeze embedding constants --- ...er-user-embedding-server-constant-set.json | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json index 7d2d23f20..70b89b877 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json @@ -1,13 +1,13 @@ { "calibration_required_values": { "capacity_retry_policy": { - "retry_after_ms": 57, + "retry_after_ms": 40, "retry_class": "after_capacity_change", "retry_condition_source": "named_condition_from_typed_capacity_response" }, "connect_timeout_ms": 2000, "election_backoff_policy": { - "initial_backoff_ms": 8, + "initial_backoff_ms": 7, "jitter": "sha256(process_start_id||attempt) modulo inclusive [initial_backoff_ms,maximum_backoff_ms]", "maximum_backoff_ms": 102 }, @@ -43,22 +43,7 @@ "query_queue_capacity": 64, "true_idle_observation_grace_ms": 2500 }, - "freeze_record": { - "calibration_bundle_sha256": "650c558b108c6473f0f244aaeb8680938f45cb51ad30cba3cc42ccaf330de004", - "calibration_freeze_digest": "14f02f80a8bf0f68d02e820b5d16d8b171839cf60ecbf643ddc6191c9f9bb40e", - "input_constant_set_sha256": "c67187b461e07bc09352cbb2b3eaaf1760780e32b284e6d127de9d49adb3c51b", - "measurement_protocol_sha256": "ce4bc4911c13d3e52438638e3abb5383e23cd9b400893805f589cdf6350e16e1", - "protocol_sha256": "f4a3fa4afb4d5bcd8e707a5e21b687cdd023dc3398b28ff6891a2318e89c5ec7", - "run_artifact_sha256s": [ - "65789b6090a68a45ab770513fc2d20ed3a228a371f1dbdd76aec6ce568dfd8e8", - "b74a95d375f6e3e37b182d91a120b2d05071f2016bf7d49dc3355bb8a2b42dec", - "bdd9340b0020be257fa2af59cb7231eed5270ee2bf3509b04bdc30cc4861da11" - ], - "selected_at": "github-actions-run:30498515242:1", - "selection_rule": "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2", - "selection_source_commit": "08451fa2cfc6665821248bebc60bd8b5636c1258", - "selection_source_tree": "2215e54a1ef49f1b1bb8f0b2dab5bcdaaec19706" - }, + "freeze_record": null, "qualification_thresholds": { "backend_observed_accelerator_residency": 1, "bulk_documents_per_second": 2, @@ -76,5 +61,5 @@ }, "schema_version": 1, "selection_protocol": "codestory-per-user-embedding-server-v1", - "status": "frozen" + "status": "unfrozen" } From c2318eb2405e16ae3a14673325af0e1918bc4efd Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Wed, 29 Jul 2026 21:18:18 -0500 Subject: [PATCH 10/20] freeze embedding server constants --- ...er-user-embedding-server-constant-set.json | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json index 70b89b877..1801527ec 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json @@ -1,7 +1,7 @@ { "calibration_required_values": { "capacity_retry_policy": { - "retry_after_ms": 40, + "retry_after_ms": 55, "retry_class": "after_capacity_change", "retry_condition_source": "named_condition_from_typed_capacity_response" }, @@ -43,7 +43,22 @@ "query_queue_capacity": 64, "true_idle_observation_grace_ms": 2500 }, - "freeze_record": null, + "freeze_record": { + "calibration_bundle_sha256": "1fddcb3e67ae5af4a09022dcbd973cc9e58f2b24ab1485783bbf51d6b8ab222d", + "calibration_freeze_digest": "3e66f41f4dbc7642512c8edfbc5230e9bc7ea2fc2073283f30ce2102832c3d70", + "input_constant_set_sha256": "c67187b461e07bc09352cbb2b3eaaf1760780e32b284e6d127de9d49adb3c51b", + "measurement_protocol_sha256": "ce4bc4911c13d3e52438638e3abb5383e23cd9b400893805f589cdf6350e16e1", + "protocol_sha256": "f4a3fa4afb4d5bcd8e707a5e21b687cdd023dc3398b28ff6891a2318e89c5ec7", + "run_artifact_sha256s": [ + "290295c5d27d513285c7ec22f4f6622b65630975a3b69e5410aef6ace9843c50", + "6123b83da3a72124a88065c9c9a5ac01d9899e746676bd4443e2f6a439837536", + "655967a0dd9ad382b01418410d9b0f9ed6ff9695c2d8105ac3343e1ec66a56e1" + ], + "selected_at": "github-actions-run:30507598793:1", + "selection_rule": "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2", + "selection_source_commit": "34154f213a411bb0c89b2fe103ea641f454ffb79", + "selection_source_tree": "5c4fa7bf948410928bd82bf7c020153c2a579abc" + }, "qualification_thresholds": { "backend_observed_accelerator_residency": 1, "bulk_documents_per_second": 2, @@ -61,5 +76,5 @@ }, "schema_version": 1, "selection_protocol": "codestory-per-user-embedding-server-v1", - "status": "unfrozen" + "status": "frozen" } From f1fb7af2458ef16129c1bca71b122f0cc9b3ef05 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Wed, 29 Jul 2026 22:24:33 -0500 Subject: [PATCH 11/20] stage cargo-linked qualification drivers --- .github/scripts/check-workflow-policy.mjs | 18 ++-- .../scripts/check-workflow-policy.test.mjs | 84 ++++++++++++++++++- .../scripts/qualification-driver-artifact.mjs | 18 +++- ...er-user-embedding-server-constant-set.json | 21 +---- 4 files changed, 115 insertions(+), 26 deletions(-) diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index 635e0abd5..540f0b12b 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -602,6 +602,8 @@ export function qualificationDriverArtifactViolations( 'binary: "codestory_embedding_qualification.exe"', 'rustTarget: "x86_64-pc-windows-msvc"', "metadata.isSymbolicLink()\n || !metadata.isFile()\n || metadata.nlink !== 1", + "function regularBuildOutput(file, label)", + "!Number.isSafeInteger(metadata.nlink)\n || metadata.nlink < 1", "metadata.isSymbolicLink() || !metadata.isDirectory()", 'fail("qualification driver helper arguments changed")', "containedRelativePath(root, candidate, label)", @@ -610,7 +612,10 @@ export function qualificationDriverArtifactViolations( 'fail(`${label} must not traverse symbolic links`)', "`codestory-cli-v${version}-${assetTarget}.${contract.archiveExtension}`", 'targetDir,\n contract.rustTarget,\n "release",\n contract.binary', + 'const sourceMetadata = regularBuildOutput(', 'fail("qualification driver artifact directory must start empty")', + "copyFileSync(source, staged)", + 'const stagedMetadata = regularFile(staged, "staged qualification driver")', "archiveBytes: archiveMetadata.size", "archiveDigest: sha256(archivePath)", "archiveFile: expectedArchiveFile", @@ -691,13 +696,14 @@ const packagedHostCompilerFinalizerDigest = "b77d8bb12c2748bfe016ab65ccb2f4581356f3ccf1d666e747306caffd6c0c46"; // The companion qualification driver is intentionally retained only inside // the private Actions package artifact. This digest pins both sides of that -// contract: the producer copies only the selected target binary and binds it -// to the exact candidate archive, while the consumer rejects symlinks, extra -// files, identity drift, and byte drift before restoring execute permission. -// Any helper edit therefore requires a policy and mutation-test review in the -// same PR as the workflow change. +// contract: the producer may read Cargo's trusted hard-linked build output, +// but retains only a new singly linked copy bound to the exact candidate +// archive. The consumer rejects symlinks, retained hardlinks, extra files, +// identity drift, and byte drift before restoring execute permission. +// Any helper edit therefore requires policy and mutation-test review in the +// same PR. const qualificationDriverArtifactDigest = - "f7946e03fa6e272ca17f12616b82579da041de4d7c30b19d24ddbc5f1c7f0063"; + "efc5126e24162d52f9da8bac38c3414b3a7492fb17eed5ff19867fadad69623e"; const draftProofCommands = [ "cargo test --locked -p codestory-llama-sys --test native_staging", "cargo test --locked -p codestory-llama-sys --test model_staging", diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index 4609f6e82..607b99ef7 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -3,6 +3,8 @@ import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { chmodSync, + linkSync, + lstatSync, mkdirSync, mkdtempSync, readFileSync, @@ -44,6 +46,10 @@ import { validateWorkflows, windowsManifestProofPolicyViolations, } from "./check-workflow-policy.mjs"; +import { + produceQualificationDriverArtifact, + verifyQualificationDriverArtifact, +} from "./qualification-driver-artifact.mjs"; const fullSha = "0123456789abcdef0123456789abcdef01234567"; const proofTopology = "proof5-v1-64015a841a2f69f33f7c9ce284f671ad27b3923a58db865fd4806d86230df6c5"; @@ -1453,7 +1459,7 @@ test("qualification driver is built once, retained privately, authenticated, and source.replace("sha256(archivePath) !== identity.archive.sha256", "false")], ["helper follows linked path ancestors", source => source.replace("lstatSync(cursor).isSymbolicLink()", "false")], - ["helper accepts hardlinked drivers", source => + ["helper accepts hardlinked retained drivers", source => source.replace("metadata.nlink !== 1", "false")], ["helper accepts extra identity fields", source => source.replace('fail(`${label} keys changed`)', "return")], @@ -1498,6 +1504,82 @@ test("qualification driver is built once, retained privately, authenticated, and } }); +test("qualification driver retention breaks a Cargo source hardlink and rejects retained hardlinks", () => { + const directory = mkdtempSync( + path.join(os.tmpdir(), "codestory-qualification-driver-"), + ); + try { + const targetDirectory = path.join(directory, "target"); + const releaseDirectory = path.join( + targetDirectory, + "x86_64-pc-windows-msvc", + "release", + ); + const depsDirectory = path.join(releaseDirectory, "deps"); + mkdirSync(depsDirectory, { recursive: true }); + const originalDriver = path.join( + depsDirectory, + "codestory_embedding_qualification-hash.exe", + ); + const cargoDriver = path.join( + releaseDirectory, + "codestory_embedding_qualification.exe", + ); + writeFileSync(originalDriver, "qualification-driver-v1"); + chmodSync(originalDriver, 0o755); + linkSync(originalDriver, cargoDriver); + assert.equal(lstatSync(cargoDriver).nlink, 2); + + const archive = path.join( + directory, + "codestory-cli-v0.16.3-windows-x64.zip", + ); + writeFileSync(archive, "candidate-archive"); + const artifactDirectory = path.join(directory, "artifact"); + const produced = produceQualificationDriverArtifact({ + archive, + assetTarget: "windows-x64", + outDir: artifactDirectory, + sourceSha: "a".repeat(40), + sourceTree: "b".repeat(40), + targetDir: targetDirectory, + trustedRoot: directory, + version: "0.16.3", + }); + assert.equal(lstatSync(produced.driver).nlink, 1); + assert.equal(readFileSync(produced.driver, "utf8"), "qualification-driver-v1"); + + writeFileSync(originalDriver, "qualification-driver-v2"); + assert.equal(readFileSync(produced.driver, "utf8"), "qualification-driver-v1"); + const verified = verifyQualificationDriverArtifact({ + archive, + artifactDir: artifactDirectory, + assetTarget: "windows-x64", + sourceSha: "a".repeat(40), + sourceTree: "b".repeat(40), + trustedRoot: directory, + version: "0.16.3", + }); + assert.equal(verified.identity.driver.sha256, produced.identity.driver.sha256); + + linkSync(produced.driver, path.join(directory, "retained-driver-alias.exe")); + assert.throws( + () => verifyQualificationDriverArtifact({ + archive, + artifactDir: artifactDirectory, + assetTarget: "windows-x64", + sourceSha: "a".repeat(40), + sourceTree: "b".repeat(40), + trustedRoot: directory, + version: "0.16.3", + }), + /qualification driver artifact must be a regular, non-symlink, singly linked file/u, + ); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + test("release workflows retain the closeout coordinator contract test", () => { assert.deepEqual(validateWorkflows(loadWorkflows()), []); for (const [file, jobName] of [ diff --git a/.github/scripts/qualification-driver-artifact.mjs b/.github/scripts/qualification-driver-artifact.mjs index 5d8cfaa90..d9f973550 100644 --- a/.github/scripts/qualification-driver-artifact.mjs +++ b/.github/scripts/qualification-driver-artifact.mjs @@ -119,6 +119,19 @@ function regularFile(file, label) { return metadata; } +function regularBuildOutput(file, label) { + const metadata = lstatSync(file); + if ( + metadata.isSymbolicLink() + || !metadata.isFile() + || !Number.isSafeInteger(metadata.nlink) + || metadata.nlink < 1 + ) { + fail(`${label} must be a regular, non-symlink build output`); + } + return metadata; +} + function regularDirectory(directory, label) { const metadata = lstatSync(directory); if (metadata.isSymbolicLink() || !metadata.isDirectory()) { @@ -270,7 +283,10 @@ export function produceQualificationDriverArtifact({ label: "qualification driver source", root: targetDir, }); - const sourceMetadata = regularFile(source, "qualification driver"); + const sourceMetadata = regularBuildOutput( + source, + "qualification driver source", + ); if (process.platform !== "win32" && (sourceMetadata.mode & 0o111) === 0) { fail("qualification driver must be executable"); } diff --git a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json index 1801527ec..70b89b877 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json @@ -1,7 +1,7 @@ { "calibration_required_values": { "capacity_retry_policy": { - "retry_after_ms": 55, + "retry_after_ms": 40, "retry_class": "after_capacity_change", "retry_condition_source": "named_condition_from_typed_capacity_response" }, @@ -43,22 +43,7 @@ "query_queue_capacity": 64, "true_idle_observation_grace_ms": 2500 }, - "freeze_record": { - "calibration_bundle_sha256": "1fddcb3e67ae5af4a09022dcbd973cc9e58f2b24ab1485783bbf51d6b8ab222d", - "calibration_freeze_digest": "3e66f41f4dbc7642512c8edfbc5230e9bc7ea2fc2073283f30ce2102832c3d70", - "input_constant_set_sha256": "c67187b461e07bc09352cbb2b3eaaf1760780e32b284e6d127de9d49adb3c51b", - "measurement_protocol_sha256": "ce4bc4911c13d3e52438638e3abb5383e23cd9b400893805f589cdf6350e16e1", - "protocol_sha256": "f4a3fa4afb4d5bcd8e707a5e21b687cdd023dc3398b28ff6891a2318e89c5ec7", - "run_artifact_sha256s": [ - "290295c5d27d513285c7ec22f4f6622b65630975a3b69e5410aef6ace9843c50", - "6123b83da3a72124a88065c9c9a5ac01d9899e746676bd4443e2f6a439837536", - "655967a0dd9ad382b01418410d9b0f9ed6ff9695c2d8105ac3343e1ec66a56e1" - ], - "selected_at": "github-actions-run:30507598793:1", - "selection_rule": "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2", - "selection_source_commit": "34154f213a411bb0c89b2fe103ea641f454ffb79", - "selection_source_tree": "5c4fa7bf948410928bd82bf7c020153c2a579abc" - }, + "freeze_record": null, "qualification_thresholds": { "backend_observed_accelerator_residency": 1, "bulk_documents_per_second": 2, @@ -76,5 +61,5 @@ }, "schema_version": 1, "selection_protocol": "codestory-per-user-embedding-server-v1", - "status": "frozen" + "status": "unfrozen" } From 150611891aebca0c9ceafbfb9f591b69b552c148 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Wed, 29 Jul 2026 22:43:56 -0500 Subject: [PATCH 12/20] freeze calibrated embedding constants --- ...er-user-embedding-server-constant-set.json | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json index 70b89b877..db3b9976d 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json @@ -1,7 +1,7 @@ { "calibration_required_values": { "capacity_retry_policy": { - "retry_after_ms": 40, + "retry_after_ms": 45, "retry_class": "after_capacity_change", "retry_condition_source": "named_condition_from_typed_capacity_response" }, @@ -9,7 +9,7 @@ "election_backoff_policy": { "initial_backoff_ms": 7, "jitter": "sha256(process_start_id||attempt) modulo inclusive [initial_backoff_ms,maximum_backoff_ms]", - "maximum_backoff_ms": 102 + "maximum_backoff_ms": 104 }, "hard_native_no_progress_ms": 385431, "request_deadlines_ms": { @@ -43,7 +43,22 @@ "query_queue_capacity": 64, "true_idle_observation_grace_ms": 2500 }, - "freeze_record": null, + "freeze_record": { + "calibration_bundle_sha256": "e0861864455fd60dc6d51f5b627f615e1d8a16825a91d029ea5940029dd0c5ea", + "calibration_freeze_digest": "0ae241b2f151c7c2232677e84097f0b4f79766115204d8410d3d5ec15fd10a15", + "input_constant_set_sha256": "c67187b461e07bc09352cbb2b3eaaf1760780e32b284e6d127de9d49adb3c51b", + "measurement_protocol_sha256": "ce4bc4911c13d3e52438638e3abb5383e23cd9b400893805f589cdf6350e16e1", + "protocol_sha256": "f4a3fa4afb4d5bcd8e707a5e21b687cdd023dc3398b28ff6891a2318e89c5ec7", + "run_artifact_sha256s": [ + "739c711ca14cd9110da279fdbb735ee6adf18629fe3fa33d4f693fb7bd4d5dea", + "c6e468a1570fc5fb4f2620df046914b0b39d54123b3ef18f5254425dc58649e1", + "dc9dc2a857e2400d3662ea9f7a111c31776296439b83da696c9aa1f982a2395f" + ], + "selected_at": "github-actions-run:30511516861:1", + "selection_rule": "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2", + "selection_source_commit": "f1fb7af2458ef16129c1bca71b122f0cc9b3ef05", + "selection_source_tree": "b084dcda7507969035353a3fc1a9ee906ce2e73d" + }, "qualification_thresholds": { "backend_observed_accelerator_residency": 1, "bulk_documents_per_second": 2, @@ -61,5 +76,5 @@ }, "schema_version": 1, "selection_protocol": "codestory-per-user-embedding-server-v1", - "status": "unfrozen" + "status": "frozen" } From 74dd8b023481ccf5e7d6036aeb5a29cdc1ee4da6 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Thu, 30 Jul 2026 04:01:06 -0500 Subject: [PATCH 13/20] bind true idle to product completion --- .../measurement_protocol_validation.py | 12 ++- .../self_test_full_stack_calibration.py | 32 ++++++++ .../scenarios/artifact/runner/measurements.rs | 4 +- .../worker/operations/measure.rs | 81 ++++++++++++++----- ...embedding-server-measurement-protocol.json | 4 +- 5 files changed, 109 insertions(+), 24 deletions(-) diff --git a/.github/scripts/packaged_agent_proof/measurement_protocol_validation.py b/.github/scripts/packaged_agent_proof/measurement_protocol_validation.py index 812b2c861..424f1f9c1 100644 --- a/.github/scripts/packaged_agent_proof/measurement_protocol_validation.py +++ b/.github/scripts/packaged_agent_proof/measurement_protocol_validation.py @@ -35,7 +35,7 @@ "calibration_workload_state_overrides", ) EXPECTED_QUALIFICATION_MEASUREMENT_SHAPE_SHA256 = ( - "1c065562adc34d0d9978187857807e491c4e6d4aa233fdd94f5636931a7b730e" + "bc78e8c0277062f1274b0ed97e9bafbef2574b2d1934cb6ab89e7f514900fef8" ) @@ -117,6 +117,11 @@ def _verify_scenario_and_metric_contracts(protocol: dict) -> tuple[set[str], dic and all(isinstance(event, str) and event for event in boundaries), f"measurement metric {metric} must have exact start and end events", ) + require( + phase_boundaries["true_idle_exit"] + == ["final_product_request_completed", "engine_and_server_absent"], + "true-idle qualification must start at final product completion", + ) metric_contracts = protocol.get("metric_contracts") require( isinstance(metric_contracts, dict) @@ -419,6 +424,11 @@ def _verify_measurement_sampling( workload.get("input_generator"), f"measurement workload {metric}.input_generator", ) + require( + workloads["true_idle_exit"].get("workload_id") + == "true_idle_after_product_completion_60000_awake_ms_v2", + "true-idle qualification workload changed its product-completion boundary", + ) sampling = protocol.get("metric_sampling") require( isinstance(sampling, dict) and set(sampling) == required_metrics, diff --git a/.github/scripts/packaged_agent_proof/self_test_full_stack_calibration.py b/.github/scripts/packaged_agent_proof/self_test_full_stack_calibration.py index 433dae637..c881c0a1e 100644 --- a/.github/scripts/packaged_agent_proof/self_test_full_stack_calibration.py +++ b/.github/scripts/packaged_agent_proof/self_test_full_stack_calibration.py @@ -33,6 +33,38 @@ def _qualification_matrix_tests(fixture: FullStackFixture) -> dict: self_measurement_protocol, require_frozen=False, ) + for label, field, value in ( + ( + "server idle epoch", + "phase", + [ + "last_queued_active_or_leased_work_ended", + "engine_and_server_absent", + ], + ), + ( + "pre-completion workload", + "workload", + "true_idle_60000_awake_ms_v1", + ), + ): + regressed_true_idle = json.loads( + json.dumps(measurement_contract["measurement_protocol"]) + ) + if field == "phase": + regressed_true_idle["phase_boundaries"]["true_idle_exit"] = value + else: + regressed_true_idle["workloads"]["true_idle_exit"]["workload_id"] = value + regressed_true_idle_path = ( + fixture.root / f"true-idle-{field}-regression.json" + ) + write_json(regressed_true_idle_path, regressed_true_idle) + try: + load_measurement_protocol(regressed_true_idle_path) + except ProofFailure: + pass + else: + raise ProofFailure(f"true-idle qualification accepted {label}") for quality_metric in ( "answer_quality", "packet_quality", diff --git a/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/measurements.rs b/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/measurements.rs index 1dff013ad..551b09332 100644 --- a/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/measurements.rs +++ b/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/measurements.rs @@ -71,7 +71,7 @@ pub(super) fn declared_phase_boundaries(metric: &str) -> Result<[&'static str; 2 ], "busy_retry_usefulness" => ["typed_retry_emitted", "named_retry_condition_became_true"], "true_idle_exit" => [ - "last_queued_active_or_leased_work_ended", + "final_product_request_completed", "engine_and_server_absent", ], "backend_observed_accelerator_residency" => [ @@ -94,7 +94,7 @@ pub(super) fn declared_workload_id(metric: &str) -> Result<&'static str> { "warm_bulk_ipc" => "warm_bulk_64x256b_v1", "bulk_documents_per_second" | "bulk_tokens_per_second" => "bulk_throughput_256x256b_v1", "busy_retry_usefulness" => "saturated_query_65th_retry_v1", - "true_idle_exit" => "true_idle_60000_awake_ms_v1", + "true_idle_exit" => "true_idle_after_product_completion_60000_awake_ms_v2", "backend_observed_accelerator_residency" => "resident_policy_identity_v1", _ => bail!("embedding_qualification_metric_workload_unknown:{metric}"), }) diff --git a/crates/codestory-cli/src/embedding_qualification/worker/operations/measure.rs b/crates/codestory-cli/src/embedding_qualification/worker/operations/measure.rs index 55b6f0c98..2f7b6927b 100644 --- a/crates/codestory-cli/src/embedding_qualification/worker/operations/measure.rs +++ b/crates/codestory-cli/src/embedding_qualification/worker/operations/measure.rs @@ -68,7 +68,7 @@ fn workload_documents(workload_id: &str, repeat: u32, count: usize, bytes: usize .collect() } -struct MeasurementSpanStart { +pub(in crate::embedding_qualification::worker) struct MeasurementSpanStart { awake_started_ns: u64, inclusive_started_ns: u64, boot_id_started: String, @@ -88,6 +88,26 @@ fn begin_span(clock: &dyn AwakeMonotonicClock) -> Result { }) } +/// Stamp product completion for `true_idle_exit`. +/// +/// The awake reading is deliberately first: once the product operation has +/// stamped completion, a delay while returning to the measurement coordinator +/// must remain inside the measured idle interval, never move its start +/// forward. The suspend-inclusive and boot witnesses follow immediately and +/// the downstream tolerance fails closed if sampling them is delayed. +fn begin_true_idle_span_at_product_completion( + clock: &dyn AwakeMonotonicClock, +) -> Result { + let awake_started_ns = clock.now_ns(); + let inclusive_started_ns = crate::embedding_server_transport::inclusive_now_ns()?; + let boot_id_started = crate::embedding_server_transport::boot_id()?; + Ok(MeasurementSpanStart { + awake_started_ns, + inclusive_started_ns, + boot_id_started, + }) +} + /// Stamp the declared end instant: awake reading first, then the /// suspend-inclusive reading and boot id. fn finish_span( @@ -467,7 +487,11 @@ fn validate_true_idle_boundary( /// bypass. pub(in crate::embedding_qualification::worker) trait TrueIdleMeasurementClient { fn observe_snapshot(&self) -> Result>; - fn complete_product_query(&self, input: &str) -> Result<()>; + fn complete_product_query_and_stamp( + &self, + input: &str, + clock: &dyn AwakeMonotonicClock, + ) -> Result; fn observe_owner_exit(&self) -> Result; } @@ -476,9 +500,13 @@ impl TrueIdleMeasurementClient for PerUserEmbeddingClient { self.observe() } - fn complete_product_query(&self, input: &str) -> Result<()> { + fn complete_product_query_and_stamp( + &self, + input: &str, + clock: &dyn AwakeMonotonicClock, + ) -> Result { let _ = self.embed_query(input)?; - Ok(()) + begin_true_idle_span_at_product_completion(clock) } fn observe_owner_exit(&self) -> Result { @@ -487,9 +515,9 @@ impl TrueIdleMeasurementClient for PerUserEmbeddingClient { } /// `true_idle_exit`: this worker completes the last product request itself, -/// stamps its return as `last_queued_active_or_leased_work_ended`, proves the -/// resident scheduler is drained, then ends at the observation that returned -/// no owner (`engine_and_server_absent`). +/// stamps `final_product_request_completed` before returning to its caller, +/// proves the resident scheduler is drained, then ends at the observation that +/// returned no owner (`engine_and_server_absent`). pub(in crate::embedding_qualification::worker) fn run_measure_true_idle( client: &dyn TrueIdleMeasurementClient, clock: &dyn AwakeMonotonicClock, @@ -500,8 +528,7 @@ pub(in crate::embedding_qualification::worker) fn run_measure_true_idle( .filter(resident_engine_generation) .ok_or_else(|| anyhow::anyhow!("embedding_qualification_true_idle_owner_missing"))?; let input = "q".repeat(input_bytes.max(1) as usize); - client.complete_product_query(&input)?; - let start = begin_span(clock)?; + let start = client.complete_product_query_and_stamp(&input, clock)?; let idle_owner = client .observe_snapshot()? .ok_or_else(|| anyhow::anyhow!("embedding_qualification_true_idle_owner_missing"))?; @@ -786,8 +813,9 @@ fn validate_vector_response( mod tests { use super::super::owner_exit::OwnerExitObservation; use super::{ - TrueIdleMeasurementClient, run_measure_true_idle, true_idle_scheduler_is_drained, - validate_true_idle_boundary, workload_input, + MeasurementSpanStart, TrueIdleMeasurementClient, + begin_true_idle_span_at_product_completion, run_measure_true_idle, + true_idle_scheduler_is_drained, validate_true_idle_boundary, workload_input, }; use anyhow::Result; use codestory_retrieval::{ @@ -801,6 +829,10 @@ mod tests { use std::sync::{Arc, Mutex}; use std::time::Duration; + const PRODUCT_COMPLETED_NS: u64 = 100; + const PRODUCT_RETURNED_NS: u64 = 10_000_000_100; + const DRAINED_OWNER_OBSERVED_NS: u64 = 10_000_000_145; + struct ScriptClock { now_ns: AtomicU64, events: Arc>>, @@ -822,7 +854,7 @@ mod tests { impl AwakeMonotonicClock for ScriptClock { fn now_ns(&self) -> u64 { let now_ns = self.now_ns.load(Ordering::Acquire); - if now_ns == 100 { + if now_ns == PRODUCT_COMPLETED_NS { self.events .lock() .expect("lock events") @@ -868,22 +900,32 @@ mod tests { let event = if remaining == 1 { "expected_owner_observed" } else { - self.clock.set(145); + self.clock.set(DRAINED_OWNER_OBSERVED_NS); "same_owner_drained_observed" }; self.events.lock().expect("lock events").push(event); Ok(Some(snapshot)) } - fn complete_product_query(&self, input: &str) -> Result<()> { + fn complete_product_query_and_stamp( + &self, + input: &str, + clock: &dyn AwakeMonotonicClock, + ) -> Result { assert_eq!(input, "q".repeat(256)); self.product_calls.fetch_add(1, Ordering::AcqRel); - self.clock.set(100); + self.clock.set(PRODUCT_COMPLETED_NS); self.events .lock() .expect("lock events") .push("product_completed"); - Ok(()) + let start = begin_true_idle_span_at_product_completion(clock)?; + self.clock.set(PRODUCT_RETURNED_NS); + self.events + .lock() + .expect("lock events") + .push("product_returned_after_delay"); + Ok(start) } fn observe_owner_exit(&self) -> Result { @@ -969,7 +1011,7 @@ mod tests { } #[test] - fn true_idle_start_is_stamped_at_product_completion_before_observation_lag() { + fn true_idle_start_is_stamped_at_product_completion_before_return_and_observation_lag() { let events = Arc::new(Mutex::new(Vec::new())); let clock = Arc::new(ScriptClock::new(Arc::clone(&events))); let client = ScriptedTrueIdleClient { @@ -990,8 +1032,8 @@ mod tests { run_measure_true_idle(&client, clock.as_ref(), 256).expect("measure true idle"); assert_eq!(client.product_calls.load(Ordering::Acquire), 1); - assert_eq!(measurement.span.awake_started_ns, 100); - assert!(measurement.span.awake_finished_ns >= 145); + assert_eq!(measurement.span.awake_started_ns, PRODUCT_COMPLETED_NS); + assert!(measurement.span.awake_finished_ns >= DRAINED_OWNER_OBSERVED_NS); assert_eq!( measurement.snapshot.process.server_instance_id, "measured-owner" @@ -1002,6 +1044,7 @@ mod tests { "expected_owner_observed", "product_completed", "span_started", + "product_returned_after_delay", "same_owner_drained_observed", "same_owner_still_present", "owner_absence_observed", diff --git a/crates/codestory-llama-sys/per-user-embedding-server-measurement-protocol.json b/crates/codestory-llama-sys/per-user-embedding-server-measurement-protocol.json index deb9c385b..d8ddd7f61 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-measurement-protocol.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-measurement-protocol.json @@ -157,7 +157,7 @@ "named_retry_condition_became_true" ], "true_idle_exit": [ - "last_queued_active_or_leased_work_ended", + "final_product_request_completed", "engine_and_server_absent" ], "total_codestory_process_memory": [ @@ -279,7 +279,7 @@ "measured_request_ordinal": 65 }, "true_idle_exit": { - "workload_id": "true_idle_60000_awake_ms_v1", + "workload_id": "true_idle_after_product_completion_60000_awake_ms_v2", "owner_state": "resident_then_absent", "operation": "observe", "input_generator": "none", From 76dfa93e6780030595789b877d292d12fc069583 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Thu, 30 Jul 2026 04:48:11 -0500 Subject: [PATCH 14/20] freeze calibrated embedding constants --- ...er-user-embedding-server-constant-set.json | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json index 36a5e49fc..f3a1c167e 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json @@ -1,21 +1,21 @@ { "calibration_required_values": { "capacity_retry_policy": { - "retry_after_ms": 40, + "retry_after_ms": 50, "retry_class": "after_capacity_change", "retry_condition_source": "named_condition_from_typed_capacity_response" }, "connect_timeout_ms": 2000, "election_backoff_policy": { - "initial_backoff_ms": 7, + "initial_backoff_ms": 9, "jitter": "sha256(process_start_id||attempt) modulo inclusive [initial_backoff_ms,maximum_backoff_ms]", - "maximum_backoff_ms": 102 + "maximum_backoff_ms": 108 }, "hard_native_no_progress_ms": 385431, "request_deadlines_ms": { "bulk_replay_success_budget_ms": 144537, "bulk_request_deadline_ms": 564239, - "query_request_deadline_ms": 10000 + "query_request_deadline_ms": 10160 }, "spawn_convergence_timeout_ms": 15000, "watchdog_cadence_ms": 19271 @@ -43,7 +43,22 @@ "query_queue_capacity": 64, "true_idle_observation_grace_ms": 2500 }, - "freeze_record": null, + "freeze_record": { + "calibration_bundle_sha256": "2d02991184e58a28440ffee38f0df50840b57b6b13e57736ec1a273e614b3f97", + "calibration_freeze_digest": "9002c6b680c8b45e4ff8b076ec73cf0d314703a359ea27d60ed3392d9c1ddb01", + "input_constant_set_sha256": "ea58d298473ddf320469d15dc7c32176a1109d615a689c15ff76b67fb337e109", + "measurement_protocol_sha256": "d1bb9b2c7eb354fe98990aa32eedc0e165b0cf804212966e0a2ee362a2f5bf8e", + "protocol_sha256": "f4a3fa4afb4d5bcd8e707a5e21b687cdd023dc3398b28ff6891a2318e89c5ec7", + "run_artifact_sha256s": [ + "65a4c4d5429ce9b83905505d6778c917c6a86d61fbd904c8950db1e0eac83ffc", + "cc5fe78225514333c8741564425c2c6f92db95e88d58df4e78ec03ac6985d6ca", + "e78409ef1925c467c178ed0b90b55f8d88269584a938e56393ead122ea835bb9" + ], + "selected_at": "github-actions-run:30531101324:1", + "selection_rule": "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2", + "selection_source_commit": "74dd8b023481ccf5e7d6036aeb5a29cdc1ee4da6", + "selection_source_tree": "1eddbce0062d298868049a0c6f4603c419195100" + }, "qualification_thresholds": { "backend_observed_accelerator_residency": 1, "bulk_documents_per_second": 2, @@ -60,5 +75,5 @@ }, "schema_version": 1, "selection_protocol": "codestory-per-user-embedding-server-v1", - "status": "unfrozen" + "status": "frozen" } From 04605eb8321dc46c7950c998b6a8eabe8c7ccb59 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Thu, 30 Jul 2026 06:06:27 -0500 Subject: [PATCH 15/20] freeze calibrated embedding constants --- ...er-user-embedding-server-constant-set.json | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json index 36a5e49fc..05a9f8a13 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json @@ -1,7 +1,7 @@ { "calibration_required_values": { "capacity_retry_policy": { - "retry_after_ms": 40, + "retry_after_ms": 45, "retry_class": "after_capacity_change", "retry_condition_source": "named_condition_from_typed_capacity_response" }, @@ -9,7 +9,7 @@ "election_backoff_policy": { "initial_backoff_ms": 7, "jitter": "sha256(process_start_id||attempt) modulo inclusive [initial_backoff_ms,maximum_backoff_ms]", - "maximum_backoff_ms": 102 + "maximum_backoff_ms": 107 }, "hard_native_no_progress_ms": 385431, "request_deadlines_ms": { @@ -43,7 +43,22 @@ "query_queue_capacity": 64, "true_idle_observation_grace_ms": 2500 }, - "freeze_record": null, + "freeze_record": { + "calibration_bundle_sha256": "f27a0dfdc256c332daa0c405b388fca2cf88cdaee0ef7fd9f56b8b08a62135e6", + "calibration_freeze_digest": "afc407e83e3192b6095f29e7083b92e2cd8f1d9590bec0345ec5a164b66cccc6", + "input_constant_set_sha256": "ea58d298473ddf320469d15dc7c32176a1109d615a689c15ff76b67fb337e109", + "measurement_protocol_sha256": "d1bb9b2c7eb354fe98990aa32eedc0e165b0cf804212966e0a2ee362a2f5bf8e", + "protocol_sha256": "f4a3fa4afb4d5bcd8e707a5e21b687cdd023dc3398b28ff6891a2318e89c5ec7", + "run_artifact_sha256s": [ + "1ffcf00054d38dadf22e4554edeb61bf9af38822513729685fb7da0e9aa46a7d", + "35dc5bd2f926ef0c791514962fc21d0a91186862ca7c6019b452bc9b81c6297b", + "ef5ec53eaee5cee7fadb657afd379820f4c1f722c72001bf5401f4284b852317" + ], + "selected_at": "github-actions-run:30536808468:1", + "selection_rule": "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2", + "selection_source_commit": "d318b485e1d1dfd3753711afa5d29f441500ce5a", + "selection_source_tree": "a9c6d4490589f1f8afcc721b476f937395fd483c" + }, "qualification_thresholds": { "backend_observed_accelerator_residency": 1, "bulk_documents_per_second": 2, @@ -60,5 +75,5 @@ }, "schema_version": 1, "selection_protocol": "codestory-per-user-embedding-server-v1", - "status": "unfrozen" + "status": "frozen" } From f265f76bf731f03ef716f0c09b0852dbd467ab45 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Thu, 30 Jul 2026 06:45:31 -0500 Subject: [PATCH 16/20] enforce atomic candidate archive admission --- .github/scripts/candidate-archive-store.mjs | 10 +++++ .../scripts/candidate-archive-store.test.mjs | 41 ++++++++++++++++++- ...er-user-embedding-server-constant-set.json | 23 ++--------- 3 files changed, 54 insertions(+), 20 deletions(-) diff --git a/.github/scripts/candidate-archive-store.mjs b/.github/scripts/candidate-archive-store.mjs index 0febeed00..f76284c18 100644 --- a/.github/scripts/candidate-archive-store.mjs +++ b/.github/scripts/candidate-archive-store.mjs @@ -874,6 +874,7 @@ function publishStoreEntry(storeRoot, inputRoot, record) { removeOwnedTemporary(temporary, paths.parent, path.basename(paths.entry)); return { admitted: false, ...concurrent }; } + const prepared = lstatSync(temporary, { bigint: true }); try { renameSync(temporary, paths.entry); } catch (error) { @@ -884,6 +885,15 @@ function publishStoreEntry(storeRoot, inputRoot, record) { removeOwnedTemporary(temporary, paths.parent, path.basename(paths.entry)); return { admitted: false, ...concurrent }; } + const published = lstatSync(paths.entry, { bigint: true }); + if ( + !published.isDirectory() + || published.isSymbolicLink() + || published.dev !== prepared.dev + || published.ino !== prepared.ino + ) { + fail("candidate archive store entry was not published by atomic directory rename"); + } return { admitted: true, ...verifyStoreEntry(storeRoot, expected) }; } catch (error) { if (existsSync(temporary)) { diff --git a/.github/scripts/candidate-archive-store.test.mjs b/.github/scripts/candidate-archive-store.test.mjs index 41548c3ee..277c853cc 100644 --- a/.github/scripts/candidate-archive-store.test.mjs +++ b/.github/scripts/candidate-archive-store.test.mjs @@ -17,7 +17,7 @@ import os from "node:os"; import path from "node:path"; import { spawnSync } from "node:child_process"; import test from "node:test"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { admitCandidateArchive, @@ -291,6 +291,45 @@ test("admission and a later hit materialize the complete exact payload as fresh } }); +test("admission rejects sequential publication beneath the final store key", async () => { + const fixture = createFixture(); + try { + const source = readFileSync(SCRIPT, "utf8"); + const atomicPublication = " renameSync(temporary, paths.entry);"; + assert.equal( + source.split(atomicPublication).length - 1, + 1, + "atomic store publication must have one mutation target", + ); + const sequentialPublication = [ + " mkdirSync(paths.entry, { mode: 0o700 });", + " renameSync(temporaryPayload, paths.payload);", + " renameSync(path.join(temporary, RECORD_FILE), paths.recordFile);", + " rmSync(temporary, { recursive: true });", + ].join("\n"); + const mutantFile = path.join(fixture.root, "candidate-archive-store-mutant.mjs"); + writeFileSync( + mutantFile, + source.replace(atomicPublication, sequentialPublication), + { flag: "wx" }, + ); + const mutant = await import(pathToFileURL(mutantFile).href); + assert.throws( + () => mutant.admitCandidateArchive({ + inputRoot: fixture.inputRoot, + outputDir: outputDir(fixture), + outputRoot: fixture.outputRoot, + record: fixture.record, + storeRoot: fixture.storeRoot, + }), + /not published by atomic directory rename/u, + ); + assert.equal(statExists(outputDir(fixture)), false); + } finally { + cleanup(fixture); + } +}); + test("the public checksum companion pair is mandatory", () => { const fixture = createFixture(); try { diff --git a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json index 05a9f8a13..36a5e49fc 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json @@ -1,7 +1,7 @@ { "calibration_required_values": { "capacity_retry_policy": { - "retry_after_ms": 45, + "retry_after_ms": 40, "retry_class": "after_capacity_change", "retry_condition_source": "named_condition_from_typed_capacity_response" }, @@ -9,7 +9,7 @@ "election_backoff_policy": { "initial_backoff_ms": 7, "jitter": "sha256(process_start_id||attempt) modulo inclusive [initial_backoff_ms,maximum_backoff_ms]", - "maximum_backoff_ms": 107 + "maximum_backoff_ms": 102 }, "hard_native_no_progress_ms": 385431, "request_deadlines_ms": { @@ -43,22 +43,7 @@ "query_queue_capacity": 64, "true_idle_observation_grace_ms": 2500 }, - "freeze_record": { - "calibration_bundle_sha256": "f27a0dfdc256c332daa0c405b388fca2cf88cdaee0ef7fd9f56b8b08a62135e6", - "calibration_freeze_digest": "afc407e83e3192b6095f29e7083b92e2cd8f1d9590bec0345ec5a164b66cccc6", - "input_constant_set_sha256": "ea58d298473ddf320469d15dc7c32176a1109d615a689c15ff76b67fb337e109", - "measurement_protocol_sha256": "d1bb9b2c7eb354fe98990aa32eedc0e165b0cf804212966e0a2ee362a2f5bf8e", - "protocol_sha256": "f4a3fa4afb4d5bcd8e707a5e21b687cdd023dc3398b28ff6891a2318e89c5ec7", - "run_artifact_sha256s": [ - "1ffcf00054d38dadf22e4554edeb61bf9af38822513729685fb7da0e9aa46a7d", - "35dc5bd2f926ef0c791514962fc21d0a91186862ca7c6019b452bc9b81c6297b", - "ef5ec53eaee5cee7fadb657afd379820f4c1f722c72001bf5401f4284b852317" - ], - "selected_at": "github-actions-run:30536808468:1", - "selection_rule": "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2", - "selection_source_commit": "d318b485e1d1dfd3753711afa5d29f441500ce5a", - "selection_source_tree": "a9c6d4490589f1f8afcc721b476f937395fd483c" - }, + "freeze_record": null, "qualification_thresholds": { "backend_observed_accelerator_residency": 1, "bulk_documents_per_second": 2, @@ -75,5 +60,5 @@ }, "schema_version": 1, "selection_protocol": "codestory-per-user-embedding-server-v1", - "status": "frozen" + "status": "unfrozen" } From e980b8205499545a61d3caa0c9937b2cf54cab48 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Thu, 30 Jul 2026 09:20:51 -0500 Subject: [PATCH 17/20] freeze embedding server constants --- ...er-user-embedding-server-constant-set.json | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json index 36a5e49fc..967aa2776 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json @@ -1,7 +1,7 @@ { "calibration_required_values": { "capacity_retry_policy": { - "retry_after_ms": 40, + "retry_after_ms": 45, "retry_class": "after_capacity_change", "retry_condition_source": "named_condition_from_typed_capacity_response" }, @@ -9,7 +9,7 @@ "election_backoff_policy": { "initial_backoff_ms": 7, "jitter": "sha256(process_start_id||attempt) modulo inclusive [initial_backoff_ms,maximum_backoff_ms]", - "maximum_backoff_ms": 102 + "maximum_backoff_ms": 109 }, "hard_native_no_progress_ms": 385431, "request_deadlines_ms": { @@ -43,7 +43,22 @@ "query_queue_capacity": 64, "true_idle_observation_grace_ms": 2500 }, - "freeze_record": null, + "freeze_record": { + "calibration_bundle_sha256": "40813f9d5fac21274301a41a70aa446aaacb3d816ceb926fcb4201ff1b5bd27b", + "calibration_freeze_digest": "d2dd1d32ed8699980f90e6c14806d3e86b0c68d7791c14dbacb57a4ee25fb975", + "input_constant_set_sha256": "ea58d298473ddf320469d15dc7c32176a1109d615a689c15ff76b67fb337e109", + "measurement_protocol_sha256": "d1bb9b2c7eb354fe98990aa32eedc0e165b0cf804212966e0a2ee362a2f5bf8e", + "protocol_sha256": "f4a3fa4afb4d5bcd8e707a5e21b687cdd023dc3398b28ff6891a2318e89c5ec7", + "run_artifact_sha256s": [ + "6083cf27e86367eec7ff5b31365e0475641df950309d41b48f1db3365c1452e6", + "a807d48bfa1b72bdb6e5eead4f0666fca49de879d8f89a4e7a49a84ae250c9d3", + "e576fdd9d2f8a1b7498edc64f78b224b8a81407183e204d011a0e750d868df18" + ], + "selected_at": "github-actions-run:30550654213:1", + "selection_rule": "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2", + "selection_source_commit": "196fd369b267e8e182c91b970fbbea40f1a679a5", + "selection_source_tree": "e555fe17511fc7bc6a6dad1545d33693bb32839f" + }, "qualification_thresholds": { "backend_observed_accelerator_residency": 1, "bulk_documents_per_second": 2, @@ -60,5 +75,5 @@ }, "schema_version": 1, "selection_protocol": "codestory-per-user-embedding-server-v1", - "status": "unfrozen" + "status": "frozen" } From 6c28714ded34b28be6f560fdb765e195f28268d9 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Thu, 30 Jul 2026 11:27:08 -0500 Subject: [PATCH 18/20] restore unfrozen calibration input --- ...er-user-embedding-server-constant-set.json | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json index 967aa2776..36a5e49fc 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json @@ -1,7 +1,7 @@ { "calibration_required_values": { "capacity_retry_policy": { - "retry_after_ms": 45, + "retry_after_ms": 40, "retry_class": "after_capacity_change", "retry_condition_source": "named_condition_from_typed_capacity_response" }, @@ -9,7 +9,7 @@ "election_backoff_policy": { "initial_backoff_ms": 7, "jitter": "sha256(process_start_id||attempt) modulo inclusive [initial_backoff_ms,maximum_backoff_ms]", - "maximum_backoff_ms": 109 + "maximum_backoff_ms": 102 }, "hard_native_no_progress_ms": 385431, "request_deadlines_ms": { @@ -43,22 +43,7 @@ "query_queue_capacity": 64, "true_idle_observation_grace_ms": 2500 }, - "freeze_record": { - "calibration_bundle_sha256": "40813f9d5fac21274301a41a70aa446aaacb3d816ceb926fcb4201ff1b5bd27b", - "calibration_freeze_digest": "d2dd1d32ed8699980f90e6c14806d3e86b0c68d7791c14dbacb57a4ee25fb975", - "input_constant_set_sha256": "ea58d298473ddf320469d15dc7c32176a1109d615a689c15ff76b67fb337e109", - "measurement_protocol_sha256": "d1bb9b2c7eb354fe98990aa32eedc0e165b0cf804212966e0a2ee362a2f5bf8e", - "protocol_sha256": "f4a3fa4afb4d5bcd8e707a5e21b687cdd023dc3398b28ff6891a2318e89c5ec7", - "run_artifact_sha256s": [ - "6083cf27e86367eec7ff5b31365e0475641df950309d41b48f1db3365c1452e6", - "a807d48bfa1b72bdb6e5eead4f0666fca49de879d8f89a4e7a49a84ae250c9d3", - "e576fdd9d2f8a1b7498edc64f78b224b8a81407183e204d011a0e750d868df18" - ], - "selected_at": "github-actions-run:30550654213:1", - "selection_rule": "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2", - "selection_source_commit": "196fd369b267e8e182c91b970fbbea40f1a679a5", - "selection_source_tree": "e555fe17511fc7bc6a6dad1545d33693bb32839f" - }, + "freeze_record": null, "qualification_thresholds": { "backend_observed_accelerator_residency": 1, "bulk_documents_per_second": 2, @@ -75,5 +60,5 @@ }, "schema_version": 1, "selection_protocol": "codestory-per-user-embedding-server-v1", - "status": "frozen" + "status": "unfrozen" } From 681ca99098b86006e7dc1f51c27158c757ac05c9 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Thu, 30 Jul 2026 11:44:27 -0500 Subject: [PATCH 19/20] prove only the frozen candidate once --- .github/scripts/check-workflow-policy.mjs | 12 +++++- .../scripts/check-workflow-policy.test.mjs | 37 +++++++++++++++++++ .github/workflows/packaged-platform-pr.yml | 2 +- .../release-evidence/fixtures/candidate.json | 6 +-- .../release-evidence/fixtures/report.json | 6 +-- release-claims.json | 2 + scripts/codestory-release-claims.mjs | 4 +- .../tests/codestory-release-claims.test.mjs | 11 ++++++ .../fixtures/release-claims/positive.json | 2 +- 9 files changed, 72 insertions(+), 10 deletions(-) diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index 37c943205..18cef670f 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -970,7 +970,7 @@ const packagedPlatformWorkflowDigest = // made advisory, parked in dead code, or followed by a payload substitution // while leaving the expected tokens in place. const packagedPlatformCoordinatorWorkflowDigest = - "464906e3cd7ec0e2f7e9195d60de035fdba76172c25d8b0861d0982f9d7dcc3e"; + "5017abab05e80355daf4618795d5ec7f09c07b4bc33cc1d52dca968a96b056bb"; const frozenCandidateQualityWorkflowDigest = "92d0a7ab0e0df63dacd5cc3ef0b58500a6578036494c329aa35279048734f173"; const macosMetalWorkflowDigest = @@ -5145,6 +5145,16 @@ function validatePackagedCoordinator(workflows, violations, graph) { INPUT_CALIBRATION_RUN_ID: "${{ inputs.calibration_bundle_run_id }}", }); requireExactResolverContract(violations, file, route, platformResolverContractDigest); + const sourceProofRequirement = namedStep( + route, + "Require successful exact-head source proof", + ); + add( + violations, + sourceProofRequirement?.if + === "steps.resolve.outputs.mode != 'integration' && steps.resolve.outputs.mode != 'calibration'", + `${file} calibration alone must skip pre-freeze source proof while every frozen-candidate mode requires it`, + ); requireStepRun(violations, file, route, "Require successful exact-head source proof", [ "actions/runs?head_sha=$HEAD_SHA", '.path == ".github/workflows/source-proof.yml"', diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index 45f48a142..f2a6fb61d 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -2941,6 +2941,43 @@ test("source proof reuse accepts only whole successful workflow runs", async (t) } }); +test("calibration precedes the sole frozen-candidate source proof", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const coordinatorFile = "packaged-platform-pr.yml"; + const mutations = [ + ["calibration regains a pre-freeze source proof", workflow => { + draftStep( + workflow.jobs.route, + "Require successful exact-head source proof", + ).if = "steps.resolve.outputs.mode != 'integration'"; + }], + ["qualification loses the frozen-head source proof", workflow => { + draftStep( + workflow.jobs.route, + "Require successful exact-head source proof", + ).if + = "steps.resolve.outputs.mode != 'integration' && steps.resolve.outputs.mode != 'calibration' && steps.resolve.outputs.mode != 'qualification'"; + }], + ["every mode loses the exact-head source proof", workflow => { + draftStep( + workflow.jobs.route, + "Require successful exact-head source proof", + ).if = "false"; + }], + ]; + + for (const [name, mutate] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(coordinatorFile)); + assert.match( + validateWorkflows(workflows).join("\n"), + /calibration alone must skip pre-freeze source proof while every frozen-candidate mode requires it/u, + ); + }); + } +}); + test("Windows package proof retains the readable native sccache executable", () => { const directory = mkdtempSync(path.join(os.tmpdir(), "codestory-windows-sccache-")); try { diff --git a/.github/workflows/packaged-platform-pr.yml b/.github/workflows/packaged-platform-pr.yml index 64d322a15..17852d590 100644 --- a/.github/workflows/packaged-platform-pr.yml +++ b/.github/workflows/packaged-platform-pr.yml @@ -166,7 +166,7 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Require successful exact-head source proof - if: steps.resolve.outputs.mode != 'integration' + if: steps.resolve.outputs.mode != 'integration' && steps.resolve.outputs.mode != 'calibration' shell: bash env: GH_TOKEN: ${{ github.token }} diff --git a/benchmarks/release-evidence/fixtures/candidate.json b/benchmarks/release-evidence/fixtures/candidate.json index c8c6a5b49..d0fd4c0e9 100644 --- a/benchmarks/release-evidence/fixtures/candidate.json +++ b/benchmarks/release-evidence/fixtures/candidate.json @@ -62,7 +62,7 @@ }, "release_claims": { "graph_schema": "codestory.release-claims/v1", - "graph_sha256": "7385b32924d42e94c14b8d1f41fcec50bded200fc0e65fec05f43036e2896087", + "graph_sha256": "b08ec2429ec708519b2b67532217379d7bb6a64dd1afbbfa2501652411cb1b4b", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "requested_claims": [ @@ -85,7 +85,7 @@ "type": "performance", "tier": "live_behavior", "status": "measured", - "graph_sha256": "7385b32924d42e94c14b8d1f41fcec50bded200fc0e65fec05f43036e2896087", + "graph_sha256": "b08ec2429ec708519b2b67532217379d7bb6a64dd1afbbfa2501652411cb1b4b", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -106,7 +106,7 @@ "type": "answer_quality", "tier": "answer_quality", "status": "pass", - "graph_sha256": "7385b32924d42e94c14b8d1f41fcec50bded200fc0e65fec05f43036e2896087", + "graph_sha256": "b08ec2429ec708519b2b67532217379d7bb6a64dd1afbbfa2501652411cb1b4b", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { diff --git a/benchmarks/release-evidence/fixtures/report.json b/benchmarks/release-evidence/fixtures/report.json index 7a0d10f04..f203f4e35 100644 --- a/benchmarks/release-evidence/fixtures/report.json +++ b/benchmarks/release-evidence/fixtures/report.json @@ -26,7 +26,7 @@ "type": "performance", "tier": "live_behavior", "status": "pass", - "graph_sha256": "7385b32924d42e94c14b8d1f41fcec50bded200fc0e65fec05f43036e2896087", + "graph_sha256": "b08ec2429ec708519b2b67532217379d7bb6a64dd1afbbfa2501652411cb1b4b", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -47,7 +47,7 @@ "type": "answer_quality", "tier": "answer_quality", "status": "pass", - "graph_sha256": "7385b32924d42e94c14b8d1f41fcec50bded200fc0e65fec05f43036e2896087", + "graph_sha256": "b08ec2429ec708519b2b67532217379d7bb6a64dd1afbbfa2501652411cb1b4b", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -69,7 +69,7 @@ "schema": "codestory.release-claim-evaluation/v1", "status": "pass", "graph_schema": "codestory.release-claims/v1", - "graph_sha256": "7385b32924d42e94c14b8d1f41fcec50bded200fc0e65fec05f43036e2896087", + "graph_sha256": "b08ec2429ec708519b2b67532217379d7bb6a64dd1afbbfa2501652411cb1b4b", "evidence_selection": "all_matching_rows_must_pass", "expected_commit": "2222222222222222222222222222222222222222", "evaluated_at": "2026-07-21T02:13:20.738Z", diff --git a/release-claims.json b/release-claims.json index e9e07b945..bf62003d8 100644 --- a/release-claims.json +++ b/release-claims.json @@ -1133,6 +1133,8 @@ "coordinator_workflow": "packaged-platform-pr.yml", "mode": "calibration", "assembly_job": "calibration-assemble", + "pre_collection_source_proof_required": false, + "source_proof_stage": "frozen_candidate_before_qualification", "required_cells": [ { "id": "protected_macos_arm64_metal", diff --git a/scripts/codestory-release-claims.mjs b/scripts/codestory-release-claims.mjs index 7ee478bd3..e0a348fb3 100644 --- a/scripts/codestory-release-claims.mjs +++ b/scripts/codestory-release-claims.mjs @@ -541,8 +541,10 @@ function validateCalibrationPolicy(value) { calibration.coordinator_workflow !== "packaged-platform-pr.yml" || calibration.mode !== "calibration" || calibration.assembly_job !== "calibration-assemble" + || calibration.pre_collection_source_proof_required !== false + || calibration.source_proof_stage !== "frozen_candidate_before_qualification" ) { - fail("workflow_policy.calibration must name the canonical calibration coordinator and assembly job"); + fail("workflow_policy.calibration must collect before the sole frozen-candidate source proof"); } if (calibration.runs_per_required_cell !== 3) { fail("workflow_policy.calibration must require exactly three clean runs per required cell"); diff --git a/scripts/tests/codestory-release-claims.test.mjs b/scripts/tests/codestory-release-claims.test.mjs index b4cbd4a3f..3e3f7a225 100644 --- a/scripts/tests/codestory-release-claims.test.mjs +++ b/scripts/tests/codestory-release-claims.test.mjs @@ -289,6 +289,11 @@ test("claim graph freezes Mac-only accelerated 3x1 constant calibration", () => assert.equal(calibration.optional_cells[0].feeds_constant_selection, false); assert.equal(calibration.runs_per_required_cell, 3); assert.equal(calibration.samples_per_metric_per_run, 1); + assert.equal(calibration.pre_collection_source_proof_required, false); + assert.equal( + calibration.source_proof_stage, + "frozen_candidate_before_qualification", + ); assert.deepEqual(calibration.forbidden_environment, [ "CODESTORY_EMBED_ALLOW_CPU=1", ]); @@ -312,6 +317,12 @@ test("claim graph freezes Mac-only accelerated 3x1 constant calibration", () => [draft => { draft.workflow_policy.calibration.samples_per_metric_per_run = 3; }, /exactly one sample per metric per run/u], + [draft => { + draft.workflow_policy.calibration.pre_collection_source_proof_required = true; + }, /sole frozen-candidate source proof/u], + [draft => { + draft.workflow_policy.calibration.source_proof_stage = "before_calibration"; + }, /sole frozen-candidate source proof/u], [draft => { draft.workflow_policy.calibration.forbidden_environment = [ "CODESTORY_EMBED_ALLOW_CPU=0", diff --git a/scripts/tests/fixtures/release-claims/positive.json b/scripts/tests/fixtures/release-claims/positive.json index 902934188..3ddc9f8f4 100644 --- a/scripts/tests/fixtures/release-claims/positive.json +++ b/scripts/tests/fixtures/release-claims/positive.json @@ -17,7 +17,7 @@ "type": "source_behavior", "tier": "source", "status": "pass", - "graph_sha256": "7385b32924d42e94c14b8d1f41fcec50bded200fc0e65fec05f43036e2896087", + "graph_sha256": "b08ec2429ec708519b2b67532217379d7bb6a64dd1afbbfa2501652411cb1b4b", "observed_at": "2026-07-16T11:00:00.000Z", "expires_at": "2026-07-17T11:00:00.000Z", "identity": { From c72c87a963b81c626530958078b9f54f8f47efc5 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Thu, 30 Jul 2026 11:51:36 -0500 Subject: [PATCH 20/20] freeze embedding server constants --- ...er-user-embedding-server-constant-set.json | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json index 36a5e49fc..33c1aed5a 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json @@ -1,7 +1,7 @@ { "calibration_required_values": { "capacity_retry_policy": { - "retry_after_ms": 40, + "retry_after_ms": 42, "retry_class": "after_capacity_change", "retry_condition_source": "named_condition_from_typed_capacity_response" }, @@ -9,7 +9,7 @@ "election_backoff_policy": { "initial_backoff_ms": 7, "jitter": "sha256(process_start_id||attempt) modulo inclusive [initial_backoff_ms,maximum_backoff_ms]", - "maximum_backoff_ms": 102 + "maximum_backoff_ms": 104 }, "hard_native_no_progress_ms": 385431, "request_deadlines_ms": { @@ -43,7 +43,22 @@ "query_queue_capacity": 64, "true_idle_observation_grace_ms": 2500 }, - "freeze_record": null, + "freeze_record": { + "calibration_bundle_sha256": "2adaaab974814cf890609bac0f1b6be54fb04cda4812aea33f06dff63a954ed0", + "calibration_freeze_digest": "511ec0e9018d73c1cfb4669de2e1b4bd722ef650cd9e6d38d7179d568913f2d5", + "input_constant_set_sha256": "ea58d298473ddf320469d15dc7c32176a1109d615a689c15ff76b67fb337e109", + "measurement_protocol_sha256": "d1bb9b2c7eb354fe98990aa32eedc0e165b0cf804212966e0a2ee362a2f5bf8e", + "protocol_sha256": "f4a3fa4afb4d5bcd8e707a5e21b687cdd023dc3398b28ff6891a2318e89c5ec7", + "run_artifact_sha256s": [ + "2332db8dfdf81057263a7db45d184e0847f576d1cb2047f2704998d0f20848c8", + "7a0a63f344b0ba0f88cc8e3a7118d8c8d3ec044ee1ea0b71ad3fbc75421ee44c", + "7bb038f4503c333d200f5f532fded590ce74acdb2cabc99cfaf7643b6930767f" + ], + "selected_at": "github-actions-run:30562970311:1", + "selection_rule": "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2", + "selection_source_commit": "681ca99098b86006e7dc1f51c27158c757ac05c9", + "selection_source_tree": "1a5494c9b82bcddc030438452285154e9d5b3e2b" + }, "qualification_thresholds": { "backend_observed_accelerator_residency": 1, "bulk_documents_per_second": 2, @@ -60,5 +75,5 @@ }, "schema_version": 1, "selection_protocol": "codestory-per-user-embedding-server-v1", - "status": "unfrozen" + "status": "frozen" }