diff --git a/.github/scripts/candidate-archive-store.mjs b/.github/scripts/candidate-archive-store.mjs index 0febeed00..f76284c18 100644 --- a/.github/scripts/candidate-archive-store.mjs +++ b/.github/scripts/candidate-archive-store.mjs @@ -874,6 +874,7 @@ function publishStoreEntry(storeRoot, inputRoot, record) { removeOwnedTemporary(temporary, paths.parent, path.basename(paths.entry)); return { admitted: false, ...concurrent }; } + const prepared = lstatSync(temporary, { bigint: true }); try { renameSync(temporary, paths.entry); } catch (error) { @@ -884,6 +885,15 @@ function publishStoreEntry(storeRoot, inputRoot, record) { removeOwnedTemporary(temporary, paths.parent, path.basename(paths.entry)); return { admitted: false, ...concurrent }; } + const published = lstatSync(paths.entry, { bigint: true }); + if ( + !published.isDirectory() + || published.isSymbolicLink() + || published.dev !== prepared.dev + || published.ino !== prepared.ino + ) { + fail("candidate archive store entry was not published by atomic directory rename"); + } return { admitted: true, ...verifyStoreEntry(storeRoot, expected) }; } catch (error) { if (existsSync(temporary)) { diff --git a/.github/scripts/candidate-archive-store.test.mjs b/.github/scripts/candidate-archive-store.test.mjs index 41548c3ee..277c853cc 100644 --- a/.github/scripts/candidate-archive-store.test.mjs +++ b/.github/scripts/candidate-archive-store.test.mjs @@ -17,7 +17,7 @@ import os from "node:os"; import path from "node:path"; import { spawnSync } from "node:child_process"; import test from "node:test"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { admitCandidateArchive, @@ -291,6 +291,45 @@ test("admission and a later hit materialize the complete exact payload as fresh } }); +test("admission rejects sequential publication beneath the final store key", async () => { + const fixture = createFixture(); + try { + const source = readFileSync(SCRIPT, "utf8"); + const atomicPublication = " renameSync(temporary, paths.entry);"; + assert.equal( + source.split(atomicPublication).length - 1, + 1, + "atomic store publication must have one mutation target", + ); + const sequentialPublication = [ + " mkdirSync(paths.entry, { mode: 0o700 });", + " renameSync(temporaryPayload, paths.payload);", + " renameSync(path.join(temporary, RECORD_FILE), paths.recordFile);", + " rmSync(temporary, { recursive: true });", + ].join("\n"); + const mutantFile = path.join(fixture.root, "candidate-archive-store-mutant.mjs"); + writeFileSync( + mutantFile, + source.replace(atomicPublication, sequentialPublication), + { flag: "wx" }, + ); + const mutant = await import(pathToFileURL(mutantFile).href); + assert.throws( + () => mutant.admitCandidateArchive({ + inputRoot: fixture.inputRoot, + outputDir: outputDir(fixture), + outputRoot: fixture.outputRoot, + record: fixture.record, + storeRoot: fixture.storeRoot, + }), + /not published by atomic directory rename/u, + ); + assert.equal(statExists(outputDir(fixture)), false); + } finally { + cleanup(fixture); + } +}); + test("the public checksum companion pair is mandatory", () => { const fixture = createFixture(); try { diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index 5fdd4935b..18cef670f 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -896,6 +896,8 @@ export function qualificationDriverArtifactViolations( 'binary: "codestory_embedding_qualification.exe"', 'rustTarget: "x86_64-pc-windows-msvc"', "metadata.isSymbolicLink()\n || !metadata.isFile()\n || metadata.nlink !== 1", + "function regularBuildOutput(file, label)", + "!Number.isSafeInteger(metadata.nlink)\n || metadata.nlink < 1", "metadata.isSymbolicLink() || !metadata.isDirectory()", 'fail("qualification driver helper arguments changed")', "containedRelativePath(root, candidate, label)", @@ -904,7 +906,10 @@ export function qualificationDriverArtifactViolations( 'fail(`${label} must not traverse symbolic links`)', "`codestory-cli-v${version}-${assetTarget}.${contract.archiveExtension}`", 'targetDir,\n contract.rustTarget,\n "release",\n contract.binary', + 'const sourceMetadata = regularBuildOutput(', 'fail("qualification driver artifact directory must start empty")', + "copyFileSync(source, staged)", + 'const stagedMetadata = regularFile(staged, "staged qualification driver")', "archiveBytes: archiveMetadata.size", "archiveDigest: sha256(archivePath)", "archiveFile: expectedArchiveFile", @@ -965,7 +970,7 @@ const packagedPlatformWorkflowDigest = // made advisory, parked in dead code, or followed by a payload substitution // while leaving the expected tokens in place. const packagedPlatformCoordinatorWorkflowDigest = - "464906e3cd7ec0e2f7e9195d60de035fdba76172c25d8b0861d0982f9d7dcc3e"; + "5017abab05e80355daf4618795d5ec7f09c07b4bc33cc1d52dca968a96b056bb"; const frozenCandidateQualityWorkflowDigest = "92d0a7ab0e0df63dacd5cc3ef0b58500a6578036494c329aa35279048734f173"; const macosMetalWorkflowDigest = @@ -987,13 +992,14 @@ const packagedHostCompilerFinalizerDigest = "b77d8bb12c2748bfe016ab65ccb2f4581356f3ccf1d666e747306caffd6c0c46"; // The companion qualification driver is intentionally retained only inside // the private Actions package artifact. This digest pins both sides of that -// contract: the producer copies only the selected target binary and binds it -// to the exact candidate archive, while the consumer rejects symlinks, extra -// files, identity drift, and byte drift before restoring execute permission. -// Any helper edit therefore requires a policy and mutation-test review in the -// same PR as the workflow change. +// contract: the producer may read Cargo's trusted hard-linked build output, +// but retains only a new singly linked copy bound to the exact candidate +// archive. The consumer rejects symlinks, retained hardlinks, extra files, +// identity drift, and byte drift before restoring execute permission. +// Any helper edit therefore requires policy and mutation-test review in the +// same PR. const qualificationDriverArtifactDigest = - "f7946e03fa6e272ca17f12616b82579da041de4d7c30b19d24ddbc5f1c7f0063"; + "efc5126e24162d52f9da8bac38c3414b3a7492fb17eed5ff19867fadad69623e"; const draftProofCommands = [ "cargo test --locked -p codestory-llama-sys --test native_staging", "cargo test --locked -p codestory-llama-sys --test model_staging", @@ -5139,6 +5145,16 @@ function validatePackagedCoordinator(workflows, violations, graph) { INPUT_CALIBRATION_RUN_ID: "${{ inputs.calibration_bundle_run_id }}", }); requireExactResolverContract(violations, file, route, platformResolverContractDigest); + const sourceProofRequirement = namedStep( + route, + "Require successful exact-head source proof", + ); + add( + violations, + sourceProofRequirement?.if + === "steps.resolve.outputs.mode != 'integration' && steps.resolve.outputs.mode != 'calibration'", + `${file} calibration alone must skip pre-freeze source proof while every frozen-candidate mode requires it`, + ); requireStepRun(violations, file, route, "Require successful exact-head source proof", [ "actions/runs?head_sha=$HEAD_SHA", '.path == ".github/workflows/source-proof.yml"', diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index ab96496d7..f2a6fb61d 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -3,6 +3,8 @@ import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { chmodSync, + linkSync, + lstatSync, mkdirSync, mkdtempSync, readFileSync, @@ -47,6 +49,10 @@ import { validateWorkflows, windowsManifestProofPolicyViolations, } from "./check-workflow-policy.mjs"; +import { + produceQualificationDriverArtifact, + verifyQualificationDriverArtifact, +} from "./qualification-driver-artifact.mjs"; const fullSha = "0123456789abcdef0123456789abcdef01234567"; const proofTopology = "proof5-v1-64015a841a2f69f33f7c9ce284f671ad27b3923a58db865fd4806d86230df6c5"; @@ -1610,7 +1616,7 @@ test("qualification driver is built once, retained privately, authenticated, and source.replace("sha256(archivePath) !== identity.archive.sha256", "false")], ["helper follows linked path ancestors", source => source.replace("lstatSync(cursor).isSymbolicLink()", "false")], - ["helper accepts hardlinked drivers", source => + ["helper accepts hardlinked retained drivers", source => source.replace("metadata.nlink !== 1", "false")], ["helper accepts extra identity fields", source => source.replace('fail(`${label} keys changed`)', "return")], @@ -1655,6 +1661,82 @@ test("qualification driver is built once, retained privately, authenticated, and } }); +test("qualification driver retention breaks a Cargo source hardlink and rejects retained hardlinks", () => { + const directory = mkdtempSync( + path.join(os.tmpdir(), "codestory-qualification-driver-"), + ); + try { + const targetDirectory = path.join(directory, "target"); + const releaseDirectory = path.join( + targetDirectory, + "x86_64-pc-windows-msvc", + "release", + ); + const depsDirectory = path.join(releaseDirectory, "deps"); + mkdirSync(depsDirectory, { recursive: true }); + const originalDriver = path.join( + depsDirectory, + "codestory_embedding_qualification-hash.exe", + ); + const cargoDriver = path.join( + releaseDirectory, + "codestory_embedding_qualification.exe", + ); + writeFileSync(originalDriver, "qualification-driver-v1"); + chmodSync(originalDriver, 0o755); + linkSync(originalDriver, cargoDriver); + assert.equal(lstatSync(cargoDriver).nlink, 2); + + const archive = path.join( + directory, + "codestory-cli-v0.16.3-windows-x64.zip", + ); + writeFileSync(archive, "candidate-archive"); + const artifactDirectory = path.join(directory, "artifact"); + const produced = produceQualificationDriverArtifact({ + archive, + assetTarget: "windows-x64", + outDir: artifactDirectory, + sourceSha: "a".repeat(40), + sourceTree: "b".repeat(40), + targetDir: targetDirectory, + trustedRoot: directory, + version: "0.16.3", + }); + assert.equal(lstatSync(produced.driver).nlink, 1); + assert.equal(readFileSync(produced.driver, "utf8"), "qualification-driver-v1"); + + writeFileSync(originalDriver, "qualification-driver-v2"); + assert.equal(readFileSync(produced.driver, "utf8"), "qualification-driver-v1"); + const verified = verifyQualificationDriverArtifact({ + archive, + artifactDir: artifactDirectory, + assetTarget: "windows-x64", + sourceSha: "a".repeat(40), + sourceTree: "b".repeat(40), + trustedRoot: directory, + version: "0.16.3", + }); + assert.equal(verified.identity.driver.sha256, produced.identity.driver.sha256); + + linkSync(produced.driver, path.join(directory, "retained-driver-alias.exe")); + assert.throws( + () => verifyQualificationDriverArtifact({ + archive, + artifactDir: artifactDirectory, + assetTarget: "windows-x64", + sourceSha: "a".repeat(40), + sourceTree: "b".repeat(40), + trustedRoot: directory, + version: "0.16.3", + }), + /qualification driver artifact must be a regular, non-symlink, singly linked file/u, + ); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + test("Windows packages one release graph into exact public and private artifacts", async (t) => { assert.deepEqual(validateWorkflows(loadWorkflows()), []); const file = "packaged-platform-proof.yml"; @@ -2859,6 +2941,43 @@ test("source proof reuse accepts only whole successful workflow runs", async (t) } }); +test("calibration precedes the sole frozen-candidate source proof", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const coordinatorFile = "packaged-platform-pr.yml"; + const mutations = [ + ["calibration regains a pre-freeze source proof", workflow => { + draftStep( + workflow.jobs.route, + "Require successful exact-head source proof", + ).if = "steps.resolve.outputs.mode != 'integration'"; + }], + ["qualification loses the frozen-head source proof", workflow => { + draftStep( + workflow.jobs.route, + "Require successful exact-head source proof", + ).if + = "steps.resolve.outputs.mode != 'integration' && steps.resolve.outputs.mode != 'calibration' && steps.resolve.outputs.mode != 'qualification'"; + }], + ["every mode loses the exact-head source proof", workflow => { + draftStep( + workflow.jobs.route, + "Require successful exact-head source proof", + ).if = "false"; + }], + ]; + + for (const [name, mutate] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(coordinatorFile)); + assert.match( + validateWorkflows(workflows).join("\n"), + /calibration alone must skip pre-freeze source proof while every frozen-candidate mode requires it/u, + ); + }); + } +}); + test("Windows package proof retains the readable native sccache executable", () => { const directory = mkdtempSync(path.join(os.tmpdir(), "codestory-windows-sccache-")); try { diff --git a/.github/scripts/packaged_agent_proof/measurement_protocol_validation.py b/.github/scripts/packaged_agent_proof/measurement_protocol_validation.py index 812b2c861..424f1f9c1 100644 --- a/.github/scripts/packaged_agent_proof/measurement_protocol_validation.py +++ b/.github/scripts/packaged_agent_proof/measurement_protocol_validation.py @@ -35,7 +35,7 @@ "calibration_workload_state_overrides", ) EXPECTED_QUALIFICATION_MEASUREMENT_SHAPE_SHA256 = ( - "1c065562adc34d0d9978187857807e491c4e6d4aa233fdd94f5636931a7b730e" + "bc78e8c0277062f1274b0ed97e9bafbef2574b2d1934cb6ab89e7f514900fef8" ) @@ -117,6 +117,11 @@ def _verify_scenario_and_metric_contracts(protocol: dict) -> tuple[set[str], dic and all(isinstance(event, str) and event for event in boundaries), f"measurement metric {metric} must have exact start and end events", ) + require( + phase_boundaries["true_idle_exit"] + == ["final_product_request_completed", "engine_and_server_absent"], + "true-idle qualification must start at final product completion", + ) metric_contracts = protocol.get("metric_contracts") require( isinstance(metric_contracts, dict) @@ -419,6 +424,11 @@ def _verify_measurement_sampling( workload.get("input_generator"), f"measurement workload {metric}.input_generator", ) + require( + workloads["true_idle_exit"].get("workload_id") + == "true_idle_after_product_completion_60000_awake_ms_v2", + "true-idle qualification workload changed its product-completion boundary", + ) sampling = protocol.get("metric_sampling") require( isinstance(sampling, dict) and set(sampling) == required_metrics, diff --git a/.github/scripts/packaged_agent_proof/self_test_full_stack_calibration.py b/.github/scripts/packaged_agent_proof/self_test_full_stack_calibration.py index 433dae637..c881c0a1e 100644 --- a/.github/scripts/packaged_agent_proof/self_test_full_stack_calibration.py +++ b/.github/scripts/packaged_agent_proof/self_test_full_stack_calibration.py @@ -33,6 +33,38 @@ def _qualification_matrix_tests(fixture: FullStackFixture) -> dict: self_measurement_protocol, require_frozen=False, ) + for label, field, value in ( + ( + "server idle epoch", + "phase", + [ + "last_queued_active_or_leased_work_ended", + "engine_and_server_absent", + ], + ), + ( + "pre-completion workload", + "workload", + "true_idle_60000_awake_ms_v1", + ), + ): + regressed_true_idle = json.loads( + json.dumps(measurement_contract["measurement_protocol"]) + ) + if field == "phase": + regressed_true_idle["phase_boundaries"]["true_idle_exit"] = value + else: + regressed_true_idle["workloads"]["true_idle_exit"]["workload_id"] = value + regressed_true_idle_path = ( + fixture.root / f"true-idle-{field}-regression.json" + ) + write_json(regressed_true_idle_path, regressed_true_idle) + try: + load_measurement_protocol(regressed_true_idle_path) + except ProofFailure: + pass + else: + raise ProofFailure(f"true-idle qualification accepted {label}") for quality_metric in ( "answer_quality", "packet_quality", diff --git a/.github/scripts/qualification-driver-artifact.mjs b/.github/scripts/qualification-driver-artifact.mjs index 5d8cfaa90..d9f973550 100644 --- a/.github/scripts/qualification-driver-artifact.mjs +++ b/.github/scripts/qualification-driver-artifact.mjs @@ -119,6 +119,19 @@ function regularFile(file, label) { return metadata; } +function regularBuildOutput(file, label) { + const metadata = lstatSync(file); + if ( + metadata.isSymbolicLink() + || !metadata.isFile() + || !Number.isSafeInteger(metadata.nlink) + || metadata.nlink < 1 + ) { + fail(`${label} must be a regular, non-symlink build output`); + } + return metadata; +} + function regularDirectory(directory, label) { const metadata = lstatSync(directory); if (metadata.isSymbolicLink() || !metadata.isDirectory()) { @@ -270,7 +283,10 @@ export function produceQualificationDriverArtifact({ label: "qualification driver source", root: targetDir, }); - const sourceMetadata = regularFile(source, "qualification driver"); + const sourceMetadata = regularBuildOutput( + source, + "qualification driver source", + ); if (process.platform !== "win32" && (sourceMetadata.mode & 0o111) === 0) { fail("qualification driver must be executable"); } diff --git a/.github/workflows/packaged-platform-pr.yml b/.github/workflows/packaged-platform-pr.yml index 64d322a15..17852d590 100644 --- a/.github/workflows/packaged-platform-pr.yml +++ b/.github/workflows/packaged-platform-pr.yml @@ -166,7 +166,7 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Require successful exact-head source proof - if: steps.resolve.outputs.mode != 'integration' + if: steps.resolve.outputs.mode != 'integration' && steps.resolve.outputs.mode != 'calibration' shell: bash env: GH_TOKEN: ${{ github.token }} diff --git a/benchmarks/release-evidence/fixtures/candidate.json b/benchmarks/release-evidence/fixtures/candidate.json index c8c6a5b49..d0fd4c0e9 100644 --- a/benchmarks/release-evidence/fixtures/candidate.json +++ b/benchmarks/release-evidence/fixtures/candidate.json @@ -62,7 +62,7 @@ }, "release_claims": { "graph_schema": "codestory.release-claims/v1", - "graph_sha256": "7385b32924d42e94c14b8d1f41fcec50bded200fc0e65fec05f43036e2896087", + "graph_sha256": "b08ec2429ec708519b2b67532217379d7bb6a64dd1afbbfa2501652411cb1b4b", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "requested_claims": [ @@ -85,7 +85,7 @@ "type": "performance", "tier": "live_behavior", "status": "measured", - "graph_sha256": "7385b32924d42e94c14b8d1f41fcec50bded200fc0e65fec05f43036e2896087", + "graph_sha256": "b08ec2429ec708519b2b67532217379d7bb6a64dd1afbbfa2501652411cb1b4b", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -106,7 +106,7 @@ "type": "answer_quality", "tier": "answer_quality", "status": "pass", - "graph_sha256": "7385b32924d42e94c14b8d1f41fcec50bded200fc0e65fec05f43036e2896087", + "graph_sha256": "b08ec2429ec708519b2b67532217379d7bb6a64dd1afbbfa2501652411cb1b4b", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { diff --git a/benchmarks/release-evidence/fixtures/report.json b/benchmarks/release-evidence/fixtures/report.json index 7a0d10f04..f203f4e35 100644 --- a/benchmarks/release-evidence/fixtures/report.json +++ b/benchmarks/release-evidence/fixtures/report.json @@ -26,7 +26,7 @@ "type": "performance", "tier": "live_behavior", "status": "pass", - "graph_sha256": "7385b32924d42e94c14b8d1f41fcec50bded200fc0e65fec05f43036e2896087", + "graph_sha256": "b08ec2429ec708519b2b67532217379d7bb6a64dd1afbbfa2501652411cb1b4b", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -47,7 +47,7 @@ "type": "answer_quality", "tier": "answer_quality", "status": "pass", - "graph_sha256": "7385b32924d42e94c14b8d1f41fcec50bded200fc0e65fec05f43036e2896087", + "graph_sha256": "b08ec2429ec708519b2b67532217379d7bb6a64dd1afbbfa2501652411cb1b4b", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -69,7 +69,7 @@ "schema": "codestory.release-claim-evaluation/v1", "status": "pass", "graph_schema": "codestory.release-claims/v1", - "graph_sha256": "7385b32924d42e94c14b8d1f41fcec50bded200fc0e65fec05f43036e2896087", + "graph_sha256": "b08ec2429ec708519b2b67532217379d7bb6a64dd1afbbfa2501652411cb1b4b", "evidence_selection": "all_matching_rows_must_pass", "expected_commit": "2222222222222222222222222222222222222222", "evaluated_at": "2026-07-21T02:13:20.738Z", diff --git a/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/measurements.rs b/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/measurements.rs index aa12723f3..551b09332 100644 --- a/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/measurements.rs +++ b/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/measurements.rs @@ -10,7 +10,7 @@ use super::analysis::{ }; use super::process::{ busy_retry_marker_timeout, busy_retry_worker_timeout, measurement_worker_timeout, - query_parameters, require_worker_success, + query_parameters, }; use super::{ RunningWorker, ScenarioRunner, WorkerOutput, opaque_constant_calibration_sample_id, push_metric, @@ -71,7 +71,7 @@ pub(super) fn declared_phase_boundaries(metric: &str) -> Result<[&'static str; 2 ], "busy_retry_usefulness" => ["typed_retry_emitted", "named_retry_condition_became_true"], "true_idle_exit" => [ - "last_queued_active_or_leased_work_ended", + "final_product_request_completed", "engine_and_server_absent", ], "backend_observed_accelerator_residency" => [ @@ -94,7 +94,7 @@ pub(super) fn declared_workload_id(metric: &str) -> Result<&'static str> { "warm_bulk_ipc" => "warm_bulk_64x256b_v1", "bulk_documents_per_second" | "bulk_tokens_per_second" => "bulk_throughput_256x256b_v1", "busy_retry_usefulness" => "saturated_query_65th_retry_v1", - "true_idle_exit" => "true_idle_60000_awake_ms_v1", + "true_idle_exit" => "true_idle_after_product_completion_60000_awake_ms_v2", "backend_observed_accelerator_residency" => "resident_policy_identity_v1", _ => bail!("embedding_qualification_metric_workload_unknown:{metric}"), }) @@ -556,14 +556,6 @@ impl<'a> ScenarioRunner<'a> { )?; } - let idle_worker = self.spawn_worker("query", query_parameters(1), None)?; - let idle_output = self.finish_worker(idle_worker, measurement_worker_timeout("query"))?; - require_worker_success(&idle_output, "true_idle_owner")?; - let idle_owner = - self.record_worker_snapshot("measurement_true_idle_owner", &idle_output)?; - if !snapshot_has_resident_generation(&idle_owner) { - bail!("embedding_qualification_true_idle_owner_not_resident"); - } let measured = self.run_measure_worker( "measure_true_idle", "true_idle_exit", @@ -846,7 +838,7 @@ fn validate_constant_engine_evidence( || identity.load_error.is_some() || identity.policy != "accelerated" || identity.backend.eq_ignore_ascii_case("cpu") - || !identity.backend.eq_ignore_ascii_case(expected_backend) + || !constant_backend_matches_expected(&identity.backend, expected_backend) || identity.model_digest != expected_model_sha256 || identity.materialized_model_sha256 != expected_model_sha256 || !identity.embedded_model @@ -857,6 +849,29 @@ fn validate_constant_engine_evidence( Ok(()) } +fn constant_backend_matches_expected(observed: &str, expected: &str) -> bool { + let Some(expected_family) = constant_backend_family(expected) else { + return false; + }; + constant_backend_family(observed) == Some(expected_family) +} + +fn constant_backend_family(value: &str) -> Option<&'static str> { + let normalized = value.trim().to_ascii_lowercase(); + match normalized.as_str() { + "metal" | "mtl" => Some("metal"), + "vulkan" => Some("vulkan"), + value + if value.strip_prefix("vulkan").is_some_and(|suffix| { + !suffix.is_empty() && suffix.bytes().all(|byte| byte.is_ascii_digit()) + }) => + { + Some("vulkan") + } + _ => None, + } +} + fn expected_materialization_reuse(run_index: u32) -> Result { match run_index { 1 => Ok(false), @@ -1093,11 +1108,42 @@ mod constant_calibration_tests { fn engine_precondition_binds_accelerated_backend_model_and_reuse_only() { let server = server_identity("server-a"); let metal = engine_identity("server-a", "Metal", false); - validate_constant_engine_evidence(&metal, &server, "metal", &"a".repeat(64), false) - .expect("Metal identity"); - let vulkan = engine_identity("server-a", "Vulkan", false); - validate_constant_engine_evidence(&vulkan, &server, "vulkan", &"a".repeat(64), false) - .expect("Vulkan identity"); + for (observed, expected) in [ + ("Metal", "metal"), + ("MTL", "metal"), + ("Vulkan", "vulkan"), + ("Vulkan0", "vulkan"), + ] { + let identity = engine_identity("server-a", observed, false); + validate_constant_engine_evidence(&identity, &server, expected, &"a".repeat(64), false) + .unwrap_or_else(|error| { + panic!("{observed} must satisfy expected GPU family {expected}: {error}") + }); + } + for (observed, expected) in [ + ("CPU", "metal"), + ("cpu_explicit", "metal"), + ("", "metal"), + ("unknown", "metal"), + ("metal-cpu", "metal"), + ("mtl0", "metal"), + ("MTL", "vulkan"), + ("Vulkan", "metal"), + ("vulkan-cpu", "vulkan"), + ] { + let identity = engine_identity("server-a", observed, false); + assert!( + validate_constant_engine_evidence( + &identity, + &server, + expected, + &"a".repeat(64), + false, + ) + .is_err(), + "{observed} must not satisfy expected GPU family {expected}" + ); + } let mut invalid = metal.clone(); invalid.policy = "cpu_explicit".into(); diff --git a/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/process.rs b/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/process.rs index 779aeedd8..477bb6eea 100644 --- a/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/process.rs +++ b/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/process.rs @@ -1,6 +1,4 @@ -use super::super::{ - CONTROL_TIMEOUT, ControlEvent, IDLE_EXIT_GRACE, POLL, QUEUE_SETUP_TIMEOUT, SNAPSHOT_TIMEOUT, -}; +use super::super::{CONTROL_TIMEOUT, ControlEvent, POLL, QUEUE_SETUP_TIMEOUT, SNAPSHOT_TIMEOUT}; use super::analysis::elapsed; use super::{ EMBEDDING_QUALIFICATION_WORKER_SCHEMA_VERSION, ProcessInvocation, RunningWorker, WorkerOutput, @@ -17,6 +15,8 @@ use std::path::{Path, PathBuf}; use std::process::{Child, ExitStatus}; use std::time::Duration; +pub(super) const MEASUREMENT_OWNER_ABSENCE_GRACE: Duration = Duration::from_secs(30); + pub(super) fn existing_control_events(directory: &Path) -> Result> { existing_control_events_for_nonce(directory, &qualification_nonce()?) } @@ -291,12 +291,17 @@ pub(super) fn stall_worker_timeout() -> Duration { pub(super) fn measurement_worker_timeout(operation: &str) -> Duration { let budgets = EmbeddingClientBudgets::current(); if operation == "measure_true_idle" { - // The idle worker first proves the resident owner quiescent (bounded - // by the snapshot allowance), then waits out the server's own idle - // deadline plus the exit grace before the absence observation. - return Duration::from_millis(PER_USER_EMBEDDING_SERVER_IDLE_TIMEOUT_MS) - .saturating_add(IDLE_EXIT_GRACE) - .saturating_add(SNAPSHOT_TIMEOUT) + // The idle worker runs the product request that starts the measured + // idle epoch itself, then waits out the server's idle deadline plus + // the exit grace before the absence observation. + return budgets + .connect + .saturating_add(budgets.spawn) + .saturating_add(budgets.query_request) + .saturating_add(Duration::from_millis( + PER_USER_EMBEDDING_SERVER_IDLE_TIMEOUT_MS, + )) + .saturating_add(MEASUREMENT_OWNER_ABSENCE_GRACE) .saturating_add(SNAPSHOT_TIMEOUT) .saturating_add(CONTROL_TIMEOUT); } diff --git a/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/tests.rs b/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/tests.rs index 2d344c217..f08625dd6 100644 --- a/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/tests.rs +++ b/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/tests.rs @@ -7,9 +7,9 @@ use super::measurements::{ declared_phase_boundaries, declared_workload_id, measurement_span_interval, }; use super::process::{ - LOAD_ESTABLISHMENT_WAITS, busy_retry_worker_timeout, dead_client_setup_timeout, - load_establishment_budget, load_establishment_timeout, measurement_worker_timeout, - published_control_events, + LOAD_ESTABLISHMENT_WAITS, MEASUREMENT_OWNER_ABSENCE_GRACE, busy_retry_worker_timeout, + dead_client_setup_timeout, load_establishment_budget, load_establishment_timeout, + measurement_worker_timeout, published_control_events, }; use super::{ScenarioEvidence, WorkerOutput, opaque_measurement_sample_id}; use crate::qualification::request::{QualificationContracts, REQUIRED_METRICS, REQUIRED_SCENARIOS}; @@ -76,15 +76,26 @@ fn measurement_worker_budgets_dominate_the_deadlines_workers_honor() { measurement_worker_timeout("measure_resident_identity"), "the scenario residency probe must carry the same bulk-deadline budget as the residency measurement" ); - // The true-idle measurement worker waits out the server's own idle - // deadline before the absence observation; its watchdog must dominate - // that self-enforced wait plus its quiescence and absence-grace waits. + // The true-idle measurement worker now performs the product request that + // starts the idle epoch itself; its watchdog must cover that whole client + // chain before the server idle deadline and absence-grace wait. + assert_eq!( + MEASUREMENT_OWNER_ABSENCE_GRACE, + Duration::from_secs(30), + "the coordinator must mirror the measurement worker's owner-absence grace" + ); assert!( measurement_worker_timeout("measure_true_idle") - >= Duration::from_millis(PER_USER_EMBEDDING_SERVER_IDLE_TIMEOUT_MS) - .saturating_add(Duration::from_secs(30)) + >= budgets + .connect + .saturating_add(budgets.spawn) + .saturating_add(budgets.query_request) + .saturating_add(Duration::from_millis( + PER_USER_EMBEDDING_SERVER_IDLE_TIMEOUT_MS, + )) + .saturating_add(MEASUREMENT_OWNER_ABSENCE_GRACE) .saturating_add(SNAPSHOT_TIMEOUT), - "true-idle measurement budget must dominate the server idle deadline plus the worker's own waits" + "true-idle measurement budget must dominate its product request, server idle deadline, and worker waits" ); // The busy-retry worker seeds the held queues (queue-setup phase), then // after release drains queries bounded by its own 120s per-request diff --git a/crates/codestory-cli/src/embedding_qualification/worker.rs b/crates/codestory-cli/src/embedding_qualification/worker.rs index 290d0942e..0f871153e 100644 --- a/crates/codestory-cli/src/embedding_qualification/worker.rs +++ b/crates/codestory-cli/src/embedding_qualification/worker.rs @@ -237,7 +237,11 @@ fn run_measure_operation( &request.parameters, ), "measure_resident_identity" => run_measure_resident_identity(runtime, clock.as_ref()), - "measure_true_idle" => run_measure_true_idle(runtime, clock.as_ref()), + "measure_true_idle" => run_measure_true_idle( + &PerUserEmbeddingClient::for_runtime(runtime)?, + clock.as_ref(), + request.parameters.input_bytes, + ), "measure_busy_retry" => { let marker = request .retry_marker diff --git a/crates/codestory-cli/src/embedding_qualification/worker/operations/measure.rs b/crates/codestory-cli/src/embedding_qualification/worker/operations/measure.rs index 160b13c34..2f7b6927b 100644 --- a/crates/codestory-cli/src/embedding_qualification/worker/operations/measure.rs +++ b/crates/codestory-cli/src/embedding_qualification/worker/operations/measure.rs @@ -19,7 +19,7 @@ use super::super::protocol::{ run_raw_protocol_exchange_with_input, validated_hello, write_protocol_frame, }; use super::ANTI_IDLE_PROTOCOL_DEADLINE_MS; -use super::owner_exit::{observe_owner_exit, wait_for_owner_exit}; +use super::owner_exit::{OwnerExitObservation, observe_owner_exit, wait_for_owner_exit}; use super::queue::{QueueOperation, run_queue_operation}; use anyhow::{Context, Result, bail}; use codestory_retrieval::{ @@ -68,7 +68,7 @@ fn workload_documents(workload_id: &str, repeat: u32, count: usize, bytes: usize .collect() } -struct MeasurementSpanStart { +pub(in crate::embedding_qualification::worker) struct MeasurementSpanStart { awake_started_ns: u64, inclusive_started_ns: u64, boot_id_started: String, @@ -88,6 +88,26 @@ fn begin_span(clock: &dyn AwakeMonotonicClock) -> Result { }) } +/// Stamp product completion for `true_idle_exit`. +/// +/// The awake reading is deliberately first: once the product operation has +/// stamped completion, a delay while returning to the measurement coordinator +/// must remain inside the measured idle interval, never move its start +/// forward. The suspend-inclusive and boot witnesses follow immediately and +/// the downstream tolerance fails closed if sampling them is delayed. +fn begin_true_idle_span_at_product_completion( + clock: &dyn AwakeMonotonicClock, +) -> Result { + let awake_started_ns = clock.now_ns(); + let inclusive_started_ns = crate::embedding_server_transport::inclusive_now_ns()?; + let boot_id_started = crate::embedding_server_transport::boot_id()?; + Ok(MeasurementSpanStart { + awake_started_ns, + inclusive_started_ns, + boot_id_started, + }) +} + /// Stamp the declared end instant: awake reading first, then the /// suspend-inclusive reading and boot id. fn finish_span( @@ -416,30 +436,103 @@ pub(in crate::embedding_qualification::worker) fn run_measure_resident_identity( }) } -/// `true_idle_exit`: span start at the observation proving the resident owner -/// carries zero queued, active, or leased work -/// (`last_queued_active_or_leased_work_ended`), span end at the observation -/// that returned no owner (`engine_and_server_absent`). +fn true_idle_scheduler_is_drained( + active_request_count: u64, + query_depth: u64, + bulk_depth: u64, + lease_count: u64, +) -> bool { + active_request_count == 0 && query_depth == 0 && bulk_depth == 0 && lease_count == 0 +} + +fn snapshot_is_true_idle_boundary(snapshot: &EmbeddingServerSnapshot) -> bool { + snapshot.lifecycle == "resident" + && resident_engine_generation(snapshot) + && true_idle_scheduler_is_drained( + snapshot.scheduler.active_request_count, + snapshot.scheduler.query_depth, + snapshot.scheduler.bulk_depth, + snapshot.scheduler.lease_count, + ) +} + +fn same_true_idle_owner( + expected_server_instance_id: &str, + observed_server_instance_id: &str, +) -> bool { + expected_server_instance_id == observed_server_instance_id +} + +fn validate_true_idle_boundary( + expected_server_instance_id: &str, + snapshot: &EmbeddingServerSnapshot, +) -> Result<()> { + if !same_true_idle_owner( + expected_server_instance_id, + &snapshot.process.server_instance_id, + ) { + bail!("embedding_qualification_true_idle_owner_changed"); + } + if !snapshot_is_true_idle_boundary(snapshot) { + bail!("embedding_qualification_true_idle_not_quiescent"); + } + Ok(()) +} + +/// The product and observation operations used by the true-idle measurement. +/// +/// The worker implements this interface with the real per-user embedding +/// client. Tests use a scripted implementation to execute this same +/// `run_measure_true_idle` algorithm rather than a helper that production can +/// bypass. +pub(in crate::embedding_qualification::worker) trait TrueIdleMeasurementClient { + fn observe_snapshot(&self) -> Result>; + fn complete_product_query_and_stamp( + &self, + input: &str, + clock: &dyn AwakeMonotonicClock, + ) -> Result; + fn observe_owner_exit(&self) -> Result; +} + +impl TrueIdleMeasurementClient for PerUserEmbeddingClient { + fn observe_snapshot(&self) -> Result> { + self.observe() + } + + fn complete_product_query_and_stamp( + &self, + input: &str, + clock: &dyn AwakeMonotonicClock, + ) -> Result { + let _ = self.embed_query(input)?; + begin_true_idle_span_at_product_completion(clock) + } + + fn observe_owner_exit(&self) -> Result { + observe_owner_exit(self) + } +} + +/// `true_idle_exit`: this worker completes the last product request itself, +/// stamps `final_product_request_completed` before returning to its caller, +/// proves the resident scheduler is drained, then ends at the observation that +/// returned no owner (`engine_and_server_absent`). pub(in crate::embedding_qualification::worker) fn run_measure_true_idle( - runtime: &SidecarRuntimeConfig, + client: &dyn TrueIdleMeasurementClient, clock: &dyn AwakeMonotonicClock, + input_bytes: u32, ) -> Result { - let client = PerUserEmbeddingClient::for_runtime(runtime)?; - let idle_owner = wait_for_observed_snapshot( - &client, - clock, - SNAPSHOT_TIMEOUT, - "embedding_qualification_true_idle_not_quiescent", - |snapshot| { - snapshot.lifecycle == "resident" - && resident_engine_generation(snapshot) - && snapshot.scheduler.active_request_count == 0 - && snapshot.scheduler.query_depth == 0 - && snapshot.scheduler.bulk_depth == 0 - && snapshot.scheduler.lease_count == 0 - }, - )?; - let start = begin_span(clock)?; + let expected_owner = client + .observe_snapshot()? + .filter(resident_engine_generation) + .ok_or_else(|| anyhow::anyhow!("embedding_qualification_true_idle_owner_missing"))?; + let input = "q".repeat(input_bytes.max(1) as usize); + let start = client.complete_product_query_and_stamp(&input, clock)?; + let idle_owner = client + .observe_snapshot()? + .ok_or_else(|| anyhow::anyhow!("embedding_qualification_true_idle_owner_missing"))?; + validate_true_idle_boundary(&expected_owner.process.server_instance_id, &idle_owner)?; let timeout = Duration::from_millis(PER_USER_EMBEDDING_SERVER_IDLE_TIMEOUT_MS) .saturating_add(OWNER_ABSENCE_GRACE); let wait_started = clock.now_ns(); @@ -448,7 +541,7 @@ pub(in crate::embedding_qualification::worker) fn run_measure_true_idle( wait_started, timeout, &idle_owner.process.server_instance_id, - || observe_owner_exit(&client), + || client.observe_owner_exit(), )?; let span = finish_span(clock, start)?; Ok(measurement(span, idle_owner)) @@ -718,7 +811,191 @@ fn validate_vector_response( #[cfg(test)] mod tests { - use super::workload_input; + use super::super::owner_exit::OwnerExitObservation; + use super::{ + MeasurementSpanStart, TrueIdleMeasurementClient, + begin_true_idle_span_at_product_completion, run_measure_true_idle, + true_idle_scheduler_is_drained, validate_true_idle_boundary, workload_input, + }; + use anyhow::Result; + use codestory_retrieval::{ + AwakeMonotonicClock, EmbeddingServerAuthoritySnapshot, EmbeddingServerClockSnapshot, + EmbeddingServerEngineSnapshot, EmbeddingServerProcessSnapshot, + EmbeddingServerProtocolSnapshot, EmbeddingServerSchedulerSnapshot, EmbeddingServerSnapshot, + PER_USER_EMBEDDING_SERVER_SNAPSHOT_SCHEMA_VERSION, + }; + use std::collections::VecDeque; + use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + + const PRODUCT_COMPLETED_NS: u64 = 100; + const PRODUCT_RETURNED_NS: u64 = 10_000_000_100; + const DRAINED_OWNER_OBSERVED_NS: u64 = 10_000_000_145; + + struct ScriptClock { + now_ns: AtomicU64, + events: Arc>>, + } + + impl ScriptClock { + fn new(events: Arc>>) -> Self { + Self { + now_ns: AtomicU64::new(0), + events, + } + } + + fn set(&self, now_ns: u64) { + self.now_ns.store(now_ns, Ordering::Release); + } + } + + impl AwakeMonotonicClock for ScriptClock { + fn now_ns(&self) -> u64 { + let now_ns = self.now_ns.load(Ordering::Acquire); + if now_ns == PRODUCT_COMPLETED_NS { + self.events + .lock() + .expect("lock events") + .push("span_started"); + } + now_ns + } + + fn sleep(&self, duration: Duration) { + self.now_ns.fetch_add( + u64::try_from(duration.as_nanos()).expect("test duration fits u64"), + Ordering::AcqRel, + ); + } + + fn snapshot(&self) -> EmbeddingServerClockSnapshot { + EmbeddingServerClockSnapshot { + domain: "awake_monotonic".into(), + api: "script_clock".into(), + boot_id: "test-boot".into(), + resolution_ns: 1, + } + } + } + + struct ScriptedTrueIdleClient { + clock: Arc, + events: Arc>>, + snapshots: Mutex>, + exit_observations: Mutex>, + product_calls: AtomicUsize, + } + + impl TrueIdleMeasurementClient for ScriptedTrueIdleClient { + fn observe_snapshot(&self) -> Result> { + let (snapshot, remaining) = { + let mut snapshots = self.snapshots.lock().expect("lock snapshots"); + let snapshot = snapshots + .pop_front() + .expect("the algorithm observed past its snapshot script"); + (snapshot, snapshots.len()) + }; + let event = if remaining == 1 { + "expected_owner_observed" + } else { + self.clock.set(DRAINED_OWNER_OBSERVED_NS); + "same_owner_drained_observed" + }; + self.events.lock().expect("lock events").push(event); + Ok(Some(snapshot)) + } + + fn complete_product_query_and_stamp( + &self, + input: &str, + clock: &dyn AwakeMonotonicClock, + ) -> Result { + assert_eq!(input, "q".repeat(256)); + self.product_calls.fetch_add(1, Ordering::AcqRel); + self.clock.set(PRODUCT_COMPLETED_NS); + self.events + .lock() + .expect("lock events") + .push("product_completed"); + let start = begin_true_idle_span_at_product_completion(clock)?; + self.clock.set(PRODUCT_RETURNED_NS); + self.events + .lock() + .expect("lock events") + .push("product_returned_after_delay"); + Ok(start) + } + + fn observe_owner_exit(&self) -> Result { + let observation = self + .exit_observations + .lock() + .expect("lock exit observations") + .pop_front() + .expect("the algorithm observed past its exit script"); + let event = match &observation { + OwnerExitObservation::Present(_) => "same_owner_still_present", + OwnerExitObservation::Lost => "owner_connection_lost", + OwnerExitObservation::Absent => "owner_absence_observed", + }; + self.events.lock().expect("lock events").push(event); + Ok(observation) + } + } + + fn true_idle_snapshot( + owner: &str, + active_request_count: u64, + query_depth: u64, + bulk_depth: u64, + lease_count: u64, + ) -> EmbeddingServerSnapshot { + EmbeddingServerSnapshot { + schema_version: PER_USER_EMBEDDING_SERVER_SNAPSHOT_SCHEMA_VERSION, + event_sequence: 1, + lifecycle: "resident".into(), + clock: EmbeddingServerClockSnapshot { + domain: "awake_monotonic".into(), + api: "script_clock".into(), + boot_id: "test-boot".into(), + resolution_ns: 1, + }, + protocol: EmbeddingServerProtocolSnapshot::current(), + authority: EmbeddingServerAuthoritySnapshot { + endpoint_namespace_id: "endpoint".into(), + lifetime_authority_id: "authority".into(), + listener_id: "listener".into(), + peer_verified: true, + }, + process: EmbeddingServerProcessSnapshot { + server_instance_id: owner.into(), + pid: 42, + process_start_id: "server-start".into(), + executable_sha256: "a".repeat(64), + executable_version: "0.16.3".into(), + }, + scheduler: EmbeddingServerSchedulerSnapshot { + query_capacity: 64, + query_depth, + bulk_capacity: 64, + bulk_depth, + connection_count: 1, + active_request_count, + lease_count, + active_request: None, + }, + engine: Some(EmbeddingServerEngineSnapshot { + engine_owner_id: owner.into(), + native_worker_id: "native-worker".into(), + load_generation: 1, + model_load_count: 1, + successful_encode_count: 1, + }), + failure: None, + } + } #[test] fn workload_inputs_are_deterministic_ascii_and_distinct_per_ordinal() { @@ -732,4 +1009,88 @@ mod tests { assert_ne!(first, other_repeat); assert_ne!(first, other_ordinal); } + + #[test] + fn true_idle_start_is_stamped_at_product_completion_before_return_and_observation_lag() { + let events = Arc::new(Mutex::new(Vec::new())); + let clock = Arc::new(ScriptClock::new(Arc::clone(&events))); + let client = ScriptedTrueIdleClient { + clock: Arc::clone(&clock), + events: Arc::clone(&events), + snapshots: Mutex::new(VecDeque::from([ + true_idle_snapshot("measured-owner", 0, 0, 0, 0), + true_idle_snapshot("measured-owner", 0, 0, 0, 0), + ])), + exit_observations: Mutex::new(VecDeque::from([ + OwnerExitObservation::Present("measured-owner".into()), + OwnerExitObservation::Absent, + ])), + product_calls: AtomicUsize::new(0), + }; + + let measurement = + run_measure_true_idle(&client, clock.as_ref(), 256).expect("measure true idle"); + + assert_eq!(client.product_calls.load(Ordering::Acquire), 1); + assert_eq!(measurement.span.awake_started_ns, PRODUCT_COMPLETED_NS); + assert!(measurement.span.awake_finished_ns >= DRAINED_OWNER_OBSERVED_NS); + assert_eq!( + measurement.snapshot.process.server_instance_id, + "measured-owner" + ); + assert_eq!( + *events.lock().expect("lock events"), + [ + "expected_owner_observed", + "product_completed", + "span_started", + "product_returned_after_delay", + "same_owner_drained_observed", + "same_owner_still_present", + "owner_absence_observed", + ] + ); + } + + #[test] + fn true_idle_boundary_rejects_each_queued_active_or_leased_shape() { + let occupied = [ + ("active", (1, 0, 0, 0)), + ("query", (0, 1, 0, 0)), + ("bulk", (0, 0, 1, 0)), + ("lease", (0, 0, 0, 1)), + ]; + assert!(true_idle_scheduler_is_drained(0, 0, 0, 0)); + for (label, state) in occupied { + assert!( + !true_idle_scheduler_is_drained(state.0, state.1, state.2, state.3), + "{label} work must keep the true-idle boundary open" + ); + let result = validate_true_idle_boundary( + "measured-owner", + &true_idle_snapshot("measured-owner", state.0, state.1, state.2, state.3), + ); + assert_eq!( + result + .expect_err("occupied scheduler must reject the sample") + .to_string(), + "embedding_qualification_true_idle_not_quiescent", + "{label} work must fail closed rather than shift the boundary" + ); + } + } + + #[test] + fn true_idle_boundary_rejects_a_different_owner_after_product_completion() { + let result = validate_true_idle_boundary( + "measured-owner", + &true_idle_snapshot("replacement-owner", 0, 0, 0, 0), + ); + assert_eq!( + result + .expect_err("replacement owner must reject the sample") + .to_string(), + "embedding_qualification_true_idle_owner_changed" + ); + } } diff --git a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json index 36a5e49fc..33c1aed5a 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-constant-set.json @@ -1,7 +1,7 @@ { "calibration_required_values": { "capacity_retry_policy": { - "retry_after_ms": 40, + "retry_after_ms": 42, "retry_class": "after_capacity_change", "retry_condition_source": "named_condition_from_typed_capacity_response" }, @@ -9,7 +9,7 @@ "election_backoff_policy": { "initial_backoff_ms": 7, "jitter": "sha256(process_start_id||attempt) modulo inclusive [initial_backoff_ms,maximum_backoff_ms]", - "maximum_backoff_ms": 102 + "maximum_backoff_ms": 104 }, "hard_native_no_progress_ms": 385431, "request_deadlines_ms": { @@ -43,7 +43,22 @@ "query_queue_capacity": 64, "true_idle_observation_grace_ms": 2500 }, - "freeze_record": null, + "freeze_record": { + "calibration_bundle_sha256": "2adaaab974814cf890609bac0f1b6be54fb04cda4812aea33f06dff63a954ed0", + "calibration_freeze_digest": "511ec0e9018d73c1cfb4669de2e1b4bd722ef650cd9e6d38d7179d568913f2d5", + "input_constant_set_sha256": "ea58d298473ddf320469d15dc7c32176a1109d615a689c15ff76b67fb337e109", + "measurement_protocol_sha256": "d1bb9b2c7eb354fe98990aa32eedc0e165b0cf804212966e0a2ee362a2f5bf8e", + "protocol_sha256": "f4a3fa4afb4d5bcd8e707a5e21b687cdd023dc3398b28ff6891a2318e89c5ec7", + "run_artifact_sha256s": [ + "2332db8dfdf81057263a7db45d184e0847f576d1cb2047f2704998d0f20848c8", + "7a0a63f344b0ba0f88cc8e3a7118d8c8d3ec044ee1ea0b71ad3fbc75421ee44c", + "7bb038f4503c333d200f5f532fded590ce74acdb2cabc99cfaf7643b6930767f" + ], + "selected_at": "github-actions-run:30562970311:1", + "selection_rule": "constant_only_three_fresh_generations_one_sample_each+slow_host_floors_v2", + "selection_source_commit": "681ca99098b86006e7dc1f51c27158c757ac05c9", + "selection_source_tree": "1a5494c9b82bcddc030438452285154e9d5b3e2b" + }, "qualification_thresholds": { "backend_observed_accelerator_residency": 1, "bulk_documents_per_second": 2, @@ -60,5 +75,5 @@ }, "schema_version": 1, "selection_protocol": "codestory-per-user-embedding-server-v1", - "status": "unfrozen" + "status": "frozen" } diff --git a/crates/codestory-llama-sys/per-user-embedding-server-measurement-protocol.json b/crates/codestory-llama-sys/per-user-embedding-server-measurement-protocol.json index deb9c385b..d8ddd7f61 100644 --- a/crates/codestory-llama-sys/per-user-embedding-server-measurement-protocol.json +++ b/crates/codestory-llama-sys/per-user-embedding-server-measurement-protocol.json @@ -157,7 +157,7 @@ "named_retry_condition_became_true" ], "true_idle_exit": [ - "last_queued_active_or_leased_work_ended", + "final_product_request_completed", "engine_and_server_absent" ], "total_codestory_process_memory": [ @@ -279,7 +279,7 @@ "measured_request_ordinal": 65 }, "true_idle_exit": { - "workload_id": "true_idle_60000_awake_ms_v1", + "workload_id": "true_idle_after_product_completion_60000_awake_ms_v2", "owner_state": "resident_then_absent", "operation": "observe", "input_generator": "none", diff --git a/release-claims.json b/release-claims.json index e9e07b945..bf62003d8 100644 --- a/release-claims.json +++ b/release-claims.json @@ -1133,6 +1133,8 @@ "coordinator_workflow": "packaged-platform-pr.yml", "mode": "calibration", "assembly_job": "calibration-assemble", + "pre_collection_source_proof_required": false, + "source_proof_stage": "frozen_candidate_before_qualification", "required_cells": [ { "id": "protected_macos_arm64_metal", diff --git a/scripts/codestory-release-claims.mjs b/scripts/codestory-release-claims.mjs index 7ee478bd3..e0a348fb3 100644 --- a/scripts/codestory-release-claims.mjs +++ b/scripts/codestory-release-claims.mjs @@ -541,8 +541,10 @@ function validateCalibrationPolicy(value) { calibration.coordinator_workflow !== "packaged-platform-pr.yml" || calibration.mode !== "calibration" || calibration.assembly_job !== "calibration-assemble" + || calibration.pre_collection_source_proof_required !== false + || calibration.source_proof_stage !== "frozen_candidate_before_qualification" ) { - fail("workflow_policy.calibration must name the canonical calibration coordinator and assembly job"); + fail("workflow_policy.calibration must collect before the sole frozen-candidate source proof"); } if (calibration.runs_per_required_cell !== 3) { fail("workflow_policy.calibration must require exactly three clean runs per required cell"); diff --git a/scripts/tests/codestory-release-claims.test.mjs b/scripts/tests/codestory-release-claims.test.mjs index b4cbd4a3f..3e3f7a225 100644 --- a/scripts/tests/codestory-release-claims.test.mjs +++ b/scripts/tests/codestory-release-claims.test.mjs @@ -289,6 +289,11 @@ test("claim graph freezes Mac-only accelerated 3x1 constant calibration", () => assert.equal(calibration.optional_cells[0].feeds_constant_selection, false); assert.equal(calibration.runs_per_required_cell, 3); assert.equal(calibration.samples_per_metric_per_run, 1); + assert.equal(calibration.pre_collection_source_proof_required, false); + assert.equal( + calibration.source_proof_stage, + "frozen_candidate_before_qualification", + ); assert.deepEqual(calibration.forbidden_environment, [ "CODESTORY_EMBED_ALLOW_CPU=1", ]); @@ -312,6 +317,12 @@ test("claim graph freezes Mac-only accelerated 3x1 constant calibration", () => [draft => { draft.workflow_policy.calibration.samples_per_metric_per_run = 3; }, /exactly one sample per metric per run/u], + [draft => { + draft.workflow_policy.calibration.pre_collection_source_proof_required = true; + }, /sole frozen-candidate source proof/u], + [draft => { + draft.workflow_policy.calibration.source_proof_stage = "before_calibration"; + }, /sole frozen-candidate source proof/u], [draft => { draft.workflow_policy.calibration.forbidden_environment = [ "CODESTORY_EMBED_ALLOW_CPU=0", diff --git a/scripts/tests/fixtures/release-claims/positive.json b/scripts/tests/fixtures/release-claims/positive.json index 902934188..3ddc9f8f4 100644 --- a/scripts/tests/fixtures/release-claims/positive.json +++ b/scripts/tests/fixtures/release-claims/positive.json @@ -17,7 +17,7 @@ "type": "source_behavior", "tier": "source", "status": "pass", - "graph_sha256": "7385b32924d42e94c14b8d1f41fcec50bded200fc0e65fec05f43036e2896087", + "graph_sha256": "b08ec2429ec708519b2b67532217379d7bb6a64dd1afbbfa2501652411cb1b4b", "observed_at": "2026-07-16T11:00:00.000Z", "expires_at": "2026-07-17T11:00:00.000Z", "identity": {