diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 492a51d..20a5b28 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -80,3 +80,15 @@ jobs: <(jq -S . compatibility/m1-corpus-baseline.json) \ <(jq -S . build/m1-corpus-report.json) jq . build/m1-corpus-report.json | tee -a "${GITHUB_STEP_SUMMARY}" + + - name: Audit whole-corpus catalog references + working-directory: toolkit + run: | + cargo run --locked --package atrinik-catalog --example corpus -- \ + --root ../corpus \ + --revision 01b1fdb65c2243df4bafe9c8109fc93229df0121 \ + >build/m2-catalog-corpus-report.json + diff -u \ + <(jq -S . compatibility/m2-catalog-corpus-baseline.json) \ + <(jq -S . build/m2-catalog-corpus-report.json) + jq . build/m2-catalog-corpus-report.json | tee -a "${GITHUB_STEP_SUMMARY}" diff --git a/Cargo.lock b/Cargo.lock index e9a2623..ead6cfc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,7 +13,9 @@ dependencies = [ name = "atrinik-catalog" version = "0.1.0" dependencies = [ + "atrinik-diagnostics", "atrinik-source", + "sha2", ] [[package]] diff --git a/README.md b/README.md index 7b86c22..ca881b9 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,30 @@ Clippy-as-errors, unit/doc/property/adversarial tests, dependency/license and vulnerability checks, provenance validation, public fixture/CLI conformance, release builds, and a release dry run. +## Stable catalog and diagnostics + +`atrinik-catalog` exposes the shared headless catalog API used by CI, CLI, and +editor integrations. `CatalogId` combines an explicit domain, namespace, and +locale-independent local ID for archetypes, maps, faces, animations, treasures, +factions, interfaces, quests, and resources. Ordered indexes resolve canonical +IDs, aliases, inheritance, and typed references. `Query` and `preview` return +bounded metadata without retaining or reparsing resource payloads. + +Catalog inputs carry their source revision and schema version. A generation is +derived deterministically from those inputs, while `update_document` and +`remove_document` report the exact changed identities and their transitive +dependents. An unchanged document is a no-op, and an incremental result is +identical to a clean build over the same documents. + +`atrinik-diagnostics` is the single structured diagnostic representation. Each +diagnostic has a stable code, severity, source span, related locations, +semantic path, message, optional fix hint, and explicit suppression state. +Catalog conflicts use `catalog.duplicate_id`, `catalog.ambiguous_alias`, +`catalog.missing_reference`, `catalog.ambiguous_reference`, and +`catalog.inheritance_cycle`. Only diagnostics explicitly marked suppressible +can be suppressed, and diagnostic count, related-location count, text size, and +semantic-path depth are bounded. + `Content toolkit corpus` checks out the authored `arch`/`maps` corpus at the immutable revision recorded in `provenance/reuse.json`, byte-round-trips every selected authored file or returns a bounded classified failure, and publishes @@ -66,6 +90,22 @@ cargo run --locked --package atrinik-source --example corpus -- \ --revision 01b1fdb65c2243df4bafe9c8109fc93229df0121 ``` +The same CI job builds the cross-file catalog from every supported semantic +document, indexes every bounded regular file as a resource and every PNG media +identity as a face, resolves supported references, and emits bounded diagnostic +counts plus a deterministic digest of their source/span/path/message identities: + +```sh +cargo run --locked --package atrinik-catalog --example corpus -- \ + --root /absolute/content/checkout \ + --revision 01b1fdb65c2243df4bafe9c8109fc93229df0121 +``` + +`compatibility/m2-catalog-corpus-baseline.json` pins the resulting generation, +domain-definition totals, severity-qualified diagnostic counts, and diagnostic +identity digest. CI fails on any drift, including a newly unresolved, +ambiguous, duplicate, or cyclic edge even when aggregate counts are unchanged. + The M1 baseline is 5,590 authored files, 61,379,370 bytes, zero diagnostics or truncation, and digest `8cc6a362dcf20ed7760ec2b9813fff9dbdb7803017247c4530e5575a20ffa5e3`. diff --git a/compatibility/m2-catalog-corpus-baseline.json b/compatibility/m2-catalog-corpus-baseline.json new file mode 100644 index 0000000..0e052e6 --- /dev/null +++ b/compatibility/m2-catalog-corpus-baseline.json @@ -0,0 +1,24 @@ +{ + "schema_version": 1, + "corpus_revision": "01b1fdb65c2243df4bafe9c8109fc93229df0121", + "generation": "c0c85cc10a39f93d28a172dfcae9b4c5196ab9277ba5d02685940bb6c2831230", + "documents": 5678, + "definitions": 33372, + "diagnostics": 254, + "diagnostic_digest": "5d77c014f0dbc27882be7655872adfe6bf017782a80de4dd1bbc1b80ff36c13e", + "codes": { + "catalog.missing_reference:error": 245, + "catalog.missing_reference:warning": 9 + }, + "domains": { + "animation": 775, + "archetype": 3784, + "face": 9413, + "faction": 56, + "interface": 67, + "map": 3666, + "quest": 16, + "resource": 15453, + "treasure": 142 + } +} diff --git a/crates/atrinik-catalog/Cargo.toml b/crates/atrinik-catalog/Cargo.toml index 5d3a3e5..538a46b 100644 --- a/crates/atrinik-catalog/Cargo.toml +++ b/crates/atrinik-catalog/Cargo.toml @@ -9,4 +9,6 @@ rust-version.workspace = true version.workspace = true [dependencies] +atrinik-diagnostics.workspace = true atrinik-source.workspace = true +sha2.workspace = true diff --git a/crates/atrinik-catalog/examples/corpus.rs b/crates/atrinik-catalog/examples/corpus.rs new file mode 100644 index 0000000..920a464 --- /dev/null +++ b/crates/atrinik-catalog/examples/corpus.rs @@ -0,0 +1,498 @@ +// Copyright 2026 The Atrinik Project +// SPDX-License-Identifier: MIT + +use std::{ + collections::{BTreeMap, BTreeSet}, + env, + error::Error, + fs::{self, File}, + io::Read, + path::{Path, PathBuf}, + sync::Arc, +}; + +use atrinik_catalog::{ + Catalog, CatalogDocument, CatalogId, CatalogLimits, Definition, Domain, EvidenceReferences, + FieldRule, LineDocumentLoader, ReferenceKind, +}; +use atrinik_diagnostics::{DiagnosticLimits, Location, Span, SuppressionPolicy}; +use atrinik_source::{Document, Limits, RecordKind, SourceId}; +use sha2::{Digest, Sha256}; + +const MAXIMUM_FILES: usize = 25_000; +const MAXIMUM_ENTRIES: usize = 25_000; +const MAXIMUM_DIAGNOSTICS: usize = 200_000; + +fn main() { + if let Err(error) = run() { + eprintln!("content catalog corpus: {error}"); + std::process::exit(1); + } +} + +fn run() -> Result<(), Box> { + let mut arguments = env::args_os().skip(1); + if arguments.next().as_deref() != Some(std::ffi::OsStr::new("--root")) { + return Err("usage: corpus --root PATH --revision COMMIT".into()); + } + let root = normalize_root(PathBuf::from( + arguments.next().ok_or("missing corpus root")?, + )); + if arguments.next().as_deref() != Some(std::ffi::OsStr::new("--revision")) { + return Err("usage: corpus --root PATH --revision COMMIT".into()); + } + let revision = arguments + .next() + .and_then(|value| value.into_string().ok()) + .ok_or("missing UTF-8 corpus revision")?; + if arguments.next().is_some() + || revision.len() != 40 + || !revision.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + return Err("invalid corpus revision or trailing argument".into()); + } + validate_root(&root)?; + + let archetypes = LineDocumentLoader::new( + Domain::Archetype, + "classic", + 1, + [ + (b"arch".to_vec(), FieldRule::EmbeddedObject), + reference( + "other_arch", + Domain::Archetype, + ReferenceKind::Archetype, + false, + ), + reference("face", Domain::Face, ReferenceKind::Face, true), + reference( + "animation", + Domain::Animation, + ReferenceKind::Animation, + true, + ), + reference( + "randomitems", + Domain::Treasure, + ReferenceKind::Treasure, + true, + ), + ], + )?; + let maps = LineDocumentLoader::new( + Domain::Map, + "classic", + 1, + [reference( + "arch", + Domain::Archetype, + ReferenceKind::Archetype, + false, + )], + )?; + let catalog_limits = CatalogLimits { + diagnostic_limits: DiagnosticLimits { + maximum_diagnostics: MAXIMUM_DIAGNOSTICS, + ..DiagnosticLimits::default() + }, + ..CatalogLimits::default() + }; + let mut documents = Vec::new(); + let mut definition_count = 0_usize; + let mut resources = BTreeSet::new(); + let mut faces = Vec::new(); + let mut paths = authored_paths(&root)?; + paths.sort(); + for path in paths { + let relative = normalized_relative(path.strip_prefix(&root)?)?; + resources.insert(relative.clone()); + if let Some(face) = face_id(&relative) { + faces.push(face); + } + let Some(domain) = classify(&relative) else { + continue; + }; + let source = Arc::<[u8]>::from(read_bounded(&path, Limits::default().maximum_file_bytes)?); + let document = Document::parse( + SourceId::new(format!("content:{relative}"))?, + source, + Limits::default(), + )?; + let evidence = evidence(&relative); + let catalog_document = (|| -> Result> { + Ok(match domain { + Domain::Archetype => archetypes.load_objects(&document, evidence)?, + Domain::Map => maps.load_single(&document, stable_path_id(&relative), evidence)?, + Domain::Animation => { + definitions_from_fields(&document, Domain::Animation, &[b"anim"], evidence)? + } + Domain::Treasure => definitions_from_fields( + &document, + Domain::Treasure, + &[b"treasure", b"treasureone"], + evidence, + )?, + Domain::Faction => { + definitions_from_fields(&document, Domain::Faction, &[b"faction"], evidence)? + } + domain @ (Domain::Interface | Domain::Quest) => CatalogDocument::new( + document.source_id().clone(), + document.revision(), + 1, + vec![ + Definition::new( + CatalogId::new(domain, "classic", stable_path_id(&relative))?, + Location::new( + document.source_id().as_str(), + Span::new(0, document.source_bytes().len()), + ), + ) + .with_evidence(evidence), + ], + ), + Domain::Resource | Domain::Face => { + return Err(format!("unsupported corpus classification: {relative}").into()); + } + }) + })() + .map_err(|error| format!("catalog adapter failed for {relative}: {error}"))?; + push_bounded_document( + &mut documents, + &mut definition_count, + catalog_document, + catalog_limits, + )?; + } + for document in [ + synthetic_document(Domain::Resource, "resources", resources)?, + synthetic_document(Domain::Face, "faces", faces)?, + ] { + push_bounded_document( + &mut documents, + &mut definition_count, + document, + catalog_limits, + )?; + } + + let catalog = Catalog::build(documents, catalog_limits, SuppressionPolicy::default())?; + if catalog.diagnostics().truncated() { + return Err("catalog corpus diagnostics were truncated".into()); + } + let mut codes = BTreeMap::<(&str, &str), usize>::new(); + for diagnostic in catalog.diagnostics().values() { + *codes + .entry((diagnostic.code, severity(diagnostic.severity))) + .or_default() += 1; + } + let diagnostic_digest = diagnostic_digest(catalog.diagnostics().values()); + let mut domains = Domain::ALL + .into_iter() + .map(|domain| (domain.as_str(), 0_usize)) + .collect::>(); + for definition in catalog.definitions() { + *domains.entry(definition.id.domain().as_str()).or_default() += 1; + } + print!( + "{{\"schema_version\":1,\"corpus_revision\":\"{revision}\",\"generation\":\"{}\",\"documents\":{},\"definitions\":{},\"diagnostics\":{},\"diagnostic_digest\":\"{diagnostic_digest}\",\"codes\":{{", + catalog.generation(), + catalog.documents().count(), + catalog.definitions().count(), + catalog.diagnostics().values().len() + ); + for (index, ((code, severity), count)) in codes.into_iter().enumerate() { + if index != 0 { + print!(","); + } + print!("\"{code}:{severity}\":{count}"); + } + print!("}},\"domains\":{{"); + for (index, (domain, count)) in domains.into_iter().enumerate() { + if index != 0 { + print!(","); + } + print!("\"{domain}\":{count}"); + } + println!("}}}}"); + Ok(()) +} + +fn severity(value: atrinik_diagnostics::Severity) -> &'static str { + match value { + atrinik_diagnostics::Severity::Info => "info", + atrinik_diagnostics::Severity::Warning => "warning", + atrinik_diagnostics::Severity::Error => "error", + } +} + +fn evidence(relative: &str) -> EvidenceReferences { + EvidenceReferences { + provenance: Some("provenance/reuse.json".to_owned()), + license: Some( + if relative.starts_with("arch/") { + "content:arch/COPYING" + } else { + "content:maps/COPYING" + } + .to_owned(), + ), + } +} + +fn definitions_from_fields( + document: &Document, + domain: Domain, + keys: &[&[u8]], + evidence: EvidenceReferences, +) -> Result> { + let mut definitions = Vec::new(); + for record in document.records() { + let RecordKind::Field { key, value } = record.kind else { + continue; + }; + if keys.contains(&document.bytes(key)?) { + let local = std::str::from_utf8(document.bytes(value)?)?; + definitions.push( + Definition::new( + CatalogId::new(domain, "classic", local)?, + Location::new(document.source_id().as_str(), value), + ) + .with_evidence(evidence.clone()), + ); + } + } + Ok(CatalogDocument::new( + document.source_id().clone(), + document.revision(), + 1, + definitions, + )) +} + +fn synthetic_document( + domain: Domain, + name: &str, + values: impl IntoIterator, +) -> Result> { + let bytes = Arc::<[u8]>::from(format!("catalog {name}\n").into_bytes()); + let document = Document::parse( + SourceId::new(format!("catalog:{name}"))?, + bytes, + Limits::default(), + )?; + let definitions = values + .into_iter() + .map(|value| { + Ok(Definition::new( + CatalogId::new(domain, "classic", value)?, + Location::new(document.source_id().as_str(), Span::new(0, 0)), + )) + }) + .collect::, atrinik_catalog::Error>>()?; + Ok(CatalogDocument::new( + document.source_id().clone(), + document.revision(), + 1, + definitions, + )) +} + +fn push_bounded_document( + documents: &mut Vec, + definition_count: &mut usize, + document: CatalogDocument, + limits: CatalogLimits, +) -> Result<(), Box> { + if documents.len() >= limits.maximum_documents { + return Err("catalog document limit exceeded".into()); + } + *definition_count = definition_count + .checked_add(document.definitions().len()) + .ok_or("catalog definition count overflow")?; + if *definition_count > limits.maximum_definitions { + return Err("catalog definition limit exceeded".into()); + } + documents.push(document); + Ok(()) +} + +fn face_id(relative: &str) -> Option { + Path::new(relative) + .extension() + .and_then(|value| value.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("png")) + .then(|| { + Path::new(relative) + .file_stem() + .and_then(|value| value.to_str()) + .map(str::to_owned) + }) + .flatten() +} + +fn diagnostic_digest(diagnostics: &[atrinik_diagnostics::Diagnostic]) -> String { + let mut identities = diagnostics + .iter() + .map(|diagnostic| { + format!( + "{}\0{}\0{}\0{}\0{}\0{}\0{}", + diagnostic.code, + severity(diagnostic.severity), + diagnostic.location.source, + diagnostic.location.span.start, + diagnostic.location.span.end, + diagnostic.semantic_path.join("\0"), + diagnostic.message + ) + }) + .collect::>(); + identities.sort(); + let mut digest = Sha256::new(); + for identity in identities { + digest.update((identity.len() as u64).to_le_bytes()); + digest.update(identity.as_bytes()); + } + digest + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn reference( + field: &str, + domain: Domain, + kind: ReferenceKind, + optional: bool, +) -> (Vec, FieldRule) { + ( + field.as_bytes().to_vec(), + FieldRule::Reference { + domain, + kind, + optional, + }, + ) +} + +fn classify(relative: &str) -> Option { + if relative.ends_with(".arc") { + Some(Domain::Archetype) + } else if relative.ends_with(".anim") { + Some(Domain::Animation) + } else if relative == "arch/treasures.trs" || relative == "arch/artifacts.art" { + Some(Domain::Treasure) + } else if relative.ends_with(".factions") { + Some(Domain::Faction) + } else if relative.starts_with("maps/interfaces/quests/") && relative.ends_with(".xml") { + Some(Domain::Quest) + } else if relative.starts_with("maps/interfaces/") && relative.ends_with(".xml") { + Some(Domain::Interface) + } else if relative.starts_with("maps/") + && (Path::new(relative).extension().is_none() + || matches!( + Path::new(relative) + .extension() + .and_then(|value| value.to_str()), + Some("arena" | "art" | "reg" | "trs") + )) + { + Some(Domain::Map) + } else { + None + } +} + +fn validate_root(root: &Path) -> Result<(), Box> { + let canonical = fs::canonicalize(root)?; + if fs::symlink_metadata(root)?.file_type().is_symlink() { + return Err("corpus root cannot be a symlink".into()); + } + for directory in [root.join("arch"), root.join("maps")] { + if fs::symlink_metadata(&directory)?.file_type().is_symlink() + || !fs::canonicalize(&directory)?.starts_with(&canonical) + { + return Err("corpus top-level directory escapes the root".into()); + } + } + Ok(()) +} + +fn normalize_root(root: PathBuf) -> PathBuf { + root.components().collect() +} + +fn authored_paths(root: &Path) -> Result, Box> { + let mut directories = vec![root.join("arch"), root.join("maps")]; + let mut paths = Vec::new(); + let mut entries_seen = 0_usize; + while let Some(directory) = directories.pop() { + let mut entries = Vec::new(); + for entry in fs::read_dir(directory)? { + entries_seen = entries_seen.checked_add(1).ok_or("entry count overflow")?; + if entries_seen > MAXIMUM_ENTRIES { + return Err("corpus entry limit exceeded".into()); + } + entries.push(entry?); + } + entries.sort_by_key(fs::DirEntry::file_name); + for entry in entries { + let file_type = entry.file_type()?; + if file_type.is_symlink() { + return Err( + format!("corpus symlink is not allowed: {}", entry.path().display()).into(), + ); + } + if file_type.is_dir() { + directories.push(entry.path()); + } else if file_type.is_file() { + if paths.len() >= MAXIMUM_FILES { + return Err("corpus file limit exceeded".into()); + } + paths.push(entry.path()); + } + } + } + Ok(paths) +} + +fn stable_path_id(path: &str) -> String { + path.bytes() + .map(|byte| { + if byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.' | b'/') { + byte as char + } else { + '_' + } + }) + .collect() +} + +fn normalized_relative(path: &Path) -> Result> { + let parts = path + .components() + .map(|component| { + component + .as_os_str() + .to_str() + .ok_or("corpus path is not UTF-8") + }) + .collect::, _>>()?; + Ok(parts.join("/")) +} + +fn read_bounded(path: &Path, maximum: usize) -> Result, Box> { + if fs::symlink_metadata(path)?.file_type().is_symlink() { + return Err("corpus file cannot be a symlink".into()); + } + let mut file = File::open(path)?; + let mut source = Vec::with_capacity((file.metadata()?.len() as usize).min(maximum)); + (&mut file) + .take((maximum as u64).saturating_add(1)) + .read_to_end(&mut source)?; + if source.len() > maximum { + return Err(format!("file byte limit exceeded: {}", path.display()).into()); + } + Ok(source) +} diff --git a/crates/atrinik-catalog/src/lib.rs b/crates/atrinik-catalog/src/lib.rs index fd7d4ec..968f756 100644 --- a/crates/atrinik-catalog/src/lib.rs +++ b/crates/atrinik-catalog/src/lib.rs @@ -3,67 +3,1915 @@ #![forbid(unsafe_code)] -use std::{collections::BTreeMap, fmt, sync::Arc}; +use std::{ + collections::{BTreeMap, BTreeSet, VecDeque}, + fmt, +}; -use atrinik_source::{Document, SourceId}; +use atrinik_diagnostics::{ + Diagnostic, DiagnosticLimits, DiagnosticSet, Location, RelatedLocation, Severity, Span, + SuppressionPolicy, +}; +use atrinik_source::{Document, RecordKind, Revision, SourceId}; +use sha2::{Digest, Sha256}; -#[derive(Clone, Debug, Default)] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum Domain { + Archetype, + Map, + Face, + Animation, + Treasure, + Faction, + Interface, + Quest, + Resource, +} + +impl Domain { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Archetype => "archetype", + Self::Map => "map", + Self::Face => "face", + Self::Animation => "animation", + Self::Treasure => "treasure", + Self::Faction => "faction", + Self::Interface => "interface", + Self::Quest => "quest", + Self::Resource => "resource", + } + } + + pub const ALL: [Self; 9] = [ + Self::Archetype, + Self::Map, + Self::Face, + Self::Animation, + Self::Treasure, + Self::Faction, + Self::Interface, + Self::Quest, + Self::Resource, + ]; +} + +impl fmt::Display for Domain { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} + +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct CatalogId { + domain: Domain, + namespace: String, + local: String, +} + +impl CatalogId { + pub fn new( + domain: Domain, + namespace: impl Into, + local: impl Into, + ) -> Result { + let namespace = namespace.into(); + let local = local.into(); + if !valid_namespace(&namespace) || !valid_local_id(&local) { + return Err(Error::InvalidIdentifier); + } + Ok(Self { + domain, + namespace, + local, + }) + } + + #[must_use] + pub const fn domain(&self) -> Domain { + self.domain + } + + #[must_use] + pub fn namespace(&self) -> &str { + &self.namespace + } + + #[must_use] + pub fn local(&self) -> &str { + &self.local + } +} + +impl fmt::Display for CatalogId { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "{}:{}/{}", + self.domain, self.namespace, self.local + ) + } +} + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum ReferenceKind { + Generic, + Archetype, + Inherits, + Map, + Face, + Animation, + Treasure, + Faction, + Interface, + Quest, + Resource, +} + +impl ReferenceKind { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Generic => "generic", + Self::Archetype => "archetype", + Self::Inherits => "inherits", + Self::Map => "map", + Self::Face => "face", + Self::Animation => "animation", + Self::Treasure => "treasure", + Self::Faction => "faction", + Self::Interface => "interface", + Self::Quest => "quest", + Self::Resource => "resource", + } + } + + #[must_use] + pub const fn expected_domain(self) -> Option { + match self { + Self::Generic | Self::Inherits => None, + Self::Archetype => Some(Domain::Archetype), + Self::Map => Some(Domain::Map), + Self::Face => Some(Domain::Face), + Self::Animation => Some(Domain::Animation), + Self::Treasure => Some(Domain::Treasure), + Self::Faction => Some(Domain::Faction), + Self::Interface => Some(Domain::Interface), + Self::Quest => Some(Domain::Quest), + Self::Resource => Some(Domain::Resource), + } + } +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct Reference { + pub target: CatalogId, + pub kind: ReferenceKind, + pub location: Location, + pub semantic_path: Vec, + pub optional: bool, +} + +impl Reference { + #[must_use] + pub fn new(target: CatalogId, kind: ReferenceKind, location: Location) -> Self { + Self { + target, + kind, + location, + semantic_path: Vec::new(), + optional: false, + } + } + + #[must_use] + pub fn with_semantic_path(mut self, path: impl IntoIterator>) -> Self { + self.semantic_path = path.into_iter().map(Into::into).collect(); + self + } + + #[must_use] + pub const fn optional(mut self, optional: bool) -> Self { + self.optional = optional; + self + } +} + +#[derive(Clone, Debug, Default, Eq, Ord, PartialEq, PartialOrd)] +pub struct PreviewMetadata { + pub label: Option, + pub summary: Option, + pub tags: BTreeSet, + pub keywords: BTreeSet, + pub media: BTreeMap, +} + +#[derive(Clone, Debug, Default, Eq, Ord, PartialEq, PartialOrd)] +pub struct EvidenceReferences { + pub provenance: Option, + pub license: Option, +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct Definition { + pub id: CatalogId, + pub location: Location, + pub aliases: BTreeSet, + pub inherits: Option, + pub references: Vec, + pub preview: PreviewMetadata, + pub evidence: EvidenceReferences, +} + +impl Definition { + #[must_use] + pub fn new(id: CatalogId, location: Location) -> Self { + Self { + id, + location, + aliases: BTreeSet::new(), + inherits: None, + references: Vec::new(), + preview: PreviewMetadata::default(), + evidence: EvidenceReferences::default(), + } + } + + #[must_use] + pub fn with_alias(mut self, alias: CatalogId) -> Self { + self.aliases.insert(alias); + self + } + + #[must_use] + pub fn with_inheritance(mut self, inheritance: Reference) -> Self { + self.inherits = Some(inheritance); + self + } + + #[must_use] + pub fn with_reference(mut self, reference: Reference) -> Self { + self.references.push(reference); + self + } + + #[must_use] + pub fn with_preview(mut self, preview: PreviewMetadata) -> Self { + self.preview = preview; + self + } + + #[must_use] + pub fn with_evidence(mut self, evidence: EvidenceReferences) -> Self { + self.evidence = evidence; + self + } + + fn all_references(&self) -> impl Iterator { + self.inherits + .iter() + .chain(&self.references) + .chain(self.preview.media.values()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CatalogDocument { + source_id: SourceId, + revision: Revision, + schema_version: u32, + definitions: Vec, +} + +impl CatalogDocument { + #[must_use] + pub fn new( + source_id: SourceId, + revision: Revision, + schema_version: u32, + definitions: Vec, + ) -> Self { + Self { + source_id, + revision, + schema_version, + definitions, + } + } + + #[must_use] + pub fn source_id(&self) -> &SourceId { + &self.source_id + } + + #[must_use] + pub const fn revision(&self) -> Revision { + self.revision + } + + #[must_use] + pub const fn schema_version(&self) -> u32 { + self.schema_version + } + + #[must_use] + pub fn definitions(&self) -> &[Definition] { + &self.definitions + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CatalogLimits { + pub maximum_documents: usize, + pub maximum_definitions_per_document: usize, + pub maximum_definitions: usize, + pub maximum_aliases_per_definition: usize, + pub maximum_references_per_definition: usize, + pub maximum_preview_values: usize, + pub maximum_string_bytes: usize, + pub maximum_semantic_depth: usize, + pub maximum_graph_work: usize, + pub maximum_invalidation: usize, + pub maximum_query_terms: usize, + pub maximum_query_work: usize, + pub diagnostic_limits: DiagnosticLimits, +} + +impl Default for CatalogLimits { + fn default() -> Self { + Self { + maximum_documents: 100_000, + maximum_definitions_per_document: 250_000, + maximum_definitions: 1_000_000, + maximum_aliases_per_definition: 64, + maximum_references_per_definition: 4096, + maximum_preview_values: 256, + maximum_string_bytes: 4096, + maximum_semantic_depth: 32, + maximum_graph_work: 8_000_000, + maximum_invalidation: 1_000_000, + maximum_query_terms: 256, + maximum_query_work: 1_000_000, + diagnostic_limits: DiagnosticLimits::default(), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct Generation([u8; 32]); + +impl Generation { + #[must_use] + pub const fn bytes(self) -> [u8; 32] { + self.0 + } +} + +impl fmt::Display for Generation { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + for byte in self.0 { + write!(formatter, "{byte:02x}")?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] pub struct Catalog { - documents: BTreeMap>, + documents: BTreeMap, + candidates: BTreeMap>, + aliases: BTreeMap>, + aliases_by_target: BTreeMap>, + dependents: BTreeMap>, + diagnostics: DiagnosticSet, + generation: Generation, + limits: CatalogLimits, + suppressions: SuppressionPolicy, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Resolution<'a> { + Found(&'a Definition), + Missing, + Ambiguous, } impl Catalog { pub fn build( - documents: impl IntoIterator>, - maximum_documents: usize, + documents: impl IntoIterator, + limits: CatalogLimits, + suppressions: SuppressionPolicy, ) -> Result { - let mut catalog = Self::default(); + let mut document_index = BTreeMap::new(); + let mut total_definitions = 0_usize; + let mut total_index_entries = 0_usize; for document in documents { - if catalog.documents.len() >= maximum_documents { - return Err(Error::LimitExceeded); - } - if catalog - .documents - .insert(document.source_id().clone(), document) - .is_some() - { + if document_index.contains_key(document.source_id()) { return Err(Error::DuplicateSource); } + if document_index.len() >= limits.maximum_documents { + return Err(Error::LimitExceeded("documents")); + } + validate_document(&document, limits)?; + total_definitions = total_definitions + .checked_add(document.definitions.len()) + .ok_or(Error::LimitExceeded("definitions"))?; + if total_definitions > limits.maximum_definitions { + return Err(Error::LimitExceeded("definitions")); + } + total_index_entries = checked_index_entries( + total_index_entries, + document.definitions(), + limits.maximum_graph_work, + )?; + document_index.insert(document.source_id().clone(), document); } + Self::from_index(document_index, limits, suppressions) + } + + fn from_index( + documents: BTreeMap, + limits: CatalogLimits, + suppressions: SuppressionPolicy, + ) -> Result { + if documents.len() > limits.maximum_documents { + return Err(Error::LimitExceeded("documents")); + } + let mut candidates: BTreeMap> = BTreeMap::new(); + let mut aliases: BTreeMap> = BTreeMap::new(); + let mut total_definitions = 0_usize; + let mut total_index_entries = 0_usize; + for document in documents.values() { + validate_document(document, limits)?; + total_definitions = total_definitions + .checked_add(document.definitions.len()) + .ok_or(Error::LimitExceeded("definitions"))?; + if total_definitions > limits.maximum_definitions { + return Err(Error::LimitExceeded("definitions")); + } + total_index_entries = checked_index_entries( + total_index_entries, + document.definitions(), + limits.maximum_graph_work, + )?; + for definition in &document.definitions { + candidates + .entry(definition.id.clone()) + .or_default() + .push(definition.clone()); + for alias in &definition.aliases { + aliases + .entry(alias.clone()) + .or_default() + .insert(definition.id.clone()); + } + } + } + for values in candidates.values_mut() { + values.sort(); + } + let aliases_by_target = reverse_aliases(&aliases); + + let mut catalog = Self { + generation: generation(&documents), + documents, + candidates, + aliases, + aliases_by_target, + dependents: BTreeMap::new(), + diagnostics: DiagnosticSet::with_limits(limits.diagnostic_limits), + limits, + suppressions, + }; + catalog.index_conflicts(); + catalog.index_references(true)?; + catalog.index_cycles()?; Ok(catalog) } + fn index_conflicts(&mut self) { + for (id, definitions) in &self.candidates { + if definitions.len() > 1 { + let mut diagnostic = Diagnostic::new( + "catalog.duplicate_id", + Severity::Error, + definitions[0].location.clone(), + format!("catalog ID `{id}` has multiple definitions"), + ) + .with_semantic_path(["definitions".to_owned(), id.to_string()]) + .with_fix_hint("rename or remove every conflicting definition"); + let maximum_related = self.limits.diagnostic_limits.maximum_related; + for definition in definitions.iter().skip(1).take(maximum_related) { + diagnostic = diagnostic.with_related(RelatedLocation::new( + definition.location.clone(), + "conflicting definition", + )); + } + if definitions.len().saturating_sub(1) > maximum_related { + self.diagnostics.mark_truncated(); + } + self.diagnostics + .push_with_policy(diagnostic, &self.suppressions); + } + } + for (alias, targets) in &self.aliases { + let shadows = self.candidates.contains_key(alias) && !targets.contains(alias); + if targets.len() > 1 || shadows { + let maximum_locations = self + .limits + .diagnostic_limits + .maximum_related + .saturating_add(1); + let locations: Vec = self + .candidates + .get(alias) + .into_iter() + .flatten() + .chain( + targets + .iter() + .filter_map(|target| self.candidates.get(target)) + .flatten(), + ) + .map(|definition| definition.location.clone()) + .take(maximum_locations) + .collect(); + let primary = locations + .first() + .cloned() + .or_else(|| { + self.candidates + .get(alias) + .and_then(|values| values.first()) + .map(|definition| definition.location.clone()) + }) + .unwrap_or_else(|| Location::new("catalog", Span::new(0, 0))); + let mut diagnostic = Diagnostic::new( + "catalog.ambiguous_alias", + Severity::Error, + primary, + format!("catalog alias `{alias}` resolves ambiguously"), + ) + .with_semantic_path(["aliases".to_owned(), alias.to_string()]) + .with_fix_hint("assign each alias to exactly one non-conflicting catalog ID"); + for location in locations.into_iter().skip(1) { + diagnostic = diagnostic + .with_related(RelatedLocation::new(location, "other alias target")); + } + let locations_truncated = self + .candidates + .get(alias) + .into_iter() + .flatten() + .chain( + targets + .iter() + .filter_map(|target| self.candidates.get(target)) + .flatten(), + ) + .map(|definition| &definition.location) + .take(maximum_locations.saturating_add(1)) + .count() + > maximum_locations; + if locations_truncated { + self.diagnostics.mark_truncated(); + } + self.diagnostics + .push_with_policy(diagnostic, &self.suppressions); + } + } + } + + fn index_references(&mut self, update_dependents: bool) -> Result<(), Error> { + let reference_count = self + .candidates + .values() + .filter(|values| values.len() == 1) + .try_fold(0_usize, |count, values| { + count + .checked_add(values[0].all_references().count()) + .ok_or(Error::LimitExceeded("graph work")) + })?; + if reference_count > self.limits.maximum_graph_work { + return Err(Error::LimitExceeded("graph work")); + } + let definitions: Vec<(CatalogId, Vec)> = self + .candidates + .values() + .filter(|values| values.len() == 1) + .map(|values| { + ( + values[0].id.clone(), + values[0].all_references().cloned().collect(), + ) + }) + .collect(); + let mut graph_work = 0_usize; + for (definition_id, mut references) in definitions { + references.sort_by(|left, right| { + left.target + .cmp(&right.target) + .then_with(|| left.kind.cmp(&right.kind)) + .then_with(|| left.location.cmp(&right.location)) + .then_with(|| left.semantic_path.cmp(&right.semantic_path)) + .then_with(|| left.optional.cmp(&right.optional)) + }); + for reference in &references { + graph_work = graph_work + .checked_add(1) + .ok_or(Error::LimitExceeded("graph work"))?; + if graph_work > self.limits.maximum_graph_work { + return Err(Error::LimitExceeded("graph work")); + } + if update_dependents { + self.dependents + .entry(reference.target.clone()) + .or_default() + .insert(definition_id.clone()); + } + match self.resolve(&reference.target) { + Resolution::Found(_) => {} + Resolution::Missing => { + let severity = if reference.optional { + Severity::Warning + } else { + Severity::Error + }; + let diagnostic = Diagnostic::new( + "catalog.missing_reference", + severity, + reference.location.clone(), + format!( + "{} reference `{}` does not resolve", + reference.kind.as_str(), + reference.target + ), + ) + .with_semantic_path(reference.semantic_path.clone()) + .with_fix_hint("define the target or update the stable catalog ID") + .suppressible(reference.optional); + self.diagnostics + .push_with_policy(diagnostic, &self.suppressions); + } + Resolution::Ambiguous => { + let diagnostic = Diagnostic::new( + "catalog.ambiguous_reference", + Severity::Error, + reference.location.clone(), + format!( + "{} reference `{}` has multiple targets", + reference.kind.as_str(), + reference.target + ), + ) + .with_semantic_path(reference.semantic_path.clone()) + .with_fix_hint("remove the duplicate ID or conflicting alias"); + self.diagnostics + .push_with_policy(diagnostic, &self.suppressions); + } + } + } + } + Ok(()) + } + + fn index_cycles(&mut self) -> Result<(), Error> { + let ids: Vec = self + .candidates + .iter() + .filter(|(_, values)| values.len() == 1) + .map(|(id, _)| id.clone()) + .collect(); + let mut state: BTreeMap = BTreeMap::new(); + let mut graph_work = 0_usize; + for start in ids { + if state.get(&start).copied().unwrap_or(0) != 0 { + continue; + } + let mut path = Vec::new(); + let mut positions = BTreeMap::new(); + let mut current = start; + loop { + graph_work = graph_work + .checked_add(1) + .ok_or(Error::LimitExceeded("graph work"))?; + if graph_work > self.limits.maximum_graph_work { + return Err(Error::LimitExceeded("graph work")); + } + match state.get(¤t).copied().unwrap_or(0) { + 2 => break, + 1 => { + if let Some(position) = positions.get(¤t).copied() { + self.push_cycle(&path[position..]); + } + break; + } + _ => {} + } + state.insert(current.clone(), 1); + positions.insert(current.clone(), path.len()); + path.push(current.clone()); + let Some(definition) = self.unique_definition(¤t) else { + break; + }; + let Some(inheritance) = &definition.inherits else { + break; + }; + let Resolution::Found(target) = self.resolve(&inheritance.target) else { + break; + }; + current = target.id.clone(); + } + for id in path { + state.insert(id, 2); + } + } + Ok(()) + } + + fn push_cycle(&mut self, cycle: &[CatalogId]) { + let Some(first) = cycle.first() else { + return; + }; + let Some(definition) = self.unique_definition(first) else { + return; + }; + let mut diagnostic = Diagnostic::new( + "catalog.inheritance_cycle", + Severity::Error, + definition.location.clone(), + format!("inheritance cycle contains `{first}`"), + ) + .with_semantic_path(["inherits"]) + .with_fix_hint("remove an inheritance edge from the cycle"); + let maximum_related = self.limits.diagnostic_limits.maximum_related; + for id in cycle.iter().skip(1).take(maximum_related) { + if let Some(definition) = self.unique_definition(id) { + diagnostic = diagnostic.with_related(RelatedLocation::new( + definition.location.clone(), + format!("cycle member `{id}`"), + )); + } + } + if cycle.len().saturating_sub(1) > maximum_related { + self.diagnostics.mark_truncated(); + } + self.diagnostics + .push_with_policy(diagnostic, &self.suppressions); + } + #[must_use] - pub fn get(&self, source: &SourceId) -> Option<&Arc> { - self.documents.get(source) + pub fn resolve(&self, id: &CatalogId) -> Resolution<'_> { + let direct = self.candidates.get(id); + let aliases = self.aliases.get(id); + match direct { + Some(values) if values.len() > 1 => Resolution::Ambiguous, + Some(values) => { + let conflicting_alias = aliases + .is_some_and(|targets| targets.iter().any(|target| target != &values[0].id)); + if conflicting_alias { + Resolution::Ambiguous + } else { + Resolution::Found(&values[0]) + } + } + None => { + let Some(targets) = aliases else { + return Resolution::Missing; + }; + if targets.len() != 1 { + return Resolution::Ambiguous; + } + let target = targets.first().expect("one alias target"); + match self.candidates.get(target) { + Some(values) if values.len() == 1 => Resolution::Found(&values[0]), + Some(_) => Resolution::Ambiguous, + None => Resolution::Missing, + } + } + } } - pub fn iter(&self) -> impl Iterator)> { - self.documents.iter() + fn unique_definition(&self, id: &CatalogId) -> Option<&Definition> { + self.candidates + .get(id) + .filter(|values| values.len() == 1) + .map(|values| &values[0]) + } + + pub fn definitions(&self) -> impl Iterator { + self.candidates + .values() + .filter(|values| values.len() == 1) + .map(|values| &values[0]) + } + + pub fn documents(&self) -> impl Iterator { + self.documents.values() + } + + #[must_use] + pub fn diagnostics(&self) -> &DiagnosticSet { + &self.diagnostics } #[must_use] - pub fn len(&self) -> usize { - self.documents.len() + pub const fn generation(&self) -> Generation { + self.generation } #[must_use] - pub fn is_empty(&self) -> bool { - self.documents.is_empty() + pub fn preview(&self, id: &CatalogId) -> Option<&PreviewMetadata> { + match self.resolve(id) { + Resolution::Found(definition) => Some(&definition.preview), + Resolution::Missing | Resolution::Ambiguous => None, + } + } + + pub fn dependents(&self, id: &CatalogId) -> impl Iterator { + self.dependents.get(id).into_iter().flatten() + } + + pub fn search<'a>( + &'a self, + query: &Query, + maximum: usize, + ) -> Result, Error> { + if maximum == 0 { + return Ok(Vec::new()); + } + if maximum > self.limits.maximum_query_terms + || query.tags.len() > self.limits.maximum_query_terms + { + return Err(Error::LimitExceeded("query terms")); + } + for value in query + .namespace + .iter() + .chain(query.text.iter()) + .chain(query.tags.iter()) + { + validate_text(value, self.limits)?; + } + let query_bytes = query + .namespace + .iter() + .chain(query.text.iter()) + .chain(query.tags.iter()) + .try_fold(0_usize, |total, term| total.checked_add(term.len())) + .ok_or(Error::LimitExceeded("query work"))?; + if query_bytes > self.limits.maximum_query_work { + return Err(Error::LimitExceeded("query work")); + } + let needle = query.text.as_ref().map(|value| value.to_lowercase()); + let mut work = query_bytes; + let mut results = Vec::with_capacity(maximum.min(64)); + for definition in self.definitions() { + let searchable_bytes = definition + .id + .namespace() + .len() + .checked_add(definition.id.local().len()) + .and_then(|value| value.checked_add(definition.id.domain().as_str().len())) + .and_then(|value| { + definition + .preview + .label + .iter() + .chain(definition.preview.summary.iter()) + .chain(definition.preview.tags.iter()) + .chain(definition.preview.keywords.iter()) + .try_fold(value, |total, term| total.checked_add(term.len())) + }) + .ok_or(Error::LimitExceeded("query work"))?; + work = work + .checked_add(searchable_bytes.max(1)) + .ok_or(Error::LimitExceeded("query work"))?; + if work > self.limits.maximum_query_work { + return Err(Error::LimitExceeded("query work")); + } + if query + .domain + .is_none_or(|domain| definition.id.domain() == domain) + && query + .namespace + .as_ref() + .is_none_or(|namespace| definition.id.namespace() == namespace) + && query.tags.is_subset(&definition.preview.tags) + && needle.as_ref().is_none_or(|needle| { + definition.id.to_string().to_lowercase().contains(needle) + || definition + .preview + .label + .as_ref() + .is_some_and(|value| value.to_lowercase().contains(needle)) + || definition + .preview + .summary + .as_ref() + .is_some_and(|value| value.to_lowercase().contains(needle)) + || definition + .preview + .keywords + .iter() + .any(|value| value.to_lowercase().contains(needle)) + }) + { + results.push(definition); + if results.len() == maximum { + break; + } + } + } + Ok(results) + } + + pub fn update_document(&self, document: CatalogDocument) -> Result { + if self.documents.get(document.source_id()) == Some(&document) { + return Ok(CatalogUpdate { + catalog: self.clone(), + invalidation: Invalidation { + source: document.source_id().clone(), + changed: BTreeSet::new(), + affected: BTreeSet::new(), + }, + }); + } + validate_document(&document, self.limits)?; + let source = document.source_id().clone(); + let old = self.documents.get(&source); + let (changed_canonical, mut changed) = + changed_identities(old, Some(&document), self.limits.maximum_invalidation)?; + let catalog = self.with_replaced_document(document, &changed_canonical)?; + expand_changed_aliases( + &mut changed, + &changed_canonical, + [&self.aliases, &catalog.aliases], + self.limits.maximum_invalidation, + )?; + let affected = self.collect_invalidation(&catalog, &changed)?; + Ok(CatalogUpdate { + catalog, + invalidation: Invalidation { + source, + changed, + affected, + }, + }) + } + + pub fn remove_document(&self, source: &SourceId) -> Result { + let Some(old) = self.documents.get(source) else { + return Err(Error::MissingSource); + }; + let (changed_canonical, mut changed) = + changed_identities(Some(old), None, self.limits.maximum_invalidation)?; + let catalog = self.without_document(source, &changed_canonical)?; + expand_changed_aliases( + &mut changed, + &changed_canonical, + [&self.aliases, &catalog.aliases], + self.limits.maximum_invalidation, + )?; + let affected = self.collect_invalidation(&catalog, &changed)?; + Ok(CatalogUpdate { + catalog, + invalidation: Invalidation { + source: source.clone(), + changed, + affected, + }, + }) + } + + fn collect_invalidation( + &self, + next: &Self, + changed: &BTreeSet, + ) -> Result, Error> { + self.validate_invalidation_seed(changed)?; + let mut affected = changed.clone(); + let mut queue = VecDeque::from_iter(changed.iter().cloned()); + while let Some(target) = queue.pop_front() { + for dependent in self + .dependents(&target) + .chain(next.dependents(&target)) + .chain(self.aliases_for(&target)) + .chain(next.aliases_for(&target)) + { + if affected.insert(dependent.clone()) { + if affected.len() > self.limits.maximum_invalidation { + return Err(Error::LimitExceeded("invalidation")); + } + queue.push_back(dependent.clone()); + } + } + } + Ok(affected) + } + + fn aliases_for(&self, id: &CatalogId) -> impl Iterator { + self.aliases_by_target.get(id).into_iter().flatten() + } + + fn validate_invalidation_seed(&self, changed: &BTreeSet) -> Result<(), Error> { + if changed.len() > self.limits.maximum_invalidation { + Err(Error::LimitExceeded("invalidation")) + } else { + Ok(()) + } + } + + fn with_replaced_document( + &self, + document: CatalogDocument, + changed_canonical: &BTreeSet, + ) -> Result { + validate_document(&document, self.limits)?; + let mut next = self.clone(); + let old = next + .documents + .insert(document.source_id().clone(), document.clone()); + next.patch_document_indexes(old.as_ref(), Some(&document), changed_canonical)?; + Ok(next) + } + + fn without_document( + &self, + source: &SourceId, + changed_canonical: &BTreeSet, + ) -> Result { + let mut next = self.clone(); + let old = next.documents.remove(source).ok_or(Error::MissingSource)?; + next.patch_document_indexes(Some(&old), None, changed_canonical)?; + Ok(next) + } + + fn patch_document_indexes( + &mut self, + old: Option<&CatalogDocument>, + new: Option<&CatalogDocument>, + impacted_ids: &BTreeSet, + ) -> Result<(), Error> { + if self.documents.len() > self.limits.maximum_documents { + return Err(Error::LimitExceeded("documents")); + } + let total_definitions = self + .documents + .values() + .try_fold(0_usize, |count, document| { + count + .checked_add(document.definitions.len()) + .ok_or(Error::LimitExceeded("definitions")) + })?; + if total_definitions > self.limits.maximum_definitions { + return Err(Error::LimitExceeded("definitions")); + } + let mut total_index_entries = 0_usize; + for document in self.documents.values() { + total_index_entries = checked_index_entries( + total_index_entries, + document.definitions(), + self.limits.maximum_graph_work, + )?; + } + + let old_edges = unique_edges(&self.candidates, impacted_ids); + if let Some(old) = old { + for definition in &old.definitions { + if !impacted_ids.contains(&definition.id) { + continue; + } + let remove_entry = if let Some(values) = self.candidates.get_mut(&definition.id) { + if let Some(position) = values.iter().position(|value| value == definition) { + values.remove(position); + } + values.is_empty() + } else { + false + }; + if remove_entry { + self.candidates.remove(&definition.id); + } + } + } + if let Some(new) = new { + for definition in &new.definitions { + if !impacted_ids.contains(&definition.id) { + continue; + } + self.candidates + .entry(definition.id.clone()) + .or_default() + .push(definition.clone()); + } + } + for id in impacted_ids { + if let Some(values) = self.candidates.get_mut(id) { + values.sort(); + } + } + + let impacted_aliases = old + .into_iter() + .chain(new) + .flat_map(CatalogDocument::definitions) + .filter(|definition| impacted_ids.contains(&definition.id)) + .flat_map(|definition| definition.aliases.iter().cloned()) + .collect::>(); + let replacement_alias_targets = collect_alias_targets( + &self.candidates, + impacted_ids, + &impacted_aliases, + self.limits.maximum_graph_work, + )?; + for alias in impacted_aliases { + let mut targets = self.aliases.remove(&alias).unwrap_or_default(); + for target in &targets { + if let Some(values) = self.aliases_by_target.get_mut(target) { + values.remove(&alias); + if values.is_empty() { + self.aliases_by_target.remove(target); + } + } + } + targets.retain(|target| !impacted_ids.contains(target)); + if let Some(replacements) = replacement_alias_targets.get(&alias) { + targets.extend(replacements.iter().cloned()); + } + if !targets.is_empty() { + for target in &targets { + self.aliases_by_target + .entry(target.clone()) + .or_default() + .insert(alias.clone()); + } + self.aliases.insert(alias, targets); + } + } + + for (target, dependent) in old_edges { + if let Some(values) = self.dependents.get_mut(&target) { + values.remove(&dependent); + if values.is_empty() { + self.dependents.remove(&target); + } + } + } + for (target, dependent) in unique_edges(&self.candidates, impacted_ids) { + self.dependents.entry(target).or_default().insert(dependent); + } + + self.generation = generation(&self.documents); + self.diagnostics = DiagnosticSet::with_limits(self.limits.diagnostic_limits); + self.index_conflicts(); + self.index_references(false)?; + self.index_cycles()?; + Ok(()) + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct Query { + pub domain: Option, + pub namespace: Option, + pub text: Option, + pub tags: BTreeSet, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Invalidation { + pub source: SourceId, + pub changed: BTreeSet, + pub affected: BTreeSet, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CatalogUpdate { + pub catalog: Catalog, + pub invalidation: Invalidation, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum FieldRule { + Alias, + Inherits, + EmbeddedObject, + Reference { + domain: Domain, + kind: ReferenceKind, + optional: bool, + }, + Label, + Summary, + Tag, + Keyword, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LineDocumentLoader { + domain: Domain, + namespace: String, + schema_version: u32, + rules: BTreeMap, FieldRule>, + limits: CatalogLimits, +} + +impl LineDocumentLoader { + pub fn new( + domain: Domain, + namespace: impl Into, + schema_version: u32, + rules: impl IntoIterator, FieldRule)>, + ) -> Result { + let namespace = namespace.into(); + if !valid_namespace(&namespace) || schema_version == 0 { + return Err(Error::InvalidDocument); + } + let mut accepted = BTreeMap::new(); + let limits = CatalogLimits::default(); + let maximum_rules = limits + .maximum_aliases_per_definition + .checked_add(limits.maximum_references_per_definition) + .and_then(|value| value.checked_add(limits.maximum_preview_values)) + .and_then(|value| value.checked_add(2)) + .ok_or(Error::LimitExceeded("loader rules"))?; + for (field, rule) in rules { + if accepted.len() >= maximum_rules + || field.len() > limits.maximum_string_bytes + || field.is_empty() + || !field.iter().all(u8::is_ascii) + || accepted.insert(field, rule).is_some() + { + return Err(Error::InvalidDocument); + } + } + Ok(Self { + domain, + namespace, + schema_version, + rules: accepted, + limits, + }) + } + + pub fn with_limits(mut self, limits: CatalogLimits) -> Result { + let maximum_rules = limits + .maximum_aliases_per_definition + .checked_add(limits.maximum_references_per_definition) + .and_then(|value| value.checked_add(limits.maximum_preview_values)) + .and_then(|value| value.checked_add(2)) + .ok_or(Error::LimitExceeded("loader rules"))?; + validate_text(&self.namespace, limits)?; + if self.rules.len() > maximum_rules + || self + .rules + .keys() + .any(|field| field.len() > limits.maximum_string_bytes) + { + return Err(Error::LimitExceeded("loader rules")); + } + self.limits = limits; + Ok(self) + } + + pub fn load_objects( + &self, + document: &Document, + evidence: EvidenceReferences, + ) -> Result { + validate_evidence(&evidence, self.limits)?; + let mut stack: Vec = Vec::new(); + let mut embedded_depth = 0_usize; + let mut definitions = Vec::new(); + for record in document.records() { + match &record.kind { + RecordKind::ObjectStart { name } => { + if embedded_depth != 0 { + return Err(Error::InvalidDocument); + } + let definition_count = definitions + .len() + .checked_add(stack.len()) + .ok_or(Error::LimitExceeded("definitions per document"))?; + if definition_count >= self.limits.maximum_definitions_per_document { + return Err(Error::LimitExceeded("definitions per document")); + } + let name_text = text(document, *name, self.limits)?; + let id = CatalogId::new(self.domain, self.namespace.clone(), name_text)?; + stack.push( + Definition::new(id, Location::new(document.source_id().as_str(), *name)) + .with_evidence(evidence.clone()), + ); + } + RecordKind::Field { key, value } => { + let Some(definition) = stack.last_mut() else { + continue; + }; + let Some(rule) = self + .rules + .get(document.bytes(*key).map_err(|_| Error::InvalidDocument)?) + else { + continue; + }; + if *rule == FieldRule::EmbeddedObject { + embedded_depth = embedded_depth + .checked_add(1) + .ok_or(Error::LimitExceeded("object depth"))?; + if embedded_depth > self.limits.maximum_semantic_depth { + return Err(Error::LimitExceeded("object depth")); + } + continue; + } + if embedded_depth != 0 { + continue; + } + let value_text = text(document, *value, self.limits)?; + let location = Location::new(document.source_id().as_str(), *value); + apply_rule( + definition, + rule, + &self.namespace, + value_text, + location, + self.limits, + )?; + } + RecordKind::ObjectEnd => { + if embedded_depth != 0 { + embedded_depth -= 1; + continue; + } + let definition = stack.pop().ok_or(Error::InvalidDocument)?; + definitions.push(definition); + } + _ => {} + } + } + if !stack.is_empty() || embedded_depth != 0 { + return Err(Error::InvalidDocument); + } + Ok(CatalogDocument::new( + document.source_id().clone(), + document.revision(), + self.schema_version, + definitions, + )) + } + + pub fn load_single( + &self, + document: &Document, + local_id: impl Into, + evidence: EvidenceReferences, + ) -> Result { + if self.limits.maximum_definitions_per_document == 0 { + return Err(Error::LimitExceeded("definitions per document")); + } + validate_evidence(&evidence, self.limits)?; + let local_id = local_id.into(); + validate_text(&local_id, self.limits)?; + let id = CatalogId::new(self.domain, self.namespace.clone(), local_id)?; + let mut definition = Definition::new( + id, + Location::new( + document.source_id().as_str(), + Span::new(0, document.source_bytes().len()), + ), + ) + .with_evidence(evidence); + for record in document.records() { + let RecordKind::Field { key, value } = &record.kind else { + continue; + }; + let Some(rule) = self + .rules + .get(document.bytes(*key).map_err(|_| Error::InvalidDocument)?) + else { + continue; + }; + apply_rule( + &mut definition, + rule, + &self.namespace, + text(document, *value, self.limits)?, + Location::new(document.source_id().as_str(), *value), + self.limits, + )?; + } + Ok(CatalogDocument::new( + document.source_id().clone(), + document.revision(), + self.schema_version, + vec![definition], + )) } } +fn apply_rule( + definition: &mut Definition, + rule: &FieldRule, + namespace: &str, + value: String, + location: Location, + limits: CatalogLimits, +) -> Result<(), Error> { + validate_text(&value, limits)?; + match rule { + FieldRule::Alias => { + let alias = CatalogId::new(definition.id.domain(), namespace, value)?; + if !definition.aliases.contains(&alias) + && definition.aliases.len() >= limits.maximum_aliases_per_definition + { + return Err(Error::LimitExceeded("aliases per definition")); + } + definition.aliases.insert(alias); + } + FieldRule::Inherits => { + if definition.inherits.is_none() + && definition_reference_count(definition)? + >= limits.maximum_references_per_definition + { + return Err(Error::LimitExceeded("references per definition")); + } + definition.inherits = Some( + Reference::new( + CatalogId::new(definition.id.domain(), namespace, value)?, + ReferenceKind::Inherits, + location, + ) + .with_semantic_path(["inherits"]), + ); + } + FieldRule::EmbeddedObject => return Err(Error::InvalidDocument), + FieldRule::Reference { + domain, + kind, + optional, + } => { + if *kind == ReferenceKind::Inherits + || kind + .expected_domain() + .is_some_and(|expected| expected != *domain) + { + return Err(Error::InvalidDocument); + } + if definition_reference_count(definition)? >= limits.maximum_references_per_definition { + return Err(Error::LimitExceeded("references per definition")); + } + definition.references.push( + Reference::new(CatalogId::new(*domain, namespace, value)?, *kind, location) + .with_semantic_path(["references", kind.as_str()]) + .optional(*optional), + ); + } + FieldRule::Label => definition.preview.label = Some(value), + FieldRule::Summary => definition.preview.summary = Some(value), + FieldRule::Tag => { + let current = definition.preview.tags.len() + definition.preview.keywords.len(); + if !definition.preview.tags.contains(&value) && current >= limits.maximum_preview_values + { + return Err(Error::LimitExceeded("preview values")); + } + definition.preview.tags.insert(value); + } + FieldRule::Keyword => { + let current = definition.preview.tags.len() + definition.preview.keywords.len(); + if !definition.preview.keywords.contains(&value) + && current >= limits.maximum_preview_values + { + return Err(Error::LimitExceeded("preview values")); + } + definition.preview.keywords.insert(value); + } + } + Ok(()) +} + +fn definition_reference_count(definition: &Definition) -> Result { + definition + .references + .len() + .checked_add(usize::from(definition.inherits.is_some())) + .and_then(|value| value.checked_add(definition.preview.media.len())) + .ok_or(Error::LimitExceeded("references per definition")) +} + +fn text(document: &Document, span: Span, limits: CatalogLimits) -> Result { + if span.len() > limits.maximum_string_bytes { + return Err(Error::LimitExceeded("string bytes")); + } + std::str::from_utf8(document.bytes(span).map_err(|_| Error::InvalidDocument)?) + .map(str::to_owned) + .map_err(|_| Error::InvalidDocument) +} + +fn validate_evidence(evidence: &EvidenceReferences, limits: CatalogLimits) -> Result<(), Error> { + for value in evidence.provenance.iter().chain(evidence.license.iter()) { + validate_text(value, limits)?; + } + Ok(()) +} + +fn validate_document(document: &CatalogDocument, limits: CatalogLimits) -> Result<(), Error> { + if document.schema_version == 0 { + return Err(Error::InvalidDocument); + } + if document.definitions.len() > limits.maximum_definitions_per_document { + return Err(Error::LimitExceeded("definitions per document")); + } + for definition in &document.definitions { + let reference_count = definition + .references + .len() + .checked_add(usize::from(definition.inherits.is_some())) + .and_then(|value| value.checked_add(definition.preview.media.len())) + .ok_or(Error::LimitExceeded("references per definition"))?; + let preview_count = definition + .preview + .tags + .len() + .checked_add(definition.preview.keywords.len()) + .and_then(|value| value.checked_add(definition.preview.media.len())) + .ok_or(Error::LimitExceeded("preview values"))?; + if definition.location.source != document.source_id.as_str() + || definition.aliases.len() > limits.maximum_aliases_per_definition + || reference_count > limits.maximum_references_per_definition + || preview_count > limits.maximum_preview_values + { + return Err(Error::InvalidDocument); + } + validate_text(definition.id.namespace(), limits)?; + validate_text(definition.id.local(), limits)?; + for alias in &definition.aliases { + if alias.domain() != definition.id.domain() { + return Err(Error::InvalidDocument); + } + validate_text(alias.namespace(), limits)?; + validate_text(alias.local(), limits)?; + } + if let Some(inheritance) = &definition.inherits { + if inheritance.kind != ReferenceKind::Inherits + || inheritance.target.domain() != definition.id.domain() + { + return Err(Error::InvalidDocument); + } + validate_reference(inheritance, document, limits)?; + } + for reference in definition + .references + .iter() + .chain(definition.preview.media.values()) + { + if reference.kind == ReferenceKind::Inherits + || reference + .kind + .expected_domain() + .is_some_and(|domain| domain != reference.target.domain()) + { + return Err(Error::InvalidDocument); + } + validate_reference(reference, document, limits)?; + } + for value in definition + .preview + .label + .iter() + .chain(definition.preview.summary.iter()) + .chain(definition.preview.tags.iter()) + .chain(definition.preview.keywords.iter()) + .chain(definition.preview.media.keys()) + .chain(definition.evidence.provenance.iter()) + .chain(definition.evidence.license.iter()) + { + validate_text(value, limits)?; + } + } + Ok(()) +} + +fn validate_reference( + reference: &Reference, + document: &CatalogDocument, + limits: CatalogLimits, +) -> Result<(), Error> { + if reference.location.source != document.source_id.as_str() + || reference.semantic_path.len() > limits.maximum_semantic_depth + { + return Err(Error::InvalidDocument); + } + validate_text(reference.target.namespace(), limits)?; + validate_text(reference.target.local(), limits)?; + for segment in &reference.semantic_path { + validate_text(segment, limits)?; + } + Ok(()) +} + +fn validate_text(value: &str, limits: CatalogLimits) -> Result<(), Error> { + if value.len() > limits.maximum_string_bytes || value.contains('\0') { + Err(Error::LimitExceeded("string bytes")) + } else { + Ok(()) + } +} + +fn unique_edges( + candidates: &BTreeMap>, + ids: &BTreeSet, +) -> BTreeSet<(CatalogId, CatalogId)> { + ids.iter() + .filter_map(|id| candidates.get(id).filter(|values| values.len() == 1)) + .flat_map(|values| { + values[0] + .all_references() + .map(|reference| (reference.target.clone(), values[0].id.clone())) + }) + .collect() +} + +fn checked_index_entries( + initial: usize, + definitions: &[Definition], + maximum: usize, +) -> Result { + let total = definitions.iter().try_fold(initial, |count, definition| { + count + .checked_add(definition.aliases.len()) + .and_then(|value| value.checked_add(definition.all_references().count())) + .ok_or(Error::LimitExceeded("index entries")) + })?; + if total > maximum { + Err(Error::LimitExceeded("index entries")) + } else { + Ok(total) + } +} + +fn collect_alias_targets( + candidates: &BTreeMap>, + ids: &BTreeSet, + aliases: &BTreeSet, + maximum_work: usize, +) -> Result>, Error> { + let mut targets = BTreeMap::>::new(); + let mut work = 0_usize; + for id in ids { + let Some(definitions) = candidates.get(id) else { + continue; + }; + for definition in definitions { + work = work + .checked_add(1) + .ok_or(Error::LimitExceeded("graph work"))?; + if work > maximum_work { + return Err(Error::LimitExceeded("graph work")); + } + for alias in &definition.aliases { + work = work + .checked_add(1) + .ok_or(Error::LimitExceeded("graph work"))?; + if work > maximum_work { + return Err(Error::LimitExceeded("graph work")); + } + if aliases.contains(alias) { + targets.entry(alias.clone()).or_default().insert(id.clone()); + } + } + } + } + Ok(targets) +} + +fn reverse_aliases( + aliases: &BTreeMap>, +) -> BTreeMap> { + let mut reverse = BTreeMap::>::new(); + for (alias, targets) in aliases { + for target in targets { + reverse + .entry(target.clone()) + .or_default() + .insert(alias.clone()); + } + } + reverse +} + +fn changed_identities( + old: Option<&CatalogDocument>, + new: Option<&CatalogDocument>, + maximum: usize, +) -> Result<(BTreeSet, BTreeSet), Error> { + let mut old_definitions: BTreeMap<&CatalogId, Vec<&Definition>> = BTreeMap::new(); + let mut new_definitions: BTreeMap<&CatalogId, Vec<&Definition>> = BTreeMap::new(); + for definition in old.into_iter().flat_map(CatalogDocument::definitions) { + old_definitions + .entry(&definition.id) + .or_default() + .push(definition); + } + for definition in new.into_iter().flat_map(CatalogDocument::definitions) { + new_definitions + .entry(&definition.id) + .or_default() + .push(definition); + } + for values in old_definitions + .values_mut() + .chain(new_definitions.values_mut()) + { + values.sort(); + } + let schema_changed = old + .zip(new) + .is_some_and(|(left, right)| left.schema_version() != right.schema_version()); + let ids = old_definitions.keys().chain(new_definitions.keys()); + let mut canonical = BTreeSet::new(); + let mut changed = BTreeSet::new(); + for id in ids { + let differs = schema_changed || old_definitions.get(id) != new_definitions.get(id); + if differs { + insert_bounded(&mut canonical, (*id).clone(), maximum)?; + insert_bounded(&mut changed, (*id).clone(), maximum)?; + for alias in old_definitions + .get(id) + .into_iter() + .chain(new_definitions.get(id)) + .flat_map(|values| values.iter()) + .flat_map(|definition| &definition.aliases) + { + insert_bounded(&mut changed, alias.clone(), maximum)?; + } + } + } + Ok((canonical, changed)) +} + +fn expand_changed_aliases<'a>( + changed: &mut BTreeSet, + canonical: &BTreeSet, + indexes: impl IntoIterator>>, + maximum: usize, +) -> Result<(), Error> { + for aliases in indexes { + for (alias, targets) in aliases { + if !targets.is_disjoint(canonical) { + insert_bounded(changed, alias.clone(), maximum)?; + } + } + } + Ok(()) +} + +fn insert_bounded( + values: &mut BTreeSet, + value: CatalogId, + maximum: usize, +) -> Result<(), Error> { + if !values.contains(&value) && values.len() >= maximum { + return Err(Error::LimitExceeded("invalidation")); + } + values.insert(value); + Ok(()) +} + +fn generation(documents: &BTreeMap) -> Generation { + let mut digest = Sha256::new(); + digest.update(b"atrinik-catalog-generation-v1\0"); + digest_usize(&mut digest, documents.len()); + for document in documents.values() { + digest_str(&mut digest, document.source_id.as_str()); + digest.update(document.revision.bytes()); + digest.update(document.schema_version.to_be_bytes()); + let mut definitions = document.definitions.clone(); + for definition in &mut definitions { + definition.references.sort(); + } + definitions.sort(); + digest_usize(&mut digest, definitions.len()); + for definition in definitions { + digest_id(&mut digest, &definition.id); + digest_location(&mut digest, &definition.location); + digest_usize(&mut digest, definition.aliases.len()); + for alias in definition.aliases { + digest_id(&mut digest, &alias); + } + digest_reference(&mut digest, definition.inherits.as_ref()); + let references = definition.references; + digest_usize(&mut digest, references.len()); + for reference in &references { + digest_reference(&mut digest, Some(reference)); + } + digest_option(&mut digest, definition.preview.label.as_deref()); + digest_option(&mut digest, definition.preview.summary.as_deref()); + for values in [&definition.preview.tags, &definition.preview.keywords] { + digest_usize(&mut digest, values.len()); + for value in values { + digest_str(&mut digest, value); + } + } + digest_usize(&mut digest, definition.preview.media.len()); + for (name, reference) in definition.preview.media { + digest_str(&mut digest, &name); + digest_reference(&mut digest, Some(&reference)); + } + digest_option(&mut digest, definition.evidence.provenance.as_deref()); + digest_option(&mut digest, definition.evidence.license.as_deref()); + } + } + Generation(digest.finalize().into()) +} + +fn digest_reference(digest: &mut Sha256, reference: Option<&Reference>) { + let Some(reference) = reference else { + digest.update([0]); + return; + }; + digest.update([1]); + digest_id(digest, &reference.target); + digest_str(digest, reference.kind.as_str()); + digest_location(digest, &reference.location); + digest.update([u8::from(reference.optional)]); + digest_usize(digest, reference.semantic_path.len()); + for segment in &reference.semantic_path { + digest_str(digest, segment); + } +} + +fn digest_location(digest: &mut Sha256, location: &Location) { + digest_str(digest, &location.source); + digest.update((location.span.start as u64).to_be_bytes()); + digest.update((location.span.end as u64).to_be_bytes()); +} + +fn digest_id(digest: &mut Sha256, id: &CatalogId) { + digest_str(digest, id.domain().as_str()); + digest_str(digest, id.namespace()); + digest_str(digest, id.local()); +} + +fn digest_option(digest: &mut Sha256, value: Option<&str>) { + match value { + Some(value) => { + digest.update([1]); + digest_str(digest, value); + } + None => { + digest.update([0]); + } + } +} + +fn digest_str(digest: &mut Sha256, value: &str) { + digest.update((value.len() as u64).to_be_bytes()); + digest.update(value.as_bytes()); +} + +fn digest_usize(digest: &mut Sha256, value: usize) { + digest.update((value as u64).to_be_bytes()); +} + +fn valid_namespace(value: &str) -> bool { + !value.is_empty() + && value.len() <= 128 + && value.as_bytes()[0].is_ascii_lowercase() + && value.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'_' | b'-' | b'.') + }) +} + +fn valid_local_id(value: &str) -> bool { + !value.is_empty() + && value.len() <= 512 + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.' | b'/')) + && !value.starts_with('/') + && !value.ends_with('/') + && !value + .split('/') + .any(|segment| segment.is_empty() || segment == "." || segment == "..") +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum Error { + InvalidIdentifier, + InvalidDocument, DuplicateSource, - LimitExceeded, + MissingSource, + LimitExceeded(&'static str), } impl fmt::Display for Error { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::InvalidIdentifier => write!(formatter, "catalog identifier is invalid"), + Self::InvalidDocument => write!(formatter, "catalog document is invalid"), Self::DuplicateSource => write!(formatter, "catalog source identity is duplicated"), - Self::LimitExceeded => write!(formatter, "catalog document limit is exceeded"), + Self::MissingSource => write!(formatter, "catalog source identity is not indexed"), + Self::LimitExceeded(limit) => write!(formatter, "catalog {limit} limit is exceeded"), } } } @@ -71,27 +1919,4 @@ impl fmt::Display for Error { impl std::error::Error for Error {} #[cfg(test)] -mod tests { - use std::sync::Arc; - - use atrinik_source::{Document, Limits, SourceId}; - - use super::{Catalog, Error}; - - #[test] - fn orders_sources_and_rejects_duplicates() { - let source = SourceId::new("fixture:a").unwrap(); - let document = Arc::new( - Document::parse( - source, - Arc::<[u8]>::from(&b"name a\n"[..]), - Limits::default(), - ) - .unwrap(), - ); - assert_eq!( - Catalog::build([document.clone(), document], 2).unwrap_err(), - Error::DuplicateSource - ); - } -} +mod tests; diff --git a/crates/atrinik-catalog/src/tests.rs b/crates/atrinik-catalog/src/tests.rs new file mode 100644 index 0000000..c9ab609 --- /dev/null +++ b/crates/atrinik-catalog/src/tests.rs @@ -0,0 +1,1114 @@ +// Copyright 2026 The Atrinik Project +// SPDX-License-Identifier: MIT + +use std::{collections::BTreeSet, sync::Arc}; + +use atrinik_diagnostics::{DiagnosticLimits, Location, Span, SuppressionPolicy}; +use atrinik_source::{Document, Limits, SourceId}; + +use super::{ + Catalog, CatalogDocument, CatalogId, CatalogLimits, Definition, Domain, Error, + EvidenceReferences, FieldRule, LineDocumentLoader, PreviewMetadata, Query, Reference, + ReferenceKind, Resolution, +}; + +fn source(name: &str, bytes: &[u8]) -> Document { + Document::parse( + SourceId::new(format!("fixture:{name}")).unwrap(), + Arc::<[u8]>::from(bytes), + Limits::default(), + ) + .unwrap() +} + +fn id(domain: Domain, name: &str) -> CatalogId { + CatalogId::new(domain, "core", name).unwrap() +} + +fn definition(document: &Document, domain: Domain, name: &str) -> Definition { + Definition::new( + id(domain, name), + Location::new(document.source_id().as_str(), Span::new(0, 1)), + ) +} + +fn input(document: &Document, definitions: Vec) -> CatalogDocument { + CatalogDocument::new( + document.source_id().clone(), + document.revision(), + 1, + definitions, + ) +} + +fn build(documents: Vec) -> Catalog { + Catalog::build( + documents, + CatalogLimits::default(), + SuppressionPolicy::default(), + ) + .unwrap() +} + +#[test] +fn covers_every_owned_domain_with_stable_order_and_evidence() { + let mut documents = Vec::new(); + for domain in Domain::ALL.into_iter().rev() { + let document = source(domain.as_str(), b"name localized\n"); + let mut preview = PreviewMetadata { + label: Some(format!("Localized {}", domain.as_str())), + ..PreviewMetadata::default() + }; + preview.tags.insert("public".to_owned()); + documents.push(input( + &document, + vec![ + definition(&document, domain, "stable-id") + .with_preview(preview) + .with_evidence(EvidenceReferences { + provenance: Some("registry:synthetic".to_owned()), + license: Some("LicenseRef-Synthetic".to_owned()), + }), + ], + )); + } + let catalog = build(documents); + let domains: Vec = catalog + .definitions() + .map(|value| value.id.domain()) + .collect(); + assert_eq!(domains, Domain::ALL); + assert!(catalog.diagnostics().values().is_empty()); + assert_eq!( + catalog + .definitions() + .next() + .unwrap() + .evidence + .license + .as_deref(), + Some("LicenseRef-Synthetic") + ); +} + +#[test] +fn reports_duplicate_alias_missing_and_ambiguous_references() { + let first = source("first", b"name first\n"); + let second = source("second", b"name second\n"); + let missing = id(Domain::Archetype, "missing"); + let shared_alias = id(Domain::Archetype, "shared"); + let duplicate = id(Domain::Archetype, "duplicate"); + let one = definition(&first, Domain::Archetype, "one") + .with_alias(shared_alias.clone()) + .with_reference( + Reference::new( + missing, + ReferenceKind::Generic, + Location::new(first.source_id().as_str(), Span::new(2, 3)), + ) + .with_semantic_path(["references", "missing"]), + ) + .with_reference( + Reference::new( + duplicate.clone(), + ReferenceKind::Generic, + Location::new(first.source_id().as_str(), Span::new(3, 4)), + ) + .with_semantic_path(["references", "ambiguous"]), + ); + let two = definition(&second, Domain::Archetype, "two").with_alias(shared_alias.clone()); + let duplicate_first = definition(&first, Domain::Archetype, "duplicate"); + let duplicate_second = definition(&second, Domain::Archetype, "duplicate"); + let catalog = build(vec![ + input(&second, vec![two, duplicate_second]), + input(&first, vec![one, duplicate_first]), + ]); + let codes: Vec<&str> = catalog + .diagnostics() + .values() + .iter() + .map(|value| value.code) + .collect(); + assert_eq!( + codes, + [ + "catalog.duplicate_id", + "catalog.ambiguous_alias", + "catalog.ambiguous_reference", + "catalog.missing_reference", + ] + ); + assert!(matches!( + catalog.resolve(&shared_alias), + Resolution::Ambiguous + )); + assert!(matches!(catalog.resolve(&duplicate), Resolution::Ambiguous)); + assert!(catalog.diagnostics().has_errors()); +} + +#[test] +fn detects_inheritance_cycles_and_resolves_aliases() { + let document = source("cycles", b"name cycles\n"); + let alias = id(Domain::Archetype, "former-a"); + let a = definition(&document, Domain::Archetype, "a") + .with_alias(alias.clone()) + .with_inheritance(Reference::new( + id(Domain::Archetype, "b"), + ReferenceKind::Inherits, + Location::new(document.source_id().as_str(), Span::new(1, 2)), + )); + let b = definition(&document, Domain::Archetype, "b").with_inheritance(Reference::new( + id(Domain::Archetype, "a"), + ReferenceKind::Inherits, + Location::new(document.source_id().as_str(), Span::new(2, 3)), + )); + let catalog = build(vec![input(&document, vec![b, a])]); + assert!(matches!(catalog.resolve(&alias), Resolution::Found(value) if value.id.local() == "a")); + assert_eq!( + catalog + .diagnostics() + .values() + .iter() + .filter(|value| value.code == "catalog.inheritance_cycle") + .count(), + 1 + ); +} + +#[test] +fn incremental_rename_invalidates_only_changed_ids_and_dependents() { + let target = source("target", b"name target\n"); + let dependent = source("dependent", b"name dependent\n"); + let unrelated = source("unrelated", b"name unrelated\n"); + let target_id = id(Domain::Quest, "old-name"); + let dependent_definition = + definition(&dependent, Domain::Interface, "journal").with_reference(Reference::new( + target_id.clone(), + ReferenceKind::Quest, + Location::new(dependent.source_id().as_str(), Span::new(0, 1)), + )); + let original = build(vec![ + input( + &target, + vec![definition(&target, Domain::Quest, "old-name")], + ), + input(&dependent, vec![dependent_definition.clone()]), + input( + &unrelated, + vec![definition(&unrelated, Domain::Map, "elsewhere")], + ), + ]); + let edited = source("target", b"name target changed\n"); + let replacement = input( + &edited, + vec![definition(&edited, Domain::Quest, "new-name")], + ); + let update = original.update_document(replacement.clone()).unwrap(); + assert_eq!( + update.invalidation.changed, + BTreeSet::from([id(Domain::Quest, "new-name"), id(Domain::Quest, "old-name"),]) + ); + assert_eq!( + update.invalidation.affected, + BTreeSet::from([ + id(Domain::Interface, "journal"), + id(Domain::Quest, "new-name"), + id(Domain::Quest, "old-name"), + ]) + ); + let clean = build(vec![ + replacement, + input(&dependent, vec![dependent_definition]), + input( + &unrelated, + vec![definition(&unrelated, Domain::Map, "elsewhere")], + ), + ]); + assert_eq!(update.catalog, clean); + assert_eq!(update.catalog.generation(), clean.generation()); +} + +#[test] +fn incremental_edit_excludes_unchanged_siblings_and_enforces_bounds() { + let document = source("multi", b"name multi\n"); + let original = build(vec![input( + &document, + vec![ + definition(&document, Domain::Quest, "changed"), + definition(&document, Domain::Quest, "untouched"), + ], + )]); + let replacement = input( + &source("multi", b"name changed revision\n"), + vec![ + definition(&document, Domain::Quest, "changed").with_preview(PreviewMetadata { + summary: Some("changed semantics".to_owned()), + ..PreviewMetadata::default() + }), + definition(&document, Domain::Quest, "untouched"), + ], + ); + let update = original.update_document(replacement).unwrap(); + assert_eq!( + update.invalidation.changed, + BTreeSet::from([id(Domain::Quest, "changed")]) + ); + assert!( + !update + .invalidation + .affected + .contains(&id(Domain::Quest, "untouched")) + ); + + let limits = CatalogLimits { + maximum_documents: 1, + maximum_invalidation: 0, + ..CatalogLimits::default() + }; + let bounded = Catalog::build( + [input( + &document, + vec![definition(&document, Domain::Quest, "one")], + )], + limits, + SuppressionPolicy::default(), + ) + .unwrap(); + let added = source("added", b"name added\n"); + assert!( + bounded + .update_document(input( + &added, + vec![definition(&added, Domain::Quest, "two")] + )) + .is_err() + ); + assert!(bounded.remove_document(document.source_id()).is_err()); + + let oversized = CatalogDocument::new( + document.source_id().clone(), + document.revision(), + 1, + vec![ + definition(&document, Domain::Quest, "one"), + definition(&document, Domain::Quest, "two"), + ], + ); + let strict = Catalog::build( + std::iter::empty(), + CatalogLimits { + maximum_definitions_per_document: 1, + ..CatalogLimits::default() + }, + SuppressionPolicy::default(), + ) + .unwrap(); + assert!(strict.update_document(oversized).is_err()); +} + +#[test] +fn incremental_alias_rebuild_is_linear_and_charges_candidate_work() { + let original = source("alias-work", b"name original\n"); + let aliases = (0..128) + .map(|index| { + definition(&original, Domain::Resource, &format!("value-{index:03}")) + .with_alias(id(Domain::Resource, &format!("old-{index:03}"))) + }) + .collect::>(); + let limits = CatalogLimits { + maximum_graph_work: 256, + ..CatalogLimits::default() + }; + let catalog = Catalog::build( + [input(&original, aliases)], + limits, + SuppressionPolicy::default(), + ) + .unwrap(); + let edited = source("alias-work", b"name edited\n"); + let replacement = input( + &edited, + (0..128) + .map(|index| { + definition(&edited, Domain::Resource, &format!("value-{index:03}")) + .with_alias(id(Domain::Resource, &format!("new-{index:03}"))) + }) + .collect(), + ); + let update = catalog.update_document(replacement.clone()).unwrap(); + let clean = + Catalog::build([replacement.clone()], limits, SuppressionPolicy::default()).unwrap(); + assert_eq!(update.catalog, clean); + + let tight_limits = CatalogLimits { + maximum_graph_work: 255, + ..CatalogLimits::default() + }; + let tight = Catalog::build( + [input( + &original, + (0..128) + .map(|index| { + definition(&original, Domain::Resource, &format!("value-{index:03}")) + .with_alias(id(Domain::Resource, &format!("old-{index:03}"))) + }) + .collect(), + )], + tight_limits, + SuppressionPolicy::default(), + ) + .unwrap(); + assert_eq!( + tight.update_document(replacement).unwrap_err(), + Error::LimitExceeded("graph work") + ); +} + +#[test] +fn invalidation_traverses_aliases_of_transitive_dependents() { + let document = source("alias-chain", b"name chain\n"); + let a = id(Domain::Archetype, "a"); + let b_alias = id(Domain::Archetype, "old-b"); + let c = id(Domain::Quest, "c"); + let values = vec![ + definition(&document, Domain::Archetype, "a"), + definition(&document, Domain::Archetype, "b") + .with_alias(b_alias.clone()) + .with_reference(Reference::new( + a, + ReferenceKind::Archetype, + Location::new(document.source_id().as_str(), Span::new(0, 1)), + )), + definition(&document, Domain::Quest, "c").with_reference(Reference::new( + b_alias, + ReferenceKind::Archetype, + Location::new(document.source_id().as_str(), Span::new(0, 1)), + )), + ]; + let catalog = build(vec![input(&document, values)]); + let edited = source("alias-chain", b"name changed chain\n"); + let replacement = input( + &edited, + vec![ + definition(&edited, Domain::Archetype, "renamed-a"), + definition(&edited, Domain::Archetype, "b") + .with_alias(id(Domain::Archetype, "old-b")) + .with_reference(Reference::new( + id(Domain::Archetype, "a"), + ReferenceKind::Archetype, + Location::new(edited.source_id().as_str(), Span::new(0, 1)), + )), + definition(&edited, Domain::Quest, "c").with_reference(Reference::new( + id(Domain::Archetype, "old-b"), + ReferenceKind::Archetype, + Location::new(edited.source_id().as_str(), Span::new(0, 1)), + )), + ], + ); + let update = catalog.update_document(replacement).unwrap(); + assert!(update.invalidation.affected.contains(&c)); +} + +#[test] +fn same_digest_and_semantics_are_an_incremental_noop() { + let document = source("noop", b"name noop\n"); + let input = input( + &document, + vec![definition(&document, Domain::Resource, "noop")], + ); + let catalog = build(vec![input.clone()]); + let update = catalog.update_document(input).unwrap(); + assert!(update.invalidation.changed.is_empty()); + assert!(update.invalidation.affected.is_empty()); + assert_eq!(update.catalog.generation(), catalog.generation()); +} + +#[test] +fn schema_evolution_and_semantics_change_generation() { + let document = source("schema", b"name schema\n"); + let definition = definition(&document, Domain::Resource, "schema"); + let first = build(vec![CatalogDocument::new( + document.source_id().clone(), + document.revision(), + 1, + vec![definition.clone()], + )]); + let second = build(vec![CatalogDocument::new( + document.source_id().clone(), + document.revision(), + 2, + vec![definition], + )]); + assert_ne!(first.generation(), second.generation()); +} + +#[test] +fn generation_is_canonical_for_tied_definitions_and_references() { + let document = source("canonical", b"name canonical\n"); + let reference = |path: &str, optional: bool| { + Reference::new( + id(Domain::Resource, "target"), + ReferenceKind::Resource, + Location::new(document.source_id().as_str(), Span::new(0, 1)), + ) + .with_semantic_path([path]) + .optional(optional) + }; + let first_definition = definition(&document, Domain::Resource, "duplicate") + .with_reference(reference("one", false)) + .with_reference(reference("two", true)); + let second_definition = + definition(&document, Domain::Resource, "duplicate").with_preview(PreviewMetadata { + label: Some("different".to_owned()), + ..PreviewMetadata::default() + }); + let target = definition(&document, Domain::Resource, "target"); + let first = build(vec![input( + &document, + vec![ + first_definition.clone(), + second_definition.clone(), + target.clone(), + ], + )]); + let second = build(vec![input( + &document, + vec![second_definition, first_definition, target], + )]); + assert_eq!(first.generation(), second.generation()); + + let reordered = definition(&document, Domain::Resource, "duplicate") + .with_reference(reference("two", true)) + .with_reference(reference("one", false)); + let other = + definition(&document, Domain::Resource, "duplicate").with_preview(PreviewMetadata { + label: Some("different".to_owned()), + ..PreviewMetadata::default() + }); + let third = build(vec![input( + &document, + vec![ + reordered, + other, + definition(&document, Domain::Resource, "target"), + ], + )]); + assert_eq!(first.generation(), third.generation()); +} + +#[test] +fn query_filter_and_preview_do_not_require_payload_access() { + let document = source("query", b"opaque payload\n"); + let mut preview = PreviewMetadata { + label: Some("Localized Silver Sword".to_owned()), + summary: Some("A preview only".to_owned()), + ..PreviewMetadata::default() + }; + preview.tags.insert("weapon".to_owned()); + preview.keywords.insert("blade".to_owned()); + let catalog = build(vec![input( + &document, + vec![definition(&document, Domain::Archetype, "silver_sword").with_preview(preview)], + )]); + let query = Query { + domain: Some(Domain::Archetype), + namespace: Some("core".to_owned()), + text: Some("blade".to_owned()), + tags: BTreeSet::from(["weapon".to_owned()]), + }; + let results = catalog.search(&query, 1).unwrap(); + assert_eq!(results[0].id.local(), "silver_sword"); + assert_eq!( + catalog.preview(&results[0].id).unwrap().label.as_deref(), + Some("Localized Silver Sword") + ); +} + +#[test] +fn query_input_and_scan_work_are_bounded() { + let document = source("query-bounds", b"name bounds\n"); + let limits = CatalogLimits { + maximum_query_terms: 1, + maximum_query_work: 0, + ..CatalogLimits::default() + }; + let catalog = Catalog::build( + [input( + &document, + vec![definition(&document, Domain::Resource, "one")], + )], + limits, + SuppressionPolicy::default(), + ) + .unwrap(); + assert!(catalog.search(&Query::default(), 2).is_err()); + assert!(catalog.search(&Query::default(), 1).is_err()); + + let mut preview = PreviewMetadata::default(); + preview.tags.insert("large-tag".repeat(20)); + let tagged = Catalog::build( + [input( + &document, + vec![definition(&document, Domain::Resource, "tiny").with_preview(preview)], + )], + CatalogLimits { + maximum_query_work: 32, + ..CatalogLimits::default() + }, + SuppressionPolicy::default(), + ) + .unwrap(); + assert!( + tagged + .search( + &Query { + tags: BTreeSet::from(["missing".to_owned()]), + ..Query::default() + }, + 1, + ) + .is_err() + ); +} + +#[test] +fn media_references_are_resolved_diagnosed_and_invalidate_consumers() { + let document = source("media", b"name media\n"); + let target = id(Domain::Face, "portrait"); + let mut preview = PreviewMetadata::default(); + preview.media.insert( + "portrait".to_owned(), + Reference::new( + target.clone(), + ReferenceKind::Face, + Location::new(document.source_id().as_str(), Span::new(0, 1)), + ) + .with_semantic_path(["preview", "media", "portrait"]), + ); + let consumer = definition(&document, Domain::Quest, "consumer").with_preview(preview); + let missing = build(vec![input(&document, vec![consumer.clone()])]); + assert_eq!( + missing.diagnostics().values()[0].code, + "catalog.missing_reference" + ); + + let face = source("face-media", b"name face\n"); + let catalog = build(vec![ + input(&document, vec![consumer]), + input(&face, vec![definition(&face, Domain::Face, "portrait")]), + ]); + assert_eq!( + catalog.dependents(&target).next(), + Some(&id(Domain::Quest, "consumer")) + ); + let update = catalog.remove_document(face.source_id()).unwrap(); + assert!( + update + .invalidation + .affected + .contains(&id(Domain::Quest, "consumer")) + ); + assert_eq!( + update.catalog.diagnostics().values()[0].code, + "catalog.missing_reference" + ); +} + +#[test] +fn rejects_cross_domain_aliases_inheritance_and_typed_references() { + let document = source("types", b"name types\n"); + let invalid_alias = + definition(&document, Domain::Archetype, "value").with_alias(id(Domain::Map, "alias")); + assert!( + Catalog::build( + [input(&document, vec![invalid_alias])], + CatalogLimits::default(), + SuppressionPolicy::default(), + ) + .is_err() + ); + let invalid_reference = + definition(&document, Domain::Archetype, "value").with_reference(Reference::new( + id(Domain::Map, "target"), + ReferenceKind::Face, + Location::new(document.source_id().as_str(), Span::new(0, 1)), + )); + assert!( + Catalog::build( + [input(&document, vec![invalid_reference])], + CatalogLimits::default(), + SuppressionPolicy::default(), + ) + .is_err() + ); +} + +#[test] +fn rejects_inheritance_edges_copied_into_other_reference_collections() { + let document = source("duplicate-inheritance", b"name child\n"); + let inheritance = Reference::new( + id(Domain::Archetype, "parent"), + ReferenceKind::Inherits, + Location::new(document.source_id().as_str(), Span::new(0, 1)), + ); + let ordinary = definition(&document, Domain::Archetype, "ordinary") + .with_inheritance(inheritance.clone()) + .with_reference(inheritance.clone()); + assert!( + Catalog::build( + [input(&document, vec![ordinary])], + CatalogLimits::default(), + SuppressionPolicy::default(), + ) + .is_err() + ); + + let mut preview = PreviewMetadata::default(); + preview + .media + .insert("parent".to_owned(), inheritance.clone()); + let media = definition(&document, Domain::Archetype, "media") + .with_inheritance(inheritance) + .with_preview(preview); + assert!( + Catalog::build( + [input(&document, vec![media])], + CatalogLimits::default(), + SuppressionPolicy::default(), + ) + .is_err() + ); +} + +#[test] +fn duplicate_transitions_invalidate_alias_consumers_from_unchanged_documents() { + let first = source("alias-owner", b"name owner\n"); + let duplicate = source("duplicate-added", b"name duplicate\n"); + let consumer_source = source("alias-consumer", b"name consumer\n"); + let canonical = id(Domain::Archetype, "canonical"); + let alias = id(Domain::Archetype, "old-canonical"); + let consumer_id = id(Domain::Quest, "consumer"); + let owner = definition(&first, Domain::Archetype, "canonical").with_alias(alias.clone()); + let consumer = + definition(&consumer_source, Domain::Quest, "consumer").with_reference(Reference::new( + alias.clone(), + ReferenceKind::Archetype, + Location::new(consumer_source.source_id().as_str(), Span::new(0, 1)), + )); + let catalog = build(vec![ + input(&first, vec![owner.clone()]), + input(&consumer_source, vec![consumer.clone()]), + ]); + let update = catalog + .update_document(input( + &duplicate, + vec![definition(&duplicate, Domain::Archetype, "canonical")], + )) + .unwrap(); + assert!(update.invalidation.changed.contains(&canonical)); + assert!(update.invalidation.changed.contains(&alias)); + assert!(update.invalidation.affected.contains(&consumer_id)); + assert!(matches!( + update.catalog.resolve(&alias), + Resolution::Ambiguous + )); + let clean = build(vec![ + input(&first, vec![owner]), + input(&consumer_source, vec![consumer]), + input( + &duplicate, + vec![definition(&duplicate, Domain::Archetype, "canonical")], + ), + ]); + assert_eq!(update.catalog, clean); +} + +#[test] +fn media_counts_toward_the_reference_limit() { + let document = source("media-limit", b"name media\n"); + let mut preview = PreviewMetadata::default(); + preview.media.insert( + "one".to_owned(), + Reference::new( + id(Domain::Face, "one"), + ReferenceKind::Face, + Location::new(document.source_id().as_str(), Span::new(0, 1)), + ), + ); + let limits = CatalogLimits { + maximum_references_per_definition: 0, + ..CatalogLimits::default() + }; + assert!( + Catalog::build( + [input( + &document, + vec![definition(&document, Domain::Archetype, "value").with_preview(preview)], + )], + limits, + SuppressionPolicy::default(), + ) + .is_err() + ); +} + +#[test] +fn filesystem_order_and_localized_labels_do_not_control_ids_or_order() { + let a = source("z-path", b"name Zulu\n"); + let b = source("a-path", b"name Alpha\n"); + let with_label = |document: &Document, name: &str, label: &str| { + definition(document, Domain::Face, name).with_preview(PreviewMetadata { + label: Some(label.to_owned()), + ..PreviewMetadata::default() + }) + }; + let first = build(vec![ + input(&a, vec![with_label(&a, "a", "Zulu")]), + input(&b, vec![with_label(&b, "b", "Alpha")]), + ]); + let second = build(vec![ + input(&b, vec![with_label(&b, "b", "Different")]), + input(&a, vec![with_label(&a, "a", "Other")]), + ]); + let ids: Vec<&str> = first.definitions().map(|value| value.id.local()).collect(); + assert_eq!(ids, ["a", "b"]); + let second_ids: Vec<&str> = second.definitions().map(|value| value.id.local()).collect(); + assert_eq!(second_ids, ["a", "b"]); +} + +#[test] +fn line_loader_exposes_one_shared_domain_loading_boundary() { + let document = source( + "loader", + b"Object child\nname Child label\nalias old_child\nparent parent\nface child.101\nend\nObject parent\nname Parent label\nend\n", + ); + let loader = LineDocumentLoader::new( + Domain::Archetype, + "core", + 1, + [ + (b"name".to_vec(), FieldRule::Label), + (b"alias".to_vec(), FieldRule::Alias), + (b"parent".to_vec(), FieldRule::Inherits), + ( + b"face".to_vec(), + FieldRule::Reference { + domain: Domain::Face, + kind: ReferenceKind::Face, + optional: false, + }, + ), + ], + ) + .unwrap(); + let loaded = loader + .load_objects(&document, EvidenceReferences::default()) + .unwrap(); + let face_document = source("face", b"name face\n"); + let catalog = build(vec![ + loaded, + input( + &face_document, + vec![definition(&face_document, Domain::Face, "child.101")], + ), + ]); + assert!(matches!( + catalog.resolve(&id(Domain::Archetype, "old_child")), + Resolution::Found(value) if value.id.local() == "child" + )); + assert!(catalog.diagnostics().values().is_empty()); +} + +#[test] +fn line_loader_rejects_unbalanced_object_boundaries() { + let loader = LineDocumentLoader::new(Domain::Archetype, "core", 1, []).unwrap(); + for (name, bytes) in [ + ("unclosed", b"Object value\n".as_slice()), + ("unmatched", b"end\n".as_slice()), + ] { + let document = source(name, bytes); + assert!( + loader + .load_objects(&document, EvidenceReferences::default()) + .is_err() + ); + } +} + +#[test] +fn line_loader_balances_explicit_embedded_objects() { + let document = source( + "embedded", + b"Object value\narch event\nface ignored\nend\nface retained\nend\n", + ); + let loader = LineDocumentLoader::new( + Domain::Archetype, + "core", + 1, + [ + (b"arch".to_vec(), FieldRule::EmbeddedObject), + ( + b"face".to_vec(), + FieldRule::Reference { + domain: Domain::Face, + kind: ReferenceKind::Face, + optional: true, + }, + ), + ], + ) + .unwrap(); + let loaded = loader + .load_objects(&document, EvidenceReferences::default()) + .unwrap(); + assert_eq!(loaded.definitions()[0].references.len(), 1); + assert_eq!( + loaded.definitions()[0].references[0].target.local(), + "retained" + ); +} + +#[test] +fn line_loader_rejects_values_before_exceeding_catalog_limits() { + let document = source( + "loader-bounds", + b"Object value\ntag one\ntag two\nface one.101\nend\n", + ); + let loader = LineDocumentLoader::new( + Domain::Archetype, + "core", + 1, + [ + (b"tag".to_vec(), FieldRule::Tag), + ( + b"face".to_vec(), + FieldRule::Reference { + domain: Domain::Face, + kind: ReferenceKind::Face, + optional: true, + }, + ), + ], + ) + .unwrap() + .with_limits(CatalogLimits { + maximum_preview_values: 1, + maximum_references_per_definition: 1, + ..CatalogLimits::default() + }) + .unwrap(); + assert!( + loader + .load_objects(&document, EvidenceReferences::default()) + .is_err() + ); + + let zero = LineDocumentLoader::new(Domain::Map, "core", 1, []).unwrap(); + assert!( + zero.with_limits(CatalogLimits { + maximum_definitions_per_document: 0, + ..CatalogLimits::default() + }) + .unwrap() + .load_single(&document, "map", EvidenceReferences::default()) + .is_err() + ); + + let duplicate = source( + "loader-duplicate", + b"Object value\nalias old\nalias old\ntag same\ntag same\nend\n", + ); + let duplicate_loader = LineDocumentLoader::new( + Domain::Archetype, + "core", + 1, + [ + (b"alias".to_vec(), FieldRule::Alias), + (b"tag".to_vec(), FieldRule::Tag), + ], + ) + .unwrap() + .with_limits(CatalogLimits { + maximum_aliases_per_definition: 1, + maximum_preview_values: 1, + ..CatalogLimits::default() + }) + .unwrap(); + assert!( + duplicate_loader + .load_objects(&duplicate, EvidenceReferences::default()) + .is_ok() + ); + + assert!( + LineDocumentLoader::new(Domain::Map, "core", 1, []) + .unwrap() + .with_limits(CatalogLimits { + maximum_string_bytes: 3, + ..CatalogLimits::default() + }) + .is_err() + ); + let short = LineDocumentLoader::new(Domain::Map, "c", 1, []) + .unwrap() + .with_limits(CatalogLimits { + maximum_string_bytes: 1, + ..CatalogLimits::default() + }) + .unwrap(); + assert!( + short + .load_single(&document, "map", EvidenceReferences::default()) + .is_err() + ); +} + +#[test] +fn line_loader_handles_the_reference_limit_without_recounting_prior_edges() { + const REFERENCES: usize = 4_096; + let mut bytes = b"Object value\n".to_vec(); + bytes.extend_from_slice(b"face portrait\n".repeat(REFERENCES).as_slice()); + bytes.extend_from_slice(b"end\n"); + let document = source("loader-reference-work", &bytes); + let loader = LineDocumentLoader::new( + Domain::Archetype, + "core", + 1, + [( + b"face".to_vec(), + FieldRule::Reference { + domain: Domain::Face, + kind: ReferenceKind::Face, + optional: true, + }, + )], + ) + .unwrap() + .with_limits(CatalogLimits { + maximum_references_per_definition: REFERENCES, + maximum_graph_work: REFERENCES, + ..CatalogLimits::default() + }) + .unwrap(); + let loaded = loader + .load_objects(&document, EvidenceReferences::default()) + .unwrap(); + assert_eq!(loaded.definitions()[0].references.len(), REFERENCES); +} + +#[test] +fn handles_large_bounded_graph_and_truncates_diagnostics() { + let document = source("large", b"name large\n"); + let mut definitions = Vec::new(); + for index in 0..4096 { + let mut value = definition(&document, Domain::Resource, &format!("node-{index:04}")); + if index != 0 { + value = value.with_reference(Reference::new( + id(Domain::Resource, &format!("node-{:04}", index - 1)), + ReferenceKind::Resource, + Location::new(document.source_id().as_str(), Span::new(index, index + 1)), + )); + } + definitions.push(value); + } + let limits = CatalogLimits { + maximum_graph_work: 20_000, + diagnostic_limits: DiagnosticLimits { + maximum_diagnostics: 2, + ..DiagnosticLimits::default() + }, + ..CatalogLimits::default() + }; + let catalog = Catalog::build( + [input(&document, definitions)], + limits, + SuppressionPolicy::default(), + ) + .unwrap(); + assert_eq!(catalog.definitions().count(), 4096); + assert!(catalog.diagnostics().values().is_empty()); + + let invalid = source("invalid", b"name invalid\n"); + let definitions = (0..8) + .map(|index| { + definition(&invalid, Domain::Quest, &format!("quest-{index}")).with_reference( + Reference::new( + id(Domain::Quest, &format!("missing-{index}")), + ReferenceKind::Quest, + Location::new(invalid.source_id().as_str(), Span::new(index, index + 1)), + ), + ) + }) + .collect(); + let catalog = Catalog::build( + [input(&invalid, definitions)], + limits, + SuppressionPolicy::default(), + ) + .unwrap(); + assert_eq!(catalog.diagnostics().values().len(), 2); + assert!(catalog.diagnostics().truncated()); +} + +#[test] +fn build_and_updates_preflight_global_definition_and_index_work() { + let first = source("global-one", b"name one\n"); + let second = source("global-two", b"name two\n"); + let definition_with_aliases = definition(&first, Domain::Resource, "one") + .with_alias(id(Domain::Resource, "old-one")) + .with_alias(id(Domain::Resource, "older-one")); + assert!( + Catalog::build( + [input(&first, vec![definition_with_aliases])], + CatalogLimits { + maximum_graph_work: 1, + ..CatalogLimits::default() + }, + SuppressionPolicy::default(), + ) + .is_err() + ); + + let bounded = Catalog::build( + [input( + &first, + vec![definition(&first, Domain::Resource, "one")], + )], + CatalogLimits { + maximum_definitions: 1, + ..CatalogLimits::default() + }, + SuppressionPolicy::default(), + ) + .unwrap(); + assert!( + bounded + .update_document(input( + &second, + vec![definition(&second, Domain::Resource, "two")], + )) + .is_err() + ); +} + +#[test] +fn optional_missing_references_follow_explicit_suppression_policy() { + let document = source("suppressed", b"name suppressed\n"); + let definition = definition(&document, Domain::Quest, "quest").with_reference( + Reference::new( + id(Domain::Map, "optional-map"), + ReferenceKind::Map, + Location::new(document.source_id().as_str(), Span::new(0, 1)), + ) + .optional(true), + ); + let policy = SuppressionPolicy::new(["catalog.missing_reference"], 8, 64).unwrap(); + let catalog = Catalog::build( + [input(&document, vec![definition])], + CatalogLimits::default(), + policy, + ) + .unwrap(); + assert!(catalog.diagnostics().values()[0].suppressed); + assert!(!catalog.diagnostics().has_errors()); +} diff --git a/crates/atrinik-diagnostics/src/lib.rs b/crates/atrinik-diagnostics/src/lib.rs index 8a3e752..5be7557 100644 --- a/crates/atrinik-diagnostics/src/lib.rs +++ b/crates/atrinik-diagnostics/src/lib.rs @@ -3,7 +3,7 @@ #![forbid(unsafe_code)] -use std::fmt; +use std::{collections::BTreeSet, fmt}; #[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd)] pub struct Span { @@ -30,41 +30,309 @@ impl Span { #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] pub enum Severity { + Info, Warning, Error, } +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct Location { + pub source: String, + pub span: Span, +} + +impl Location { + #[must_use] + pub fn new(source: impl Into, span: Span) -> Self { + Self { + source: source.into(), + span, + } + } +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct RelatedLocation { + pub location: Location, + pub message: String, +} + +impl RelatedLocation { + #[must_use] + pub fn new(location: Location, message: impl Into) -> Self { + Self { + location, + message: message.into(), + } + } +} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct Diagnostic { pub code: &'static str, pub severity: Severity, - pub span: Span, - pub message: &'static str, + pub location: Location, + pub related: Vec, + pub semantic_path: Vec, + pub message: String, + pub fix_hint: Option, + pub suppressible: bool, + pub suppressed: bool, } +impl Diagnostic { + #[must_use] + pub fn new( + code: &'static str, + severity: Severity, + location: Location, + message: impl Into, + ) -> Self { + debug_assert!(valid_code(code)); + Self { + code, + severity, + location, + related: Vec::new(), + semantic_path: Vec::new(), + message: message.into(), + fix_hint: None, + suppressible: false, + suppressed: false, + } + } + + #[must_use] + pub fn with_related(mut self, related: RelatedLocation) -> Self { + self.related.push(related); + self + } + + #[must_use] + pub fn with_semantic_path(mut self, path: impl IntoIterator>) -> Self { + self.semantic_path = path.into_iter().map(Into::into).collect(); + self + } + + #[must_use] + pub fn with_fix_hint(mut self, hint: impl Into) -> Self { + self.fix_hint = Some(hint.into()); + self + } + + #[must_use] + pub const fn suppressible(mut self, suppressible: bool) -> Self { + self.suppressible = suppressible; + self + } + + #[must_use] + pub const fn is_active(&self) -> bool { + !self.suppressed + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct DiagnosticLimits { + pub maximum_diagnostics: usize, + pub maximum_related: usize, + pub maximum_semantic_depth: usize, + pub maximum_text_bytes: usize, +} + +impl Default for DiagnosticLimits { + fn default() -> Self { + Self { + maximum_diagnostics: 256, + maximum_related: 16, + maximum_semantic_depth: 32, + maximum_text_bytes: 4096, + } + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct SuppressionPolicy { + codes: BTreeSet, +} + +impl SuppressionPolicy { + pub fn new( + codes: impl IntoIterator>, + maximum_codes: usize, + maximum_code_bytes: usize, + ) -> Result { + let mut accepted = BTreeSet::new(); + for (index, code) in codes.into_iter().enumerate() { + if index >= maximum_codes { + return Err(SuppressionError::LimitExceeded); + } + let code = code.into(); + if code.len() > maximum_code_bytes || !valid_code(&code) { + return Err(SuppressionError::InvalidCode); + } + accepted.insert(code); + } + Ok(Self { codes: accepted }) + } + + #[must_use] + pub fn is_suppressed(&self, code: &str) -> bool { + self.codes.contains(code) + } + + pub fn codes(&self) -> impl Iterator { + self.codes.iter().map(String::as_str) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SuppressionError { + InvalidCode, + LimitExceeded, +} + +impl fmt::Display for SuppressionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidCode => write!(formatter, "suppression code is invalid"), + Self::LimitExceeded => write!(formatter, "suppression code limit is exceeded"), + } + } +} + +impl std::error::Error for SuppressionError {} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct DiagnosticSet { - maximum: usize, + limits: DiagnosticLimits, values: Vec, truncated: bool, + omitted_error: bool, + suppressed_slots: BTreeSet, + warning_slots: BTreeSet, } impl DiagnosticSet { #[must_use] pub fn new(maximum: usize) -> Self { + Self::with_limits(DiagnosticLimits { + maximum_diagnostics: maximum, + ..DiagnosticLimits::default() + }) + } + + #[must_use] + pub fn with_limits(limits: DiagnosticLimits) -> Self { Self { - maximum, - values: Vec::with_capacity(maximum.min(64)), + limits, + values: Vec::with_capacity(limits.maximum_diagnostics.min(64)), truncated: false, + omitted_error: false, + suppressed_slots: BTreeSet::new(), + warning_slots: BTreeSet::new(), } } pub fn push(&mut self, diagnostic: Diagnostic) { - if self.values.len() < self.maximum { - self.values.push(diagnostic); - } else { + self.push_with_policy(diagnostic, &SuppressionPolicy::default()); + } + + pub fn push_with_policy(&mut self, mut diagnostic: Diagnostic, policy: &SuppressionPolicy) { + diagnostic.suppressed = diagnostic.suppressible && policy.is_suppressed(diagnostic.code); + if diagnostic.related.len() > self.limits.maximum_related { + diagnostic.related.truncate(self.limits.maximum_related); + self.truncated = true; + } + if diagnostic.semantic_path.len() > self.limits.maximum_semantic_depth { + diagnostic + .semantic_path + .truncate(self.limits.maximum_semantic_depth); self.truncated = true; } + if truncate_utf8( + &mut diagnostic.location.source, + self.limits.maximum_text_bytes, + ) | truncate_utf8(&mut diagnostic.message, self.limits.maximum_text_bytes) + { + self.truncated = true; + } + for related in &mut diagnostic.related { + if truncate_utf8(&mut related.location.source, self.limits.maximum_text_bytes) + | truncate_utf8(&mut related.message, self.limits.maximum_text_bytes) + { + self.truncated = true; + } + } + for segment in &mut diagnostic.semantic_path { + if truncate_utf8(segment, self.limits.maximum_text_bytes) { + self.truncated = true; + } + } + if let Some(hint) = diagnostic.fix_hint.as_mut() + && truncate_utf8(hint, self.limits.maximum_text_bytes) + { + self.truncated = true; + } + if self.values.len() >= self.limits.maximum_diagnostics { + self.truncated = true; + if diagnostic.suppressed { + return; + } + let replacement = self.suppressed_slots.last().copied().or_else(|| { + (diagnostic.severity == Severity::Error) + .then(|| self.warning_slots.last().copied()) + .flatten() + }); + let Some(position) = replacement else { + if diagnostic.severity == Severity::Error { + self.omitted_error = true; + } + return; + }; + // Once a lower-priority tier is interleaved with retained values, + // compact that entire tier. Subsequent lower-priority values are + // appended at the end and can be evicted without repeated shifts. + if position + 1 == self.values.len() { + self.values.pop(); + } else if self.values[position].suppressed { + self.values.retain(|value| !value.suppressed); + } else { + self.values + .retain(|value| value.suppressed || value.severity == Severity::Error); + } + self.values.push(diagnostic); + self.rebuild_slots(); + return; + } + let position = self.values.len(); + self.values.push(diagnostic); + self.record_slot(position); + } + + fn record_slot(&mut self, position: usize) { + let value = &self.values[position]; + if value.suppressed { + self.suppressed_slots.insert(position); + } else if value.severity != Severity::Error { + self.warning_slots.insert(position); + } + } + + fn rebuild_slots(&mut self) { + self.suppressed_slots.clear(); + self.warning_slots.clear(); + for (position, value) in self.values.iter().enumerate() { + if value.suppressed { + self.suppressed_slots.insert(position); + } else if value.severity != Severity::Error { + self.warning_slots.insert(position); + } + } + } + + pub fn mark_truncated(&mut self) { + self.truncated = true; } #[must_use] @@ -72,6 +340,12 @@ impl DiagnosticSet { &self.values } + pub fn active_values(&self) -> impl Iterator { + self.values + .iter() + .filter(|diagnostic| diagnostic.is_active()) + } + #[must_use] pub const fn truncated(&self) -> bool { self.truncated @@ -79,9 +353,10 @@ impl DiagnosticSet { #[must_use] pub fn has_errors(&self) -> bool { - self.values - .iter() - .any(|diagnostic| diagnostic.severity == Severity::Error) + self.omitted_error + || self + .active_values() + .any(|diagnostic| diagnostic.severity == Severity::Error) } } @@ -89,29 +364,207 @@ impl fmt::Display for Diagnostic { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { write!( formatter, - "{} {:?} at {}..{}: {}", - self.code, self.severity, self.span.start, self.span.end, self.message - ) + "{} {:?} at {}:{}..{}: {}", + self.code, + self.severity, + self.location.source, + self.location.span.start, + self.location.span.end, + self.message + )?; + if let Some(hint) = &self.fix_hint { + write!(formatter, " (hint: {hint})")?; + } + if self.suppressed { + write!(formatter, " [suppressed]")?; + } + Ok(()) + } +} + +fn valid_code(code: &str) -> bool { + let mut segments = code.split('.'); + let Some(first) = segments.next() else { + return false; + }; + valid_code_segment(first) && segments.all(valid_code_segment) +} + +fn valid_code_segment(segment: &str) -> bool { + !segment.is_empty() + && segment.as_bytes()[0].is_ascii_lowercase() + && segment + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') +} + +fn truncate_utf8(value: &mut String, maximum: usize) -> bool { + if value.len() <= maximum { + return false; } + let mut boundary = maximum; + while !value.is_char_boundary(boundary) { + boundary -= 1; + } + value.truncate(boundary); + true } #[cfg(test)] mod tests { - use super::{Diagnostic, DiagnosticSet, Severity, Span}; + use super::{ + Diagnostic, DiagnosticLimits, DiagnosticSet, Location, RelatedLocation, Severity, Span, + SuppressionError, SuppressionPolicy, + }; + + fn value(code: &'static str) -> Diagnostic { + Diagnostic::new( + code, + Severity::Error, + Location::new("fixture:test", Span::new(1, 2)), + "test diagnostic", + ) + } #[test] fn bounds_diagnostics_without_reordering() { let mut diagnostics = DiagnosticSet::new(1); - let value = Diagnostic { - code: "test", - severity: Severity::Error, - span: Span::new(1, 2), - message: "test diagnostic", - }; + let value = value("test.error"); diagnostics.push(value.clone()); diagnostics.push(value.clone()); assert_eq!(diagnostics.values(), &[value]); assert!(diagnostics.truncated()); assert!(diagnostics.has_errors()); } + + #[test] + fn preserves_structured_context_with_deterministic_bounds() { + let mut diagnostics = DiagnosticSet::with_limits(DiagnosticLimits { + maximum_diagnostics: 2, + maximum_related: 1, + maximum_semantic_depth: 1, + maximum_text_bytes: 8, + }); + diagnostics.push( + value("catalog.missing") + .with_related(RelatedLocation::new( + Location::new("fixture:related-long", Span::new(3, 4)), + "first related message", + )) + .with_related(RelatedLocation::new( + Location::new("fixture:second", Span::new(5, 6)), + "second", + )) + .with_semantic_path(["references", "target"]) + .with_fix_hint("define the target"), + ); + let value = &diagnostics.values()[0]; + assert_eq!(value.location.source, "fixture:"); + assert_eq!(value.related.len(), 1); + assert_eq!(value.related[0].location.source, "fixture:"); + assert_eq!(value.semantic_path, ["referenc"]); + assert_eq!(value.fix_hint.as_deref(), Some("define t")); + assert!(diagnostics.truncated()); + } + + #[test] + fn suppresses_only_diagnostics_that_explicitly_allow_it() { + let policy = SuppressionPolicy::new(["catalog.missing"], 4, 64).unwrap(); + let mut diagnostics = DiagnosticSet::new(4); + diagnostics.push_with_policy(value("catalog.missing").suppressible(true), &policy); + diagnostics.push_with_policy(value("catalog.conflict"), &policy); + assert!(diagnostics.values()[0].suppressed); + assert!(!diagnostics.values()[1].suppressed); + assert!(diagnostics.has_errors()); + assert_eq!(diagnostics.active_values().count(), 1); + } + + #[test] + fn active_error_displaces_a_suppressed_warning_at_capacity() { + let policy = SuppressionPolicy::new(["catalog.optional"], 4, 64).unwrap(); + let mut diagnostics = DiagnosticSet::new(1); + diagnostics.push_with_policy(value("catalog.optional").suppressible(true), &policy); + diagnostics.push_with_policy(value("catalog.required"), &policy); + assert_eq!(diagnostics.values()[0].code, "catalog.required"); + assert!(diagnostics.has_errors()); + assert!(diagnostics.truncated()); + } + + #[test] + fn active_error_displaces_warning_or_fails_closed_at_zero_capacity() { + let mut diagnostics = DiagnosticSet::new(2); + let mut warning = value("catalog.warning"); + warning.severity = Severity::Warning; + diagnostics.push(warning); + diagnostics.push(value("catalog.second")); + diagnostics.push(value("catalog.required")); + assert_eq!(diagnostics.values()[0].code, "catalog.second"); + assert_eq!(diagnostics.values()[1].code, "catalog.required"); + assert!(diagnostics.has_errors()); + + let mut zero = DiagnosticSet::new(0); + zero.push(value("catalog.required")); + assert!(zero.values().is_empty()); + assert!(zero.has_errors()); + assert!(zero.truncated()); + } + + #[test] + fn sustained_error_overflow_remains_bounded_and_fails_closed() { + let mut diagnostics = DiagnosticSet::new(2); + for _ in 0..10_000 { + diagnostics.push(value("catalog.required")); + } + assert_eq!(diagnostics.values().len(), 2); + assert!(diagnostics.has_errors()); + assert!(diagnostics.truncated()); + } + + #[test] + fn warning_then_error_overflow_preserves_producer_order_without_repeated_shifts() { + const MAXIMUM: usize = 4096; + let mut diagnostics = DiagnosticSet::new(MAXIMUM); + for index in 0..MAXIMUM { + let mut warning = value("catalog.warning"); + warning.severity = Severity::Warning; + warning.message = format!("warning-{index}"); + diagnostics.push(warning); + } + for index in 0..MAXIMUM { + let mut error = value("catalog.required"); + error.message = format!("error-{index}"); + diagnostics.push(error); + } + + assert_eq!(diagnostics.values().len(), MAXIMUM); + assert!( + diagnostics + .values() + .iter() + .all(|diagnostic| diagnostic.severity == Severity::Error) + ); + assert_eq!(diagnostics.values()[0].message, "error-0"); + assert_eq!( + diagnostics.values()[MAXIMUM - 1].message, + format!("error-{}", MAXIMUM - 1) + ); + assert!(diagnostics.has_errors()); + assert!(diagnostics.truncated()); + } + + #[test] + fn rejects_unbounded_or_malformed_suppression_input() { + assert_eq!( + SuppressionPolicy::new(["Bad Code"], 1, 64), + Err(SuppressionError::InvalidCode) + ); + assert_eq!( + SuppressionPolicy::new(["one.code", "two.code"], 1, 64), + Err(SuppressionError::LimitExceeded) + ); + assert_eq!( + SuppressionPolicy::new(["one.code", "one.code"], 1, 64), + Err(SuppressionError::LimitExceeded) + ); + } } diff --git a/crates/atrinik-schema/src/lib.rs b/crates/atrinik-schema/src/lib.rs index 1539e47..925d726 100644 --- a/crates/atrinik-schema/src/lib.rs +++ b/crates/atrinik-schema/src/lib.rs @@ -5,7 +5,7 @@ use std::{collections::BTreeSet, fmt}; -use atrinik_diagnostics::{Diagnostic, DiagnosticSet, Severity, Span}; +use atrinik_diagnostics::{Diagnostic, DiagnosticSet, Location, Severity, Span}; use atrinik_source::Document; pub const FOUNDATION_ARTIFACT_SCHEMA: &str = @@ -71,12 +71,16 @@ impl Schema { let mut diagnostics = DiagnosticSet::new(maximum_diagnostics); for required in &self.required_fields { if !present.contains(required.as_slice()) { - diagnostics.push(Diagnostic { - code: "schema.required_field", - severity: Severity::Error, - span: Span::new(0, 0), - message: "a required field is absent", - }); + diagnostics.push( + Diagnostic::new( + "schema.required_field", + Severity::Error, + Location::new(document.source_id().as_str(), Span::new(0, 0)), + "a required field is absent", + ) + .with_semantic_path([String::from_utf8_lossy(required).into_owned()]) + .with_fix_hint("add the required field"), + ); } } diagnostics diff --git a/crates/atrinik-source/src/lib.rs b/crates/atrinik-source/src/lib.rs index 59e9c83..8984a13 100644 --- a/crates/atrinik-source/src/lib.rs +++ b/crates/atrinik-source/src/lib.rs @@ -6,7 +6,7 @@ use std::{fmt, sync::Arc}; -use atrinik_diagnostics::{Diagnostic, DiagnosticSet, Severity, Span}; +use atrinik_diagnostics::{Diagnostic, DiagnosticSet, Location, Severity, Span}; use sha2::{Digest, Sha256}; #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -166,9 +166,12 @@ impl Document { start, content_end, newline, - &mut nesting, - &mut raw_block, - &mut diagnostics, + &mut ParseState { + source_id: source_id.as_str(), + nesting: &mut nesting, + raw_block: &mut raw_block, + diagnostics: &mut diagnostics, + }, ); let value_bytes = match record.kind { RecordKind::Field { value, .. } => value.len(), @@ -192,12 +195,12 @@ impl Document { } if raw_block { - diagnostics.push(Diagnostic { - code: "source.unclosed_raw_block", - severity: Severity::Error, - span: Span::new(source.len(), source.len()), - message: "msg block has no matching endmsg record", - }); + diagnostics.push(Diagnostic::new( + "source.unclosed_raw_block", + Severity::Error, + Location::new(source_id.as_str(), Span::new(source.len(), source.len())), + "msg block has no matching endmsg record", + )); } let revision = Revision(Sha256::digest(&source).into()); @@ -400,6 +403,13 @@ struct Preflight { newline_style: NewlineStyle, } +struct ParseState<'a> { + source_id: &'a str, + nesting: &'a mut usize, + raw_block: &'a mut bool, + diagnostics: &'a mut DiagnosticSet, +} + fn preflight(source: &[u8], limits: Limits) -> Result { if source.len() > limits.maximum_file_bytes { return Err(Error::LimitExceeded("file bytes")); @@ -462,15 +472,13 @@ fn parse_record( start: usize, content_end: usize, end: usize, - nesting: &mut usize, - raw_block: &mut bool, - diagnostics: &mut DiagnosticSet, + state: &mut ParseState<'_>, ) -> Record { let content = Span::new(start, content_end); let span = Span::new(start, end); - if *raw_block { + if *state.raw_block { if &source[start..content_end] == b"endmsg" { - *raw_block = false; + *state.raw_block = false; return Record { span, content, @@ -516,8 +524,8 @@ fn parse_record( }; } if &source[first..content_end] == b"end" { - if *nesting != 0 { - *nesting -= 1; + if *state.nesting != 0 { + *state.nesting -= 1; } return Record { span, @@ -540,12 +548,12 @@ fn parse_record( .count() + key_end; if !valid_key(&source[first..key_end]) { - diagnostics.push(Diagnostic { - code: "source.invalid_record", - severity: Severity::Error, - span: content, - message: "record must contain an ASCII key and a separated value", - }); + state.diagnostics.push(Diagnostic::new( + "source.invalid_record", + Severity::Error, + Location::new(state.source_id, content), + "record must contain an ASCII key and a separated value", + )); return Record { span, content, @@ -560,20 +568,20 @@ fn parse_record( if key_end == content_end { let key = Span::new(first, key_end); let kind = if &source[first..key_end] == b"msg" { - *raw_block = true; + *state.raw_block = true; RecordKind::RawBlockStart } else if &source[first..key_end] == b"Object" { - *nesting += 1; + *state.nesting += 1; RecordKind::ObjectStart { name: Span::new(content_end, content_end), } } else if &source[first..key_end] == b"endmsg" { - diagnostics.push(Diagnostic { - code: "source.unexpected_endmsg", - severity: Severity::Error, - span: key, - message: "endmsg has no matching msg record", - }); + state.diagnostics.push(Diagnostic::new( + "source.unexpected_endmsg", + Severity::Error, + Location::new(state.source_id, key), + "endmsg has no matching msg record", + )); RecordKind::RawBlockEnd } else { RecordKind::Directive @@ -606,7 +614,7 @@ fn parse_record( }, ]; let kind = if &source[first..key_end] == b"Object" { - *nesting += 1; + *state.nesting += 1; tokens[0].kind = TokenKind::ObjectStart; RecordKind::ObjectStart { name: value } } else { diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 88fbcfd..400737d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -28,6 +28,47 @@ bytes remain untouched. Malformed records are preserved with classified, bounded diagnostics. Semantic edits replace only a selected value span and must match the source SHA-256 revision. +## Catalog boundary + +The catalog consumes already parsed `CatalogDocument` values; it neither owns +resource payloads nor reads paths. A document records its `SourceId`, SHA-256 +revision, schema version, definitions, aliases, inheritance, typed reference +edges, bounded preview metadata, and opaque provenance/license evidence +references. Evidence references identify records for later policy checks; the +catalog makes no legal or ownership determination. + +Stable IDs have the form `domain:namespace/local-id`. Their domain and local ID +are never derived from directory enumeration, display text, or localization. +All externally observable traversal uses ordered maps and sets. Catalog +generations hash canonical semantic input order, including schema versions and +source revisions, so the same input is reproducible regardless of discovery +order. + +`LineDocumentLoader` is the common adapter boundary for all nine owned domains. +It maps selected parsed fields to aliases, inheritance, typed references, and +preview metadata without a second parse. Integrations can also construct +`CatalogDocument` directly when a domain has a different source grammar. +`resolve`, `search`, `preview`, and `dependents` are the shared query surface for +CLI, CI, and editor consumers. + +Document replacement and removal return a new immutable catalog plus an +`Invalidation`: changed canonical IDs and aliases, followed through both old +and new reverse-reference indexes to obtain transitive dependents. Identical +documents return the existing generation with empty invalidation. Limits cap +documents, definitions, aliases, references, preview values, strings, graph +work, diagnostics, and invalidation breadth before unbounded work can occur. + +## Diagnostic boundary + +Parser, schema, and catalog layers share `atrinik-diagnostics::Diagnostic`. +Stable machine codes accompany severity, an exact source span, related +locations, a semantic path, a human message, and an optional fix hint. +Suppression is allowlisted by code and only affects diagnostics whose producer +explicitly permits suppression; conflicts, ambiguity, and required missing +references remain active errors. A `DiagnosticSet` preserves deterministic +insertion order and reports truncation when any configured count, text, or +depth bound is reached. + The CLI requires explicit input, source identity, and new output paths. It preflights and bounded-reads inputs, validates before output, writes and syncs a same-directory temporary file, then atomically hard-links it into a previously