diff --git a/CHANGELOG.md b/CHANGELOG.md index a6f2417..90de4d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,15 @@ deprecations follow [ADR-14](docs/adr/ADR-14-deprecation-policy.md). ### Added +- Native, backend-neutral `LossSpec`, `MetricSpec`, role-reference and generic + implementation-descriptor contracts under ADR-22. New standalone v1 schemas, + strict TCV1 fingerprints, versioned built-in catalogs, compile-time loss + resolution, negative fixtures, an independent semantic oracle, an exact + conformance pack and CLI validators. A typed metric task/result protocol now + dispatches both built-in and host-local providers through one registry, + validates provider identity, finite values, scope and coverage, and reduces + results before native score persistence. This additive contract publication + does not change the public C ABI. - ADR-20 and W0 JSON contracts for conformal calibration ownership, `ParameterPatch`, `OutputBinding`, `TrainingInfluenceManifest`, complete `TrainingOutcome`/`ReplayOutcome` payloads, and the existing execution-bundle diff --git a/crates/dag-ml-cli/src/main.rs b/crates/dag-ml-cli/src/main.rs index cd3a6f2..cae7688 100644 --- a/crates/dag-ml-cli/src/main.rs +++ b/crates/dag-ml-cli/src/main.rs @@ -27,14 +27,15 @@ use dag_ml_core::{ ControllerRegistry, DagMlError, DataRequestPartition, ExecutionBundle, ExplanationBlock, ExternalDataPlanEnvelope, FileArtifactManifestStore, FileArtifactPayloadStore, FilePredictionCacheStore, GraphSpec, HandleKind, HandleRef, InMemoryArtifactStore, - InMemoryDataProvider, LineageId, LineageRecord, MetricObjective, NodeId, NodeResult, NodeTask, - OofCampaign, OperatorVariantModel, ParallelScheduler, Phase, PipelineDslSpec, - PortablePredictorPackage, PredictionBlock, PredictionLevel, PredictionPartition, - PredictionUnitId, RefitArtifactRecord, RegressionMetricKind, RegressionMetricReport, - RegressionTargetBlock, ReplayPhaseRequest, ResearchProvenancePackage, RunContext, RunId, - RuntimeArtifactStore, RuntimeController, RuntimeControllerRegistry, RuntimeDataProvider, - RuntimePredictionCacheStore, SampleId, ScoreSet, SelectionDecision, SelectionMetric, - SelectionPolicy, SequentialScheduler, TrainingRequest, VariantId, SCORE_SET_SCHEMA_VERSION, + InMemoryDataProvider, LineageId, LineageRecord, LossSpec, MetricObjective, MetricSpec, NodeId, + NodeResult, NodeTask, OofCampaign, OperatorVariantModel, ParallelScheduler, Phase, + PipelineDslSpec, PortablePredictorPackage, PredictionBlock, PredictionLevel, + PredictionPartition, PredictionUnitId, RefitArtifactRecord, RegressionMetricKind, + RegressionMetricReport, RegressionTargetBlock, ReplayPhaseRequest, ResearchProvenancePackage, + RunContext, RunId, RuntimeArtifactStore, RuntimeController, RuntimeControllerRegistry, + RuntimeDataProvider, RuntimePredictionCacheStore, SampleId, ScoreSet, SelectionDecision, + SelectionMetric, SelectionPolicy, SequentialScheduler, TrainingRequest, VariantId, + SCORE_SET_SCHEMA_VERSION, }; use serde::{Deserialize, Serialize}; @@ -220,6 +221,14 @@ enum Command { ValidateGraph { path: PathBuf, }, + /// Strictly validate a versioned training-loss semantic contract. + ValidateLossSpec { + path: PathBuf, + }, + /// Strictly validate a versioned metric semantic contract. + ValidateMetricSpec { + path: PathBuf, + }, /// Strictly parse, fingerprint and semantically validate a W1 TrainingRequest. ValidateTrainingRequest { path: PathBuf, @@ -882,6 +891,14 @@ fn main() -> Result<()> { let graph = read_external_contract(&path, "graph", GraphSpec::from_json)?; println!("valid graph: {}", graph.id); } + Command::ValidateLossSpec { path } => { + let spec = read_external_contract(&path, "loss spec", LossSpec::from_json)?; + println!("valid loss spec: {}", spec.loss_id); + } + Command::ValidateMetricSpec { path } => { + let spec = read_external_contract(&path, "metric spec", MetricSpec::from_json)?; + println!("valid metric spec: {}", spec.metric_id); + } Command::ValidateTrainingRequest { path } => { let json = std::fs::read_to_string(&path).with_context(|| { format!("failed to read training request JSON at {}", path.display()) diff --git a/crates/dag-ml-cli/tests/cli_contracts.rs b/crates/dag-ml-cli/tests/cli_contracts.rs index 6e83745..2d2d6a9 100644 --- a/crates/dag-ml-cli/tests/cli_contracts.rs +++ b/crates/dag-ml-cli/tests/cli_contracts.rs @@ -26,6 +26,80 @@ fn sha256_hex(bytes: &[u8]) -> String { out } +#[test] +fn cli_validates_loss_and_metric_specs_and_rejects_tampering() { + let root = repo_root(); + let fixture: serde_json::Value = serde_json::from_slice( + &std::fs::read(root.join("examples/fixtures/criteria/criteria_contracts.v1.json")) + .expect("criteria fixture is readable"), + ) + .expect("criteria fixture is JSON"); + + for (command, key, expected) in [ + ( + "validate-loss-spec", + "loss_spec", + "valid loss spec: example.loss.asymmetric@1", + ), + ( + "validate-metric-spec", + "metric_spec", + "valid metric spec: example.metric.bias@1", + ), + ] { + let path = std::env::temp_dir().join(format!( + "dag_ml_cli_{key}_{}_{}.json", + std::process::id(), + unique_suffix() + )); + std::fs::write( + &path, + serde_json::to_vec_pretty(&fixture["valid"][key]).unwrap(), + ) + .expect("write valid criteria spec"); + let output = Command::new(cli()) + .current_dir(&root) + .args([command, path.to_str().expect("temp path is valid utf-8")]) + .output() + .expect("run criteria validation command"); + assert!( + output.status.success(), + "{command} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains(expected)); + std::fs::remove_file(path).ok(); + } + + let invalid = fixture["invalid"] + .as_array() + .unwrap() + .iter() + .find(|case| case["id"] == "loss_mismatched_fingerprint") + .unwrap(); + let path = std::env::temp_dir().join(format!( + "dag_ml_cli_invalid_loss_{}_{}.json", + std::process::id(), + unique_suffix() + )); + std::fs::write( + &path, + serde_json::to_vec_pretty(&invalid["document"]).unwrap(), + ) + .expect("write invalid loss spec"); + let output = Command::new(cli()) + .current_dir(&root) + .args([ + "validate-loss-spec", + path.to_str().expect("temp path is valid utf-8"), + ]) + .output() + .expect("run invalid loss validation command"); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("fingerprint mismatch")); + std::fs::remove_file(path).ok(); +} + #[test] fn cli_compiles_pipeline_dsl_to_graph() { let root = repo_root(); diff --git a/crates/dag-ml-core/src/criteria.rs b/crates/dag-ml-core/src/criteria.rs new file mode 100644 index 0000000..81c9fe2 --- /dev/null +++ b/crates/dag-ml-core/src/criteria.rs @@ -0,0 +1,1547 @@ +//! Backend-neutral training-loss and metric semantic contracts. +//! +//! This module owns serializable semantics and implementation identity only. +//! Host controllers retain model objects, tensors, autodiff graphs and callback +//! objects. Process-local executable implementations are resolved by binding +//! registries from opaque [`ImplementationDescriptor::registry_key`] values. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; + +use crate::canonical::{ + deserialize_external_contract, parse_typed_json, validate_typed_serde_value, +}; +use crate::error::{DagMlError, Result}; +use crate::ids::NodeId; +use crate::oof::PredictionPartition; +use crate::phase::Phase; +use crate::policy::PredictionLevel; +use crate::selection::MetricObjective; +use crate::training::PredictionKind; + +pub const LOSS_SPEC_SCHEMA_VERSION: u32 = 1; +pub const METRIC_SPEC_SCHEMA_VERSION: u32 = 1; +pub const IMPLEMENTATION_DESCRIPTOR_SCHEMA_VERSION: u32 = 1; +pub const LOSS_ROLE_SCHEMA_VERSION: u32 = 1; +pub const METRIC_ROLE_SCHEMA_VERSION: u32 = 1; + +pub const LOSS_SPEC_SCHEMA_ID: &str = + "https://github.com/GBeurier/dag-ml/schemas/loss_spec.v1.schema.json"; +pub const METRIC_SPEC_SCHEMA_ID: &str = + "https://github.com/GBeurier/dag-ml/schemas/metric_spec.v1.schema.json"; +pub const IMPLEMENTATION_DESCRIPTOR_SCHEMA_ID: &str = + "https://github.com/GBeurier/dag-ml/schemas/implementation_descriptor.v1.schema.json"; +pub const TRAINING_LOSS_ROLE_SCHEMA_ID: &str = + "https://github.com/GBeurier/dag-ml/schemas/training_loss_role.v1.schema.json"; +pub const METRIC_ROLE_SCHEMA_ID: &str = + "https://github.com/GBeurier/dag-ml/schemas/metric_role.v1.schema.json"; + +const FINGERPRINT_LEN: usize = 64; +const FORBIDDEN_EXECUTABLE_KEYS: &[&str] = &[ + "bytecode", + "callable", + "code", + "function_source", + "import_path", + "module_path", + "pickle", + "serialized_callable", + "source_code", +]; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SemanticSpecKind { + BuiltIn, + Custom, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LearningTaskKind { + Regression, + BinaryClassification, + MulticlassClassification, + MultilabelClassification, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CriterionInput { + Target, + Prediction, + SampleWeight, + MissingMask, + Group, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LossReduction { + Mean, + Sum, + WeightedMean, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LossCapability { + Differentiable, + DistributedReduction, + PerOutput, + SupportsMissingMask, + SupportsSampleWeights, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MetricReduction { + Global, + Mean, + Sum, + WeightedMean, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MetricDecomposition { + Global, + PerOutput, + PerUnit, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MetricCapability { + Decomposable, + DistributedReduction, + SupportsMissingMask, + SupportsSampleWeights, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ImplementationSemanticKind { + Loss, + Metric, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ImplementationCapability { + Deterministic, + Differentiable, + DistributedReduction, + NeedsGil, + ProcessSafe, + SupportsMissingMask, + SupportsSampleWeights, + ThreadSafe, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PortabilityClass { + HostLocal, + PortableRegistered, + PortableBuiltIn, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReplayabilityClass { + ProcessLocal, + RegistryRequired, + Detached, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LossSpec { + pub schema_version: u32, + pub loss_id: String, + pub kind: SemanticSpecKind, + pub task_kinds: BTreeSet, + pub prediction_kinds: BTreeSet, + pub objective: MetricObjective, + pub reduction: LossReduction, + pub required_inputs: BTreeSet, + pub capabilities: BTreeSet, + pub parameters: serde_json::Value, + pub spec_fingerprint: String, +} + +impl LossSpec { + #[allow(clippy::too_many_arguments)] + pub fn new( + loss_id: impl Into, + kind: SemanticSpecKind, + task_kinds: BTreeSet, + prediction_kinds: BTreeSet, + reduction: LossReduction, + required_inputs: BTreeSet, + capabilities: BTreeSet, + parameters: serde_json::Value, + ) -> Result { + let mut spec = Self { + schema_version: LOSS_SPEC_SCHEMA_VERSION, + loss_id: loss_id.into(), + kind, + task_kinds, + prediction_kinds, + objective: MetricObjective::Minimize, + reduction, + required_inputs, + capabilities, + parameters, + spec_fingerprint: String::new(), + }; + spec.spec_fingerprint = spec.compute_fingerprint()?; + spec.validate()?; + Ok(spec) + } + + pub fn from_json(json: &str) -> Result { + let spec: Self = + deserialize_external_contract(json, "loss spec", DagMlError::CampaignValidation)?; + spec.validate()?; + Ok(spec) + } + + pub fn compute_fingerprint(&self) -> Result { + fingerprint_without(self, "spec_fingerprint", "loss spec") + } + + pub fn validate(&self) -> Result<()> { + validate_schema_version("loss spec", self.schema_version, LOSS_SPEC_SCHEMA_VERSION)?; + validate_versioned_id("loss", &self.loss_id)?; + validate_nonempty_set("loss task_kinds", &self.task_kinds)?; + validate_nonempty_set("loss prediction_kinds", &self.prediction_kinds)?; + if self.objective != MetricObjective::Minimize { + return contract_error("loss objective must be minimize in schema version 1"); + } + validate_required_target_prediction("loss", &self.required_inputs)?; + validate_parameters("loss", &self.parameters)?; + if self.reduction == LossReduction::WeightedMean + && !self.required_inputs.contains(&CriterionInput::SampleWeight) + { + return contract_error("weighted_mean loss requires sample_weight input"); + } + if self.required_inputs.contains(&CriterionInput::SampleWeight) + && !self + .capabilities + .contains(&LossCapability::SupportsSampleWeights) + { + return contract_error( + "loss requiring sample_weight must declare supports_sample_weights", + ); + } + if self.required_inputs.contains(&CriterionInput::MissingMask) + && !self + .capabilities + .contains(&LossCapability::SupportsMissingMask) + { + return contract_error( + "loss requiring missing_mask must declare supports_missing_mask", + ); + } + validate_fingerprint("loss spec", &self.spec_fingerprint)?; + let expected = self.compute_fingerprint()?; + if self.spec_fingerprint != expected { + return contract_error(format!( + "loss spec fingerprint mismatch: declared {}, expected {expected}", + self.spec_fingerprint + )); + } + Ok(()) + } + + pub fn validate_compatibility( + &self, + task_kind: LearningTaskKind, + prediction_kind: PredictionKind, + ) -> Result<()> { + self.validate()?; + if !self.task_kinds.contains(&task_kind) + || !self.prediction_kinds.contains(&prediction_kind) + { + return contract_error(format!( + "loss `{}` is not compatible with task {task_kind:?} and prediction {prediction_kind:?}", + self.loss_id + )); + } + Ok(()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MetricSpec { + pub schema_version: u32, + pub metric_id: String, + pub kind: SemanticSpecKind, + pub task_kinds: BTreeSet, + pub prediction_kinds: BTreeSet, + pub objective: MetricObjective, + pub supported_levels: BTreeSet, + pub decomposition: MetricDecomposition, + pub reduction: MetricReduction, + pub required_inputs: BTreeSet, + pub capabilities: BTreeSet, + pub parameters: serde_json::Value, + pub spec_fingerprint: String, +} + +impl MetricSpec { + #[allow(clippy::too_many_arguments)] + pub fn new( + metric_id: impl Into, + kind: SemanticSpecKind, + task_kinds: BTreeSet, + prediction_kinds: BTreeSet, + objective: MetricObjective, + supported_levels: BTreeSet, + decomposition: MetricDecomposition, + reduction: MetricReduction, + required_inputs: BTreeSet, + capabilities: BTreeSet, + parameters: serde_json::Value, + ) -> Result { + let mut spec = Self { + schema_version: METRIC_SPEC_SCHEMA_VERSION, + metric_id: metric_id.into(), + kind, + task_kinds, + prediction_kinds, + objective, + supported_levels, + decomposition, + reduction, + required_inputs, + capabilities, + parameters, + spec_fingerprint: String::new(), + }; + spec.spec_fingerprint = spec.compute_fingerprint()?; + spec.validate()?; + Ok(spec) + } + + pub fn from_json(json: &str) -> Result { + let spec: Self = + deserialize_external_contract(json, "metric spec", DagMlError::CampaignValidation)?; + spec.validate()?; + Ok(spec) + } + + pub fn compute_fingerprint(&self) -> Result { + fingerprint_without(self, "spec_fingerprint", "metric spec") + } + + pub fn validate(&self) -> Result<()> { + validate_schema_version( + "metric spec", + self.schema_version, + METRIC_SPEC_SCHEMA_VERSION, + )?; + validate_versioned_id("metric", &self.metric_id)?; + validate_nonempty_set("metric task_kinds", &self.task_kinds)?; + validate_nonempty_set("metric prediction_kinds", &self.prediction_kinds)?; + validate_nonempty_set("metric supported_levels", &self.supported_levels)?; + validate_required_target_prediction("metric", &self.required_inputs)?; + validate_parameters("metric", &self.parameters)?; + match (self.decomposition, self.reduction) { + (MetricDecomposition::Global, MetricReduction::Global) => {} + (MetricDecomposition::Global, _) => { + return contract_error("global metric decomposition requires global reduction"); + } + (_, MetricReduction::Global) => { + return contract_error("decomposed metric cannot use global reduction"); + } + _ => {} + } + if self.reduction == MetricReduction::WeightedMean + && !self.required_inputs.contains(&CriterionInput::SampleWeight) + { + return contract_error("weighted_mean metric requires sample_weight input"); + } + if self.reduction == MetricReduction::WeightedMean + && self.decomposition != MetricDecomposition::PerUnit + { + return contract_error("weighted_mean metric requires per_unit decomposition"); + } + if self.decomposition != MetricDecomposition::Global + && !self.capabilities.contains(&MetricCapability::Decomposable) + { + return contract_error("decomposed metric must declare decomposable capability"); + } + if self.required_inputs.contains(&CriterionInput::SampleWeight) + && !self + .capabilities + .contains(&MetricCapability::SupportsSampleWeights) + { + return contract_error( + "metric requiring sample_weight must declare supports_sample_weights", + ); + } + if self.required_inputs.contains(&CriterionInput::MissingMask) + && !self + .capabilities + .contains(&MetricCapability::SupportsMissingMask) + { + return contract_error( + "metric requiring missing_mask must declare supports_missing_mask", + ); + } + validate_fingerprint("metric spec", &self.spec_fingerprint)?; + let expected = self.compute_fingerprint()?; + if self.spec_fingerprint != expected { + return contract_error(format!( + "metric spec fingerprint mismatch: declared {}, expected {expected}", + self.spec_fingerprint + )); + } + Ok(()) + } + + pub fn validate_compatibility( + &self, + task_kind: LearningTaskKind, + prediction_kind: PredictionKind, + level: PredictionLevel, + ) -> Result<()> { + self.validate()?; + if !self.task_kinds.contains(&task_kind) + || !self.prediction_kinds.contains(&prediction_kind) + || !self.supported_levels.contains(&level) + { + return contract_error(format!( + "metric `{}` is not compatible with task {task_kind:?}, prediction {prediction_kind:?}, and level {level:?}", + self.metric_id + )); + } + Ok(()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ImplementationDescriptor { + pub schema_version: u32, + pub semantic_kind: ImplementationSemanticKind, + pub semantic_id: String, + pub semantic_fingerprint: String, + pub provider_id: String, + pub binding_id: String, + pub implementation_version: String, + pub implementation_fingerprint: String, + pub supported_controller_families: BTreeSet, + pub runtime_requirements: BTreeSet, + pub capabilities: BTreeSet, + pub portability: PortabilityClass, + pub replayability: ReplayabilityClass, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub registry_key: Option, + pub descriptor_fingerprint: String, +} + +impl ImplementationDescriptor { + #[allow(clippy::too_many_arguments)] + pub fn new( + semantic_kind: ImplementationSemanticKind, + semantic_id: impl Into, + semantic_fingerprint: impl Into, + provider_id: impl Into, + binding_id: impl Into, + implementation_version: impl Into, + implementation_fingerprint: impl Into, + supported_controller_families: BTreeSet, + runtime_requirements: BTreeSet, + capabilities: BTreeSet, + portability: PortabilityClass, + replayability: ReplayabilityClass, + registry_key: Option, + ) -> Result { + let mut descriptor = Self { + schema_version: IMPLEMENTATION_DESCRIPTOR_SCHEMA_VERSION, + semantic_kind, + semantic_id: semantic_id.into(), + semantic_fingerprint: semantic_fingerprint.into(), + provider_id: provider_id.into(), + binding_id: binding_id.into(), + implementation_version: implementation_version.into(), + implementation_fingerprint: implementation_fingerprint.into(), + supported_controller_families, + runtime_requirements, + capabilities, + portability, + replayability, + registry_key, + descriptor_fingerprint: String::new(), + }; + descriptor.descriptor_fingerprint = descriptor.compute_fingerprint()?; + descriptor.validate()?; + Ok(descriptor) + } + + pub fn from_json(json: &str) -> Result { + let descriptor: Self = deserialize_external_contract( + json, + "implementation descriptor", + DagMlError::CampaignValidation, + )?; + descriptor.validate()?; + Ok(descriptor) + } + + pub fn compute_fingerprint(&self) -> Result { + fingerprint_without(self, "descriptor_fingerprint", "implementation descriptor") + } + + pub fn validate(&self) -> Result<()> { + validate_schema_version( + "implementation descriptor", + self.schema_version, + IMPLEMENTATION_DESCRIPTOR_SCHEMA_VERSION, + )?; + validate_versioned_id("implementation semantic", &self.semantic_id)?; + validate_fingerprint("implementation semantic", &self.semantic_fingerprint)?; + validate_token("provider_id", &self.provider_id)?; + validate_token("binding_id", &self.binding_id)?; + validate_token("implementation_version", &self.implementation_version)?; + validate_fingerprint("implementation", &self.implementation_fingerprint)?; + validate_string_set( + "supported_controller_families", + &self.supported_controller_families, + )?; + validate_string_set("runtime_requirements", &self.runtime_requirements)?; + if let Some(registry_key) = &self.registry_key { + validate_token("registry_key", registry_key)?; + } + match self.portability { + PortabilityClass::HostLocal => { + if self.registry_key.is_none() { + return contract_error("host_local implementation requires registry_key"); + } + if self.replayability == ReplayabilityClass::Detached { + return contract_error( + "host_local implementation cannot be detached-replayable", + ); + } + } + PortabilityClass::PortableRegistered => { + if self.registry_key.is_none() { + return contract_error( + "portable_registered implementation requires registry_key", + ); + } + if self.replayability != ReplayabilityClass::RegistryRequired { + return contract_error( + "portable_registered implementation requires registry_required replay", + ); + } + } + PortabilityClass::PortableBuiltIn => { + if self.registry_key.is_some() { + return contract_error("portable_builtin implementation forbids registry_key"); + } + if self.replayability != ReplayabilityClass::Detached { + return contract_error( + "portable_builtin implementation must be detached-replayable", + ); + } + } + } + validate_fingerprint("implementation descriptor", &self.descriptor_fingerprint)?; + let expected = self.compute_fingerprint()?; + if self.descriptor_fingerprint != expected { + return contract_error(format!( + "implementation descriptor fingerprint mismatch: declared {}, expected {expected}", + self.descriptor_fingerprint + )); + } + Ok(()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LossReference { + pub spec: LossSpec, + pub implementation: ImplementationDescriptor, +} + +impl LossReference { + pub fn validate(&self) -> Result<()> { + self.spec.validate()?; + self.implementation.validate()?; + validate_descriptor_semantic( + &self.implementation, + ImplementationSemanticKind::Loss, + &self.spec.loss_id, + &self.spec.spec_fingerprint, + )?; + for (required, provided, label) in [ + ( + self.spec + .capabilities + .contains(&LossCapability::Differentiable), + self.implementation + .capabilities + .contains(&ImplementationCapability::Differentiable), + "differentiable", + ), + ( + self.spec + .capabilities + .contains(&LossCapability::SupportsSampleWeights), + self.implementation + .capabilities + .contains(&ImplementationCapability::SupportsSampleWeights), + "supports_sample_weights", + ), + ( + self.spec + .capabilities + .contains(&LossCapability::SupportsMissingMask), + self.implementation + .capabilities + .contains(&ImplementationCapability::SupportsMissingMask), + "supports_missing_mask", + ), + ( + self.spec + .capabilities + .contains(&LossCapability::DistributedReduction), + self.implementation + .capabilities + .contains(&ImplementationCapability::DistributedReduction), + "distributed_reduction", + ), + ] { + if required && !provided { + return contract_error(format!( + "loss implementation lacks required `{label}` capability" + )); + } + } + Ok(()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MetricReference { + pub spec: MetricSpec, + pub implementation: ImplementationDescriptor, +} + +impl MetricReference { + pub fn validate(&self) -> Result<()> { + self.spec.validate()?; + self.implementation.validate()?; + validate_descriptor_semantic( + &self.implementation, + ImplementationSemanticKind::Metric, + &self.spec.metric_id, + &self.spec.spec_fingerprint, + )?; + for (required, provided, label) in [ + ( + self.spec + .capabilities + .contains(&MetricCapability::SupportsSampleWeights), + self.implementation + .capabilities + .contains(&ImplementationCapability::SupportsSampleWeights), + "supports_sample_weights", + ), + ( + self.spec + .capabilities + .contains(&MetricCapability::SupportsMissingMask), + self.implementation + .capabilities + .contains(&ImplementationCapability::SupportsMissingMask), + "supports_missing_mask", + ), + ( + self.spec + .capabilities + .contains(&MetricCapability::DistributedReduction), + self.implementation + .capabilities + .contains(&ImplementationCapability::DistributedReduction), + "distributed_reduction", + ), + ] { + if required && !provided { + return contract_error(format!( + "metric implementation lacks required `{label}` capability" + )); + } + } + Ok(()) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TrainingLossRoleReference { + pub schema_version: u32, + pub node_id: NodeId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_id: Option, + pub phases: BTreeSet, + pub loss: LossReference, +} + +impl TrainingLossRoleReference { + pub fn from_json(json: &str) -> Result { + let role: Self = deserialize_external_contract( + json, + "training loss role", + DagMlError::CampaignValidation, + )?; + role.validate()?; + Ok(role) + } + + pub fn validate(&self) -> Result<()> { + validate_schema_version( + "training loss role", + self.schema_version, + LOSS_ROLE_SCHEMA_VERSION, + )?; + if let Some(output_id) = &self.output_id { + validate_token("loss output_id", output_id)?; + } + if self.phases.is_empty() + || self + .phases + .iter() + .any(|phase| !matches!(phase, Phase::FitCv | Phase::Refit)) + { + return contract_error( + "training loss phases must be a non-empty subset of FIT_CV/REFIT", + ); + } + self.loss.validate() + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MetricRoleKind { + EarlyStopping, + Selection, + Reporting, + Tuning, + Pruning, + Threshold, + EnsembleWeighting, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MissingMetricPolicy { + Error, + Skip, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MetricRoleReference { + pub schema_version: u32, + pub role_id: String, + pub role: MetricRoleKind, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_id: Option, + pub partition: PredictionPartition, + pub level: PredictionLevel, + pub missing_value_policy: MissingMetricPolicy, + pub metric: MetricReference, +} + +impl MetricRoleReference { + pub fn from_json(json: &str) -> Result { + let role: Self = + deserialize_external_contract(json, "metric role", DagMlError::CampaignValidation)?; + role.validate()?; + Ok(role) + } + + pub fn validate(&self) -> Result<()> { + validate_schema_version( + "metric role", + self.schema_version, + METRIC_ROLE_SCHEMA_VERSION, + )?; + validate_token("metric role_id", &self.role_id)?; + if let Some(output_id) = &self.output_id { + validate_token("metric output_id", output_id)?; + } + if self.missing_value_policy == MissingMetricPolicy::Skip + && self.role != MetricRoleKind::Reporting + { + return contract_error("only reporting metrics may skip missing values"); + } + if !self.metric.spec.supported_levels.contains(&self.level) { + return contract_error(format!( + "metric role level {:?} is not supported by `{}`", + self.level, self.metric.spec.metric_id + )); + } + self.metric.validate() + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LossResolutionSource { + ExplicitNodeOutput, + ControllerProfile, + CampaignDefault, + TaskFamilyDefault, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LossResolutionRequest { + pub task_kind: LearningTaskKind, + pub prediction_kind: PredictionKind, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub explicit_node_output: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub controller_profile: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub campaign_default: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub task_family_default: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ResolvedLossSpec { + pub source: LossResolutionSource, + pub spec: LossSpec, +} + +impl LossResolutionRequest { + pub fn resolve(&self) -> Result { + let candidates = [ + ( + LossResolutionSource::ExplicitNodeOutput, + self.explicit_node_output.as_ref(), + ), + ( + LossResolutionSource::ControllerProfile, + self.controller_profile.as_ref(), + ), + ( + LossResolutionSource::CampaignDefault, + self.campaign_default.as_ref(), + ), + ( + LossResolutionSource::TaskFamilyDefault, + self.task_family_default.as_ref(), + ), + ]; + let Some((source, spec)) = candidates + .into_iter() + .find_map(|(source, spec)| spec.map(|spec| (source, spec))) + else { + return contract_error(format!( + "no loss resolves for task {:?} and prediction {:?}", + self.task_kind, self.prediction_kind + )); + }; + spec.validate_compatibility(self.task_kind, self.prediction_kind)?; + Ok(ResolvedLossSpec { + source, + spec: spec.clone(), + }) + } +} + +pub fn builtin_loss_catalog() -> Result> { + let target_prediction = BTreeSet::from([CriterionInput::Target, CriterionInput::Prediction]); + let differentiable = BTreeSet::from([LossCapability::Differentiable]); + let regression = LossSpec::new( + "dagml.loss.mse@1", + SemanticSpecKind::BuiltIn, + BTreeSet::from([LearningTaskKind::Regression]), + BTreeSet::from([PredictionKind::RegressionPoint]), + LossReduction::Mean, + target_prediction.clone(), + differentiable.clone(), + empty_parameters(), + )?; + let binary = LossSpec::new( + "dagml.loss.binary_cross_entropy@1", + SemanticSpecKind::BuiltIn, + BTreeSet::from([ + LearningTaskKind::BinaryClassification, + LearningTaskKind::MultilabelClassification, + ]), + BTreeSet::from([PredictionKind::ClassLabel, PredictionKind::ClassProbability]), + LossReduction::Mean, + target_prediction.clone(), + differentiable.clone(), + empty_parameters(), + )?; + let multiclass = LossSpec::new( + "dagml.loss.sparse_categorical_cross_entropy@1", + SemanticSpecKind::BuiltIn, + BTreeSet::from([LearningTaskKind::MulticlassClassification]), + BTreeSet::from([PredictionKind::ClassLabel, PredictionKind::ClassProbability]), + LossReduction::Mean, + target_prediction, + differentiable, + empty_parameters(), + )?; + Ok([regression, binary, multiclass] + .into_iter() + .map(|spec| (spec.loss_id.clone(), spec)) + .collect()) +} + +pub fn builtin_metric_catalog() -> Result> { + let all_levels = BTreeSet::from([ + PredictionLevel::Observation, + PredictionLevel::Sample, + PredictionLevel::Target, + PredictionLevel::Group, + ]); + let target_prediction = BTreeSet::from([CriterionInput::Target, CriterionInput::Prediction]); + let decomposable = BTreeSet::from([MetricCapability::Decomposable]); + let mut specs = Vec::new(); + for (name, objective) in [ + ("mse", MetricObjective::Minimize), + ("rmse", MetricObjective::Minimize), + ("mae", MetricObjective::Minimize), + ("r2", MetricObjective::Maximize), + ] { + specs.push(MetricSpec::new( + format!("dagml.metric.{name}@1"), + SemanticSpecKind::BuiltIn, + BTreeSet::from([LearningTaskKind::Regression]), + BTreeSet::from([PredictionKind::RegressionPoint]), + objective, + all_levels.clone(), + MetricDecomposition::PerOutput, + MetricReduction::Mean, + target_prediction.clone(), + decomposable.clone(), + empty_parameters(), + )?); + } + for (name, tasks) in [ + ( + "accuracy", + BTreeSet::from([ + LearningTaskKind::BinaryClassification, + LearningTaskKind::MulticlassClassification, + LearningTaskKind::MultilabelClassification, + ]), + ), + ( + "balanced_accuracy", + BTreeSet::from([ + LearningTaskKind::BinaryClassification, + LearningTaskKind::MulticlassClassification, + ]), + ), + ] { + specs.push(MetricSpec::new( + format!("dagml.metric.{name}@1"), + SemanticSpecKind::BuiltIn, + tasks, + BTreeSet::from([PredictionKind::ClassLabel]), + MetricObjective::Maximize, + all_levels.clone(), + MetricDecomposition::PerOutput, + MetricReduction::Mean, + target_prediction.clone(), + decomposable.clone(), + empty_parameters(), + )?); + } + Ok(specs + .into_iter() + .map(|spec| (spec.metric_id.clone(), spec)) + .collect()) +} + +fn empty_parameters() -> serde_json::Value { + serde_json::Value::Object(serde_json::Map::new()) +} + +fn validate_schema_version(label: &str, actual: u32, expected: u32) -> Result<()> { + if actual != expected { + return contract_error(format!( + "{label} schema_version {actual} is unsupported (expected {expected})" + )); + } + Ok(()) +} + +fn validate_versioned_id(label: &str, value: &str) -> Result<()> { + validate_token(&format!("{label}_id"), value)?; + let Some((base, version)) = value.rsplit_once('@') else { + return contract_error(format!("{label} id `{value}` is not versioned")); + }; + if base.is_empty() + || version.is_empty() + || version.starts_with('0') + || !version.bytes().all(|byte| byte.is_ascii_digit()) + || base.contains('@') + { + return contract_error(format!( + "{label} id `{value}` must end in exactly one positive `@` suffix" + )); + } + Ok(()) +} + +pub(crate) fn validate_token(label: &str, value: &str) -> Result<()> { + if value.is_empty() + || value.trim() != value + || value.chars().any(char::is_whitespace) + || value.chars().any(char::is_control) + { + return contract_error(format!("{label} must be non-blank canonical text")); + } + Ok(()) +} + +fn validate_string_set(label: &str, values: &BTreeSet) -> Result<()> { + for value in values { + validate_token(label, value)?; + } + Ok(()) +} + +fn validate_nonempty_set(label: &str, values: &BTreeSet) -> Result<()> { + if values.is_empty() { + return contract_error(format!("{label} must be non-empty")); + } + Ok(()) +} + +fn validate_required_target_prediction( + label: &str, + required_inputs: &BTreeSet, +) -> Result<()> { + if !required_inputs.contains(&CriterionInput::Target) + || !required_inputs.contains(&CriterionInput::Prediction) + { + return contract_error(format!("{label} requires target and prediction inputs")); + } + Ok(()) +} + +fn validate_parameters(label: &str, parameters: &serde_json::Value) -> Result<()> { + if !parameters.is_object() { + return contract_error(format!("{label} parameters must be a JSON object")); + } + validate_typed_serde_value(parameters).map_err(|error| { + DagMlError::CampaignValidation(format!( + "{label} parameters are outside strict TCV1: {error}" + )) + })?; + validate_no_executable_payload(parameters, &format!("{label}.parameters")) +} + +fn validate_no_executable_payload(value: &serde_json::Value, path: &str) -> Result<()> { + match value { + serde_json::Value::Object(entries) => { + for (key, value) in entries { + let normalized_key = key.to_ascii_lowercase(); + if FORBIDDEN_EXECUTABLE_KEYS.contains(&normalized_key.as_str()) { + return contract_error(format!( + "{path}.{key} is an executable-code payload field" + )); + } + validate_no_executable_payload(value, &format!("{path}.{key}"))?; + } + } + serde_json::Value::Array(values) => { + for (index, value) in values.iter().enumerate() { + validate_no_executable_payload(value, &format!("{path}[{index}]"))?; + } + } + _ => {} + } + Ok(()) +} + +fn validate_descriptor_semantic( + descriptor: &ImplementationDescriptor, + expected_kind: ImplementationSemanticKind, + expected_id: &str, + expected_fingerprint: &str, +) -> Result<()> { + if descriptor.semantic_kind != expected_kind + || descriptor.semantic_id != expected_id + || descriptor.semantic_fingerprint != expected_fingerprint + { + return contract_error(format!( + "implementation descriptor semantic identity does not match {expected_kind:?} `{expected_id}`" + )); + } + Ok(()) +} + +pub(crate) fn validate_fingerprint(label: &str, value: &str) -> Result<()> { + if value.len() != FINGERPRINT_LEN + || !value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return contract_error(format!("{label} fingerprint must be lowercase sha256 hex")); + } + Ok(()) +} + +pub(crate) fn fingerprint_without( + value: &T, + field: &str, + label: &str, +) -> Result { + let json = serde_json::to_string(value)?; + parse_typed_json(&json) + .and_then(|value| value.fingerprint_without(field)) + .map_err(|error| { + DagMlError::CampaignValidation(format!( + "cannot compute {label} TCV1 fingerprint: {error}" + )) + }) +} + +fn contract_error(message: impl Into) -> Result { + Err(DagMlError::CampaignValidation(message.into())) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn custom_loss() -> LossSpec { + LossSpec::new( + "example.loss.asymmetric@1", + SemanticSpecKind::Custom, + BTreeSet::from([LearningTaskKind::Regression]), + BTreeSet::from([PredictionKind::RegressionPoint]), + LossReduction::Mean, + BTreeSet::from([CriterionInput::Target, CriterionInput::Prediction]), + BTreeSet::from([LossCapability::Differentiable]), + json!({"under_weight": 2.0, "over_weight": 1.0}), + ) + .unwrap() + } + + fn host_local_descriptor( + kind: ImplementationSemanticKind, + semantic_id: &str, + semantic_fingerprint: &str, + ) -> ImplementationDescriptor { + ImplementationDescriptor::new( + kind, + semantic_id, + semantic_fingerprint, + "provider:python-local", + "binding:python", + "1.0.0", + "1f4c71b0b758c5ed25b4e38b132b9ad56fb2f5ff2cf490f7eb8786c4350a62f7", + BTreeSet::new(), + BTreeSet::new(), + BTreeSet::from([ + ImplementationCapability::Deterministic, + ImplementationCapability::Differentiable, + ImplementationCapability::NeedsGil, + ]), + PortabilityClass::HostLocal, + ReplayabilityClass::RegistryRequired, + Some("loss:run-123:asymmetric".to_string()), + ) + .unwrap() + } + + #[test] + fn built_in_catalogs_are_versioned_and_self_fingerprinted() { + let losses = builtin_loss_catalog().unwrap(); + assert_eq!(losses.len(), 3); + assert!(losses.contains_key("dagml.loss.mse@1")); + assert!(losses.values().all(|spec| spec.validate().is_ok())); + + let metrics = builtin_metric_catalog().unwrap(); + assert_eq!(metrics.len(), 6); + assert_eq!( + metrics["dagml.metric.rmse@1"].objective, + MetricObjective::Minimize + ); + assert_eq!( + metrics["dagml.metric.balanced_accuracy@1"].objective, + MetricObjective::Maximize + ); + assert!(metrics.values().all(|spec| spec.validate().is_ok())); + } + + #[test] + fn loss_resolution_uses_declared_priority_and_never_falls_back_on_incompatibility() { + let builtins = builtin_loss_catalog().unwrap(); + let explicit = custom_loss(); + let request = LossResolutionRequest { + task_kind: LearningTaskKind::Regression, + prediction_kind: PredictionKind::RegressionPoint, + explicit_node_output: Some(explicit.clone()), + controller_profile: Some(builtins["dagml.loss.mse@1"].clone()), + campaign_default: None, + task_family_default: None, + }; + let resolved = request.resolve().unwrap(); + assert_eq!(resolved.source, LossResolutionSource::ExplicitNodeOutput); + assert_eq!(resolved.spec, explicit); + + let mut incompatible = request; + incompatible.task_kind = LearningTaskKind::BinaryClassification; + let error = incompatible.resolve().unwrap_err().to_string(); + assert!(error.contains("not compatible")); + } + + #[test] + fn empty_unversioned_and_tampered_loss_specs_are_rejected() { + let mut spec = custom_loss(); + spec.loss_id = "example.loss.asymmetric".to_string(); + assert!(spec + .validate() + .unwrap_err() + .to_string() + .contains("not versioned")); + + let mut spec = custom_loss(); + spec.task_kinds.clear(); + assert!(spec + .validate() + .unwrap_err() + .to_string() + .contains("non-empty")); + + let mut spec = custom_loss(); + spec.spec_fingerprint = "0".repeat(64); + assert!(spec + .validate() + .unwrap_err() + .to_string() + .contains("fingerprint mismatch")); + } + + #[test] + fn versioned_ids_are_canonical_decimal_without_an_artificial_width_limit() { + let mut leading_zero = custom_loss(); + leading_zero.loss_id = "example.loss.asymmetric@01".to_string(); + leading_zero.spec_fingerprint = leading_zero.compute_fingerprint().unwrap(); + assert!(leading_zero + .validate() + .unwrap_err() + .to_string() + .contains("positive `@` suffix")); + + let mut large_version = custom_loss(); + large_version.loss_id = "example.loss.asymmetric@4294967296".to_string(); + large_version.spec_fingerprint = large_version.compute_fingerprint().unwrap(); + large_version.validate().unwrap(); + } + + #[test] + fn executable_payloads_and_nfc_colliding_parameters_are_rejected() { + let error = LossSpec::new( + "example.loss.code@1", + SemanticSpecKind::Custom, + BTreeSet::from([LearningTaskKind::Regression]), + BTreeSet::from([PredictionKind::RegressionPoint]), + LossReduction::Mean, + BTreeSet::from([CriterionInput::Target, CriterionInput::Prediction]), + BTreeSet::new(), + json!({"callable": "lambda y, p: 0"}), + ) + .unwrap_err() + .to_string(); + assert!(error.contains("executable-code payload")); + + let mut values = serde_json::Map::new(); + values.insert("é".to_string(), json!(1)); + values.insert("e\u{301}".to_string(), json!(2)); + let error = LossSpec::new( + "example.loss.nfc@1", + SemanticSpecKind::Custom, + BTreeSet::from([LearningTaskKind::Regression]), + BTreeSet::from([PredictionKind::RegressionPoint]), + LossReduction::Mean, + BTreeSet::from([CriterionInput::Target, CriterionInput::Prediction]), + BTreeSet::new(), + serde_json::Value::Object(values), + ) + .unwrap_err() + .to_string(); + assert!(error.contains("NFC-colliding")); + } + + #[test] + fn reduction_and_input_capabilities_are_consistent() { + let error = LossSpec::new( + "example.loss.weighted@1", + SemanticSpecKind::Custom, + BTreeSet::from([LearningTaskKind::Regression]), + BTreeSet::from([PredictionKind::RegressionPoint]), + LossReduction::WeightedMean, + BTreeSet::from([CriterionInput::Target, CriterionInput::Prediction]), + BTreeSet::new(), + empty_parameters(), + ) + .unwrap_err() + .to_string(); + assert!(error.contains("requires sample_weight")); + + let error = MetricSpec::new( + "example.metric.bad-decomposition@1", + SemanticSpecKind::Custom, + BTreeSet::from([LearningTaskKind::Regression]), + BTreeSet::from([PredictionKind::RegressionPoint]), + MetricObjective::Minimize, + BTreeSet::from([PredictionLevel::Sample]), + MetricDecomposition::PerUnit, + MetricReduction::Global, + BTreeSet::from([CriterionInput::Target, CriterionInput::Prediction]), + BTreeSet::from([MetricCapability::Decomposable]), + empty_parameters(), + ) + .unwrap_err() + .to_string(); + assert!(error.contains("cannot use global reduction")); + + let error = MetricSpec::new( + "example.metric.bad-weighted-output@1", + SemanticSpecKind::Custom, + BTreeSet::from([LearningTaskKind::Regression]), + BTreeSet::from([PredictionKind::RegressionPoint]), + MetricObjective::Minimize, + BTreeSet::from([PredictionLevel::Sample]), + MetricDecomposition::PerOutput, + MetricReduction::WeightedMean, + BTreeSet::from([ + CriterionInput::Target, + CriterionInput::Prediction, + CriterionInput::SampleWeight, + ]), + BTreeSet::from([ + MetricCapability::Decomposable, + MetricCapability::SupportsSampleWeights, + ]), + empty_parameters(), + ) + .unwrap_err() + .to_string(); + assert!(error.contains("requires per_unit decomposition")); + } + + #[test] + fn one_descriptor_contract_binds_loss_and_metric_without_merging_semantics() { + let loss = custom_loss(); + let loss_reference = LossReference { + implementation: host_local_descriptor( + ImplementationSemanticKind::Loss, + &loss.loss_id, + &loss.spec_fingerprint, + ), + spec: loss, + }; + loss_reference.validate().unwrap(); + + let metric = MetricSpec::new( + "example.metric.bias@1", + SemanticSpecKind::Custom, + BTreeSet::from([LearningTaskKind::Regression]), + BTreeSet::from([PredictionKind::RegressionPoint]), + MetricObjective::Minimize, + BTreeSet::from([PredictionLevel::Sample]), + MetricDecomposition::Global, + MetricReduction::Global, + BTreeSet::from([CriterionInput::Target, CriterionInput::Prediction]), + BTreeSet::new(), + empty_parameters(), + ) + .unwrap(); + let metric_reference = MetricReference { + implementation: host_local_descriptor( + ImplementationSemanticKind::Metric, + &metric.metric_id, + &metric.spec_fingerprint, + ), + spec: metric, + }; + metric_reference.validate().unwrap(); + assert_ne!( + loss_reference.implementation.semantic_kind, + metric_reference.implementation.semantic_kind + ); + } + + #[test] + fn local_descriptor_serializes_only_an_opaque_registry_key() { + let loss = custom_loss(); + let descriptor = host_local_descriptor( + ImplementationSemanticKind::Loss, + &loss.loss_id, + &loss.spec_fingerprint, + ); + let serialized = serde_json::to_string(&descriptor).unwrap(); + assert!(serialized.contains("loss:run-123:asymmetric")); + for forbidden in FORBIDDEN_EXECUTABLE_KEYS { + assert!(!serialized.contains(&format!("\"{forbidden}\""))); + } + } + + #[test] + fn semantic_reference_rejects_wrong_kind_or_fingerprint() { + let loss = custom_loss(); + let mut descriptor = host_local_descriptor( + ImplementationSemanticKind::Loss, + &loss.loss_id, + &loss.spec_fingerprint, + ); + descriptor.semantic_kind = ImplementationSemanticKind::Metric; + descriptor.descriptor_fingerprint = descriptor.compute_fingerprint().unwrap(); + let error = LossReference { + spec: loss, + implementation: descriptor, + } + .validate() + .unwrap_err() + .to_string(); + assert!(error.contains("semantic identity")); + } + + #[test] + fn roles_keep_training_loss_and_metric_policy_distinct() { + let loss = custom_loss(); + let loss_role = TrainingLossRoleReference { + schema_version: LOSS_ROLE_SCHEMA_VERSION, + node_id: NodeId::new("model:custom").unwrap(), + output_id: Some("prediction".to_string()), + phases: BTreeSet::from([Phase::FitCv, Phase::Refit]), + loss: LossReference { + implementation: host_local_descriptor( + ImplementationSemanticKind::Loss, + &loss.loss_id, + &loss.spec_fingerprint, + ), + spec: loss, + }, + }; + loss_role.validate().unwrap(); + + let metric = builtin_metric_catalog().unwrap()["dagml.metric.rmse@1"].clone(); + let metric_role = MetricRoleReference { + schema_version: METRIC_ROLE_SCHEMA_VERSION, + role_id: "selection:rmse".to_string(), + role: MetricRoleKind::Selection, + output_id: Some("prediction".to_string()), + partition: PredictionPartition::Validation, + level: PredictionLevel::Sample, + missing_value_policy: MissingMetricPolicy::Error, + metric: MetricReference { + implementation: ImplementationDescriptor::new( + ImplementationSemanticKind::Metric, + &metric.metric_id, + &metric.spec_fingerprint, + "provider:dag-ml-core", + "binding:rust", + "1.0.0", + "4991854599d650fd613dfd02b10d90a649ad7fec85f20a027d5e7b2a553f628b", + BTreeSet::new(), + BTreeSet::new(), + BTreeSet::from([ImplementationCapability::Deterministic]), + PortabilityClass::PortableBuiltIn, + ReplayabilityClass::Detached, + None, + ) + .unwrap(), + spec: metric, + }, + }; + metric_role.validate().unwrap(); + + let mut invalid_missing = metric_role; + invalid_missing.missing_value_policy = MissingMetricPolicy::Skip; + assert!(invalid_missing + .validate() + .unwrap_err() + .to_string() + .contains("only reporting")); + } + + #[test] + fn strict_json_rejects_duplicate_keys_before_deserialization() { + let valid = serde_json::to_string(&custom_loss()).unwrap(); + LossSpec::from_json(&valid).unwrap(); + + let duplicate = valid.replacen( + "\"schema_version\":1", + "\"schema_version\":1,\"schema_version\":1", + 1, + ); + assert!(LossSpec::from_json(&duplicate) + .unwrap_err() + .to_string() + .contains("duplicate JSON object key")); + } + + #[test] + fn published_criteria_fixture_matches_rust_contracts_and_negative_cases() { + let fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../../examples/fixtures/criteria/criteria_contracts.v1.json" + )) + .unwrap(); + let valid = fixture["valid"].as_object().unwrap(); + + let loss = LossSpec::from_json(&valid["loss_spec"].to_string()).unwrap(); + assert_eq!( + loss.spec_fingerprint, + "cf661225cc7137ab5ef9b87871ed5736a8479dd21587b7e17150c442b1e43eb0" + ); + let metric = MetricSpec::from_json(&valid["metric_spec"].to_string()).unwrap(); + assert_eq!( + metric.spec_fingerprint, + "be54e6824479b10c398c229169bd324387c2fbb932e8fa879e37eba7d8821006" + ); + for key in ["loss_implementation", "metric_implementation"] { + ImplementationDescriptor::from_json(&valid[key].to_string()).unwrap(); + } + TrainingLossRoleReference::from_json(&valid["training_loss_role"].to_string()).unwrap(); + MetricRoleReference::from_json(&valid["metric_role"].to_string()).unwrap(); + + for case in fixture["invalid"].as_array().unwrap() { + let document = case["document"].to_string(); + let result = match case["contract"].as_str().unwrap() { + "loss_spec" => LossSpec::from_json(&document).map(|_| ()), + "metric_spec" => MetricSpec::from_json(&document).map(|_| ()), + "implementation_descriptor" => { + ImplementationDescriptor::from_json(&document).map(|_| ()) + } + "training_loss_role" => TrainingLossRoleReference::from_json(&document).map(|_| ()), + "metric_role" => MetricRoleReference::from_json(&document).map(|_| ()), + contract => panic!("unknown criteria fixture contract `{contract}`"), + }; + assert!( + result.is_err(), + "negative case `{}` was accepted", + case["id"] + ); + } + } +} diff --git a/crates/dag-ml-core/src/lib.rs b/crates/dag-ml-core/src/lib.rs index d5aff3b..156e4e4 100644 --- a/crates/dag-ml-core/src/lib.rs +++ b/crates/dag-ml-core/src/lib.rs @@ -13,6 +13,7 @@ pub mod conformal; pub mod controller; pub mod controller_adapter; pub mod controller_registry; +pub mod criteria; pub mod data; pub mod dsl; pub mod error; @@ -20,6 +21,7 @@ pub mod fold; pub mod generation; pub mod graph; pub mod ids; +pub mod metric_provider; pub mod metrics; pub mod observability; pub mod oof; @@ -44,6 +46,7 @@ pub use conformal::*; pub use controller::*; pub use controller_adapter::*; pub use controller_registry::*; +pub use criteria::*; pub use data::*; pub use dsl::*; pub use error::{DagMlError, DagMlErrorDescriptor, Result}; @@ -51,6 +54,7 @@ pub use fold::*; pub use generation::*; pub use graph::*; pub use ids::*; +pub use metric_provider::*; pub use metrics::*; pub use observability::*; pub use oof::*; diff --git a/crates/dag-ml-core/src/metric_provider.rs b/crates/dag-ml-core/src/metric_provider.rs new file mode 100644 index 0000000..7c00d39 --- /dev/null +++ b/crates/dag-ml-core/src/metric_provider.rs @@ -0,0 +1,890 @@ +//! Typed metric-provider dispatch shared by native and binding-local implementations. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; + +use crate::aggregation::PredictionUnitId; +use crate::criteria::{ + builtin_metric_catalog, fingerprint_without, validate_fingerprint, validate_token, + CriterionInput, ImplementationCapability, ImplementationDescriptor, ImplementationSemanticKind, + LearningTaskKind, MetricDecomposition, MetricReduction, MetricReference, PortabilityClass, + ReplayabilityClass, +}; +use crate::error::{DagMlError, Result}; +use crate::ids::{FoldId, GroupId, NodeId, ObservationId, SampleId, TargetId, VariantId}; +use crate::metrics::{compute_metric_per_target, RegressionMetricKind}; +use crate::oof::PredictionPartition; +use crate::policy::PredictionLevel; +use crate::training::PredictionKind; + +pub const METRIC_EVALUATION_TASK_SCHEMA_VERSION: u32 = 1; +pub const METRIC_EVALUATION_RESULT_SCHEMA_VERSION: u32 = 1; +pub const METRIC_EVALUATION_TASK_SCHEMA_ID: &str = + "https://github.com/GBeurier/dag-ml/schemas/metric_evaluation_task.v1.schema.json"; +pub const METRIC_EVALUATION_RESULT_SCHEMA_ID: &str = + "https://github.com/GBeurier/dag-ml/schemas/metric_evaluation_result.v1.schema.json"; + +const BUILTIN_METRIC_IMPLEMENTATION_VERSION: &str = "metrics-v1"; +const BUILTIN_METRIC_IMPLEMENTATION_FINGERPRINT: &str = + "0aa68bb906c00c9fa433f411f44004d46b5a7f196f9932ca9c313fd184d81d19"; + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde( + rename_all = "snake_case", + tag = "level", + content = "id", + deny_unknown_fields +)] +pub enum MetricUnitId { + Observation(ObservationId), + Sample(SampleId), + Target(TargetId), + Group(GroupId), +} + +impl MetricUnitId { + pub fn level(&self) -> PredictionLevel { + match self { + Self::Observation(_) => PredictionLevel::Observation, + Self::Sample(_) => PredictionLevel::Sample, + Self::Target(_) => PredictionLevel::Target, + Self::Group(_) => PredictionLevel::Group, + } + } +} + +impl From<&PredictionUnitId> for MetricUnitId { + fn from(value: &PredictionUnitId) -> Self { + match value { + PredictionUnitId::Sample(id) => Self::Sample(id.clone()), + PredictionUnitId::Target(id) => Self::Target(id.clone()), + PredictionUnitId::Group(id) => Self::Group(id.clone()), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MetricEvaluationScope { + pub producer_node: NodeId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub producer_port: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prediction_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub variant_id: Option, + pub partition: PredictionPartition, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fold_id: Option, + pub level: PredictionLevel, +} + +impl MetricEvaluationScope { + fn validate(&self) -> Result<()> { + for (label, value) in [ + ("metric producer_port", self.producer_port.as_deref()), + ("metric prediction_id", self.prediction_id.as_deref()), + ] { + if let Some(value) = value { + validate_token(label, value)?; + } + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MetricEvaluationTask { + pub schema_version: u32, + pub request_id: String, + pub metric: MetricReference, + pub task_kind: LearningTaskKind, + pub prediction_kind: PredictionKind, + pub scope: MetricEvaluationScope, + pub unit_ids: Vec, + pub predictions: Vec>, + pub targets: Vec>, + pub output_ids: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sample_weights: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub missing_mask: Option>>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub group_ids: Option>, + pub task_fingerprint: String, +} + +impl MetricEvaluationTask { + #[allow(clippy::too_many_arguments)] + pub fn new( + request_id: impl Into, + metric: MetricReference, + task_kind: LearningTaskKind, + prediction_kind: PredictionKind, + scope: MetricEvaluationScope, + unit_ids: Vec, + predictions: Vec>, + targets: Vec>, + output_ids: Vec, + sample_weights: Option>, + missing_mask: Option>>, + group_ids: Option>, + ) -> Result { + let mut task = Self { + schema_version: METRIC_EVALUATION_TASK_SCHEMA_VERSION, + request_id: request_id.into(), + metric, + task_kind, + prediction_kind, + scope, + unit_ids, + predictions, + targets, + output_ids, + sample_weights, + missing_mask, + group_ids, + task_fingerprint: String::new(), + }; + task.task_fingerprint = task.compute_fingerprint()?; + task.validate()?; + Ok(task) + } + + pub fn from_json(json: &str) -> Result { + let task: Self = crate::canonical::deserialize_external_contract( + json, + "metric evaluation task", + DagMlError::CampaignValidation, + )?; + task.validate()?; + Ok(task) + } + + pub fn compute_fingerprint(&self) -> Result { + fingerprint_without(self, "task_fingerprint", "metric evaluation task") + } + + pub fn validate(&self) -> Result<()> { + if self.schema_version != METRIC_EVALUATION_TASK_SCHEMA_VERSION { + return task_error(format!( + "metric evaluation task schema_version {} is unsupported", + self.schema_version + )); + } + validate_token("metric request_id", &self.request_id)?; + self.metric.validate()?; + self.metric.spec.validate_compatibility( + self.task_kind, + self.prediction_kind, + self.scope.level, + )?; + self.scope.validate()?; + let row_count = self.unit_ids.len(); + if row_count == 0 { + return task_error("metric evaluation task has no units"); + } + if self + .unit_ids + .iter() + .any(|unit| unit.level() != self.scope.level) + { + return task_error("metric evaluation unit level does not match scope"); + } + if self.unit_ids.iter().collect::>().len() != row_count { + return task_error("metric evaluation task contains duplicate unit ids"); + } + let prediction_width = + validate_finite_matrix("metric predictions", &self.predictions, row_count)?; + let target_width = validate_finite_matrix("metric targets", &self.targets, row_count)?; + if self.output_ids.len() != target_width || self.output_ids.is_empty() { + return task_error(format!( + "metric output_ids length {} does not match target width {target_width}", + self.output_ids.len() + )); + } + let mut outputs = BTreeSet::new(); + for output_id in &self.output_ids { + validate_token("metric output_id", output_id)?; + if !outputs.insert(output_id) { + return task_error(format!("duplicate metric output_id `{output_id}`")); + } + } + if matches!( + self.prediction_kind, + PredictionKind::RegressionPoint | PredictionKind::ClassLabel + ) && prediction_width != target_width + { + return task_error(format!( + "metric prediction width {prediction_width} does not match target width {target_width}" + )); + } + validate_optional_inputs(self, row_count, target_width)?; + validate_fingerprint("metric evaluation task", &self.task_fingerprint)?; + let expected = self.compute_fingerprint()?; + if self.task_fingerprint != expected { + return task_error(format!( + "metric evaluation task fingerprint mismatch: declared {}, expected {expected}", + self.task_fingerprint + )); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MetricEvaluationValue { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub unit_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_id: Option, + pub value: f64, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MetricEvaluationResult { + pub schema_version: u32, + pub request_id: String, + pub semantic_id: String, + pub semantic_fingerprint: String, + pub implementation_fingerprint: String, + pub descriptor_fingerprint: String, + pub scope: MetricEvaluationScope, + pub values: Vec, + pub result_fingerprint: String, +} + +impl MetricEvaluationResult { + pub fn for_task( + task: &MetricEvaluationTask, + values: Vec, + ) -> Result { + let mut result = Self { + schema_version: METRIC_EVALUATION_RESULT_SCHEMA_VERSION, + request_id: task.request_id.clone(), + semantic_id: task.metric.spec.metric_id.clone(), + semantic_fingerprint: task.metric.spec.spec_fingerprint.clone(), + implementation_fingerprint: task + .metric + .implementation + .implementation_fingerprint + .clone(), + descriptor_fingerprint: task.metric.implementation.descriptor_fingerprint.clone(), + scope: task.scope.clone(), + values, + result_fingerprint: String::new(), + }; + result.result_fingerprint = result.compute_fingerprint()?; + result.validate_against(task)?; + Ok(result) + } + + pub fn from_json_for_task(json: &str, task: &MetricEvaluationTask) -> Result { + let result: Self = crate::canonical::deserialize_external_contract( + json, + "metric evaluation result", + DagMlError::RuntimeValidation, + )?; + result.validate_against(task)?; + Ok(result) + } + + pub fn compute_fingerprint(&self) -> Result { + fingerprint_without(self, "result_fingerprint", "metric evaluation result") + } + + pub fn validate_against(&self, task: &MetricEvaluationTask) -> Result<()> { + task.validate()?; + if self.schema_version != METRIC_EVALUATION_RESULT_SCHEMA_VERSION { + return result_error(format!( + "metric evaluation result schema_version {} is unsupported", + self.schema_version + )); + } + if self.request_id != task.request_id + || self.semantic_id != task.metric.spec.metric_id + || self.semantic_fingerprint != task.metric.spec.spec_fingerprint + || self.implementation_fingerprint + != task.metric.implementation.implementation_fingerprint + || self.descriptor_fingerprint != task.metric.implementation.descriptor_fingerprint + { + return result_error("metric provider identity/fingerprint does not match task"); + } + if self.scope != task.scope { + return result_error("metric provider result scope does not match task"); + } + if self.values.is_empty() { + return result_error("metric provider returned no values"); + } + if self.values.iter().any(|value| !value.value.is_finite()) { + return result_error("metric provider returned a non-finite value"); + } + validate_result_coverage(self, task)?; + validate_fingerprint("metric evaluation result", &self.result_fingerprint) + .map_err(|error| DagMlError::RuntimeValidation(error.to_string()))?; + let expected = self.compute_fingerprint()?; + if self.result_fingerprint != expected { + return result_error(format!( + "metric evaluation result fingerprint mismatch: declared {}, expected {expected}", + self.result_fingerprint + )); + } + Ok(()) + } + + fn reduce(&self, task: &MetricEvaluationTask) -> Result { + let value = match task.metric.spec.reduction { + MetricReduction::Global => self.values[0].value, + MetricReduction::Mean => { + self.values.iter().map(|value| value.value).sum::() / self.values.len() as f64 + } + MetricReduction::Sum => self.values.iter().map(|value| value.value).sum(), + MetricReduction::WeightedMean => { + let weights = task.sample_weights.as_ref().ok_or_else(|| { + DagMlError::RuntimeValidation( + "weighted metric reduction has no sample weights".to_string(), + ) + })?; + let weighted_sum = self + .values + .iter() + .zip(weights) + .map(|(value, weight)| value.value * weight) + .sum::(); + weighted_sum / weights.iter().sum::() + } + }; + if !value.is_finite() { + return result_error("metric reduction produced a non-finite value"); + } + Ok(value) + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ValidatedMetricEvaluation { + pub result: MetricEvaluationResult, + pub aggregate: f64, +} + +pub trait MetricProvider: Send + Sync { + fn evaluate(&self, task: &MetricEvaluationTask) -> Result; +} + +struct RegisteredMetricProvider { + descriptor: ImplementationDescriptor, + provider: Arc, +} + +#[derive(Default)] +pub struct MetricProviderRegistry { + providers: BTreeMap, +} + +impl MetricProviderRegistry { + pub fn register( + &mut self, + descriptor: ImplementationDescriptor, + provider: Arc, + ) -> Result<()> { + descriptor.validate()?; + if descriptor.semantic_kind != ImplementationSemanticKind::Metric { + return task_error("metric provider registry rejects non-metric descriptor"); + } + let key = provider_dispatch_key(&descriptor); + if self.providers.contains_key(&key) { + return task_error(format!("duplicate metric provider registry key `{key}`")); + } + self.providers.insert( + key, + RegisteredMetricProvider { + descriptor, + provider, + }, + ); + Ok(()) + } + + pub fn evaluate(&self, task: &MetricEvaluationTask) -> Result { + task.validate()?; + let key = provider_dispatch_key(&task.metric.implementation); + let registered = self.providers.get(&key).ok_or_else(|| { + DagMlError::RuntimeValidation(format!( + "metric provider registry has no implementation for `{key}`" + )) + })?; + if registered.descriptor != task.metric.implementation { + return result_error("registered metric provider descriptor does not match task"); + } + let result = registered.provider.evaluate(task)?; + result.validate_against(task)?; + let aggregate = result.reduce(task)?; + Ok(ValidatedMetricEvaluation { result, aggregate }) + } +} + +fn provider_dispatch_key(descriptor: &ImplementationDescriptor) -> String { + descriptor + .registry_key + .clone() + .unwrap_or_else(|| format!("portable_builtin:{}", descriptor.descriptor_fingerprint)) +} + +pub fn builtin_metric_reference(metric: RegressionMetricKind) -> Result { + let metric_id = format!("dagml.metric.{}@1", metric.name()); + let spec = builtin_metric_catalog()? + .remove(&metric_id) + .ok_or_else(|| { + DagMlError::CampaignValidation(format!("missing `{metric_id}` catalog entry")) + })?; + let implementation = ImplementationDescriptor::new( + ImplementationSemanticKind::Metric, + &spec.metric_id, + &spec.spec_fingerprint, + "provider:dag-ml-core", + "binding:rust", + BUILTIN_METRIC_IMPLEMENTATION_VERSION, + BUILTIN_METRIC_IMPLEMENTATION_FINGERPRINT, + BTreeSet::new(), + BTreeSet::new(), + BTreeSet::from([ImplementationCapability::Deterministic]), + PortabilityClass::PortableBuiltIn, + ReplayabilityClass::Detached, + None, + )?; + let reference = MetricReference { + spec, + implementation, + }; + reference.validate()?; + Ok(reference) +} + +pub fn builtin_metric_registry() -> Result { + let mut registry = MetricProviderRegistry::default(); + for metric in [ + RegressionMetricKind::Mse, + RegressionMetricKind::Rmse, + RegressionMetricKind::Mae, + RegressionMetricKind::R2, + RegressionMetricKind::Accuracy, + RegressionMetricKind::BalancedAccuracy, + ] { + let reference = builtin_metric_reference(metric)?; + registry.register( + reference.implementation, + Arc::new(BuiltinMetricProvider { metric }), + )?; + } + Ok(registry) +} + +struct BuiltinMetricProvider { + metric: RegressionMetricKind, +} + +impl MetricProvider for BuiltinMetricProvider { + fn evaluate(&self, task: &MetricEvaluationTask) -> Result { + let expected_id = format!("dagml.metric.{}@1", self.metric.name()); + if task.metric.spec.metric_id != expected_id { + return result_error(format!( + "built-in provider `{}` cannot evaluate `{}`", + self.metric.name(), + task.metric.spec.metric_id + )); + } + let predictions = task + .predictions + .iter() + .map(Vec::as_slice) + .collect::>(); + let targets = task.targets.iter().map(Vec::as_slice).collect::>(); + let values = + compute_metric_per_target(self.metric, task.output_ids.len(), &predictions, &targets) + .into_iter() + .zip(&task.output_ids) + .map(|(value, output_id)| MetricEvaluationValue { + unit_id: None, + output_id: Some(output_id.clone()), + value, + }) + .collect(); + MetricEvaluationResult::for_task(task, values) + } +} + +fn validate_finite_matrix(label: &str, values: &[Vec], expected_rows: usize) -> Result { + if values.len() != expected_rows { + return task_error(format!( + "{label} has {} rows for {expected_rows} units", + values.len() + )); + } + let width = values.first().map_or(0, Vec::len); + if width == 0 || values.iter().any(|row| row.len() != width) { + return task_error(format!("{label} is empty or ragged")); + } + if values.iter().flatten().any(|value| !value.is_finite()) { + return task_error(format!("{label} contains non-finite values")); + } + Ok(width) +} + +fn validate_optional_inputs( + task: &MetricEvaluationTask, + row_count: usize, + target_width: usize, +) -> Result<()> { + let required = &task.metric.spec.required_inputs; + match &task.sample_weights { + Some(weights) => { + if !task + .metric + .spec + .capabilities + .contains(&crate::criteria::MetricCapability::SupportsSampleWeights) + { + return task_error("metric task supplies unsupported sample weights"); + } + if weights.len() != row_count + || weights + .iter() + .any(|weight| !weight.is_finite() || *weight < 0.0) + || weights.iter().sum::() <= 0.0 + { + return task_error("metric sample weights are invalid"); + } + } + None if required.contains(&CriterionInput::SampleWeight) => { + return task_error("metric task is missing required sample weights"); + } + None => {} + } + match &task.missing_mask { + Some(mask) => { + if !task + .metric + .spec + .capabilities + .contains(&crate::criteria::MetricCapability::SupportsMissingMask) + { + return task_error("metric task supplies unsupported missing mask"); + } + if mask.len() != row_count || mask.iter().any(|row| row.len() != target_width) { + return task_error("metric missing mask shape does not match targets"); + } + } + None if required.contains(&CriterionInput::MissingMask) => { + return task_error("metric task is missing required missing mask"); + } + None => {} + } + match &task.group_ids { + Some(group_ids) => { + if !required.contains(&CriterionInput::Group) { + return task_error("metric task supplies undeclared group ids"); + } + if group_ids.len() != row_count { + return task_error("metric group_ids length does not match units"); + } + for group_id in group_ids { + validate_token("metric group_id", group_id)?; + } + } + None if required.contains(&CriterionInput::Group) => { + return task_error("metric task is missing required group ids"); + } + None => {} + } + Ok(()) +} + +fn validate_result_coverage( + result: &MetricEvaluationResult, + task: &MetricEvaluationTask, +) -> Result<()> { + match task.metric.spec.decomposition { + MetricDecomposition::Global => { + if result.values.len() != 1 + || result.values[0].unit_id.is_some() + || result.values[0].output_id.is_some() + { + return result_error("global metric provider result has wrong coverage"); + } + } + MetricDecomposition::PerOutput => { + if result.values.len() != task.output_ids.len() { + return result_error("per-output metric provider result has wrong coverage"); + } + for (value, output_id) in result.values.iter().zip(&task.output_ids) { + if value.unit_id.is_some() || value.output_id.as_ref() != Some(output_id) { + return result_error("per-output metric provider result has wrong scope/order"); + } + } + } + MetricDecomposition::PerUnit => { + if result.values.len() != task.unit_ids.len() { + return result_error("per-unit metric provider result has wrong coverage"); + } + for (value, unit_id) in result.values.iter().zip(&task.unit_ids) { + if value.output_id.is_some() || value.unit_id.as_ref() != Some(unit_id) { + return result_error("per-unit metric provider result has wrong scope/order"); + } + } + } + } + Ok(()) +} + +fn task_error(message: impl Into) -> Result { + Err(DagMlError::CampaignValidation(message.into())) +} + +fn result_error(message: impl Into) -> Result { + Err(DagMlError::RuntimeValidation(message.into())) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + use crate::criteria::{ + ImplementationSemanticKind, MetricCapability, MetricSpec, SemanticSpecKind, + }; + use crate::selection::MetricObjective; + + fn sample_scope() -> MetricEvaluationScope { + MetricEvaluationScope { + producer_node: NodeId::new("model:custom").unwrap(), + producer_port: Some("prediction".to_string()), + prediction_id: Some("prediction:validation".to_string()), + variant_id: None, + partition: PredictionPartition::Validation, + fold_id: Some(FoldId::new("fold:0").unwrap()), + level: PredictionLevel::Sample, + } + } + + fn custom_bias_reference() -> MetricReference { + let spec = MetricSpec::new( + "example.metric.bias@1", + SemanticSpecKind::Custom, + BTreeSet::from([LearningTaskKind::Regression]), + BTreeSet::from([PredictionKind::RegressionPoint]), + MetricObjective::Minimize, + BTreeSet::from([PredictionLevel::Sample]), + MetricDecomposition::PerUnit, + MetricReduction::Mean, + BTreeSet::from([CriterionInput::Target, CriterionInput::Prediction]), + BTreeSet::from([MetricCapability::Decomposable]), + json!({}), + ) + .unwrap(); + let implementation = ImplementationDescriptor::new( + ImplementationSemanticKind::Metric, + &spec.metric_id, + &spec.spec_fingerprint, + "provider:rust-local", + "binding:rust", + "1.0.0", + "4991854599d650fd613dfd02b10d90a649ad7fec85f20a027d5e7b2a553f628b", + BTreeSet::new(), + BTreeSet::new(), + BTreeSet::from([ImplementationCapability::Deterministic]), + PortabilityClass::HostLocal, + ReplayabilityClass::RegistryRequired, + Some("metric:run-123:bias".to_string()), + ) + .unwrap(); + MetricReference { + spec, + implementation, + } + } + + fn custom_task() -> MetricEvaluationTask { + MetricEvaluationTask::new( + "metric-request:bias", + custom_bias_reference(), + LearningTaskKind::Regression, + PredictionKind::RegressionPoint, + sample_scope(), + vec![ + MetricUnitId::Sample(SampleId::new("sample:0").unwrap()), + MetricUnitId::Sample(SampleId::new("sample:1").unwrap()), + ], + vec![vec![2.0], vec![5.0]], + vec![vec![1.0], vec![3.0]], + vec!["target".to_string()], + None, + None, + None, + ) + .unwrap() + } + + struct BiasProvider; + + impl MetricProvider for BiasProvider { + fn evaluate(&self, task: &MetricEvaluationTask) -> Result { + let values = task + .unit_ids + .iter() + .zip(task.predictions.iter().zip(&task.targets)) + .map(|(unit_id, (prediction, target))| MetricEvaluationValue { + unit_id: Some(unit_id.clone()), + output_id: None, + value: prediction[0] - target[0], + }) + .collect(); + MetricEvaluationResult::for_task(task, values) + } + } + + #[test] + fn custom_metric_registry_executes_and_reduces_provider_values() { + let task = custom_task(); + let mut registry = MetricProviderRegistry::default(); + registry + .register(task.metric.implementation.clone(), Arc::new(BiasProvider)) + .unwrap(); + let evaluation = registry.evaluate(&task).unwrap(); + assert_eq!(evaluation.aggregate, 1.5); + assert_eq!(evaluation.result.values.len(), 2); + } + + #[test] + fn task_rejects_custom_metric_without_objective() { + let task = custom_task(); + let mut value = serde_json::to_value(task).unwrap(); + value["metric"]["spec"] + .as_object_mut() + .unwrap() + .remove("objective"); + let error = MetricEvaluationTask::from_json(&value.to_string()) + .unwrap_err() + .to_string(); + assert!(error.contains("objective")); + } + + #[test] + fn provider_result_rejects_nonfinite_wrong_scope_coverage_and_fingerprint() { + let task = custom_task(); + let valid = BiasProvider.evaluate(&task).unwrap(); + + let mut nonfinite = valid.clone(); + nonfinite.values[0].value = f64::NAN; + assert!(nonfinite + .validate_against(&task) + .unwrap_err() + .to_string() + .contains("non-finite")); + + let mut wrong_scope = valid.clone(); + wrong_scope.scope.partition = PredictionPartition::Test; + wrong_scope.result_fingerprint = wrong_scope.compute_fingerprint().unwrap(); + assert!(wrong_scope + .validate_against(&task) + .unwrap_err() + .to_string() + .contains("scope")); + + let mut wrong_coverage = valid.clone(); + wrong_coverage.values.pop(); + wrong_coverage.result_fingerprint = wrong_coverage.compute_fingerprint().unwrap(); + assert!(wrong_coverage + .validate_against(&task) + .unwrap_err() + .to_string() + .contains("coverage")); + + let mut wrong_fingerprint = valid; + wrong_fingerprint.implementation_fingerprint = "0".repeat(64); + wrong_fingerprint.result_fingerprint = wrong_fingerprint.compute_fingerprint().unwrap(); + assert!(wrong_fingerprint + .validate_against(&task) + .unwrap_err() + .to_string() + .contains("identity/fingerprint")); + } + + #[test] + fn built_in_registry_uses_existing_metric_kernel_and_per_output_reduction() { + let reference = builtin_metric_reference(RegressionMetricKind::Rmse).unwrap(); + let task = MetricEvaluationTask::new( + "metric-request:rmse", + reference, + LearningTaskKind::Regression, + PredictionKind::RegressionPoint, + sample_scope(), + vec![ + MetricUnitId::Sample(SampleId::new("sample:0").unwrap()), + MetricUnitId::Sample(SampleId::new("sample:1").unwrap()), + ], + vec![vec![2.0, 4.0], vec![4.0, 8.0]], + vec![vec![1.0, 2.0], vec![3.0, 6.0]], + vec!["a".to_string(), "b".to_string()], + None, + None, + None, + ) + .unwrap(); + let evaluation = builtin_metric_registry().unwrap().evaluate(&task).unwrap(); + assert_eq!(evaluation.result.values[0].value, 1.0); + assert_eq!(evaluation.result.values[1].value, 2.0); + assert_eq!(evaluation.aggregate, 1.5); + } + + #[test] + fn registry_rejects_descriptor_substitution_even_with_same_registry_key() { + let task = custom_task(); + let mut substituted = task.metric.implementation.clone(); + substituted.implementation_version = "2.0.0".to_string(); + substituted.descriptor_fingerprint = substituted.compute_fingerprint().unwrap(); + let mut registry = MetricProviderRegistry::default(); + registry + .register(substituted, Arc::new(BiasProvider)) + .unwrap(); + assert!(registry + .evaluate(&task) + .unwrap_err() + .to_string() + .contains("descriptor")); + } + + #[test] + fn published_provider_fixture_matches_rust_task_and_result_contracts() { + let fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../../examples/fixtures/criteria/metric_provider_contracts.v1.json" + )) + .unwrap(); + let task = MetricEvaluationTask::from_json(&fixture["valid"]["task"].to_string()).unwrap(); + let result = MetricEvaluationResult::from_json_for_task( + &fixture["valid"]["result"].to_string(), + &task, + ) + .unwrap(); + assert_eq!( + result.reduce(&task).unwrap(), + fixture["valid"]["aggregate"].as_f64().unwrap() + ); + + for case in fixture["invalid"].as_array().unwrap() { + let document = case["document"].to_string(); + let rejected = match case["contract"].as_str().unwrap() { + "metric_evaluation_task" => MetricEvaluationTask::from_json(&document).is_err(), + "metric_evaluation_result" => { + MetricEvaluationResult::from_json_for_task(&document, &task).is_err() + } + contract => panic!("unknown metric-provider fixture contract `{contract}`"), + }; + assert!(rejected, "negative case `{}` was accepted", case["id"]); + } + } +} diff --git a/crates/dag-ml-core/src/metrics.rs b/crates/dag-ml-core/src/metrics.rs index 5a95ebe..8a72e94 100644 --- a/crates/dag-ml-core/src/metrics.rs +++ b/crates/dag-ml-core/src/metrics.rs @@ -8,9 +8,14 @@ use crate::aggregation::{ use crate::error::{DagMlError, Result}; use crate::fold::FoldPartitionMode; use crate::ids::{FoldId, NodeId, SampleId, VariantId}; +use crate::metric_provider::{ + builtin_metric_reference, builtin_metric_registry, MetricEvaluationScope, MetricEvaluationTask, + MetricUnitId, +}; use crate::oof::{validate_producer_oof_coverage, PredictionBlock, PredictionPartition}; use crate::policy::PredictionLevel; use crate::selection::{CandidateScore, MetricObjective}; +use crate::{LearningTaskKind, PredictionKind}; #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -596,17 +601,69 @@ fn score_regression_rows( targets.target_names.clone() }; let metric_suffixes = target_metric_names(predictions.width, &target_names); + let provider_output_ids = (0..predictions.width) + .map(|index| format!("output:{index}")) + .collect::>(); + let metric_units = predictions + .unit_ids + .iter() + .map(MetricUnitId::from) + .collect::>(); + let prediction_values = aligned_predictions + .iter() + .map(|row| row.to_vec()) + .collect::>(); + let target_values = aligned_targets + .iter() + .map(|row| row.to_vec()) + .collect::>(); + let scope = MetricEvaluationScope { + producer_node: predictions.origin.producer_node.clone(), + producer_port: predictions.origin.producer_port.clone(), + prediction_id: predictions.origin.prediction_id.clone(), + variant_id: None, + partition: predictions.origin.partition.clone(), + fold_id: predictions.origin.fold_id.clone(), + level: predictions.level, + }; + let registry = builtin_metric_registry()?; let mut values = BTreeMap::new(); for metric in metrics { - let per_target = compute_metric_per_target( - *metric, - predictions.width, - &aligned_predictions, - &aligned_targets, - ); - values.insert(metric.name().to_string(), macro_mean(&per_target)); - for (name, value) in metric_suffixes.iter().zip(per_target) { - values.insert(format!("{}:{name}", metric.name()), value); + let (task_kind, prediction_kind) = match metric { + RegressionMetricKind::Mse + | RegressionMetricKind::Rmse + | RegressionMetricKind::Mae + | RegressionMetricKind::R2 => ( + LearningTaskKind::Regression, + PredictionKind::RegressionPoint, + ), + RegressionMetricKind::Accuracy | RegressionMetricKind::BalancedAccuracy => ( + LearningTaskKind::MulticlassClassification, + PredictionKind::ClassLabel, + ), + }; + let task = MetricEvaluationTask::new( + format!( + "metric:{}:{}", + predictions.origin.producer_node, + metric.name() + ), + builtin_metric_reference(*metric)?, + task_kind, + prediction_kind, + scope.clone(), + metric_units.clone(), + prediction_values.clone(), + target_values.clone(), + provider_output_ids.clone(), + None, + None, + None, + )?; + let evaluation = registry.evaluate(&task)?; + values.insert(metric.name().to_string(), evaluation.aggregate); + for (component, suffix) in evaluation.result.values.into_iter().zip(&metric_suffixes) { + values.insert(format!("{}:{suffix}", metric.name()), component.value); } } @@ -632,7 +689,7 @@ fn validate_sample_prediction_block(block: &PredictionBlock) -> Result { block.validate_content() } -fn compute_metric_per_target( +pub(crate) fn compute_metric_per_target( metric: RegressionMetricKind, width: usize, predictions: &[&[f64]], @@ -747,10 +804,6 @@ fn r2_for_target(target_idx: usize, predictions: &[&[f64]], targets: &[&[f64]]) } } -fn macro_mean(values: &[f64]) -> f64 { - values.iter().sum::() / values.len() as f64 -} - fn target_metric_names(width: usize, target_names: &[String]) -> Vec { if target_names.is_empty() { (0..width).map(|idx| format!("target_{idx}")).collect() @@ -1126,6 +1179,35 @@ mod tests { assert_eq!(candidate.metadata["target_names"], serde_json::json!(["y"])); } + #[test] + fn provider_adapter_preserves_display_target_names_with_spaces() { + let predictions = PredictionBlock { + prediction_id: None, + producer_node: NodeId::new("model:pls").unwrap(), + producer_port: Some("prediction".to_string()), + partition: PredictionPartition::Validation, + fold_id: None, + sample_ids: vec![sid("sample:1"), sid("sample:2")], + values: vec![vec![2.0], vec![4.0]], + target_names: vec!["protein content".to_string()], + }; + let targets = RegressionTargetBlock { + level: PredictionLevel::Sample, + unit_ids: vec![sample_unit("sample:1"), sample_unit("sample:2")], + values: vec![vec![1.0], vec![5.0]], + target_names: vec!["protein content".to_string()], + }; + + let report = score_regression_prediction_block( + &predictions, + &targets, + &[RegressionMetricKind::Rmse], + ) + .unwrap(); + assert_close(report.metrics["rmse"], 1.0); + assert_close(report.metrics["rmse:protein content"], 1.0); + } + #[test] fn scores_target_and_group_prediction_blocks() { let predictions = AggregatedPredictionBlock { diff --git a/crates/dag-ml-py/python/dag_ml/_dag_ml.abi3.so b/crates/dag-ml-py/python/dag_ml/_dag_ml.abi3.so index 1e6466f..9f20673 100755 Binary files a/crates/dag-ml-py/python/dag_ml/_dag_ml.abi3.so and b/crates/dag-ml-py/python/dag_ml/_dag_ml.abi3.so differ diff --git a/docs/CRITERIA_CONTRACTS.md b/docs/CRITERIA_CONTRACTS.md new file mode 100644 index 0000000..be54fd8 --- /dev/null +++ b/docs/CRITERIA_CONTRACTS.md @@ -0,0 +1,41 @@ +# Loss and Metric Contracts + +The native owner is `dag_ml_core::criteria`. The published v1 wire family is: + +- `docs/contracts/loss_spec.schema.json`; +- `docs/contracts/metric_spec.schema.json`; +- `docs/contracts/implementation_descriptor.schema.json`; +- `docs/contracts/training_loss_role.schema.json`; +- `docs/contracts/metric_role.schema.json`; +- `docs/contracts/metric_evaluation_task.schema.json`; +- `docs/contracts/metric_evaluation_result.schema.json`. + +The canonical positive and negative fixtures are +`examples/fixtures/criteria/criteria_contracts.v1.json` and +`examples/fixtures/criteria/metric_provider_contracts.v1.json`. The exact +artifact closure is pinned by +`docs/contracts/criteria_conformance_pack.v1.json` and is validated +independently by `parity/criteria/oracle.py`. + +Metric providers receive a self-fingerprinted `MetricEvaluationTask` containing +the semantic and implementation references, exact scope, unit identities, +prediction/target matrices and declared optional inputs. They return a +self-fingerprinted `MetricEvaluationResult`. DAG-ML rejects identity, +implementation, scope, coverage and non-finite-value mismatches before applying +the declared reduction. Built-in metrics use this same registry and delegate to +the existing native kernels; binding-local metrics register an implementation +under the descriptor's opaque `registry_key`. + +`LossSpec` defines optimizer-objective semantics; `MetricSpec` defines +evaluation semantics and objective direction. Pipeline roles remain separate, +so selection, reporting, early stopping and training cannot silently exchange +meanings. Both specs reference the same generic implementation-descriptor +shape, which carries only provider identity, capabilities and lifecycle. Local +callbacks are resolved through opaque registry keys; executable code and import +instructions are forbidden in canonical JSON. + +These are new standalone v1 contracts, not fields added to an existing wire +shape, so there is no previous-version read window. Future incompatible changes +publish new schema ids and Rust readers. The L1 publication is additive and +does not add or modify a C ABI symbol, macro or struct layout; C callback +registration is intentionally deferred to roadmap L5. diff --git a/docs/LOSS_AND_METRIC_INVENTORY.md b/docs/LOSS_AND_METRIC_INVENTORY.md index 8a84828..a7f2b6e 100644 --- a/docs/LOSS_AND_METRIC_INVENTORY.md +++ b/docs/LOSS_AND_METRIC_INVENTORY.md @@ -17,9 +17,17 @@ The baseline revisions are: | `nirs4all` | `16c8f070` | clean committed `main` | | `nirs4all-core` | `83c246c` | clean committed `main` | +Integration addendum (2026-07-15): the completed DAG-ML training/runtime/TCV1 +work was preserved at `3097da5`, repaired and integrated at `87785fc` on +`agent/loss-runtime-integration`, and published in draft PR #20. The announced +score-provider effort completed without any discoverable branch, commit, PR, +worktree implementation or provider API. After explicit user transfer, the +native loss-contract branch also owns `MetricSpec`, the shared implementation +descriptor, provider dispatch and the typed metric evaluation task. + The private `nirs4all-drafts` and `nirs4all-lab` repositories are out of scope. -Line references below are pinned to the revisions above unless a row is -explicitly labelled as uncommitted concurrent work. +Line references below are pinned to the revisions above unless superseded by +the integration addendum. ## Summary @@ -93,7 +101,7 @@ currently supplied by host surfaces: | Host surface | Default/behavior | Source | Test status | | --- | --- | --- | --- | | CLI | `rmse`; accepted names are clap-enumerated | `dag-ml-cli/src/main.rs:107-119`, `:506` | `dag-ml-cli/tests/cli_contracts.rs` | -| Python in-process | `accuracy` and `balanced_accuracy` map explicitly; every other value maps to `rmse` | `dag-ml-py/src/in_process.rs:167-175` | missing negative characterization; owned by score-provider integration | +| Python in-process | `accuracy` and `balanced_accuracy` map explicitly; every other value maps to `rmse` | `dag-ml-py/src/in_process.rs:167-175` | missing negative characterization; owned by native metric-provider integration | | C ABI serde scorer | known enum names only; unknown values fail deserialization | `dag-ml-capi/src/lib.rs:1774-1839` | C ABI scorer tests | | policy metric level | sample level | `dag-ml-core/src/policy.rs:280-295` | policy tests | @@ -107,27 +115,18 @@ currently supplied by host surfaces: independently selected metric (`aggregation.rs:1305-1491`). - No committed threshold-optimization metric contract exists. -### Concurrent work boundary +### Integrated work boundary -The original dag-ml checkout contains a large active, unstaged training/runtime -change. At inventory time it has no published branch, commit or PR. It adds -uncommitted `training.rs`, `training_runtime.rs`, `canonical.rs`, conformal and -replay contracts, plus modifications to metrics, tasks, controllers, schemas and -bindings. In particular, its TCV1 implementation and native training request are -not available from clean `main` and must not be copied or treated as landed API. -This training/TCV1 worktree is distinct from the separately announced -score-provider effort, for which no published integration artifact is available. +At inventory time the original dag-ml checkout contained a large active, +unstaged training/runtime change. That work is now preserved at `3097da5` and +integrated in draft PR #20 at `87785fc`. Its TCV1 implementation is the sole +canonicalization API for new loss and metric contracts. No `MetricProvider`, provider registry, implementation descriptor or typed -custom-metric evaluation task was discoverable in that worktree. Consequently: - -- loss-only planning may continue; -- L1 source work requiring TCV1 waits for the concurrent artifact instead of - duplicating canonicalization; -- metric/shared-schema implementation remains blocked until a branch, commit, - PR and API map are published; -- the loss roadmap must consume the provider descriptor if it is generic enough, - rather than adding a parallel descriptor type. +custom-metric evaluation task was discoverable in that worktree, any local ref, +remote head or open PR after the score-provider agent completed. Metric work is +therefore unblocked under the explicit ownership amendment: L1 introduces one +generic provider descriptor and registry contract shared by losses and metrics. ## nirs4all Python baseline @@ -227,7 +226,7 @@ instead of adding six more independent semantics. | ID | Severity | Current behavior | Required disposition | | --- | --- | --- | --- | | `F-LOSS-001` | high | unknown nirs4all PyTorch loss silently becomes MSE | error explicitly; the reviewed draft fix in `nirs4all#46` still requires merge | -| `F-METRIC-001` | high | unknown dag-ml Python in-process selection metric silently becomes RMSE | score-provider-owned resolver must error before execution | +| `F-METRIC-001` | high | unknown dag-ml Python in-process selection metric silently becomes RMSE | native metric resolver must error before execution | | `F-DIR-001` | high | unknown nirs4all metric silently means minimize/ascending | require explicit objective for custom metrics and error for unresolved built-ins | | `F-REPORT-001` | medium | `eval_list` converts metric failures to `None` | reporting policy must make missing/error behavior explicit | | `D-METRIC-001` | high | RMSE calculation and argmin duplicated in five bindings plus oracle | replace with native DAG-ML metric/selection contract; bindings adapt only | @@ -257,7 +256,7 @@ Missing characterization tests required before changing each surface: | PyTorch callable invocation in CV and refit | `nirs4all` after DAG contract | gradient/counter integration test; resolution-only tests already exist in PR #46 | | JAX task loss and monitor defaults | `nirs4all` | extracted resolver/monitor characterization before adding registry support | | shared selection defaults | `nirs4all` | direct tests for `get_best_score_metric` and effective refit resolution | -| Python metric unknown-name rejection | `dag-ml` score-provider branch | negative binding test replacing silent RMSE fallback | +| Python metric unknown-name rejection | `dag-ml` native contract branch | negative binding test replacing silent RMSE fallback | | local custom loss lifecycle | `dag-ml` bindings | one registry/callback lifetime test per official language | ## L0 status @@ -267,9 +266,9 @@ Missing characterization tests required before changing each surface: `dag-ml#18`. - The PyTorch silent-loss fallback fix is published in draft PR `nirs4all#46`, and covered by focused resolution tests. -- The baseline inventory is complete, but L0 remains open until the concurrent - score-provider work publishes its branch, commit, PR and API map, the - training/TCV1 work publishes its branch, commit or PR and canonicalization API, - and the missing characterization tests above land in their owning repositories. -- No source or shared-schema file in the active dag-ml worktree was modified by - this inventory effort. +- The training/TCV1 dependency is integrated in draft PR #20 and its canonical + API is mapped in the roadmap. +- The score-provider dependency is closed by the explicit ownership amendment; + no external implementation artifact exists to integrate. +- L0 remains open only for the missing characterization tests in their owning + repositories and independent review of this ownership amendment. diff --git a/docs/LOSS_AND_METRIC_ROADMAP.md b/docs/LOSS_AND_METRIC_ROADMAP.md index 0a92f27..d28e644 100644 --- a/docs/LOSS_AND_METRIC_ROADMAP.md +++ b/docs/LOSS_AND_METRIC_ROADMAP.md @@ -63,7 +63,7 @@ cross-repository criteria. The authoritative evidence is: | Criterion | Owning repository | Closing evidence | | --- | --- | --- | | semantic specs, planning, attestation, cache/replay and native metrics | `dag-ml` | full dag-ml gate listed after L2 | -| custom metric provider contract and typed evaluation task | score-provider work in `dag-ml` | provider-targeted tests plus the full dag-ml gate; branch, commit and PR recorded in L0 | +| custom metric provider contract and typed evaluation task | native contract work in `dag-ml` | provider-targeted tests plus the full dag-ml gate | | Python registry transport | `dag-ml` (`dag-ml-py`) | targeted binding tests plus full dag-ml gate | | Python TensorFlow/PyTorch controllers and legacy API migration | `nirs4all` | Ruff, mypy, unit/integration tests and examples gate listed after L4 | | Rust closure and C ABI callback | `dag-ml` | core/C ABI targeted tests plus full dag-ml gate | @@ -74,47 +74,32 @@ The cross-language rows are tracked-but-external from dag-ml's perspective. Their owning repository must publish its own reviewed PR and green gate before the ecosystem criterion can be marked complete. -## Ownership and concurrent dependencies +## Ownership and integrated dependencies -A concurrent agent is implementing a score provider. Until that work lands, the -following ownership boundary prevents duplicate or conflicting abstractions. +The announced score-provider agent completed without publishing a branch, +commit, PR or implementation artifact. A repository, ref and worktree audit on +2026-07-15 found no `MetricSpec`, provider registry, implementation descriptor +or typed custom-metric evaluation task. The user then explicitly transferred +that scope to this native DAG-ML effort. L1 therefore owns the generic provider +contract for both metrics and losses; there is no concurrent source boundary. -**Dependency state at roadmap authoring**: the score-provider work is active but -has no published branch or commit available in this worktree. Metric-contract -implementation is therefore **blocked**. L0 must replace this paragraph with the -exact branch/commit/PR and a checked-in API map before any metric-related L1/L2 -source or shared-schema edit begins. Loss-only contract design may proceed -because it does not define metric provider execution. +The completed training/runtime/TCV1 work was preserved at checkpoint `3097da5` +and integrated on `agent/loss-runtime-integration` at `87785fc` in draft PR #20. +Loss and metric fingerprints use `dag_ml_core::canonical` directly: +`parse_typed_json`, `validate_typed_serde_value`, `tcv1_sha256`, +`TypedCanonicalValue::fingerprint_without` and +`deserialize_external_contract`. TCV1 normalization is frozen to Unicode 17. +No second canonicalizer may be introduced. -The active dag-ml checkout separately contains an unpublished TCV1 -canonicalization implementation as part of concurrent training/runtime work. -L0 must also record that work's branch, commit or PR and canonicalization API -before L1 source work computes or validates fingerprints. Loss work must consume -that API and must not copy the dirty checkout or create a second canonical JSON -or fingerprint implementation. - -| Surface | Loss roadmap ownership | Score-provider ownership | Integration rule | -| --- | --- | --- | --- | -| Training objective semantics | `LossSpec`, loss implementation descriptor, reduction and required inputs | none | loss work leads | -| Metric semantics and provider execution | metric role requirements only | `MetricSpec`, metric implementation descriptor, provider registry/dispatch, typed evaluation task and score result | consume the landed provider API; do not create parallel types | -| Built-in scoring | no semantic rewrite | native metric calculation | preserve core ownership | -| `ControllerCapability` | loss-specific capabilities | metric-provider capabilities | additive merge after provider review | -| `NodeTask` / `NodeResult` | fit loss resolution and attestation | metric task/result fields | update once, with both contracts represented | -| Contract schemas | loss-specific schemas initially | score-provider schemas | shared schemas change only after branch comparison | -| C ABI callbacks | loss callback and lifecycle | metric callback if required | share lifecycle/versioning conventions | - -Before editing a shared surface, the implementer must inspect the current branch, -the score-provider branch or worktree, and the target diff. If the provider API -already supplies a generic implementation descriptor, the loss work extends it -instead of creating `LossImplementationDescriptor` in parallel. - -The loss roadmap does not create `MetricSpec` or -`MetricImplementationDescriptor`. Their authoritative definitions and typed -custom-metric evaluation task must land with the score-provider work. If that -work does not provide them, L0 must record an explicit ownership amendment and -receive independent review before L1 resumes. A shared implementation descriptor -is preferred only if it can represent loss and metric semantics without moving -objective direction into the provider descriptor. +| Surface | Native contract ownership | Integration rule | +| --- | --- | --- | +| Training objective semantics | `LossSpec`, reduction, required inputs and loss role references | backend-neutral core contract | +| Metric semantics and provider execution | `MetricSpec`, provider registry/dispatch, typed evaluation task and score result | one provider path for built-in and local custom metrics | +| Implementation identity | one generic implementation descriptor used by loss and metric refs | semantic objective/direction stays in the spec, not the descriptor | +| Built-in scoring | existing native metric calculation | adapt behind `MetricSpec`; do not duplicate numerics | +| `ControllerCapability`, `NodeTask`, `NodeResult` | loss application and metric evaluation protocols | evolve shared surfaces once with explicit versioning | +| Contract schemas | loss, metric, role and implementation schemas | one conformance family and TCV1 profile | +| Binding callbacks | language-local loss and metric registries | share lifecycle and error conventions; never serialize code | ## Contract model @@ -133,11 +118,9 @@ objective direction into the provider descriptor. - capability requirements such as differentiability and distributed reduction; - canonical spec fingerprint. -The score-provider-owned `MetricSpec` contains the equivalent metric semantics, -plus objective direction, supported unit levels and decomposition/reduction -behavior. It does not encode whether the metric is used for selection, reporting -or another policy. This roadmap consumes that contract after its provider branch -lands; it does not define a competing wire type. +`MetricSpec` contains the equivalent metric semantics, plus objective direction, +supported unit levels and decomposition/reduction behavior. It does not encode +whether the metric is used for selection, reporting or another policy. ### Implementation descriptors @@ -200,16 +183,15 @@ complete. - inventory every current training-loss, selection, reporting, early-stopping and tuning default by controller and language; - record every unknown-name fallback and every duplicated metric implementation; -- map concurrent score-provider types and reserve shared integration points; -- map the concurrent training/TCV1 artifact and reserve its canonicalization API; +- record the score-provider ownership transfer and the absence of an artifact; +- map the integrated training/TCV1 artifact and canonicalization API; - add focused characterization tests for current defaults before behavior changes. **Acceptance evidence**: - inventory links each default to source and a characterization test; -- the score-provider branch/commit/PR and API map are recorded in this roadmap; -- the training/TCV1 branch/commit/PR and canonicalization API are recorded; -- no source changes overlap the active score-provider worktree; +- the score-provider ownership amendment is independently reviewed; +- the training/TCV1 branch, commit, PR and canonicalization API are recorded; - independent documentation review confirms ownership and terminology. ### L1 - Native loss contracts and metric integration gate @@ -226,15 +208,10 @@ complete. - versioned built-in loss catalog descriptors without importing ML frameworks; - compile-time default-resolution contract. -Fingerprint-related L1 source work remains blocked until the training/TCV1 -artifact identified in L0 is published. Contract design can continue, but no -provisional canonicalizer or incompatible fingerprint format may be committed. - -Metric work in this batch is integration-only and remains blocked until the -score-provider artifact identified in L0 lands. Once unblocked, L1 consumes its -`MetricSpec`, metric implementation descriptor and typed evaluation task, adds -pipeline-role references, and verifies that native aggregation, selection and -score persistence consume provider results without duplication. +L1 defines `MetricSpec`, the generic implementation descriptor, provider +registry contract and typed metric evaluation task alongside loss contracts. +Native aggregation, selection and score persistence consume provider results +without adding a second scoring path. **Required negative cases**: @@ -245,7 +222,7 @@ score persistence consume provider results without duplication. - callable/code payload present in canonical JSON; - mismatched embedded fingerprint. -The score-provider contract must independently test a custom metric without an +The generic provider protocol must independently test a custom metric without an objective, non-finite provider output, wrong scope/coverage and mismatched provider fingerprint before metric integration is accepted here. @@ -260,8 +237,7 @@ wire shape where possible, explicit version/read-window decision, updated schema and fixtures, conformance-pack entry, CHANGELOG note and C ABI decision. **Independent review focus**: backend neutrality, schema evolution, canonical -fingerprints, absence of executable code, compatibility with the score-provider -descriptor. +fingerprints, absence of executable code and one shared provider descriptor. ### L2 - Controller resolution and attestation protocol @@ -277,7 +253,7 @@ descriptor. - explicit `controller_internal` and `not_configurable` modes; - early-stopping record distinct from final OOF scoring. -L2 also integrates the score-provider-owned typed custom-metric task and verifies +L2 integrates the typed custom-metric task and verifies finite values, scope, sample coverage and implementation fingerprint before native aggregation or selection. L2 does not implement a second provider path. @@ -309,8 +285,8 @@ python3 scripts/validate_contracts.py ``` **Independent review focus**: lifecycle invariants, spoof-resistant attestation, -cache/replay correctness, no feature-buffer or tensor crossing, score-provider -merge quality. +cache/replay correctness, no feature-buffer or tensor crossing, and one generic +provider protocol. ### L3 - Python reference binding @@ -464,7 +440,7 @@ review note with: - reviewed commit SHA and scope; - findings ordered by severity with file/line references; - contract and backward-compatibility assessment; -- concurrency assessment against the score-provider branch; +- assessment that loss and metric execution share one provider abstraction; - tests inspected or rerun; - explicit `approved`, `approved_with_followups` or `changes_requested` result. @@ -507,6 +483,5 @@ draft until their required dag-ml contract version is available. No PR may claim cross-language custom-loss completion before every local-binding acceptance test listed above is green. -The loss-only L1 branch may publish while the score-provider dependency is -unpublished. Metric-related contract, task, schema or provider work may not be -committed on that branch until L0 names and reviews the provider artifact. +L1 publishes loss and metric semantic contracts together because the previously +announced score-provider effort produced no integration artifact. diff --git a/docs/adr/ADR-22-training-loss-and-metric-ownership.md b/docs/adr/ADR-22-training-loss-and-metric-ownership.md index ff9545e..e2d44b0 100644 --- a/docs/adr/ADR-22-training-loss-and-metric-ownership.md +++ b/docs/adr/ADR-22-training-loss-and-metric-ownership.md @@ -2,7 +2,7 @@ # ADR-22: Training-loss and metric contract ownership -**Status**: proposed (2026-07-15) +**Status**: accepted (2026-07-15) **Blocks**: loss/metric roadmap phases L1-L7, nirs4all DAG-ML backend migration, local custom-loss support in every official binding. ## Context @@ -75,13 +75,12 @@ portable to another language. least one configurable controller complete the conformance gate. - Algorithms with fixed analytical objectives remain valid but must expose the limitation explicitly. -- Shared changes to metric providers, controller capabilities, task contracts or - schemas require coordination with concurrent score-provider work. Neither - implementation may introduce a duplicate provider descriptor or registry. -- The concurrent score-provider work owns `MetricSpec`, its implementation - descriptor and the typed custom-metric evaluation task. Loss work consumes - those contracts after their branch/commit is recorded and reviewed; it must - not create provisional metric types while that dependency is unpublished. +- Losses and metrics share one generic implementation descriptor and provider + lifecycle. Semantic objective/direction remains in `LossSpec` or `MetricSpec`. +- The previously announced score-provider effort completed without a published + artifact. After repository/ref/worktree audit and explicit user transfer, + this native effort owns `MetricSpec`, provider dispatch and the typed metric + evaluation task together with `LossSpec`. - Every versioned controller/task/result/schema change follows ADR-02 in the same batch, including fixtures, read-window/version decision, CHANGELOG and C ABI impact. @@ -89,11 +88,8 @@ portable to another language. ## Blocks No binding-specific public custom-loss API should be stabilized before L1 and -L2 freeze the semantic contracts and controller attestation protocol. The ADR -must be accepted before the nirs4all compatibility layer becomes authoritative. -Metric-related L1/L2 source changes are additionally blocked until the active -score-provider work has a named branch/commit/PR and its API is mapped against -this ADR. Loss-only semantic design may proceed independently, but source work -that computes or validates fingerprints must consume the concurrent -training/TCV1 canonicalization API after its branch, commit or PR is published; -it must not copy the dirty worktree or introduce a second canonicalizer. +L2 freeze the semantic contracts and controller attestation protocol. +nirs4all's legacy-argument translation layer remains downstream of those +contracts and cannot become a second semantic registry. Fingerprints must use +the integrated `dag_ml_core::canonical` TCV1 API from draft PR #20; no binding +or provider may introduce another canonicalizer. diff --git a/docs/contracts/criteria_conformance_pack.v1.json b/docs/contracts/criteria_conformance_pack.v1.json new file mode 100644 index 0000000..c0ec904 --- /dev/null +++ b/docs/contracts/criteria_conformance_pack.v1.json @@ -0,0 +1,119 @@ +{ + "pack_id": "dag-ml.criteria-conformance.v1", + "schema_version": 1, + "hash_algorithm": "sha256-file-bytes", + "fingerprint_profile": "DAGML-TCV1-unicode-17.0.0", + "artifacts": [ + { + "path": "crates/dag-ml-cli/src/main.rs", + "sha256": "9278c608571b098fde52f2d6507dc5dd7150938a78fe9de7276e47413bd884a4", + "kind": "cli_validator" + }, + { + "path": "crates/dag-ml-cli/tests/cli_contracts.rs", + "sha256": "43f1f2d584826c37fdba9d7c2b9b6974a29ebe6347054b0d936bfb7b9561efae", + "kind": "cli_test" + }, + { + "path": "crates/dag-ml-core/src/criteria.rs", + "sha256": "cacd70df9e57a2f04a98f6f83b4eeb62aa90b768e0edca6306b596de83359e56", + "kind": "rust_contract" + }, + { + "path": "crates/dag-ml-core/src/metric_provider.rs", + "sha256": "73f997881c015e429b75e30a9e0f8092ec10569ccb63d01116c8715e6b16f14c", + "kind": "rust_provider_contract" + }, + { + "path": "crates/dag-ml-core/src/metrics.rs", + "sha256": "2fd5b10eaa6f198e450ea1a84d954b6a44c95facd7a87700a7d6b958e880b78a", + "kind": "native_metric_adapter" + }, + { + "path": "docs/CRITERIA_CONTRACTS.md", + "sha256": "8e0ee62145503c49881eae583c1988f9c1fbb17bd227fe04c7585ae973441f85", + "kind": "contract_documentation" + }, + { + "path": "docs/contracts/implementation_descriptor.schema.json", + "sha256": "7fc37cd6dda7f7fcdb778facd691a5b27e64c825051965dd0948cfdb09092210", + "kind": "schema" + }, + { + "path": "docs/contracts/loss_spec.schema.json", + "sha256": "18a6cfc7bf74c42927ce3ba60d31b5c6bc8e5dd447b7c96a011ab26b3e282954", + "kind": "schema" + }, + { + "path": "docs/contracts/metric_evaluation_result.schema.json", + "sha256": "9e965989e878d34a63f7ef4d76e33e79927362b96a322c517932df15c4a9190c", + "kind": "schema" + }, + { + "path": "docs/contracts/metric_evaluation_task.schema.json", + "sha256": "72e7405eebb3a78e5489b31a61187d15ef81c2dcc3fbcba5ac4b70a0d52bd13a", + "kind": "schema" + }, + { + "path": "docs/contracts/metric_role.schema.json", + "sha256": "b530ee58934571d0e82373f16ae92f86b6521c308479bef80d86040b4946f5a5", + "kind": "schema" + }, + { + "path": "docs/contracts/metric_spec.schema.json", + "sha256": "d463d99eab6b2399aecefdab92b0e0a7912e9e85cc09710015b9167393ca3523", + "kind": "schema" + }, + { + "path": "docs/contracts/training_loss_role.schema.json", + "sha256": "078886aa7f622c05c86aa22b4eaeaeeff857e2ad68da1137f62fe4f1510d02e3", + "kind": "schema" + }, + { + "path": "examples/fixtures/criteria/criteria_contracts.v1.json", + "sha256": "07284ac6c366c87ffc23af805410f79cd1d4875f2449fcb3dc2c4c3e45c1e5be", + "kind": "fixture" + }, + { + "path": "examples/fixtures/criteria/metric_provider_contracts.v1.json", + "sha256": "940eb1961335e600568bc03e74b9ef71751f8e6b5429a95c32e39af344d4a725", + "kind": "fixture" + }, + { + "path": "parity/criteria/generate_fixtures.py", + "sha256": "5d420161c10293d7f594d176ad38e39d064dda1f19e69572ee2262e4c5032ad6", + "kind": "generator" + }, + { + "path": "parity/criteria/oracle.py", + "sha256": "03508d1815c897a31423ec46eef611757b00ba45a353b256b646f45833e7ddbc", + "kind": "independent_validator" + }, + { + "path": "parity/criteria/tests/test_criteria_contracts.py", + "sha256": "457d485773542c815132a1bd62725f462ed141e43dfb72f8678dd0b8e8aa1fbb", + "kind": "parity_test" + } + ], + "required_negative_cases": [ + "loss_unversioned_id", + "loss_leading_zero_version", + "loss_c1_control_id", + "loss_uppercase_callable_payload", + "loss_unknown_task", + "loss_nested_callable_payload", + "loss_weighted_without_weight_input", + "loss_mismatched_fingerprint", + "metric_without_objective", + "host_local_descriptor_without_registry_key", + "selection_metric_skips_missing_values", + "custom_metric_without_objective", + "provider_result_wrong_scope", + "provider_result_wrong_coverage", + "provider_result_mismatched_implementation_fingerprint" + ], + "runtime_only_negative_cases": [ + "provider_result_non_finite" + ], + "pack_checksum": "ee34c0acb4960e86a26ee6ffd53e95c9c6fb79f0e5ea019d2c58940db8ae8e7b" +} diff --git a/docs/contracts/implementation_descriptor.schema.json b/docs/contracts/implementation_descriptor.schema.json new file mode 100644 index 0000000..276a951 --- /dev/null +++ b/docs/contracts/implementation_descriptor.schema.json @@ -0,0 +1,110 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GBeurier/dag-ml/schemas/implementation_descriptor.v1.schema.json", + "title": "DAG-ML ImplementationDescriptor v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "semantic_kind", + "semantic_id", + "semantic_fingerprint", + "provider_id", + "binding_id", + "implementation_version", + "implementation_fingerprint", + "supported_controller_families", + "runtime_requirements", + "capabilities", + "portability", + "replayability", + "descriptor_fingerprint" + ], + "properties": { + "schema_version": { "const": 1 }, + "semantic_kind": { "enum": ["loss", "metric"] }, + "semantic_id": { "$ref": "#/$defs/versioned_id" }, + "semantic_fingerprint": { "$ref": "#/$defs/sha256" }, + "provider_id": { "$ref": "#/$defs/token" }, + "binding_id": { "$ref": "#/$defs/token" }, + "implementation_version": { "$ref": "#/$defs/token" }, + "implementation_fingerprint": { "$ref": "#/$defs/sha256" }, + "supported_controller_families": { + "type": "array", + "uniqueItems": true, + "items": { "$ref": "#/$defs/token" } + }, + "runtime_requirements": { + "type": "array", + "uniqueItems": true, + "items": { "$ref": "#/$defs/token" } + }, + "capabilities": { + "type": "array", + "uniqueItems": true, + "items": { + "enum": [ + "deterministic", + "differentiable", + "distributed_reduction", + "needs_gil", + "process_safe", + "supports_missing_mask", + "supports_sample_weights", + "thread_safe" + ] + } + }, + "portability": { + "enum": ["host_local", "portable_registered", "portable_built_in"] + }, + "replayability": { + "enum": ["process_local", "registry_required", "detached"] + }, + "registry_key": { "$ref": "#/$defs/token" }, + "descriptor_fingerprint": { "$ref": "#/$defs/sha256" } + }, + "allOf": [ + { + "if": { "properties": { "portability": { "const": "host_local" } } }, + "then": { + "required": ["registry_key"], + "properties": { + "replayability": { "enum": ["process_local", "registry_required"] } + } + } + }, + { + "if": { + "properties": { "portability": { "const": "portable_registered" } } + }, + "then": { + "required": ["registry_key"], + "properties": { "replayability": { "const": "registry_required" } } + } + }, + { + "if": { + "properties": { "portability": { "const": "portable_built_in" } } + }, + "then": { + "not": { "required": ["registry_key"] }, + "properties": { "replayability": { "const": "detached" } } + } + } + ], + "$defs": { + "versioned_id": { + "type": "string", + "pattern": "^[^@\\s\\u0000-\\u001f\\u007f-\\u009f]+@[1-9][0-9]*$" + }, + "token": { + "type": "string", + "pattern": "^[^\\s\\u0000-\\u001f\\u007f-\\u009f]+$" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } +} diff --git a/docs/contracts/loss_spec.schema.json b/docs/contracts/loss_spec.schema.json new file mode 100644 index 0000000..decc578 --- /dev/null +++ b/docs/contracts/loss_spec.schema.json @@ -0,0 +1,136 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GBeurier/dag-ml/schemas/loss_spec.v1.schema.json", + "title": "DAG-ML LossSpec v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "loss_id", + "kind", + "task_kinds", + "prediction_kinds", + "objective", + "reduction", + "required_inputs", + "capabilities", + "parameters", + "spec_fingerprint" + ], + "properties": { + "schema_version": { "const": 1 }, + "loss_id": { "$ref": "#/$defs/versioned_id" }, + "kind": { "enum": ["built_in", "custom"] }, + "task_kinds": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/task_kind" } + }, + "prediction_kinds": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/prediction_kind" } + }, + "objective": { "const": "minimize" }, + "reduction": { "enum": ["mean", "sum", "weighted_mean"] }, + "required_inputs": { + "type": "array", + "minItems": 2, + "uniqueItems": true, + "contains": { "const": "target" }, + "items": { "$ref": "#/$defs/criterion_input" }, + "allOf": [{ "contains": { "const": "prediction" } }] + }, + "capabilities": { + "type": "array", + "uniqueItems": true, + "items": { + "enum": [ + "differentiable", + "distributed_reduction", + "per_output", + "supports_missing_mask", + "supports_sample_weights" + ] + } + }, + "parameters": { "$ref": "#/$defs/parameters" }, + "spec_fingerprint": { "$ref": "#/$defs/sha256" } + }, + "allOf": [ + { + "if": { "properties": { "reduction": { "const": "weighted_mean" } } }, + "then": { + "properties": { + "required_inputs": { "contains": { "const": "sample_weight" } }, + "capabilities": { "contains": { "const": "supports_sample_weights" } } + } + } + }, + { + "if": { + "properties": { + "required_inputs": { "contains": { "const": "missing_mask" } } + } + }, + "then": { + "properties": { + "capabilities": { "contains": { "const": "supports_missing_mask" } } + } + } + } + ], + "$defs": { + "versioned_id": { + "type": "string", + "pattern": "^[^@\\s\\u0000-\\u001f\\u007f-\\u009f]+@[1-9][0-9]*$" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "task_kind": { + "enum": [ + "regression", + "binary_classification", + "multiclass_classification", + "multilabel_classification" + ] + }, + "prediction_kind": { + "enum": [ + "regression_point", + "class_label", + "class_probability", + "decision_score" + ] + }, + "criterion_input": { + "enum": ["target", "prediction", "sample_weight", "missing_mask", "group"] + }, + "parameters": { + "type": "object", + "propertyNames": { + "not": { + "pattern": "^(?:[bB][yY][tT][eE][cC][oO][dD][eE]|[cC][aA][lL][lL][aA][bB][lL][eE]|[cC][oO][dD][eE]|[fF][uU][nN][cC][tT][iI][oO][nN]_[sS][oO][uU][rR][cC][eE]|[iI][mM][pP][oO][rR][tT]_[pP][aA][tT][hH]|[mM][oO][dD][uU][lL][eE]_[pP][aA][tT][hH]|[pP][iI][cC][kK][lL][eE]|[sS][eE][rR][iI][aA][lL][iI][zZ][eE][dD]_[cC][aA][lL][lL][aA][bB][lL][eE]|[sS][oO][uU][rR][cC][eE]_[cC][oO][dD][eE])$" + } + }, + "additionalProperties": { "$ref": "#/$defs/parameter_value" } + }, + "parameter_value": { + "oneOf": [ + { "type": "null" }, + { "type": "boolean" }, + { "type": "number" }, + { "type": "string" }, + { + "type": "array", + "items": { "$ref": "#/$defs/parameter_value" } + }, + { "$ref": "#/$defs/parameters" } + ] + } + } +} diff --git a/docs/contracts/metric_evaluation_result.schema.json b/docs/contracts/metric_evaluation_result.schema.json new file mode 100644 index 0000000..e74ab0a --- /dev/null +++ b/docs/contracts/metric_evaluation_result.schema.json @@ -0,0 +1,61 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GBeurier/dag-ml/schemas/metric_evaluation_result.v1.schema.json", + "title": "DAG-ML MetricEvaluationResult v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "request_id", + "semantic_id", + "semantic_fingerprint", + "implementation_fingerprint", + "descriptor_fingerprint", + "scope", + "values", + "result_fingerprint" + ], + "properties": { + "schema_version": { "const": 1 }, + "request_id": { "$ref": "#/$defs/token" }, + "semantic_id": { "$ref": "#/$defs/versioned_id" }, + "semantic_fingerprint": { "$ref": "#/$defs/sha256" }, + "implementation_fingerprint": { "$ref": "#/$defs/sha256" }, + "descriptor_fingerprint": { "$ref": "#/$defs/sha256" }, + "scope": { + "$ref": "https://github.com/GBeurier/dag-ml/schemas/metric_evaluation_task.v1.schema.json#/$defs/scope" + }, + "values": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/value" } + }, + "result_fingerprint": { "$ref": "#/$defs/sha256" } + }, + "$defs": { + "value": { + "type": "object", + "additionalProperties": false, + "required": ["value"], + "properties": { + "unit_id": { + "$ref": "https://github.com/GBeurier/dag-ml/schemas/metric_evaluation_task.v1.schema.json#/$defs/unit_id" + }, + "output_id": { "$ref": "#/$defs/token" }, + "value": { "type": "number" } + } + }, + "versioned_id": { + "type": "string", + "pattern": "^[^@\\s\\u0000-\\u001f\\u007f-\\u009f]+@[1-9][0-9]*$" + }, + "token": { + "type": "string", + "pattern": "^[^\\s\\u0000-\\u001f\\u007f-\\u009f]+$" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } +} diff --git a/docs/contracts/metric_evaluation_task.schema.json b/docs/contracts/metric_evaluation_task.schema.json new file mode 100644 index 0000000..e599e69 --- /dev/null +++ b/docs/contracts/metric_evaluation_task.schema.json @@ -0,0 +1,136 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GBeurier/dag-ml/schemas/metric_evaluation_task.v1.schema.json", + "title": "DAG-ML MetricEvaluationTask v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "request_id", + "metric", + "task_kind", + "prediction_kind", + "scope", + "unit_ids", + "predictions", + "targets", + "output_ids", + "task_fingerprint" + ], + "properties": { + "schema_version": { "const": 1 }, + "request_id": { "$ref": "#/$defs/token" }, + "metric": { "$ref": "#/$defs/metric_reference" }, + "task_kind": { + "enum": [ + "regression", + "binary_classification", + "multiclass_classification", + "multilabel_classification" + ] + }, + "prediction_kind": { + "enum": [ + "regression_point", + "class_label", + "class_probability", + "decision_score" + ] + }, + "scope": { "$ref": "#/$defs/scope" }, + "unit_ids": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/unit_id" } + }, + "predictions": { "$ref": "#/$defs/matrix" }, + "targets": { "$ref": "#/$defs/matrix" }, + "output_ids": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/token" } + }, + "sample_weights": { + "type": "array", + "minItems": 1, + "items": { "type": "number", "minimum": 0 } + }, + "missing_mask": { + "type": "array", + "minItems": 1, + "items": { + "type": "array", + "minItems": 1, + "items": { "type": "boolean" } + } + }, + "group_ids": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/token" } + }, + "task_fingerprint": { "$ref": "#/$defs/sha256" } + }, + "$defs": { + "metric_reference": { + "type": "object", + "additionalProperties": false, + "required": ["spec", "implementation"], + "properties": { + "spec": { + "$ref": "https://github.com/GBeurier/dag-ml/schemas/metric_spec.v1.schema.json" + }, + "implementation": { + "$ref": "https://github.com/GBeurier/dag-ml/schemas/implementation_descriptor.v1.schema.json" + } + } + }, + "scope": { + "type": "object", + "additionalProperties": false, + "required": ["producer_node", "partition", "level"], + "properties": { + "producer_node": { "$ref": "#/$defs/identifier" }, + "producer_port": { "$ref": "#/$defs/token" }, + "prediction_id": { "$ref": "#/$defs/token" }, + "variant_id": { "$ref": "#/$defs/identifier" }, + "partition": { "enum": ["train", "validation", "test", "final"] }, + "fold_id": { "$ref": "#/$defs/identifier" }, + "level": { "enum": ["observation", "sample", "target", "group"] } + } + }, + "unit_id": { + "type": "object", + "additionalProperties": false, + "required": ["level", "id"], + "properties": { + "level": { "enum": ["observation", "sample", "target", "group"] }, + "id": { "$ref": "#/$defs/identifier" } + } + }, + "matrix": { + "type": "array", + "minItems": 1, + "items": { + "type": "array", + "minItems": 1, + "items": { "type": "number" } + } + }, + "identifier": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9_.:-]+$" + }, + "token": { + "type": "string", + "pattern": "^[^\\s\\u0000-\\u001f\\u007f-\\u009f]+$" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } +} diff --git a/docs/contracts/metric_role.schema.json b/docs/contracts/metric_role.schema.json new file mode 100644 index 0000000..18dcea2 --- /dev/null +++ b/docs/contracts/metric_role.schema.json @@ -0,0 +1,64 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GBeurier/dag-ml/schemas/metric_role.v1.schema.json", + "title": "DAG-ML MetricRoleReference v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "role_id", + "role", + "partition", + "level", + "missing_value_policy", + "metric" + ], + "properties": { + "schema_version": { "const": 1 }, + "role_id": { "$ref": "#/$defs/token" }, + "output_id": { "$ref": "#/$defs/token" }, + "role": { + "enum": [ + "early_stopping", + "selection", + "reporting", + "tuning", + "pruning", + "threshold", + "ensemble_weighting" + ] + }, + "partition": { "enum": ["train", "validation", "test", "final"] }, + "level": { "enum": ["observation", "sample", "target", "group"] }, + "missing_value_policy": { "enum": ["error", "skip"] }, + "metric": { "$ref": "#/$defs/metric_reference" } + }, + "allOf": [ + { + "if": { + "properties": { "missing_value_policy": { "const": "skip" } }, + "required": ["missing_value_policy"] + }, + "then": { "properties": { "role": { "const": "reporting" } } } + } + ], + "$defs": { + "metric_reference": { + "type": "object", + "additionalProperties": false, + "required": ["spec", "implementation"], + "properties": { + "spec": { + "$ref": "https://github.com/GBeurier/dag-ml/schemas/metric_spec.v1.schema.json" + }, + "implementation": { + "$ref": "https://github.com/GBeurier/dag-ml/schemas/implementation_descriptor.v1.schema.json" + } + } + }, + "token": { + "type": "string", + "pattern": "^[^\\s\\u0000-\\u001f\\u007f-\\u009f]+$" + } + } +} diff --git a/docs/contracts/metric_spec.schema.json b/docs/contracts/metric_spec.schema.json new file mode 100644 index 0000000..08b0fdd --- /dev/null +++ b/docs/contracts/metric_spec.schema.json @@ -0,0 +1,155 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GBeurier/dag-ml/schemas/metric_spec.v1.schema.json", + "title": "DAG-ML MetricSpec v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "metric_id", + "kind", + "task_kinds", + "prediction_kinds", + "objective", + "supported_levels", + "decomposition", + "reduction", + "required_inputs", + "capabilities", + "parameters", + "spec_fingerprint" + ], + "properties": { + "schema_version": { "const": 1 }, + "metric_id": { "$ref": "#/$defs/versioned_id" }, + "kind": { "enum": ["built_in", "custom"] }, + "task_kinds": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/task_kind" } + }, + "prediction_kinds": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/prediction_kind" } + }, + "objective": { "enum": ["minimize", "maximize"] }, + "supported_levels": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "enum": ["observation", "sample", "target", "group"] } + }, + "decomposition": { "enum": ["global", "per_output", "per_unit"] }, + "reduction": { "enum": ["global", "mean", "sum", "weighted_mean"] }, + "required_inputs": { + "type": "array", + "minItems": 2, + "uniqueItems": true, + "contains": { "const": "target" }, + "items": { "$ref": "#/$defs/criterion_input" }, + "allOf": [{ "contains": { "const": "prediction" } }] + }, + "capabilities": { + "type": "array", + "uniqueItems": true, + "items": { + "enum": [ + "decomposable", + "distributed_reduction", + "supports_missing_mask", + "supports_sample_weights" + ] + } + }, + "parameters": { "$ref": "#/$defs/parameters" }, + "spec_fingerprint": { "$ref": "#/$defs/sha256" } + }, + "allOf": [ + { + "if": { "properties": { "decomposition": { "const": "global" } } }, + "then": { "properties": { "reduction": { "const": "global" } } }, + "else": { + "properties": { + "reduction": { "enum": ["mean", "sum", "weighted_mean"] }, + "capabilities": { "contains": { "const": "decomposable" } } + } + } + }, + { + "if": { "properties": { "reduction": { "const": "weighted_mean" } } }, + "then": { + "properties": { + "decomposition": { "const": "per_unit" }, + "required_inputs": { "contains": { "const": "sample_weight" } }, + "capabilities": { "contains": { "const": "supports_sample_weights" } } + } + } + }, + { + "if": { + "properties": { + "required_inputs": { "contains": { "const": "missing_mask" } } + } + }, + "then": { + "properties": { + "capabilities": { "contains": { "const": "supports_missing_mask" } } + } + } + } + ], + "$defs": { + "versioned_id": { + "type": "string", + "pattern": "^[^@\\s\\u0000-\\u001f\\u007f-\\u009f]+@[1-9][0-9]*$" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "task_kind": { + "enum": [ + "regression", + "binary_classification", + "multiclass_classification", + "multilabel_classification" + ] + }, + "prediction_kind": { + "enum": [ + "regression_point", + "class_label", + "class_probability", + "decision_score" + ] + }, + "criterion_input": { + "enum": ["target", "prediction", "sample_weight", "missing_mask", "group"] + }, + "parameters": { + "type": "object", + "propertyNames": { + "not": { + "pattern": "^(?:[bB][yY][tT][eE][cC][oO][dD][eE]|[cC][aA][lL][lL][aA][bB][lL][eE]|[cC][oO][dD][eE]|[fF][uU][nN][cC][tT][iI][oO][nN]_[sS][oO][uU][rR][cC][eE]|[iI][mM][pP][oO][rR][tT]_[pP][aA][tT][hH]|[mM][oO][dD][uU][lL][eE]_[pP][aA][tT][hH]|[pP][iI][cC][kK][lL][eE]|[sS][eE][rR][iI][aA][lL][iI][zZ][eE][dD]_[cC][aA][lL][lL][aA][bB][lL][eE]|[sS][oO][uU][rR][cC][eE]_[cC][oO][dD][eE])$" + } + }, + "additionalProperties": { "$ref": "#/$defs/parameter_value" } + }, + "parameter_value": { + "oneOf": [ + { "type": "null" }, + { "type": "boolean" }, + { "type": "number" }, + { "type": "string" }, + { + "type": "array", + "items": { "$ref": "#/$defs/parameter_value" } + }, + { "$ref": "#/$defs/parameters" } + ] + } + } +} diff --git a/docs/contracts/training_contract_conformance_pack.v1.json b/docs/contracts/training_contract_conformance_pack.v1.json index 3d21081..d8f01c9 100644 --- a/docs/contracts/training_contract_conformance_pack.v1.json +++ b/docs/contracts/training_contract_conformance_pack.v1.json @@ -86,7 +86,7 @@ }, { "path": "crates/dag-ml-core/src/metrics.rs", - "sha256": "deb1c458c2d0bd69b15badcec28b2f94f294e12d37fec7883dcc47057f3fb657", + "sha256": "2fd5b10eaa6f198e450ea1a84d954b6a44c95facd7a87700a7d6b958e880b78a", "kind": "typed_cache_dependency" }, { @@ -484,5 +484,5 @@ "d2_no_refit_without_refit_support", "d2_no_refit_missing_portable_oof_payload" ], - "pack_checksum": "fd1584c2cd60d566d743cf398e9859dbf3550e50a37e8e0f04013f1ae1f1c86f" + "pack_checksum": "6def60a5f247ae2e8586a74eec3b5b228769ed89d809fd83be82ace2ffa19043" } diff --git a/docs/contracts/training_loss_role.schema.json b/docs/contracts/training_loss_role.schema.json new file mode 100644 index 0000000..af26b29 --- /dev/null +++ b/docs/contracts/training_loss_role.schema.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GBeurier/dag-ml/schemas/training_loss_role.v1.schema.json", + "title": "DAG-ML TrainingLossRoleReference v1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "node_id", "phases", "loss"], + "properties": { + "schema_version": { "const": 1 }, + "node_id": { "$ref": "#/$defs/node_id" }, + "output_id": { "$ref": "#/$defs/token" }, + "phases": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "enum": ["FIT_CV", "REFIT"] } + }, + "loss": { "$ref": "#/$defs/loss_reference" } + }, + "$defs": { + "loss_reference": { + "type": "object", + "additionalProperties": false, + "required": ["spec", "implementation"], + "properties": { + "spec": { + "$ref": "https://github.com/GBeurier/dag-ml/schemas/loss_spec.v1.schema.json" + }, + "implementation": { + "$ref": "https://github.com/GBeurier/dag-ml/schemas/implementation_descriptor.v1.schema.json" + } + } + }, + "node_id": { + "type": "string", + "maxLength": 128, + "pattern": "^[A-Za-z0-9_.:-]+$" + }, + "token": { + "type": "string", + "pattern": "^[^\\s\\u0000-\\u001f\\u007f-\\u009f]+$" + } + } +} diff --git a/docs/contracts/training_replay_contract_conformance_pack.v1.json b/docs/contracts/training_replay_contract_conformance_pack.v1.json index 7bfffdc..953d6ac 100644 --- a/docs/contracts/training_replay_contract_conformance_pack.v1.json +++ b/docs/contracts/training_replay_contract_conformance_pack.v1.json @@ -3,8 +3,8 @@ "pack_id": "dag-ml.training-replay-contracts.v1", "mode": "current", "base_pack_id": "dag-ml.training-contracts.v1", - "base_pack_sha256": "7cc19a167332978670ebe8f24f597ef3b0b6ae80a873b2234eb2a642d466fb55", - "base_pack_checksum": "fd1584c2cd60d566d743cf398e9859dbf3550e50a37e8e0f04013f1ae1f1c86f", + "base_pack_sha256": "18074d90c040fb681b4780e5db46529ffd5ac80f744b1c9e0bd166abd7e77345", + "base_pack_checksum": "6def60a5f247ae2e8586a74eec3b5b228769ed89d809fd83be82ace2ffa19043", "base_pack_mode": "current", "hash_algorithm": "sha256-file-bytes", "canonical_profile": "DAG-ML TCV1", @@ -96,12 +96,12 @@ }, { "path": "crates/dag-ml-core/src/lib.rs", - "sha256": "430edfd613fe13abe683979f70fe1eb3fb04492d2955be11dc94d640cb7e59f0", + "sha256": "a991c357253b6bac2bc8274de71ef5a0bc46ad81b094ef37587edd98461d94ed", "kind": "training_replay_public_export" }, { "path": "crates/dag-ml-core/src/metrics.rs", - "sha256": "deb1c458c2d0bd69b15badcec28b2f94f294e12d37fec7883dcc47057f3fb657", + "sha256": "2fd5b10eaa6f198e450ea1a84d954b6a44c95facd7a87700a7d6b958e880b78a", "kind": "base_typed_cache_dependency" }, { @@ -381,7 +381,7 @@ }, { "path": "docs/contracts/training_contract_conformance_pack.v1.json", - "sha256": "7cc19a167332978670ebe8f24f597ef3b0b6ae80a873b2234eb2a642d466fb55", + "sha256": "18074d90c040fb681b4780e5db46529ffd5ac80f744b1c9e0bd166abd7e77345", "kind": "base_conformance_pack" }, { @@ -416,7 +416,7 @@ }, { "path": "docs/index.md", - "sha256": "a720e94be335d122b0ce197b1359ad26af8c991806482b0a474371176f0bcd35", + "sha256": "3161816302c8d2fc13592f8a277cb28f330421c0f5389a0ae5a2a18be8fb51e1", "kind": "documentation_index" }, { @@ -591,7 +591,7 @@ }, { "path": "parity/training/generate_training_replay_fixtures.py", - "sha256": "640629f910e8c9a0830684ba4f99fa35af689fc4e37242d042369b0e9f394981", + "sha256": "b508c3647fb6ccb40bf1b5782249affb4b0d1ec8a418e649c49cfade4e1131c0", "kind": "generator" }, { @@ -606,7 +606,7 @@ }, { "path": "parity/training/tests/test_training_replay_contracts.py", - "sha256": "e5c2a5ac53845dfdd8c84ddb13004835af10884cf2d90d4538b797bf0655fca0", + "sha256": "cd13138e21ac36e7250d36b090fdc1ee094c3a9b291b2d0fb7e76eb6bee7a1d0", "kind": "test" }, { @@ -641,7 +641,7 @@ }, { "path": "scripts/validate_training_replay_contracts.py", - "sha256": "a467a70d9569f5214f876565282cf9a8f65a37d3fd6fab721c76b5ee6235a52c", + "sha256": "21b18efe86e5f4c10b3d4105891ffcb09ebef137987fb9ba13e2a36b58fa06ec", "kind": "production_validator" } ], @@ -720,5 +720,5 @@ "d4_replay_legacy_wrapper_rejects_explicit_port", "d4_score_set_v2_duplicate_port_aware_key" ], - "pack_checksum": "71aac3537571bf504e5a0d11f69c523e2f9391d8726d9e3b4683cd1350fb9f3c" + "pack_checksum": "ba6e10a2122d4f84a544111310d9a42620a3cacdfea36b7e137b2e1223b68c67" } diff --git a/docs/index.md b/docs/index.md index 111a803..266fcde 100644 --- a/docs/index.md +++ b/docs/index.md @@ -22,6 +22,7 @@ wire nirs4all directly. | Map nirs4all parity capabilities | [Capability matrix](CAPABILITY_MATRIX.md) | | Inspect shared contracts | [Contract manifests](contracts/README.md) | | Use native training/fine-tuning contracts | [Training contracts](TRAINING_CONTRACTS.md) | +| Define loss, metric and provider contracts | [Loss and metric contracts](CRITERIA_CONTRACTS.md) | | Review public training replay syntax and migration | [Training replay contracts](TRAINING_REPLAY_CONTRACTS.md) | | Review conformal/robustness W0 contracts | [Conformal contract foundation](contracts/README.md#conformal-prediction-and-robustness-foundation-v1) | | Review conformal ownership and lifecycle modes | [ADR-20](adr/ADR-20-conformal-calibration-ownership.md) | @@ -92,6 +93,7 @@ CAPABILITY_MATRIX :hidden: contracts/README +CRITERIA_CONTRACTS TRAINING_CONTRACTS TRAINING_REPLAY_CONTRACTS adr/README diff --git a/examples/fixtures/criteria/criteria_contracts.v1.json b/examples/fixtures/criteria/criteria_contracts.v1.json new file mode 100644 index 0000000..5381bc4 --- /dev/null +++ b/examples/fixtures/criteria/criteria_contracts.v1.json @@ -0,0 +1,523 @@ +{ + "profile": "dagml.criteria.contracts.v1", + "canonicalization": "TCV1-unicode-17.0.0", + "valid": { + "loss_spec": { + "schema_version": 1, + "loss_id": "example.loss.asymmetric@1", + "kind": "custom", + "task_kinds": [ + "regression" + ], + "prediction_kinds": [ + "regression_point" + ], + "objective": "minimize", + "reduction": "mean", + "required_inputs": [ + "target", + "prediction" + ], + "capabilities": [ + "differentiable" + ], + "parameters": { + "over_weight": 1.0, + "under_weight": 2.0 + }, + "spec_fingerprint": "cf661225cc7137ab5ef9b87871ed5736a8479dd21587b7e17150c442b1e43eb0" + }, + "metric_spec": { + "schema_version": 1, + "metric_id": "example.metric.bias@1", + "kind": "custom", + "task_kinds": [ + "regression" + ], + "prediction_kinds": [ + "regression_point" + ], + "objective": "minimize", + "supported_levels": [ + "sample" + ], + "decomposition": "global", + "reduction": "global", + "required_inputs": [ + "target", + "prediction" + ], + "capabilities": [], + "parameters": {}, + "spec_fingerprint": "be54e6824479b10c398c229169bd324387c2fbb932e8fa879e37eba7d8821006" + }, + "loss_implementation": { + "schema_version": 1, + "semantic_kind": "loss", + "semantic_id": "example.loss.asymmetric@1", + "semantic_fingerprint": "cf661225cc7137ab5ef9b87871ed5736a8479dd21587b7e17150c442b1e43eb0", + "provider_id": "provider:python-local", + "binding_id": "binding:python", + "implementation_version": "1.0.0", + "implementation_fingerprint": "1f4c71b0b758c5ed25b4e38b132b9ad56fb2f5ff2cf490f7eb8786c4350a62f7", + "supported_controller_families": [], + "runtime_requirements": [], + "capabilities": [ + "deterministic", + "differentiable", + "needs_gil" + ], + "portability": "host_local", + "replayability": "registry_required", + "registry_key": "loss:run-123:asymmetric", + "descriptor_fingerprint": "031c7b120740620dade9ba14a5f2e142831ecb1be47ea7181f68511dfafdd807" + }, + "metric_implementation": { + "schema_version": 1, + "semantic_kind": "metric", + "semantic_id": "example.metric.bias@1", + "semantic_fingerprint": "be54e6824479b10c398c229169bd324387c2fbb932e8fa879e37eba7d8821006", + "provider_id": "provider:rust-local", + "binding_id": "binding:rust", + "implementation_version": "1.0.0", + "implementation_fingerprint": "4991854599d650fd613dfd02b10d90a649ad7fec85f20a027d5e7b2a553f628b", + "supported_controller_families": [], + "runtime_requirements": [], + "capabilities": [ + "deterministic" + ], + "portability": "host_local", + "replayability": "registry_required", + "registry_key": "metric:run-123:bias", + "descriptor_fingerprint": "33f6d7946a6eebd21b4e6b6578a78643eaaca8ba7725aaf3f6d84fd8d3b13077" + }, + "training_loss_role": { + "schema_version": 1, + "node_id": "model:custom", + "output_id": "prediction", + "phases": [ + "FIT_CV", + "REFIT" + ], + "loss": { + "spec": { + "schema_version": 1, + "loss_id": "example.loss.asymmetric@1", + "kind": "custom", + "task_kinds": [ + "regression" + ], + "prediction_kinds": [ + "regression_point" + ], + "objective": "minimize", + "reduction": "mean", + "required_inputs": [ + "target", + "prediction" + ], + "capabilities": [ + "differentiable" + ], + "parameters": { + "over_weight": 1.0, + "under_weight": 2.0 + }, + "spec_fingerprint": "cf661225cc7137ab5ef9b87871ed5736a8479dd21587b7e17150c442b1e43eb0" + }, + "implementation": { + "schema_version": 1, + "semantic_kind": "loss", + "semantic_id": "example.loss.asymmetric@1", + "semantic_fingerprint": "cf661225cc7137ab5ef9b87871ed5736a8479dd21587b7e17150c442b1e43eb0", + "provider_id": "provider:python-local", + "binding_id": "binding:python", + "implementation_version": "1.0.0", + "implementation_fingerprint": "1f4c71b0b758c5ed25b4e38b132b9ad56fb2f5ff2cf490f7eb8786c4350a62f7", + "supported_controller_families": [], + "runtime_requirements": [], + "capabilities": [ + "deterministic", + "differentiable", + "needs_gil" + ], + "portability": "host_local", + "replayability": "registry_required", + "registry_key": "loss:run-123:asymmetric", + "descriptor_fingerprint": "031c7b120740620dade9ba14a5f2e142831ecb1be47ea7181f68511dfafdd807" + } + } + }, + "metric_role": { + "schema_version": 1, + "role_id": "selection:bias", + "role": "selection", + "output_id": "prediction", + "partition": "validation", + "level": "sample", + "missing_value_policy": "error", + "metric": { + "spec": { + "schema_version": 1, + "metric_id": "example.metric.bias@1", + "kind": "custom", + "task_kinds": [ + "regression" + ], + "prediction_kinds": [ + "regression_point" + ], + "objective": "minimize", + "supported_levels": [ + "sample" + ], + "decomposition": "global", + "reduction": "global", + "required_inputs": [ + "target", + "prediction" + ], + "capabilities": [], + "parameters": {}, + "spec_fingerprint": "be54e6824479b10c398c229169bd324387c2fbb932e8fa879e37eba7d8821006" + }, + "implementation": { + "schema_version": 1, + "semantic_kind": "metric", + "semantic_id": "example.metric.bias@1", + "semantic_fingerprint": "be54e6824479b10c398c229169bd324387c2fbb932e8fa879e37eba7d8821006", + "provider_id": "provider:rust-local", + "binding_id": "binding:rust", + "implementation_version": "1.0.0", + "implementation_fingerprint": "4991854599d650fd613dfd02b10d90a649ad7fec85f20a027d5e7b2a553f628b", + "supported_controller_families": [], + "runtime_requirements": [], + "capabilities": [ + "deterministic" + ], + "portability": "host_local", + "replayability": "registry_required", + "registry_key": "metric:run-123:bias", + "descriptor_fingerprint": "33f6d7946a6eebd21b4e6b6578a78643eaaca8ba7725aaf3f6d84fd8d3b13077" + } + } + } + }, + "invalid": [ + { + "id": "loss_unversioned_id", + "contract": "loss_spec", + "document": { + "schema_version": 1, + "loss_id": "example.loss.asymmetric", + "kind": "custom", + "task_kinds": [ + "regression" + ], + "prediction_kinds": [ + "regression_point" + ], + "objective": "minimize", + "reduction": "mean", + "required_inputs": [ + "target", + "prediction" + ], + "capabilities": [ + "differentiable" + ], + "parameters": {}, + "spec_fingerprint": "cf661225cc7137ab5ef9b87871ed5736a8479dd21587b7e17150c442b1e43eb0" + } + }, + { + "id": "loss_leading_zero_version", + "contract": "loss_spec", + "document": { + "schema_version": 1, + "loss_id": "example.loss.asymmetric@01", + "kind": "custom", + "task_kinds": [ + "regression" + ], + "prediction_kinds": [ + "regression_point" + ], + "objective": "minimize", + "reduction": "mean", + "required_inputs": [ + "target", + "prediction" + ], + "capabilities": [], + "parameters": {}, + "spec_fingerprint": "0000000000000000000000000000000000000000000000000000000000000000" + } + }, + { + "id": "loss_c1_control_id", + "contract": "loss_spec", + "document": { + "schema_version": 1, + "loss_id": "example.loss.\u0085unsafe@1", + "kind": "custom", + "task_kinds": [ + "regression" + ], + "prediction_kinds": [ + "regression_point" + ], + "objective": "minimize", + "reduction": "mean", + "required_inputs": [ + "target", + "prediction" + ], + "capabilities": [], + "parameters": {}, + "spec_fingerprint": "0000000000000000000000000000000000000000000000000000000000000000" + } + }, + { + "id": "loss_uppercase_callable_payload", + "contract": "loss_spec", + "document": { + "schema_version": 1, + "loss_id": "example.loss.unsafe@1", + "kind": "custom", + "task_kinds": [ + "regression" + ], + "prediction_kinds": [ + "regression_point" + ], + "objective": "minimize", + "reduction": "mean", + "required_inputs": [ + "target", + "prediction" + ], + "capabilities": [], + "parameters": { + "nested": { + "CallAble": "lambda y, p: 0" + } + }, + "spec_fingerprint": "0000000000000000000000000000000000000000000000000000000000000000" + } + }, + { + "id": "loss_unknown_task", + "contract": "loss_spec", + "document": { + "schema_version": 1, + "loss_id": "example.loss.ranking@1", + "kind": "custom", + "task_kinds": [ + "ranking" + ], + "prediction_kinds": [ + "decision_score" + ], + "objective": "minimize", + "reduction": "mean", + "required_inputs": [ + "target", + "prediction" + ], + "capabilities": [], + "parameters": {}, + "spec_fingerprint": "0000000000000000000000000000000000000000000000000000000000000000" + } + }, + { + "id": "loss_nested_callable_payload", + "contract": "loss_spec", + "document": { + "schema_version": 1, + "loss_id": "example.loss.unsafe@1", + "kind": "custom", + "task_kinds": [ + "regression" + ], + "prediction_kinds": [ + "regression_point" + ], + "objective": "minimize", + "reduction": "mean", + "required_inputs": [ + "target", + "prediction" + ], + "capabilities": [], + "parameters": { + "implementation": { + "callable": "lambda y, p: 0" + } + }, + "spec_fingerprint": "0000000000000000000000000000000000000000000000000000000000000000" + } + }, + { + "id": "loss_weighted_without_weight_input", + "contract": "loss_spec", + "document": { + "schema_version": 1, + "loss_id": "example.loss.weighted@1", + "kind": "custom", + "task_kinds": [ + "regression" + ], + "prediction_kinds": [ + "regression_point" + ], + "objective": "minimize", + "reduction": "weighted_mean", + "required_inputs": [ + "target", + "prediction" + ], + "capabilities": [], + "parameters": {}, + "spec_fingerprint": "0000000000000000000000000000000000000000000000000000000000000000" + } + }, + { + "id": "loss_mismatched_fingerprint", + "contract": "loss_spec", + "document": { + "schema_version": 1, + "loss_id": "example.loss.asymmetric@1", + "kind": "custom", + "task_kinds": [ + "regression" + ], + "prediction_kinds": [ + "regression_point" + ], + "objective": "minimize", + "reduction": "mean", + "required_inputs": [ + "target", + "prediction" + ], + "capabilities": [ + "differentiable" + ], + "parameters": { + "over_weight": 1.0, + "under_weight": 2.0 + }, + "spec_fingerprint": "0000000000000000000000000000000000000000000000000000000000000000" + } + }, + { + "id": "metric_without_objective", + "contract": "metric_spec", + "document": { + "schema_version": 1, + "metric_id": "example.metric.bias@1", + "kind": "custom", + "task_kinds": [ + "regression" + ], + "prediction_kinds": [ + "regression_point" + ], + "supported_levels": [ + "sample" + ], + "decomposition": "global", + "reduction": "global", + "required_inputs": [ + "target", + "prediction" + ], + "capabilities": [], + "parameters": {}, + "spec_fingerprint": "be54e6824479b10c398c229169bd324387c2fbb932e8fa879e37eba7d8821006" + } + }, + { + "id": "host_local_descriptor_without_registry_key", + "contract": "implementation_descriptor", + "document": { + "schema_version": 1, + "semantic_kind": "loss", + "semantic_id": "example.loss.asymmetric@1", + "semantic_fingerprint": "cf661225cc7137ab5ef9b87871ed5736a8479dd21587b7e17150c442b1e43eb0", + "provider_id": "provider:python-local", + "binding_id": "binding:python", + "implementation_version": "1.0.0", + "implementation_fingerprint": "1f4c71b0b758c5ed25b4e38b132b9ad56fb2f5ff2cf490f7eb8786c4350a62f7", + "supported_controller_families": [], + "runtime_requirements": [], + "capabilities": [ + "deterministic", + "differentiable", + "needs_gil" + ], + "portability": "host_local", + "replayability": "registry_required", + "descriptor_fingerprint": "031c7b120740620dade9ba14a5f2e142831ecb1be47ea7181f68511dfafdd807" + } + }, + { + "id": "selection_metric_skips_missing_values", + "contract": "metric_role", + "document": { + "schema_version": 1, + "role_id": "selection:bias", + "role": "selection", + "partition": "validation", + "level": "sample", + "missing_value_policy": "skip", + "metric": { + "spec": { + "schema_version": 1, + "metric_id": "example.metric.bias@1", + "kind": "custom", + "task_kinds": [ + "regression" + ], + "prediction_kinds": [ + "regression_point" + ], + "objective": "minimize", + "supported_levels": [ + "sample" + ], + "decomposition": "global", + "reduction": "global", + "required_inputs": [ + "target", + "prediction" + ], + "capabilities": [], + "parameters": {}, + "spec_fingerprint": "be54e6824479b10c398c229169bd324387c2fbb932e8fa879e37eba7d8821006" + }, + "implementation": { + "schema_version": 1, + "semantic_kind": "metric", + "semantic_id": "example.metric.bias@1", + "semantic_fingerprint": "be54e6824479b10c398c229169bd324387c2fbb932e8fa879e37eba7d8821006", + "provider_id": "provider:rust-local", + "binding_id": "binding:rust", + "implementation_version": "1.0.0", + "implementation_fingerprint": "4991854599d650fd613dfd02b10d90a649ad7fec85f20a027d5e7b2a553f628b", + "supported_controller_families": [], + "runtime_requirements": [], + "capabilities": [ + "deterministic" + ], + "portability": "host_local", + "replayability": "registry_required", + "registry_key": "metric:run-123:bias", + "descriptor_fingerprint": "33f6d7946a6eebd21b4e6b6578a78643eaaca8ba7725aaf3f6d84fd8d3b13077" + } + } + } + } + ] +} diff --git a/examples/fixtures/criteria/metric_provider_contracts.v1.json b/examples/fixtures/criteria/metric_provider_contracts.v1.json new file mode 100644 index 0000000..9980874 --- /dev/null +++ b/examples/fixtures/criteria/metric_provider_contracts.v1.json @@ -0,0 +1,286 @@ +{ + "profile": "dagml.metric-provider.contracts.v1", + "canonicalization": "TCV1-unicode-17.0.0", + "valid": { + "task": { + "schema_version": 1, + "request_id": "metric-request:bias", + "metric": { + "spec": { + "schema_version": 1, + "metric_id": "example.metric.bias@1", + "kind": "custom", + "task_kinds": [ + "regression" + ], + "prediction_kinds": [ + "regression_point" + ], + "objective": "minimize", + "supported_levels": [ + "sample" + ], + "decomposition": "global", + "reduction": "global", + "required_inputs": [ + "target", + "prediction" + ], + "capabilities": [], + "parameters": {}, + "spec_fingerprint": "be54e6824479b10c398c229169bd324387c2fbb932e8fa879e37eba7d8821006" + }, + "implementation": { + "schema_version": 1, + "semantic_kind": "metric", + "semantic_id": "example.metric.bias@1", + "semantic_fingerprint": "be54e6824479b10c398c229169bd324387c2fbb932e8fa879e37eba7d8821006", + "provider_id": "provider:rust-local", + "binding_id": "binding:rust", + "implementation_version": "1.0.0", + "implementation_fingerprint": "4991854599d650fd613dfd02b10d90a649ad7fec85f20a027d5e7b2a553f628b", + "supported_controller_families": [], + "runtime_requirements": [], + "capabilities": [ + "deterministic" + ], + "portability": "host_local", + "replayability": "registry_required", + "registry_key": "metric:run-123:bias", + "descriptor_fingerprint": "33f6d7946a6eebd21b4e6b6578a78643eaaca8ba7725aaf3f6d84fd8d3b13077" + } + }, + "task_kind": "regression", + "prediction_kind": "regression_point", + "scope": { + "producer_node": "model:custom", + "producer_port": "prediction", + "prediction_id": "prediction:validation", + "partition": "validation", + "fold_id": "fold:0", + "level": "sample" + }, + "unit_ids": [ + { + "level": "sample", + "id": "sample:0" + }, + { + "level": "sample", + "id": "sample:1" + } + ], + "predictions": [ + [ + 2.0 + ], + [ + 5.0 + ] + ], + "targets": [ + [ + 1.0 + ], + [ + 3.0 + ] + ], + "output_ids": [ + "target" + ], + "task_fingerprint": "b49dc3751e0c2a30489eee92a995384ab366bc1cc628e613b1bcea3633657430" + }, + "result": { + "schema_version": 1, + "request_id": "metric-request:bias", + "semantic_id": "example.metric.bias@1", + "semantic_fingerprint": "be54e6824479b10c398c229169bd324387c2fbb932e8fa879e37eba7d8821006", + "implementation_fingerprint": "4991854599d650fd613dfd02b10d90a649ad7fec85f20a027d5e7b2a553f628b", + "descriptor_fingerprint": "33f6d7946a6eebd21b4e6b6578a78643eaaca8ba7725aaf3f6d84fd8d3b13077", + "scope": { + "producer_node": "model:custom", + "producer_port": "prediction", + "prediction_id": "prediction:validation", + "partition": "validation", + "fold_id": "fold:0", + "level": "sample" + }, + "values": [ + { + "value": 1.5 + } + ], + "result_fingerprint": "183392e42edb031021d6ba1b25d6019509597d955b29cbb6d9b3c4a695c631f4" + }, + "aggregate": 1.5 + }, + "invalid": [ + { + "id": "custom_metric_without_objective", + "contract": "metric_evaluation_task", + "document": { + "schema_version": 1, + "request_id": "metric-request:bias", + "metric": { + "spec": { + "schema_version": 1, + "metric_id": "example.metric.bias@1", + "kind": "custom", + "task_kinds": [ + "regression" + ], + "prediction_kinds": [ + "regression_point" + ], + "supported_levels": [ + "sample" + ], + "decomposition": "global", + "reduction": "global", + "required_inputs": [ + "target", + "prediction" + ], + "capabilities": [], + "parameters": {}, + "spec_fingerprint": "be54e6824479b10c398c229169bd324387c2fbb932e8fa879e37eba7d8821006" + }, + "implementation": { + "schema_version": 1, + "semantic_kind": "metric", + "semantic_id": "example.metric.bias@1", + "semantic_fingerprint": "be54e6824479b10c398c229169bd324387c2fbb932e8fa879e37eba7d8821006", + "provider_id": "provider:rust-local", + "binding_id": "binding:rust", + "implementation_version": "1.0.0", + "implementation_fingerprint": "4991854599d650fd613dfd02b10d90a649ad7fec85f20a027d5e7b2a553f628b", + "supported_controller_families": [], + "runtime_requirements": [], + "capabilities": [ + "deterministic" + ], + "portability": "host_local", + "replayability": "registry_required", + "registry_key": "metric:run-123:bias", + "descriptor_fingerprint": "33f6d7946a6eebd21b4e6b6578a78643eaaca8ba7725aaf3f6d84fd8d3b13077" + } + }, + "task_kind": "regression", + "prediction_kind": "regression_point", + "scope": { + "producer_node": "model:custom", + "producer_port": "prediction", + "prediction_id": "prediction:validation", + "partition": "validation", + "fold_id": "fold:0", + "level": "sample" + }, + "unit_ids": [ + { + "level": "sample", + "id": "sample:0" + }, + { + "level": "sample", + "id": "sample:1" + } + ], + "predictions": [ + [ + 2.0 + ], + [ + 5.0 + ] + ], + "targets": [ + [ + 1.0 + ], + [ + 3.0 + ] + ], + "output_ids": [ + "target" + ], + "task_fingerprint": "b49dc3751e0c2a30489eee92a995384ab366bc1cc628e613b1bcea3633657430" + } + }, + { + "id": "provider_result_wrong_scope", + "contract": "metric_evaluation_result", + "document": { + "schema_version": 1, + "request_id": "metric-request:bias", + "semantic_id": "example.metric.bias@1", + "semantic_fingerprint": "be54e6824479b10c398c229169bd324387c2fbb932e8fa879e37eba7d8821006", + "implementation_fingerprint": "4991854599d650fd613dfd02b10d90a649ad7fec85f20a027d5e7b2a553f628b", + "descriptor_fingerprint": "33f6d7946a6eebd21b4e6b6578a78643eaaca8ba7725aaf3f6d84fd8d3b13077", + "scope": { + "producer_node": "model:custom", + "producer_port": "prediction", + "prediction_id": "prediction:validation", + "partition": "test", + "fold_id": "fold:0", + "level": "sample" + }, + "values": [ + { + "value": 1.5 + } + ], + "result_fingerprint": "89b5cee0f5efbe5df74773e7bbcf9f5ba141911b5bd948214d4a6bb4b6766699" + } + }, + { + "id": "provider_result_wrong_coverage", + "contract": "metric_evaluation_result", + "document": { + "schema_version": 1, + "request_id": "metric-request:bias", + "semantic_id": "example.metric.bias@1", + "semantic_fingerprint": "be54e6824479b10c398c229169bd324387c2fbb932e8fa879e37eba7d8821006", + "implementation_fingerprint": "4991854599d650fd613dfd02b10d90a649ad7fec85f20a027d5e7b2a553f628b", + "descriptor_fingerprint": "33f6d7946a6eebd21b4e6b6578a78643eaaca8ba7725aaf3f6d84fd8d3b13077", + "scope": { + "producer_node": "model:custom", + "producer_port": "prediction", + "prediction_id": "prediction:validation", + "partition": "validation", + "fold_id": "fold:0", + "level": "sample" + }, + "values": [], + "result_fingerprint": "97147dc7cd9e7572f87f4d9430542c0f0834c7d18694c58b430ef79c9f34f6fb" + } + }, + { + "id": "provider_result_mismatched_implementation_fingerprint", + "contract": "metric_evaluation_result", + "document": { + "schema_version": 1, + "request_id": "metric-request:bias", + "semantic_id": "example.metric.bias@1", + "semantic_fingerprint": "be54e6824479b10c398c229169bd324387c2fbb932e8fa879e37eba7d8821006", + "implementation_fingerprint": "0000000000000000000000000000000000000000000000000000000000000000", + "descriptor_fingerprint": "33f6d7946a6eebd21b4e6b6578a78643eaaca8ba7725aaf3f6d84fd8d3b13077", + "scope": { + "producer_node": "model:custom", + "producer_port": "prediction", + "prediction_id": "prediction:validation", + "partition": "validation", + "fold_id": "fold:0", + "level": "sample" + }, + "values": [ + { + "value": 1.5 + } + ], + "result_fingerprint": "676653f80194b5aa75b2cb8043662c2c8d27db2e6017564ada07a709526b129c" + } + } + ] +} diff --git a/parity/criteria/__init__.py b/parity/criteria/__init__.py new file mode 100644 index 0000000..619c3d7 --- /dev/null +++ b/parity/criteria/__init__.py @@ -0,0 +1 @@ +"""Independent parity oracle for loss and metric contracts.""" diff --git a/parity/criteria/generate_fixtures.py b/parity/criteria/generate_fixtures.py new file mode 100644 index 0000000..ec9ca31 --- /dev/null +++ b/parity/criteria/generate_fixtures.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Regenerate criterion fingerprints and the exact L1 conformance pack.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + +from parity.conformal.oracle import fingerprint_without # noqa: E402 + + +FIXTURE = ROOT / "examples/fixtures/criteria/criteria_contracts.v1.json" +PROVIDER_FIXTURE = ROOT / "examples/fixtures/criteria/metric_provider_contracts.v1.json" +PACK = ROOT / "docs/contracts/criteria_conformance_pack.v1.json" +ARTIFACTS = { + "crates/dag-ml-cli/src/main.rs": "cli_validator", + "crates/dag-ml-cli/tests/cli_contracts.rs": "cli_test", + "crates/dag-ml-core/src/criteria.rs": "rust_contract", + "crates/dag-ml-core/src/metric_provider.rs": "rust_provider_contract", + "crates/dag-ml-core/src/metrics.rs": "native_metric_adapter", + "docs/CRITERIA_CONTRACTS.md": "contract_documentation", + "docs/contracts/implementation_descriptor.schema.json": "schema", + "docs/contracts/loss_spec.schema.json": "schema", + "docs/contracts/metric_evaluation_result.schema.json": "schema", + "docs/contracts/metric_evaluation_task.schema.json": "schema", + "docs/contracts/metric_role.schema.json": "schema", + "docs/contracts/metric_spec.schema.json": "schema", + "docs/contracts/training_loss_role.schema.json": "schema", + "examples/fixtures/criteria/criteria_contracts.v1.json": "fixture", + "examples/fixtures/criteria/metric_provider_contracts.v1.json": "fixture", + "parity/criteria/oracle.py": "independent_validator", + "parity/criteria/generate_fixtures.py": "generator", + "parity/criteria/tests/test_criteria_contracts.py": "parity_test", +} + + +def load(path: Path) -> dict[str, Any]: + with path.open(encoding="utf-8") as handle: + return json.load(handle) + + +def write(path: Path, document: dict[str, Any]) -> None: + path.write_text( + json.dumps(document, indent=2, ensure_ascii=True) + "\n", encoding="utf-8" + ) + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def build_provider_fixture(valid: dict[str, Any]) -> dict[str, Any]: + scope = { + "producer_node": "model:custom", + "producer_port": "prediction", + "prediction_id": "prediction:validation", + "partition": "validation", + "fold_id": "fold:0", + "level": "sample", + } + task = { + "schema_version": 1, + "request_id": "metric-request:bias", + "metric": { + "spec": copy.deepcopy(valid["metric_spec"]), + "implementation": copy.deepcopy(valid["metric_implementation"]), + }, + "task_kind": "regression", + "prediction_kind": "regression_point", + "scope": scope, + "unit_ids": [ + {"level": "sample", "id": "sample:0"}, + {"level": "sample", "id": "sample:1"}, + ], + "predictions": [[2.0], [5.0]], + "targets": [[1.0], [3.0]], + "output_ids": ["target"], + "task_fingerprint": "", + } + task["task_fingerprint"] = fingerprint_without(task, "task_fingerprint") + result = { + "schema_version": 1, + "request_id": task["request_id"], + "semantic_id": task["metric"]["spec"]["metric_id"], + "semantic_fingerprint": task["metric"]["spec"]["spec_fingerprint"], + "implementation_fingerprint": task["metric"]["implementation"][ + "implementation_fingerprint" + ], + "descriptor_fingerprint": task["metric"]["implementation"][ + "descriptor_fingerprint" + ], + "scope": copy.deepcopy(scope), + "values": [{"value": 1.5}], + "result_fingerprint": "", + } + result["result_fingerprint"] = fingerprint_without(result, "result_fingerprint") + + missing_objective = copy.deepcopy(task) + missing_objective["metric"]["spec"].pop("objective") + wrong_scope = copy.deepcopy(result) + wrong_scope["scope"]["partition"] = "test" + wrong_scope["result_fingerprint"] = fingerprint_without( + wrong_scope, "result_fingerprint" + ) + wrong_coverage = copy.deepcopy(result) + wrong_coverage["values"] = [] + wrong_coverage["result_fingerprint"] = fingerprint_without( + wrong_coverage, "result_fingerprint" + ) + wrong_implementation = copy.deepcopy(result) + wrong_implementation["implementation_fingerprint"] = "0" * 64 + wrong_implementation["result_fingerprint"] = fingerprint_without( + wrong_implementation, "result_fingerprint" + ) + return { + "profile": "dagml.metric-provider.contracts.v1", + "canonicalization": "TCV1-unicode-17.0.0", + "valid": {"task": task, "result": result, "aggregate": 1.5}, + "invalid": [ + { + "id": "custom_metric_without_objective", + "contract": "metric_evaluation_task", + "document": missing_objective, + }, + { + "id": "provider_result_wrong_scope", + "contract": "metric_evaluation_result", + "document": wrong_scope, + }, + { + "id": "provider_result_wrong_coverage", + "contract": "metric_evaluation_result", + "document": wrong_coverage, + }, + { + "id": "provider_result_mismatched_implementation_fingerprint", + "contract": "metric_evaluation_result", + "document": wrong_implementation, + }, + ], + } + + +def main() -> None: + fixture = load(FIXTURE) + valid = fixture["valid"] + for key in ("loss_spec", "metric_spec"): + valid[key]["spec_fingerprint"] = fingerprint_without(valid[key], "spec_fingerprint") + valid["loss_implementation"]["semantic_fingerprint"] = valid["loss_spec"]["spec_fingerprint"] + valid["metric_implementation"]["semantic_fingerprint"] = valid["metric_spec"]["spec_fingerprint"] + for key in ("loss_implementation", "metric_implementation"): + valid[key]["descriptor_fingerprint"] = fingerprint_without( + valid[key], "descriptor_fingerprint" + ) + valid["training_loss_role"]["loss"] = { + "spec": valid["loss_spec"], + "implementation": valid["loss_implementation"], + } + valid["metric_role"]["metric"] = { + "spec": valid["metric_spec"], + "implementation": valid["metric_implementation"], + } + write(FIXTURE, fixture) + provider_fixture = build_provider_fixture(valid) + write(PROVIDER_FIXTURE, provider_fixture) + + pack: dict[str, Any] = { + "pack_id": "dag-ml.criteria-conformance.v1", + "schema_version": 1, + "hash_algorithm": "sha256-file-bytes", + "fingerprint_profile": "DAGML-TCV1-unicode-17.0.0", + "artifacts": [ + {"path": path, "sha256": sha256(ROOT / path), "kind": kind} + for path, kind in sorted(ARTIFACTS.items()) + ], + "required_negative_cases": [ + case["id"] + for case in fixture["invalid"] + provider_fixture["invalid"] + ], + "runtime_only_negative_cases": ["provider_result_non_finite"], + "pack_checksum": "", + } + pack["pack_checksum"] = fingerprint_without(pack, "pack_checksum") + write(PACK, pack) + + +if __name__ == "__main__": + main() diff --git a/parity/criteria/oracle.py b/parity/criteria/oracle.py new file mode 100644 index 0000000..3b26a25 --- /dev/null +++ b/parity/criteria/oracle.py @@ -0,0 +1,335 @@ +"""Independent semantic validator for DAG-ML criterion contracts.""" + +from __future__ import annotations + +import math +import re +from typing import Any, Callable + +from parity.conformal.oracle import fingerprint_without + + +class CriteriaContractError(ValueError): + """Raised when a criterion contract violates semantic invariants.""" + + +FORBIDDEN_EXECUTABLE_KEYS = { + "bytecode", + "callable", + "code", + "function_source", + "import_path", + "module_path", + "pickle", + "serialized_callable", + "source_code", +} +VERSIONED_ID = re.compile(r"^[^@\s\x00-\x1f\x7f-\x9f]+@[1-9][0-9]*$") +TOKEN = re.compile(r"^[^\s\x00-\x1f\x7f-\x9f]+$") +SHA256 = re.compile(r"^[0-9a-f]{64}$") + + +def require(condition: bool, message: str) -> None: + if not condition: + raise CriteriaContractError(message) + + +def _require_fingerprint(document: dict[str, Any], field: str) -> None: + declared = document.get(field) + require(isinstance(declared, str) and SHA256.fullmatch(declared) is not None, field) + require(declared == fingerprint_without(document, field), f"{field} mismatch") + + +def _reject_executable_payload(value: Any, path: str) -> None: + if isinstance(value, dict): + for key, member in value.items(): + require(key.lower() not in FORBIDDEN_EXECUTABLE_KEYS, f"{path}.{key}") + _reject_executable_payload(member, f"{path}.{key}") + elif isinstance(value, list): + for index, member in enumerate(value): + _reject_executable_payload(member, f"{path}[{index}]") + + +def _validate_common_spec(document: dict[str, Any], id_field: str) -> None: + require(document.get("schema_version") == 1, "schema_version") + identifier = document.get(id_field) + require(isinstance(identifier, str) and VERSIONED_ID.fullmatch(identifier) is not None, id_field) + require(bool(document.get("task_kinds")), "task_kinds") + require(bool(document.get("prediction_kinds")), "prediction_kinds") + inputs = set(document.get("required_inputs", [])) + require({"target", "prediction"} <= inputs, "required_inputs") + parameters = document.get("parameters") + require(isinstance(parameters, dict), "parameters") + _reject_executable_payload(parameters, "parameters") + + +def validate_loss_spec(document: dict[str, Any]) -> None: + _validate_common_spec(document, "loss_id") + require(document.get("objective") == "minimize", "loss objective") + inputs = set(document["required_inputs"]) + capabilities = set(document.get("capabilities", [])) + if document.get("reduction") == "weighted_mean": + require("sample_weight" in inputs, "weighted loss input") + require("supports_sample_weights" in capabilities, "weighted loss capability") + if "sample_weight" in inputs: + require("supports_sample_weights" in capabilities, "sample weight capability") + if "missing_mask" in inputs: + require("supports_missing_mask" in capabilities, "missing mask capability") + _require_fingerprint(document, "spec_fingerprint") + + +def validate_metric_spec(document: dict[str, Any]) -> None: + _validate_common_spec(document, "metric_id") + require(document.get("objective") in {"minimize", "maximize"}, "metric objective") + require(bool(document.get("supported_levels")), "supported_levels") + decomposition = document.get("decomposition") + reduction = document.get("reduction") + capabilities = set(document.get("capabilities", [])) + inputs = set(document["required_inputs"]) + if decomposition == "global": + require(reduction == "global", "global metric reduction") + else: + require(reduction != "global", "decomposed metric reduction") + require("decomposable" in capabilities, "decomposable capability") + if reduction == "weighted_mean": + require(decomposition == "per_unit", "weighted metric decomposition") + require("sample_weight" in inputs, "weighted metric input") + require("supports_sample_weights" in capabilities, "weighted metric capability") + if "sample_weight" in inputs: + require("supports_sample_weights" in capabilities, "sample weight capability") + if "missing_mask" in inputs: + require("supports_missing_mask" in capabilities, "missing mask capability") + _require_fingerprint(document, "spec_fingerprint") + + +def validate_implementation_descriptor(document: dict[str, Any]) -> None: + require(document.get("schema_version") == 1, "schema_version") + semantic_id = document.get("semantic_id") + require( + isinstance(semantic_id, str) and VERSIONED_ID.fullmatch(semantic_id) is not None, + "semantic_id", + ) + for field in ("semantic_fingerprint", "implementation_fingerprint"): + value = document.get(field) + require(isinstance(value, str) and SHA256.fullmatch(value) is not None, field) + for field in ("provider_id", "binding_id", "implementation_version"): + value = document.get(field) + require(isinstance(value, str) and TOKEN.fullmatch(value) is not None, field) + portability = document.get("portability") + replayability = document.get("replayability") + registry_key = document.get("registry_key") + if registry_key is not None: + require( + isinstance(registry_key, str) and TOKEN.fullmatch(registry_key) is not None, + "registry_key", + ) + if portability == "host_local": + require(bool(registry_key), "host_local registry_key") + require(replayability != "detached", "host_local replayability") + elif portability == "portable_registered": + require(bool(registry_key), "portable_registered registry_key") + require(replayability == "registry_required", "portable_registered replayability") + elif portability == "portable_built_in": + require(registry_key is None, "portable_built_in registry_key") + require(replayability == "detached", "portable_built_in replayability") + else: + raise CriteriaContractError("portability") + _require_fingerprint(document, "descriptor_fingerprint") + + +def _validate_reference( + reference: dict[str, Any], + semantic_kind: str, + id_field: str, + spec_validator: Callable[[dict[str, Any]], None], +) -> None: + spec = reference["spec"] + implementation = reference["implementation"] + spec_validator(spec) + validate_implementation_descriptor(implementation) + require(implementation["semantic_kind"] == semantic_kind, "semantic_kind") + require(implementation["semantic_id"] == spec[id_field], "semantic_id") + require( + implementation["semantic_fingerprint"] == spec["spec_fingerprint"], + "semantic_fingerprint", + ) + + +def validate_training_loss_role(document: dict[str, Any]) -> None: + require(document.get("schema_version") == 1, "schema_version") + phases = document.get("phases") + require(bool(phases) and set(phases) <= {"FIT_CV", "REFIT"}, "loss phases") + _validate_reference(document["loss"], "loss", "loss_id", validate_loss_spec) + + +def validate_metric_role(document: dict[str, Any]) -> None: + require(document.get("schema_version") == 1, "schema_version") + if document.get("missing_value_policy") == "skip": + require(document.get("role") == "reporting", "missing value policy") + _validate_reference(document["metric"], "metric", "metric_id", validate_metric_spec) + require(document["level"] in document["metric"]["spec"]["supported_levels"], "level") + + +def _matrix_shape(values: Any, label: str) -> tuple[int, int]: + require(isinstance(values, list) and bool(values), label) + require(all(isinstance(row, list) and bool(row) for row in values), label) + width = len(values[0]) + require(all(len(row) == width for row in values), f"{label} ragged") + require( + all( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + for row in values + for value in row + ), + f"{label} non-finite", + ) + return len(values), width + + +def validate_metric_evaluation_task(document: dict[str, Any]) -> None: + require(document.get("schema_version") == 1, "task schema_version") + metric = document["metric"] + _validate_reference(metric, "metric", "metric_id", validate_metric_spec) + spec = metric["spec"] + require(document.get("task_kind") in spec["task_kinds"], "task kind") + require( + document.get("prediction_kind") in spec["prediction_kinds"], + "prediction kind", + ) + scope = document["scope"] + require(scope.get("level") in spec["supported_levels"], "scope level") + units = document.get("unit_ids") + require(isinstance(units, list) and bool(units), "unit_ids") + coordinates = [(unit.get("level"), unit.get("id")) for unit in units] + require(len(coordinates) == len(set(coordinates)), "duplicate unit_ids") + require(all(level == scope["level"] for level, _ in coordinates), "unit level") + prediction_rows, prediction_width = _matrix_shape( + document.get("predictions"), "predictions" + ) + target_rows, target_width = _matrix_shape(document.get("targets"), "targets") + require(prediction_rows == len(units) == target_rows, "row coverage") + output_ids = document.get("output_ids") + require( + isinstance(output_ids, list) + and len(output_ids) == target_width + and len(output_ids) == len(set(output_ids)), + "output_ids", + ) + if document.get("prediction_kind") in {"regression_point", "class_label"}: + require(prediction_width == target_width, "prediction width") + required = set(spec["required_inputs"]) + weights = document.get("sample_weights") + if "sample_weight" in required: + require(weights is not None, "sample_weights") + if weights is not None: + require( + "supports_sample_weights" in spec["capabilities"], + "sample weight capability", + ) + require( + len(weights) == len(units) + and all(math.isfinite(value) and value >= 0 for value in weights) + and sum(weights) > 0, + "sample_weights", + ) + mask = document.get("missing_mask") + if "missing_mask" in required: + require(mask is not None, "missing_mask") + if mask is not None: + require("supports_missing_mask" in spec["capabilities"], "mask capability") + require( + len(mask) == len(units) and all(len(row) == target_width for row in mask), + "missing_mask", + ) + groups = document.get("group_ids") + if "group" in required: + require(groups is not None, "group_ids") + if groups is not None: + require("group" in required, "undeclared group_ids") + require(len(groups) == len(units), "group_ids") + _require_fingerprint(document, "task_fingerprint") + + +def validate_metric_evaluation_result( + document: dict[str, Any], task: dict[str, Any] +) -> float: + validate_metric_evaluation_task(task) + require(document.get("schema_version") == 1, "result schema_version") + spec = task["metric"]["spec"] + implementation = task["metric"]["implementation"] + require(document.get("request_id") == task["request_id"], "request_id") + require(document.get("semantic_id") == spec["metric_id"], "semantic_id") + require( + document.get("semantic_fingerprint") == spec["spec_fingerprint"], + "semantic_fingerprint", + ) + require( + document.get("implementation_fingerprint") + == implementation["implementation_fingerprint"], + "implementation_fingerprint", + ) + require( + document.get("descriptor_fingerprint") + == implementation["descriptor_fingerprint"], + "descriptor_fingerprint", + ) + require(document.get("scope") == task["scope"], "result scope") + values = document.get("values") + require(isinstance(values, list) and bool(values), "result values") + require( + all( + isinstance(value.get("value"), (int, float)) + and not isinstance(value["value"], bool) + and math.isfinite(value["value"]) + for value in values + ), + "non-finite provider value", + ) + decomposition = spec["decomposition"] + if decomposition == "global": + require( + len(values) == 1 + and "unit_id" not in values[0] + and "output_id" not in values[0], + "global result coverage", + ) + elif decomposition == "per_output": + require( + [value.get("output_id") for value in values] == task["output_ids"] + and all("unit_id" not in value for value in values), + "per-output result coverage", + ) + else: + require( + [value.get("unit_id") for value in values] == task["unit_ids"] + and all("output_id" not in value for value in values), + "per-unit result coverage", + ) + _require_fingerprint(document, "result_fingerprint") + raw_values = [value["value"] for value in values] + reduction = spec["reduction"] + if reduction == "global": + aggregate = raw_values[0] + elif reduction == "mean": + aggregate = sum(raw_values) / len(raw_values) + elif reduction == "sum": + aggregate = sum(raw_values) + else: + weights = task["sample_weights"] + aggregate = sum(value * weight for value, weight in zip(raw_values, weights)) / sum( + weights + ) + require(math.isfinite(aggregate), "non-finite metric reduction") + return aggregate + + +VALIDATORS: dict[str, Callable[[dict[str, Any]], None]] = { + "loss_spec": validate_loss_spec, + "metric_spec": validate_metric_spec, + "implementation_descriptor": validate_implementation_descriptor, + "training_loss_role": validate_training_loss_role, + "metric_role": validate_metric_role, + "metric_evaluation_task": validate_metric_evaluation_task, +} diff --git a/parity/criteria/tests/test_criteria_contracts.py b/parity/criteria/tests/test_criteria_contracts.py new file mode 100644 index 0000000..8fe787c --- /dev/null +++ b/parity/criteria/tests/test_criteria_contracts.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import sys +from pathlib import Path +from typing import Any + +import pytest +from jsonschema import Draft202012Validator + + +ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(ROOT)) + +from parity.conformal.oracle import fingerprint_without # noqa: E402 +from parity.criteria.oracle import ( # noqa: E402 + VALIDATORS, + CriteriaContractError, + validate_metric_evaluation_result, +) +from scripts.validate_contracts import build_local_schema_registry # noqa: E402 + + +FIXTURE = ROOT / "examples/fixtures/criteria/criteria_contracts.v1.json" +PACK = ROOT / "docs/contracts/criteria_conformance_pack.v1.json" +PROVIDER_FIXTURE = ROOT / "examples/fixtures/criteria/metric_provider_contracts.v1.json" +SCHEMA_IDS = { + "loss_spec": "https://github.com/GBeurier/dag-ml/schemas/loss_spec.v1.schema.json", + "metric_spec": "https://github.com/GBeurier/dag-ml/schemas/metric_spec.v1.schema.json", + "implementation_descriptor": "https://github.com/GBeurier/dag-ml/schemas/implementation_descriptor.v1.schema.json", + "training_loss_role": "https://github.com/GBeurier/dag-ml/schemas/training_loss_role.v1.schema.json", + "metric_role": "https://github.com/GBeurier/dag-ml/schemas/metric_role.v1.schema.json", + "metric_evaluation_task": "https://github.com/GBeurier/dag-ml/schemas/metric_evaluation_task.v1.schema.json", + "metric_evaluation_result": "https://github.com/GBeurier/dag-ml/schemas/metric_evaluation_result.v1.schema.json", +} +VALID_CONTRACTS = { + "loss_spec": "loss_spec", + "metric_spec": "metric_spec", + "loss_implementation": "implementation_descriptor", + "metric_implementation": "implementation_descriptor", + "training_loss_role": "training_loss_role", + "metric_role": "metric_role", +} + + +def load(path: Path) -> dict[str, Any]: + with path.open(encoding="utf-8") as handle: + return json.load(handle) + + +def schema_errors(contract: str, document: dict[str, Any]) -> list[Any]: + registry, schemas = build_local_schema_registry() + validator = Draft202012Validator(schemas[SCHEMA_IDS[contract]], registry=registry) + return list(validator.iter_errors(document)) + + +def test_valid_fixture_matches_all_schemas_and_independent_semantics() -> None: + fixture = load(FIXTURE) + for key, contract in VALID_CONTRACTS.items(): + document = fixture["valid"][key] + assert schema_errors(contract, document) == [], key + VALIDATORS[contract](document) + + +def test_positive_fingerprints_are_frozen_tcv1_values() -> None: + valid = load(FIXTURE)["valid"] + for key, field in ( + ("loss_spec", "spec_fingerprint"), + ("metric_spec", "spec_fingerprint"), + ("loss_implementation", "descriptor_fingerprint"), + ("metric_implementation", "descriptor_fingerprint"), + ): + assert valid[key][field] == fingerprint_without(valid[key], field), key + + +def test_required_negative_fixture_cases_are_rejected() -> None: + cases = load(FIXTURE)["invalid"] + assert {case["id"] for case in cases} == { + "host_local_descriptor_without_registry_key", + "loss_mismatched_fingerprint", + "loss_nested_callable_payload", + "loss_c1_control_id", + "loss_leading_zero_version", + "loss_uppercase_callable_payload", + "loss_unknown_task", + "loss_unversioned_id", + "loss_weighted_without_weight_input", + "metric_without_objective", + "selection_metric_skips_missing_values", + } + schema_required = { + "loss_c1_control_id", + "loss_leading_zero_version", + "loss_uppercase_callable_payload", + } + for case in cases: + errors = schema_errors(case["contract"], case["document"]) + if case["id"] in schema_required: + assert errors, f"schema accepted parity case {case['id']}" + try: + VALIDATORS[case["contract"]](case["document"]) + except (CriteriaContractError, KeyError): + semantic_rejected = True + else: + semantic_rejected = False + assert errors or semantic_rejected, case["id"] + + +def test_nested_executable_payload_is_rejected_semantically_case_insensitively() -> None: + document = load(FIXTURE)["valid"]["loss_spec"] + document["parameters"] = {"nested": [{"CallAble": "not serialized code"}]} + document["spec_fingerprint"] = fingerprint_without(document, "spec_fingerprint") + with pytest.raises(CriteriaContractError, match="CallAble"): + VALIDATORS["loss_spec"](document) + + +def test_large_decimal_version_is_accepted_by_schema_and_semantics() -> None: + document = load(FIXTURE)["valid"]["loss_spec"] + document["loss_id"] = "example.loss.asymmetric@4294967296" + document["spec_fingerprint"] = fingerprint_without(document, "spec_fingerprint") + assert schema_errors("loss_spec", document) == [] + VALIDATORS["loss_spec"](document) + + +def test_metric_provider_fixture_has_independent_task_result_and_refusal_parity() -> None: + fixture = load(PROVIDER_FIXTURE) + task = fixture["valid"]["task"] + result = fixture["valid"]["result"] + assert schema_errors("metric_evaluation_task", task) == [] + assert schema_errors("metric_evaluation_result", result) == [] + VALIDATORS["metric_evaluation_task"](task) + assert validate_metric_evaluation_result(result, task) == fixture["valid"]["aggregate"] + + for case in fixture["invalid"]: + errors = schema_errors(case["contract"], case["document"]) + try: + if case["contract"] == "metric_evaluation_task": + VALIDATORS[case["contract"]](case["document"]) + else: + validate_metric_evaluation_result(case["document"], task) + except (CriteriaContractError, KeyError): + semantic_rejected = True + else: + semantic_rejected = False + assert errors or semantic_rejected, case["id"] + + nonfinite = copy.deepcopy(result) + nonfinite["values"][0]["value"] = float("nan") + with pytest.raises(CriteriaContractError, match="non-finite provider value"): + validate_metric_evaluation_result(nonfinite, task) + + +def test_conformance_pack_is_exact_and_self_fingerprinted() -> None: + pack = load(PACK) + assert pack["pack_checksum"] == fingerprint_without(pack, "pack_checksum") + paths = [artifact["path"] for artifact in pack["artifacts"]] + assert paths == sorted(set(paths)) + for artifact in pack["artifacts"]: + digest = hashlib.sha256((ROOT / artifact["path"]).read_bytes()).hexdigest() + assert digest == artifact["sha256"], artifact["path"] + assert pack["required_negative_cases"] == [ + case["id"] + for case in load(FIXTURE)["invalid"] + load(PROVIDER_FIXTURE)["invalid"] + ] + assert pack["runtime_only_negative_cases"] == ["provider_result_non_finite"] diff --git a/parity/training/generate_training_replay_fixtures.py b/parity/training/generate_training_replay_fixtures.py index 55932b5..40021d1 100644 --- a/parity/training/generate_training_replay_fixtures.py +++ b/parity/training/generate_training_replay_fixtures.py @@ -43,8 +43,8 @@ ROOT / "docs" / "contracts" / "training_replay_contract_conformance_pack.v1.json" ) BASE_PACK_ID = "dag-ml.training-contracts.v1" -BASE_PACK_SHA256 = "7cc19a167332978670ebe8f24f597ef3b0b6ae80a873b2234eb2a642d466fb55" -BASE_PACK_CHECKSUM = "fd1584c2cd60d566d743cf398e9859dbf3550e50a37e8e0f04013f1ae1f1c86f" +BASE_PACK_SHA256 = "18074d90c040fb681b4780e5db46529ffd5ac80f744b1c9e0bd166abd7e77345" +BASE_PACK_CHECKSUM = "6def60a5f247ae2e8586a74eec3b5b228769ed89d809fd83be82ace2ffa19043" serde_json_sha256 = _serde_sha256 LEGACY_AUTHORITY_SHA256 = { "docs/contracts/replay_outcome.schema.json": "c57279e8c76e4e2467af0eca5eb59804a2f7bb97bec6cce9d8b23975f223c36a", diff --git a/parity/training/tests/test_training_replay_contracts.py b/parity/training/tests/test_training_replay_contracts.py index cd7f49d..b87d55d 100644 --- a/parity/training/tests/test_training_replay_contracts.py +++ b/parity/training/tests/test_training_replay_contracts.py @@ -50,10 +50,10 @@ def _sha256(path: Path) -> str: def test_base_pack_remains_byte_current() -> None: pack = load_json(BASE_PACK) assert _sha256(BASE_PACK) == ( - "7cc19a167332978670ebe8f24f597ef3b0b6ae80a873b2234eb2a642d466fb55" + "18074d90c040fb681b4780e5db46529ffd5ac80f744b1c9e0bd166abd7e77345" ) assert pack["pack_checksum"] == ( - "fd1584c2cd60d566d743cf398e9859dbf3550e50a37e8e0f04013f1ae1f1c86f" + "6def60a5f247ae2e8586a74eec3b5b228769ed89d809fd83be82ace2ffa19043" ) assert len(pack["artifacts"]) == 82 assert all( diff --git a/scripts/validate_training_replay_contracts.py b/scripts/validate_training_replay_contracts.py index 831c3c6..5545c03 100644 --- a/scripts/validate_training_replay_contracts.py +++ b/scripts/validate_training_replay_contracts.py @@ -148,8 +148,8 @@ def validate_output_binding_against_plan( BASE_PACK_PATH = ROOT / "docs/contracts/training_contract_conformance_pack.v1.json" TRAINING_FIXTURE_ROOT = ROOT / "examples/fixtures/training" FIXTURE_ROOT = TRAINING_FIXTURE_ROOT / "replay" -BASE_PACK_SHA256 = "7cc19a167332978670ebe8f24f597ef3b0b6ae80a873b2234eb2a642d466fb55" -BASE_PACK_CHECKSUM = "fd1584c2cd60d566d743cf398e9859dbf3550e50a37e8e0f04013f1ae1f1c86f" +BASE_PACK_SHA256 = "18074d90c040fb681b4780e5db46529ffd5ac80f744b1c9e0bd166abd7e77345" +BASE_PACK_CHECKSUM = "6def60a5f247ae2e8586a74eec3b5b228769ed89d809fd83be82ace2ffa19043" LEGACY_AUTHORITY_SHA256 = { "docs/contracts/replay_outcome.schema.json": "c57279e8c76e4e2467af0eca5eb59804a2f7bb97bec6cce9d8b23975f223c36a", "examples/fixtures/estimator/replay_outcome_predict.v1.json": "037fad7f3cb907f3474cce4f51526538f2c4d6fcad3af93a320c6d282ce470c5",