Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,15 @@ deprecations follow [ADR-14](docs/adr/ADR-14-deprecation-policy.md).

### Added

- Native, backend-neutral `LossSpec`, `MetricSpec`, role-reference and generic
implementation-descriptor contracts under ADR-22. New standalone v1 schemas,
strict TCV1 fingerprints, versioned built-in catalogs, compile-time loss
resolution, negative fixtures, an independent semantic oracle, an exact
conformance pack and CLI validators. A typed metric task/result protocol now
dispatches both built-in and host-local providers through one registry,
validates provider identity, finite values, scope and coverage, and reduces
results before native score persistence. This additive contract publication
does not change the public C ABI.
- ADR-20 and W0 JSON contracts for conformal calibration ownership,
`ParameterPatch`, `OutputBinding`, `TrainingInfluenceManifest`, complete
`TrainingOutcome`/`ReplayOutcome` payloads, and the existing execution-bundle
Expand Down
33 changes: 25 additions & 8 deletions crates/dag-ml-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,15 @@ use dag_ml_core::{
ControllerRegistry, DagMlError, DataRequestPartition, ExecutionBundle, ExplanationBlock,
ExternalDataPlanEnvelope, FileArtifactManifestStore, FileArtifactPayloadStore,
FilePredictionCacheStore, GraphSpec, HandleKind, HandleRef, InMemoryArtifactStore,
InMemoryDataProvider, LineageId, LineageRecord, MetricObjective, NodeId, NodeResult, NodeTask,
OofCampaign, OperatorVariantModel, ParallelScheduler, Phase, PipelineDslSpec,
PortablePredictorPackage, PredictionBlock, PredictionLevel, PredictionPartition,
PredictionUnitId, RefitArtifactRecord, RegressionMetricKind, RegressionMetricReport,
RegressionTargetBlock, ReplayPhaseRequest, ResearchProvenancePackage, RunContext, RunId,
RuntimeArtifactStore, RuntimeController, RuntimeControllerRegistry, RuntimeDataProvider,
RuntimePredictionCacheStore, SampleId, ScoreSet, SelectionDecision, SelectionMetric,
SelectionPolicy, SequentialScheduler, TrainingRequest, VariantId, SCORE_SET_SCHEMA_VERSION,
InMemoryDataProvider, LineageId, LineageRecord, LossSpec, MetricObjective, MetricSpec, NodeId,
NodeResult, NodeTask, OofCampaign, OperatorVariantModel, ParallelScheduler, Phase,
PipelineDslSpec, PortablePredictorPackage, PredictionBlock, PredictionLevel,
PredictionPartition, PredictionUnitId, RefitArtifactRecord, RegressionMetricKind,
RegressionMetricReport, RegressionTargetBlock, ReplayPhaseRequest, ResearchProvenancePackage,
RunContext, RunId, RuntimeArtifactStore, RuntimeController, RuntimeControllerRegistry,
RuntimeDataProvider, RuntimePredictionCacheStore, SampleId, ScoreSet, SelectionDecision,
SelectionMetric, SelectionPolicy, SequentialScheduler, TrainingRequest, VariantId,
SCORE_SET_SCHEMA_VERSION,
};
use serde::{Deserialize, Serialize};

Expand Down Expand Up @@ -220,6 +221,14 @@ enum Command {
ValidateGraph {
path: PathBuf,
},
/// Strictly validate a versioned training-loss semantic contract.
ValidateLossSpec {
path: PathBuf,
},
/// Strictly validate a versioned metric semantic contract.
ValidateMetricSpec {
path: PathBuf,
},
/// Strictly parse, fingerprint and semantically validate a W1 TrainingRequest.
ValidateTrainingRequest {
path: PathBuf,
Expand Down Expand Up @@ -882,6 +891,14 @@ fn main() -> Result<()> {
let graph = read_external_contract(&path, "graph", GraphSpec::from_json)?;
println!("valid graph: {}", graph.id);
}
Command::ValidateLossSpec { path } => {
let spec = read_external_contract(&path, "loss spec", LossSpec::from_json)?;
println!("valid loss spec: {}", spec.loss_id);
}
Command::ValidateMetricSpec { path } => {
let spec = read_external_contract(&path, "metric spec", MetricSpec::from_json)?;
println!("valid metric spec: {}", spec.metric_id);
}
Command::ValidateTrainingRequest { path } => {
let json = std::fs::read_to_string(&path).with_context(|| {
format!("failed to read training request JSON at {}", path.display())
Expand Down
74 changes: 74 additions & 0 deletions crates/dag-ml-cli/tests/cli_contracts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,80 @@ fn sha256_hex(bytes: &[u8]) -> String {
out
}

#[test]
fn cli_validates_loss_and_metric_specs_and_rejects_tampering() {
let root = repo_root();
let fixture: serde_json::Value = serde_json::from_slice(
&std::fs::read(root.join("examples/fixtures/criteria/criteria_contracts.v1.json"))
.expect("criteria fixture is readable"),
)
.expect("criteria fixture is JSON");

for (command, key, expected) in [
(
"validate-loss-spec",
"loss_spec",
"valid loss spec: example.loss.asymmetric@1",
),
(
"validate-metric-spec",
"metric_spec",
"valid metric spec: example.metric.bias@1",
),
] {
let path = std::env::temp_dir().join(format!(
"dag_ml_cli_{key}_{}_{}.json",
std::process::id(),
unique_suffix()
));
std::fs::write(
&path,
serde_json::to_vec_pretty(&fixture["valid"][key]).unwrap(),
)
.expect("write valid criteria spec");
let output = Command::new(cli())
.current_dir(&root)
.args([command, path.to_str().expect("temp path is valid utf-8")])
.output()
.expect("run criteria validation command");
assert!(
output.status.success(),
"{command} failed: {}",
String::from_utf8_lossy(&output.stderr)
);
assert!(String::from_utf8_lossy(&output.stdout).contains(expected));
std::fs::remove_file(path).ok();
}

let invalid = fixture["invalid"]
.as_array()
.unwrap()
.iter()
.find(|case| case["id"] == "loss_mismatched_fingerprint")
.unwrap();
let path = std::env::temp_dir().join(format!(
"dag_ml_cli_invalid_loss_{}_{}.json",
std::process::id(),
unique_suffix()
));
std::fs::write(
&path,
serde_json::to_vec_pretty(&invalid["document"]).unwrap(),
)
.expect("write invalid loss spec");
let output = Command::new(cli())
.current_dir(&root)
.args([
"validate-loss-spec",
path.to_str().expect("temp path is valid utf-8"),
])
.output()
.expect("run invalid loss validation command");
assert!(!output.status.success());
assert!(String::from_utf8_lossy(&output.stderr).contains("fingerprint mismatch"));
std::fs::remove_file(path).ok();
}

#[test]
fn cli_compiles_pipeline_dsl_to_graph() {
let root = repo_root();
Expand Down
Loading