diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 111a8109e..3ff49293a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,5 +1,10 @@ # Dependabot keeps the SHA-pinned GitHub Actions (and Python deps) patched, so # the pins in .github/workflows/*.yml do not silently rot (#684 release hardening). +# +# All updates target `dev` (not the default `main`): bumps should flow through the +# integration branch and the normal dev->main promotion, not land on `main` where +# a release is being cut. NOTE: dependabot reads this file from the DEFAULT branch +# (`main`), so `target-branch` only takes effect once this change is on `main`. version: 2 updates: # GitHub Actions used across all workflows (release.yml pins are publishing- @@ -7,6 +12,7 @@ updates: # bumps arrives as one reviewable PR. - package-ecosystem: "github-actions" directory: "/" + target-branch: "dev" schedule: interval: "weekly" open-pull-requests-limit: 5 @@ -17,6 +23,7 @@ updates: # Python package dependencies for the aces-sdl distribution. - package-ecosystem: "pip" directory: "/implementations/python" + target-branch: "dev" schedule: interval: "weekly" open-pull-requests-limit: 5 diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 19ee807a9..4a403c7df 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.19.0" + ".": "0.19.1" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e3f621c6..a5d408506 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 PRs do **not** edit this file directly. release-please maintains it from the Conventional Commit history on `main` (#684). +## [0.19.1](https://github.com/Brad-Edwards/aces/compare/v0.19.0...v0.19.1) (2026-07-06) + + +### Bug Fixes + +* publish the project README as the PyPI description ([#701](https://github.com/Brad-Edwards/aces/issues/701)) ([6a56f93](https://github.com/Brad-Edwards/aces/commit/6a56f93d35e50a2362c13a81700f332f8a2709ca)) + ## [0.19.0](https://github.com/Brad-Edwards/aces/compare/v0.18.0...v0.19.0) (2026-07-06) diff --git a/changelog.d/682.changed.md b/changelog.d/682.changed.md new file mode 100644 index 000000000..ed0daaf3f --- /dev/null +++ b/changelog.d/682.changed.md @@ -0,0 +1,4 @@ +Narrow SDL objective success to observable state (ADR-073, SEM-206): +`objectives.success` now references `conditions` only. ADR-002's objective-success +clause is amended accordingly, and the published `sdl-authoring-input-v1` and +`instantiated-scenario-v1` schemas are updated with change-ledger entries. diff --git a/changelog.d/682.removed.md b/changelog.d/682.removed.md new file mode 100644 index 000000000..dffea0e29 --- /dev/null +++ b/changelog.d/682.removed.md @@ -0,0 +1,6 @@ +Remove the OCR-inherited scoring/reward surfaces from the SDL (ADR-073, SEM-206): +the `metrics`, `evaluations`, `tlos`, and `goals` sections, the +`agents.reward_calculator` label, `entities.tlos`, `injects.tlos`, and the +scoring references on workflow predicates. Scenarios using any of these now fail +parsing with a migration pointer. Graded scoring, reward, and evaluation outputs +live in the experiment/evaluator plane (ADR-055/064/069), not the SDL. diff --git a/contracts/concept-authority/controlled-vocabularies-v1.json b/contracts/concept-authority/controlled-vocabularies-v1.json index b95e465cb..df5ce81c1 100644 --- a/contracts/concept-authority/controlled-vocabularies-v1.json +++ b/contracts/concept-authority/controlled-vocabularies-v1.json @@ -724,22 +724,6 @@ "title": "Conditions", "description": "Condition section support." }, - "metrics": { - "title": "Metrics", - "description": "Metric section support." - }, - "evaluations": { - "title": "Evaluations", - "description": "Evaluation section support." - }, - "tlos": { - "title": "TLOs", - "description": "TLO section support." - }, - "goals": { - "title": "Goals", - "description": "Goal section support." - }, "objectives": { "title": "Objectives", "description": "Objective section support." diff --git a/contracts/fixtures/backend-manifest/backend-manifest-v2/valid/feature-support-bounded.json b/contracts/fixtures/backend-manifest/backend-manifest-v2/valid/feature-support-bounded.json index 8918c8417..d7ffc660f 100644 --- a/contracts/fixtures/backend-manifest/backend-manifest-v2/valid/feature-support-bounded.json +++ b/contracts/fixtures/backend-manifest/backend-manifest-v2/valid/feature-support-bounded.json @@ -159,11 +159,7 @@ "name": "stub-evaluator", "supported_sections": [ "conditions", - "evaluations", - "goals", - "metrics", - "objectives", - "tlos" + "objectives" ], "supports_scoring": true, "supports_objectives": true, diff --git a/contracts/fixtures/backend-manifest/backend-manifest-v2/valid/stub.json b/contracts/fixtures/backend-manifest/backend-manifest-v2/valid/stub.json index 0bed3f69c..77ff3b130 100644 --- a/contracts/fixtures/backend-manifest/backend-manifest-v2/valid/stub.json +++ b/contracts/fixtures/backend-manifest/backend-manifest-v2/valid/stub.json @@ -175,11 +175,7 @@ "name": "stub-evaluator", "supported_sections": [ "conditions", - "evaluations", - "goals", - "metrics", - "objectives", - "tlos" + "objectives" ], "supports_scoring": true, "supports_objectives": true, diff --git a/contracts/fixtures/concept-authority/controlled-vocabularies-v1/valid/reference.json b/contracts/fixtures/concept-authority/controlled-vocabularies-v1/valid/reference.json index b95e465cb..df5ce81c1 100644 --- a/contracts/fixtures/concept-authority/controlled-vocabularies-v1/valid/reference.json +++ b/contracts/fixtures/concept-authority/controlled-vocabularies-v1/valid/reference.json @@ -724,22 +724,6 @@ "title": "Conditions", "description": "Condition section support." }, - "metrics": { - "title": "Metrics", - "description": "Metric section support." - }, - "evaluations": { - "title": "Evaluations", - "description": "Evaluation section support." - }, - "tlos": { - "title": "TLOs", - "description": "TLO section support." - }, - "goals": { - "title": "Goals", - "description": "Goal section support." - }, "objectives": { "title": "Objectives", "description": "Objective section support." diff --git a/contracts/schema-publication-manifest.json b/contracts/schema-publication-manifest.json index 79a9dd264..1f5156a7e 100644 --- a/contracts/schema-publication-manifest.json +++ b/contracts/schema-publication-manifest.json @@ -156,10 +156,10 @@ "contract_id": "instantiated-scenario-v1", "schema_path": "contracts/schemas/sdl/instantiated-scenario-v1.json", "stability": "draft", - "content_hash": "21b5882778f3db00ee6ac4c7825e12b52ddbc9e7c14230bcf0fd4dc70b0303da", + "content_hash": "e0a28239e30855be948b4c3718ae34811de231b3f8d8e146d149d862af0b7cc5", "last_change": { - "summary": "Constrained instantiated SDL variable map keys to the declared variable-name grammar.", - "content_hash": "21b5882778f3db00ee6ac4c7825e12b52ddbc9e7c14230bcf0fd4dc70b0303da" + "summary": "Removed the OCR scoring pipeline (metrics/evaluations/tlos/goals), agents.reward_calculator, entities.tlos, injects.tlos, and workflow-predicate/objective-success scoring references; narrowed objectives.success to conditions (observable state) per ADR-073.", + "content_hash": "e0a28239e30855be948b4c3718ae34811de231b3f8d8e146d149d862af0b7cc5" } }, { @@ -352,10 +352,10 @@ "contract_id": "sdl-authoring-input-v1", "schema_path": "contracts/schemas/sdl/sdl-authoring-input-v1.json", "stability": "draft", - "content_hash": "1cf332317657cc3dee87f98b42c02f8fb98400396de0119e0a6d2c9e93700668", + "content_hash": "820fbd6619a100e382d722890edcdde01da977158fd86f958f4928a9d32ad51a", "last_change": { - "summary": "Constrained authored SDL variable map keys to the declared variable-name grammar.", - "content_hash": "1cf332317657cc3dee87f98b42c02f8fb98400396de0119e0a6d2c9e93700668" + "summary": "Removed the OCR scoring pipeline (metrics/evaluations/tlos/goals), agents.reward_calculator, entities.tlos, injects.tlos, and workflow-predicate/objective-success scoring references; narrowed objectives.success to conditions (observable state) per ADR-073.", + "content_hash": "820fbd6619a100e382d722890edcdde01da977158fd86f958f4928a9d32ad51a" } }, { diff --git a/contracts/schemas/sdl/instantiated-scenario-v1.json b/contracts/schemas/sdl/instantiated-scenario-v1.json index 2dee13b83..516c6a4c8 100644 --- a/contracts/schemas/sdl/instantiated-scenario-v1.json +++ b/contracts/schemas/sdl/instantiated-scenario-v1.json @@ -213,7 +213,7 @@ }, "Agent": { "additionalProperties": false, - "description": "An autonomous participant in the scenario.\n\nAgents reference existing scenario elements:\n\n- ``entity`` links to the entities section (team/role) and supplies\n identity and role per ADR-020\n- ``starting_accounts`` links to the accounts section\n- ``allowed_subnets`` links to infrastructure entries\n- ``initial_knowledge`` references nodes and infrastructure\n- ``starting_conditions`` links to the conditions section, giving the\n authoring surface a declarative hook for participant-relevant\n precondition checks (ACT-601)\n- ``authority_anchors`` links to declared SDL elements (entities,\n relationships, content, etc.) that anchor what the participant is\n allowed or expected to do in scenario meaning (ACT-601, ADR-020)\n- ``operating_scope`` links to targetable named scenario elements\n (subnets, hosts, services, content) defining where the participant\n may act or observe (ACT-601, ADR-020)\n- ``observation_boundaries`` links to declared participant observation\n boundaries that define participant-specific projections of world and\n evidence state (SEM-208)", + "description": "An autonomous participant in the scenario.\n\nAgents reference existing scenario elements:\n\n- ``entity`` links to the entities section (team/role) and supplies\n identity and role per ADR-020\n- ``starting_accounts`` links to the accounts section\n- ``allowed_subnets`` links to infrastructure entries\n- ``initial_knowledge`` references nodes and infrastructure\n- ``starting_conditions`` links to the conditions section, giving the\n authoring surface a declarative hook for participant-relevant\n precondition checks (ACT-601)\n- ``authority_anchors`` links to declared SDL elements (entities,\n relationships, content, etc.) that anchor what the participant is\n allowed or expected to do in scenario meaning (ACT-601, ADR-020)\n- ``operating_scope`` links to targetable named scenario elements\n (subnets, hosts, services, content) defining where the participant\n may act or observe (ACT-601, ADR-020)\n- ``observation_boundaries`` links to declared participant observation\n boundaries that define participant-specific projections of world and\n evidence state (SEM-208)\n\nPer ADR-073 the CybORG-inherited ``reward_calculator`` label was removed;\nit was an unbound, unvalidated string and graded reward lives in the\nexperiment/evaluator plane (ADR-055/064/069).", "properties": { "actions": { "items": { @@ -292,14 +292,6 @@ "title": "Operating Scope", "type": "array" }, - "reward_calculator": { - "default": "", - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "title": "Reward Calculator", - "type": "string" - }, "starting_accounts": { "items": { "not": { @@ -2559,16 +2551,6 @@ "default": null, "title": "Role" }, - "tlos": { - "items": { - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "type": "string" - }, - "title": "Tlos", - "type": "array" - }, "vulnerabilities": { "items": { "not": { @@ -2583,48 +2565,6 @@ "title": "Entity", "type": "object" }, - "Evaluation": { - "additionalProperties": false, - "description": "A group of metrics with a pass/fail threshold.", - "properties": { - "description": { - "default": "", - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "title": "Description", - "type": "string" - }, - "metrics": { - "items": { - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "type": "string" - }, - "minItems": 1, - "title": "Metrics", - "type": "array" - }, - "min_score": { - "$ref": "#/$defs/MinScore" - }, - "name": { - "default": "", - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "title": "Name", - "type": "string" - } - }, - "required": [ - "metrics", - "min_score" - ], - "title": "Evaluation", - "type": "object" - }, "Event": { "additionalProperties": false, "description": "A triggered action combining conditions and injects.", @@ -3147,44 +3087,6 @@ "title": "FeatureType", "type": "string" }, - "Goal": { - "additionalProperties": false, - "description": "High-level goal composed of TLOs.", - "properties": { - "description": { - "default": "", - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "title": "Description", - "type": "string" - }, - "name": { - "default": "", - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "title": "Name", - "type": "string" - }, - "tlos": { - "items": { - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "type": "string" - }, - "minItems": 1, - "title": "Tlos", - "type": "array" - } - }, - "required": [ - "tlos" - ], - "title": "Goal", - "type": "object" - }, "ImageAttestation": { "additionalProperties": false, "description": "Observed build-attestation availability and verification result.\n\nAttestation *availability* (``status``) and *verification result*\n(``verification``) are deliberately separate facts: a mutable local image\ntag with no registry-visible OCI/in-toto/SLSA attestation is not the same\nstate as a failed verification (ADR-023 \u00a75).", @@ -3866,16 +3768,6 @@ ], "default": null }, - "tlos": { - "items": { - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "type": "string" - }, - "title": "Tlos", - "type": "array" - }, "to_entities": { "items": { "not": { @@ -3890,137 +3782,6 @@ "title": "Inject", "type": "object" }, - "Metric": { - "additionalProperties": false, - "description": "A scoring metric \u2014 either manual or conditional.\n\nManual metrics may require artifact submission. Conditional\nmetrics reference a condition that produces the score.", - "properties": { - "artifact": { - "anyOf": [ - { - "type": "boolean" - }, - { - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Artifact" - }, - "condition": { - "anyOf": [ - { - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Condition" - }, - "description": { - "default": "", - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "title": "Description", - "type": "string" - }, - "max_score": { - "anyOf": [ - { - "type": "integer" - }, - { - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "type": "string" - } - ], - "title": "Max Score" - }, - "name": { - "default": "", - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "title": "Name", - "type": "string" - }, - "type": { - "$ref": "#/$defs/MetricType" - } - }, - "required": [ - "type", - "max_score" - ], - "title": "Metric", - "type": "object" - }, - "MetricType": { - "description": "How a metric is scored.", - "enum": [ - "manual", - "conditional" - ], - "title": "MetricType", - "type": "string" - }, - "MinScore": { - "additionalProperties": false, - "description": "Pass/fail threshold \u2014 either absolute points or percentage.\n\nShorthand: ``min-score: 50`` (interpreted as percentage).\nLonghand: ``min-score: {absolute: 50}`` or ``{percentage: 75}``.", - "properties": { - "absolute": { - "anyOf": [ - { - "type": "integer" - }, - { - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Absolute" - }, - "percentage": { - "anyOf": [ - { - "type": "integer" - }, - { - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Percentage" - } - }, - "title": "MinScore", - "type": "object" - }, "ModuleDescriptor": { "additionalProperties": false, "description": "Published module metadata for SDL composition.", @@ -4334,7 +4095,7 @@ }, "ObjectiveSuccess": { "additionalProperties": false, - "description": "Declarative success criteria for an objective.", + "description": "Declarative success criteria for an objective.\n\nPer ADR-073, objective success references observable state (``conditions``)\nonly. The OCR-inherited scoring pipeline (``metrics`` / ``evaluations`` /\n``tlos`` / ``goals``) was removed from the SDL; graded scoring and reward\nlive in the experiment/evaluator plane (ADR-055/064/069).", "properties": { "conditions": { "items": { @@ -4346,36 +4107,6 @@ "title": "Conditions", "type": "array" }, - "evaluations": { - "items": { - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "type": "string" - }, - "title": "Evaluations", - "type": "array" - }, - "goals": { - "items": { - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "type": "string" - }, - "title": "Goals", - "type": "array" - }, - "metrics": { - "items": { - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "type": "string" - }, - "title": "Metrics", - "type": "array" - }, "mode": { "anyOf": [ { @@ -4390,16 +4121,6 @@ ], "default": "all_of", "title": "Mode" - }, - "tlos": { - "items": { - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "type": "string" - }, - "title": "Tlos", - "type": "array" } }, "title": "ObjectiveSuccess", @@ -20971,40 +20692,6 @@ "title": "SuccessMode", "type": "string" }, - "TLO": { - "additionalProperties": false, - "description": "Training Learning Objective \u2014 linked to an evaluation.", - "properties": { - "description": { - "default": "", - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "title": "Description", - "type": "string" - }, - "evaluation": { - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "title": "Evaluation", - "type": "string" - }, - "name": { - "default": "", - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "title": "Name", - "type": "string" - } - }, - "required": [ - "evaluation" - ], - "title": "TLO", - "type": "object" - }, "Variable": { "additionalProperties": false, "description": "A named variable for scenario parameterization.\n\nVariables define configurable parameters with types, defaults,\nand optional value constraints. They're referenced in other\nsections via ``${variable_name}`` syntax.", @@ -21257,7 +20944,7 @@ }, "WorkflowPredicate": { "additionalProperties": false, - "description": "Branch predicate over runtime evaluation data and prior step state.", + "description": "Branch predicate over observable state, objectives, and prior step state.\n\nPer ADR-073 the OCR scoring references (``metrics`` / ``evaluations`` /\n``tlos`` / ``goals``) were removed; a predicate branches on observable\n``conditions``, declared ``objectives``, and prior workflow ``steps``.", "properties": { "conditions": { "items": { @@ -21269,36 +20956,6 @@ "title": "Conditions", "type": "array" }, - "evaluations": { - "items": { - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "type": "string" - }, - "title": "Evaluations", - "type": "array" - }, - "goals": { - "items": { - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "type": "string" - }, - "title": "Goals", - "type": "array" - }, - "metrics": { - "items": { - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "type": "string" - }, - "title": "Metrics", - "type": "array" - }, "objectives": { "items": { "not": { @@ -21315,16 +20972,6 @@ }, "title": "Steps", "type": "array" - }, - "tlos": { - "items": { - "not": { - "pattern": "\\$\\{([A-Za-z_][A-Za-z0-9_-]*)\\}" - }, - "type": "string" - }, - "title": "Tlos", - "type": "array" } }, "title": "WorkflowPredicate", @@ -21674,13 +21321,6 @@ "title": "Entities", "type": "object" }, - "evaluations": { - "additionalProperties": { - "$ref": "#/$defs/Evaluation" - }, - "title": "Evaluations", - "type": "object" - }, "events": { "additionalProperties": { "$ref": "#/$defs/Event" @@ -21709,13 +21349,6 @@ "title": "Forwarding Agents", "type": "array" }, - "goals": { - "additionalProperties": { - "$ref": "#/$defs/Goal" - }, - "title": "Goals", - "type": "object" - }, "imports": { "items": { "$ref": "#/$defs/ImportDecl" @@ -21737,13 +21370,6 @@ "title": "Injects", "type": "object" }, - "metrics": { - "additionalProperties": { - "$ref": "#/$defs/Metric" - }, - "title": "Metrics", - "type": "object" - }, "module": { "anyOf": [ { @@ -21811,13 +21437,6 @@ "title": "Stories", "type": "object" }, - "tlos": { - "additionalProperties": { - "$ref": "#/$defs/TLO" - }, - "title": "Tlos", - "type": "object" - }, "variables": { "additionalProperties": false, "patternProperties": { diff --git a/contracts/schemas/sdl/sdl-authoring-input-v1.json b/contracts/schemas/sdl/sdl-authoring-input-v1.json index 8b8410fee..12631467b 100644 --- a/contracts/schemas/sdl/sdl-authoring-input-v1.json +++ b/contracts/schemas/sdl/sdl-authoring-input-v1.json @@ -156,7 +156,7 @@ }, "Agent": { "additionalProperties": false, - "description": "An autonomous participant in the scenario.\n\nAgents reference existing scenario elements:\n\n- ``entity`` links to the entities section (team/role) and supplies\n identity and role per ADR-020\n- ``starting_accounts`` links to the accounts section\n- ``allowed_subnets`` links to infrastructure entries\n- ``initial_knowledge`` references nodes and infrastructure\n- ``starting_conditions`` links to the conditions section, giving the\n authoring surface a declarative hook for participant-relevant\n precondition checks (ACT-601)\n- ``authority_anchors`` links to declared SDL elements (entities,\n relationships, content, etc.) that anchor what the participant is\n allowed or expected to do in scenario meaning (ACT-601, ADR-020)\n- ``operating_scope`` links to targetable named scenario elements\n (subnets, hosts, services, content) defining where the participant\n may act or observe (ACT-601, ADR-020)\n- ``observation_boundaries`` links to declared participant observation\n boundaries that define participant-specific projections of world and\n evidence state (SEM-208)", + "description": "An autonomous participant in the scenario.\n\nAgents reference existing scenario elements:\n\n- ``entity`` links to the entities section (team/role) and supplies\n identity and role per ADR-020\n- ``starting_accounts`` links to the accounts section\n- ``allowed_subnets`` links to infrastructure entries\n- ``initial_knowledge`` references nodes and infrastructure\n- ``starting_conditions`` links to the conditions section, giving the\n authoring surface a declarative hook for participant-relevant\n precondition checks (ACT-601)\n- ``authority_anchors`` links to declared SDL elements (entities,\n relationships, content, etc.) that anchor what the participant is\n allowed or expected to do in scenario meaning (ACT-601, ADR-020)\n- ``operating_scope`` links to targetable named scenario elements\n (subnets, hosts, services, content) defining where the participant\n may act or observe (ACT-601, ADR-020)\n- ``observation_boundaries`` links to declared participant observation\n boundaries that define participant-specific projections of world and\n evidence state (SEM-208)\n\nPer ADR-073 the CybORG-inherited ``reward_calculator`` label was removed;\nit was an unbound, unvalidated string and graded reward lives in the\nexperiment/evaluator plane (ADR-055/064/069).", "properties": { "actions": { "items": { @@ -214,11 +214,6 @@ "title": "Operating Scope", "type": "array" }, - "reward_calculator": { - "default": "", - "title": "Reward Calculator", - "type": "string" - }, "starting_accounts": { "items": { "type": "string" @@ -2082,13 +2077,6 @@ "default": null, "title": "Role" }, - "tlos": { - "items": { - "type": "string" - }, - "title": "Tlos", - "type": "array" - }, "vulnerabilities": { "items": { "type": "string" @@ -2100,39 +2088,6 @@ "title": "Entity", "type": "object" }, - "Evaluation": { - "additionalProperties": false, - "description": "A group of metrics with a pass/fail threshold.", - "properties": { - "description": { - "default": "", - "title": "Description", - "type": "string" - }, - "metrics": { - "items": { - "type": "string" - }, - "minItems": 1, - "title": "Metrics", - "type": "array" - }, - "min_score": { - "$ref": "#/$defs/MinScore" - }, - "name": { - "default": "", - "title": "Name", - "type": "string" - } - }, - "required": [ - "metrics", - "min_score" - ], - "title": "Evaluation", - "type": "object" - }, "Event": { "additionalProperties": false, "description": "A triggered action combining conditions and injects.", @@ -2550,35 +2505,6 @@ "title": "FeatureType", "type": "string" }, - "Goal": { - "additionalProperties": false, - "description": "High-level goal composed of TLOs.", - "properties": { - "description": { - "default": "", - "title": "Description", - "type": "string" - }, - "name": { - "default": "", - "title": "Name", - "type": "string" - }, - "tlos": { - "items": { - "type": "string" - }, - "minItems": 1, - "title": "Tlos", - "type": "array" - } - }, - "required": [ - "tlos" - ], - "title": "Goal", - "type": "object" - }, "ImageAttestation": { "additionalProperties": false, "description": "Observed build-attestation availability and verification result.\n\nAttestation *availability* (``status``) and *verification result*\n(``verification``) are deliberately separate facts: a mutable local image\ntag with no registry-visible OCI/in-toto/SLSA attestation is not the same\nstate as a failed verification (ADR-023 \u00a75).", @@ -3101,13 +3027,6 @@ ], "default": null }, - "tlos": { - "items": { - "type": "string" - }, - "title": "Tlos", - "type": "array" - }, "to_entities": { "items": { "type": "string" @@ -3119,116 +3038,6 @@ "title": "Inject", "type": "object" }, - "Metric": { - "additionalProperties": false, - "description": "A scoring metric \u2014 either manual or conditional.\n\nManual metrics may require artifact submission. Conditional\nmetrics reference a condition that produces the score.", - "properties": { - "artifact": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Artifact" - }, - "condition": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Condition" - }, - "description": { - "default": "", - "title": "Description", - "type": "string" - }, - "max_score": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "string" - } - ], - "title": "Max Score" - }, - "name": { - "default": "", - "title": "Name", - "type": "string" - }, - "type": { - "$ref": "#/$defs/MetricType" - } - }, - "required": [ - "type", - "max_score" - ], - "title": "Metric", - "type": "object" - }, - "MetricType": { - "description": "How a metric is scored.", - "enum": [ - "manual", - "conditional" - ], - "title": "MetricType", - "type": "string" - }, - "MinScore": { - "additionalProperties": false, - "description": "Pass/fail threshold \u2014 either absolute points or percentage.\n\nShorthand: ``min-score: 50`` (interpreted as percentage).\nLonghand: ``min-score: {absolute: 50}`` or ``{percentage: 75}``.", - "properties": { - "absolute": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Absolute" - }, - "percentage": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Percentage" - } - }, - "title": "MinScore", - "type": "object" - }, "ModuleDescriptor": { "additionalProperties": false, "description": "Published module metadata for SDL composition.", @@ -3485,7 +3294,7 @@ }, "ObjectiveSuccess": { "additionalProperties": false, - "description": "Declarative success criteria for an objective.", + "description": "Declarative success criteria for an objective.\n\nPer ADR-073, objective success references observable state (``conditions``)\nonly. The OCR-inherited scoring pipeline (``metrics`` / ``evaluations`` /\n``tlos`` / ``goals``) was removed from the SDL; graded scoring and reward\nlive in the experiment/evaluator plane (ADR-055/064/069).", "properties": { "conditions": { "items": { @@ -3494,27 +3303,6 @@ "title": "Conditions", "type": "array" }, - "evaluations": { - "items": { - "type": "string" - }, - "title": "Evaluations", - "type": "array" - }, - "goals": { - "items": { - "type": "string" - }, - "title": "Goals", - "type": "array" - }, - "metrics": { - "items": { - "type": "string" - }, - "title": "Metrics", - "type": "array" - }, "mode": { "anyOf": [ { @@ -3526,13 +3314,6 @@ ], "default": "all_of", "title": "Mode" - }, - "tlos": { - "items": { - "type": "string" - }, - "title": "Tlos", - "type": "array" } }, "title": "ObjectiveSuccess", @@ -16885,31 +16666,6 @@ "title": "SuccessMode", "type": "string" }, - "TLO": { - "additionalProperties": false, - "description": "Training Learning Objective \u2014 linked to an evaluation.", - "properties": { - "description": { - "default": "", - "title": "Description", - "type": "string" - }, - "evaluation": { - "title": "Evaluation", - "type": "string" - }, - "name": { - "default": "", - "title": "Name", - "type": "string" - } - }, - "required": [ - "evaluation" - ], - "title": "TLO", - "type": "object" - }, "Variable": { "additionalProperties": false, "description": "A named variable for scenario parameterization.\n\nVariables define configurable parameters with types, defaults,\nand optional value constraints. They're referenced in other\nsections via ``${variable_name}`` syntax.", @@ -17132,7 +16888,7 @@ }, "WorkflowPredicate": { "additionalProperties": false, - "description": "Branch predicate over runtime evaluation data and prior step state.", + "description": "Branch predicate over observable state, objectives, and prior step state.\n\nPer ADR-073 the OCR scoring references (``metrics`` / ``evaluations`` /\n``tlos`` / ``goals``) were removed; a predicate branches on observable\n``conditions``, declared ``objectives``, and prior workflow ``steps``.", "properties": { "conditions": { "items": { @@ -17141,27 +16897,6 @@ "title": "Conditions", "type": "array" }, - "evaluations": { - "items": { - "type": "string" - }, - "title": "Evaluations", - "type": "array" - }, - "goals": { - "items": { - "type": "string" - }, - "title": "Goals", - "type": "array" - }, - "metrics": { - "items": { - "type": "string" - }, - "title": "Metrics", - "type": "array" - }, "objectives": { "items": { "type": "string" @@ -17175,13 +16910,6 @@ }, "title": "Steps", "type": "array" - }, - "tlos": { - "items": { - "type": "string" - }, - "title": "Tlos", - "type": "array" } }, "title": "WorkflowPredicate", @@ -17471,13 +17199,6 @@ "title": "Entities", "type": "object" }, - "evaluations": { - "additionalProperties": { - "$ref": "#/$defs/Evaluation" - }, - "title": "Evaluations", - "type": "object" - }, "events": { "additionalProperties": { "$ref": "#/$defs/Event" @@ -17506,13 +17227,6 @@ "title": "Forwarding Agents", "type": "array" }, - "goals": { - "additionalProperties": { - "$ref": "#/$defs/Goal" - }, - "title": "Goals", - "type": "object" - }, "imports": { "items": { "$ref": "#/$defs/ImportDecl" @@ -17534,13 +17248,6 @@ "title": "Injects", "type": "object" }, - "metrics": { - "additionalProperties": { - "$ref": "#/$defs/Metric" - }, - "title": "Metrics", - "type": "object" - }, "module": { "anyOf": [ { @@ -17605,13 +17312,6 @@ "title": "Stories", "type": "object" }, - "tlos": { - "additionalProperties": { - "$ref": "#/$defs/TLO" - }, - "title": "Tlos", - "type": "object" - }, "variables": { "additionalProperties": false, "patternProperties": { diff --git a/docs/api/sdl.rst b/docs/api/sdl.rst index 8f04a6e91..fb2b92a14 100644 --- a/docs/api/sdl.rst +++ b/docs/api/sdl.rst @@ -137,12 +137,6 @@ Objectives .. automodule:: aces_sdl.objectives :members: -Scoring -~~~~~~~ - -.. automodule:: aces_sdl.scoring - :members: - Variables ~~~~~~~~~ diff --git a/docs/decisions/adrs/README.md b/docs/decisions/adrs/README.md index 320700364..1f75263dc 100644 --- a/docs/decisions/adrs/README.md +++ b/docs/decisions/adrs/README.md @@ -195,4 +195,4 @@ adr-073-scoring-reward-language-scope | [070](adr-070-realization-envelope-semantics.md) | Realization Envelope Semantics | proposed | 2026-07-04 | | [071](adr-071-reusable-asset-trust-and-integrity-policy.md) | Reusable Asset Trust and Integrity Policy | accepted | 2026-07-05 | | [072](adr-072-validation-and-admission-profiles.md) | Validation and Admission Profiles | proposed | 2026-07-05 | -| [073](adr-073-scoring-reward-language-scope.md) | Scoring and Reward Language Scope in the SDL | proposed | 2026-07-05 | +| [073](adr-073-scoring-reward-language-scope.md) | Scoring and Reward Language Scope in the SDL | accepted | 2026-07-05 | diff --git a/docs/decisions/adrs/adr-002-declarative-sdl-objectives.md b/docs/decisions/adrs/adr-002-declarative-sdl-objectives.md index 3e324264d..909f30587 100644 --- a/docs/decisions/adrs/adr-002-declarative-sdl-objectives.md +++ b/docs/decisions/adrs/adr-002-declarative-sdl-objectives.md @@ -88,3 +88,9 @@ This ADR refines ADR-001's SDL boundary by making declarative objectives part of - Future agent-support work could accidentally collapse participant-exposure or participant-implementation concerns into objectives unless those boundaries remain explicit. + +## Amendments + +| Date | Commit/PR | Summary | +|------|-----------|---------| +| 2026-07-05 | #682 | Per [ADR-073](adr-073-scoring-reward-language-scope.md), narrowed the objective-success clause: `objectives.success` references observable state (`conditions`) only. The OCR scoring pipeline (`metrics` / `evaluations` / `tlos` / `goals`) this ADR preserved was removed from the SDL; graded scoring and reward now live in the experiment/evaluator plane (ADR-055/064/069). | diff --git a/docs/decisions/adrs/adr-073-scoring-reward-language-scope.md b/docs/decisions/adrs/adr-073-scoring-reward-language-scope.md index d4700f293..6a6545e7b 100644 --- a/docs/decisions/adrs/adr-073-scoring-reward-language-scope.md +++ b/docs/decisions/adrs/adr-073-scoring-reward-language-scope.md @@ -2,7 +2,7 @@ ## Status -proposed +accepted ## Date @@ -261,3 +261,9 @@ ratification. The ADR is proposed; implementation is spawned on acceptance. - If only `reward_calculator` is removed and the follow-through lapses, the duplicated grading pipeline persists. The decision commits to the full removal via staged migration, and SEM-206 tracks completion. + +## Amendments + +| Date | Commit/PR | Summary | +|------|-----------|---------| +| 2026-07-06 | #682 | Accepted (proposed → accepted) and realized under SEM-206: the SDL scoring/reward surfaces (`metrics`/`evaluations`/`tlos`/`goals` and `agents.reward_calculator`) were removed, `objectives.success` narrowed to `conditions`, ADR-002's objective-success clause amended, and the published SDL schemas updated with change-ledger entries. The staged deprecation window in §5 was collapsed into one change: all in-repo consumers were migrated together, with the downstream `Brad-Edwards/aptl#606` tracked to follow. | diff --git a/docs/decisions/adrs/adr-index.yaml b/docs/decisions/adrs/adr-index.yaml index e123dc6a0..6e59d1ebb 100644 --- a/docs/decisions/adrs/adr-index.yaml +++ b/docs/decisions/adrs/adr-index.yaml @@ -16,6 +16,10 @@ adrs: - id: ADR-002 path: docs/decisions/adrs/adr-002-declarative-sdl-objectives.md pin: 7550c868306aa4f6ff7a1522796b71ae8797aa0015d7faec37c8f7011e60c536 + amendments: + - date: 2026-07-05 + ref: "#682" + summary: "Per ADR-073, narrowed the objective-success clause to reference observable state (conditions) only; the OCR scoring pipeline (metrics/evaluations/tlos/goals) was removed from the SDL and graded scoring/reward moved to the experiment/evaluator plane (ADR-055/064/069)." - id: ADR-003 path: docs/decisions/adrs/adr-003-workflows-targetable-subobjects-and-enum-variables.md pin: 0039bea0de4e4d5d08ccf300a7fb7516a0ea01977f38f1e2e111409a13e108df @@ -281,3 +285,10 @@ adrs: - id: ADR-071 path: docs/decisions/adrs/adr-071-reusable-asset-trust-and-integrity-policy.md pin: d0f0d15945a91e97870d794986d7c7453cea7fade5c74c673d271ceaf05a6b7e + - id: ADR-073 + path: docs/decisions/adrs/adr-073-scoring-reward-language-scope.md + pin: 5c0f6a0fe961bd9fc3f0b268ee99e1bdf6e6b576fbbe470dd3969a543dc07356 + amendments: + - date: 2026-07-06 + ref: "#682" + summary: "Accepted (proposed → accepted) and realized under SEM-206: removed the SDL scoring/reward surfaces, narrowed objectives.success to conditions, amended ADR-002, and updated the published SDL schemas." diff --git a/docs/explain/getting-started.md b/docs/explain/getting-started.md index 69b0bf911..84c48b706 100644 --- a/docs/explain/getting-started.md +++ b/docs/explain/getting-started.md @@ -167,8 +167,9 @@ Current status: - Participant behavior has a reusable action-contract and observation-boundary template validated through current SDL semantics. - Tasks, runs, and studies have reusable templates and patterns that map those - concepts onto current objectives, workflows, timing, scoring, and evidence - references. + concepts onto current objectives, workflows, timing, conditions, and evidence + references. Graded scoring/reward is an experiment/evaluator-plane concern + (ADR-073), not an SDL section. - Evidence and provenance concerns are documented at architecture and limitation surfaces, but not fully materialized as published runtime contracts. diff --git a/docs/explain/reference/assessment-semantics.md b/docs/explain/reference/assessment-semantics.md index d79e217b0..9b0b45bf4 100644 --- a/docs/explain/reference/assessment-semantics.md +++ b/docs/explain/reference/assessment-semantics.md @@ -1,163 +1,69 @@ -# Assessment Semantics Preflight - -This note records architecture guardrails for `SEM-206`. It is not an -implementation plan. - -## Scope Boundary - -`SEM-206` covers the assessment pipeline semantics for SDL conditions, metrics, -evaluations, TLOs, goals, and their relationship to declarative objectives. The -current incumbent pipeline is: - -```text -condition bindings -> metrics -> evaluations -> TLOs -> goals -> objectives -``` - -The SDL authoring layer owns section shape and static reference rules. The -processor layer owns compiled evaluation resources, dependency semantics, -capability checks, and runtime result contracts. The backend/evaluator boundary -owns only portable evaluator result envelopes and history streams, not -backend-native scoring state. - -Issue #671 reopens whether OCR-style scoring and CybORG-style reward labels -belong in authored SDL at all. Until an ADR resolves that question, treat the -existing pipeline as the compatibility surface to preserve, not as permission to -expand scoring language. A reward, score, return, leaderboard value, or training -signal is SDL meaning only when it changes the experiment within its horizon or -is consumed by a participant in-run. Researcher-only scoring, model-training -reward, leaderboard ranking, and downstream statistical analysis belong in -experiment/evidence/derived-measure contracts or adapter-private evidence, not -in a new SDL assessment shortcut. - -## Incumbents To Reuse - -- SDL shape: `aces_sdl.conditions`, `aces_sdl.scoring`, and - `aces_sdl.objectives`. -- Static validation: `SemanticValidator._verify_conditions()`, - `_verify_metrics()`, `_verify_evaluations()`, `_verify_tlos()`, - `_verify_goals()`, and `_verify_objectives()`. -- Instantiation: `instantiate_scenario()` must rerun semantic validation after - parameter substitution. -- Compilation: `compile_runtime_model()` must remain the source of canonical - `evaluation.*` addresses and compiled `EvaluationResultContract` / - `EvaluationExecutionContract` payloads. -- Planning: `aces_processor.semantics.planner` owns ordering and refresh - dependency interpretation for compiled evaluation resources. -- Runtime boundary: `EvaluationExecutionState`, `EvaluationHistoryEvent`, - `validate_evaluation_result()`, and - `RuntimeManager`'s evaluation result contract diagnostics are the shared - enforcement path for evaluator payloads. -- Participant outcome boundary: `ParticipantOutcomeReport` carriers and - SEM-215 interpretation rules remain relationship/provenance records. They do - not carry score, reward, or objective-success fields. -- Experiment/evidence boundary: ADR-055 experiment tasks/runs/studies and - ADR-064 capture/evidence/derived-measure contracts own researcher-facing - metrics, analysis plans, evidence records, and derived measures outside live - SDL scenario meaning. -- Contracts: `aces_contracts.contracts.ContractModel`, `schema_bundle()`, and - generated `contracts/schemas/control-plane/evaluation-*-v1.json` remain the - external shape authority. -- Capability authority: evaluator support must flow through - `EvaluatorCapabilities`, `EvaluatorCapabilitiesModel`, controlled vocabulary - terms for `capabilities.evaluator.supported_sections`, and the existing - `supports_scoring` / `supports_objectives` split. - -## Guardrails - -- Keep condition templates distinct from condition bindings. Authoring names a - reusable condition; runtime scoring works against node-bound - `evaluation.condition..` addresses. -- Keep scoring resources distinct from objectives. Metrics, evaluations, TLOs, - and goals define assessment structure; objectives bind actors, targets, - success criteria, dependencies, and optional windows. -- Keep participant reward/return signals distinct from SDL scoring resources. - `agents.reward_calculator` is an inherited source label, not a semantic - authority for objective success or evaluator aggregation. If a future ADR - retains it, the implementation must bind it through governed participant - runtime or experiment/evaluator contracts instead of interpreting the string - locally. -- Keep ordering dependencies and refresh dependencies separate. Assessment - aggregation uses ordering edges from prerequisite assessment resources; - objective windows and condition-driven changes create refresh edges where - appropriate. -- Use the existing fail-closed reference behavior for missing, ambiguous, or - out-of-scope references. Do not add local string parsing in compiler, - planner, runtime manager, or backend adapters when a model/helper already - resolves the reference. -- Preserve the existing evaluator payload contract: `metric` reports score - fields, while `condition-binding`, `evaluation`, `tlo`, `goal`, and - `objective` report `passed`. Any additional portable aggregation rule must - compile into the contract or a governed contract version, not into - backend-private convention. -- Treat `detail`, `details`, and `evidence_refs` as observation metadata, not - as hidden scoring authority. They must not contain secrets, tokens, raw - credentials, backend-private object dumps, or full tracebacks. -- Extend controlled vocabulary, semantic profile, or contract authority only - when portable comparison requires it. A local evaluator implementation detail - does not belong in those surfaces. -- Migrate or deprecate existing scenario scoring sections only under an - ADR-backed compatibility rule. Do not rewrite `metrics` / `evaluations` / - `tlos` / `goals` fixture by fixture to settle the design question locally. - -## Required Gates - -- Parser/model gate: assessment fields stay closed Pydantic models derived from - `SDLModel`; `${var}` placeholders may parameterize values but may not create - mapping keys or semantic identities. -- Validation gate: `SemanticValidator` remains the static semantic choke point - and raises `SDLValidationError` with collected authoring errors. -- Instantiation gate: concrete scenarios pass `instantiate_scenario()` and - concrete semantic revalidation before compilation. -- Compiler/planner gate: compiled evaluation resources use canonical - `evaluation.*` addresses and shared planner dependency helpers. -- Manifest/profile gate: evaluator sections and scoring/objective capability - declarations resolve through the existing apparatus-manifest and controlled - vocabulary helpers. -- Runtime contract gate: evaluator payloads pass `EvaluationExecutionState`, - `EvaluationHistoryEvent`, `EvaluationResultContract`, - `EvaluationExecutionContract`, and `validate_evaluation_result()` before - entering snapshots. -- HTTP/control-plane gate: any API exposure uses the existing control-plane - request-size, authentication, authorization, idempotency, audit, response - model, and redacted-error behavior. -- Persistence/OS exposure gate: snapshots, operation records, audit details, - diagnostics, history `details`, and evidence references stay plain-data and - non-secret; bearer tokens and credentials must not appear in command-line - arguments, logs, diagnostics, fixtures, or persisted envelopes. - -## Extension Boundary - -The extension seam is a pure assessment semantic helper under -`aces_sdl.semantics` when the same aggregation or reference rule must be used -by validation, compilation, planning, runtime contract checks, and tests. The -helper should operate on structured inputs and return normalized references, -derived dependency roles, and machine-readable issues; callers may translate -those issues into their local error or diagnostic envelope. - -Portable assessment variations belong in versioned contract/profile or -controlled-vocabulary authority only when external implementations need to -compare them. They should not be hard-coded as evaluator-specific strings. - -## Anti-Patterns - -- Duplicating scoring schemas outside `aces_sdl.scoring` or external contract - schemas. -- Creating a second assessment registry beside the scenario model, semantic - profile, and concept-authority stack. -- Recomputing aggregation semantics independently in validator, compiler, - planner, manager, and backend stubs. -- Letting backend-native evaluator payloads become the observation contract. -- Treating `agents.reward_calculator`, reward arrays, cumulative return, or - leaderboard score as a shortcut for SDL objective success. -- Treating objectives as just another aggregation node, or treating goals/TLOs - as actor-bound objectives. -- Editing generated schemas under `contracts/schemas/` directly. -- Introducing SEM-206-specific exception, logging, persistence, audit, or - error-envelope stacks. - -## Non-Goals - -This note does not implement new assessment rules, change the current scoring -pipeline, publish a new contract version, add evaluator capabilities, define -evidence/provenance semantics, or transition `SEM-206`. Those belong to the -implementation run that follows. +# Assessment Semantics + +Implementer-facing reference for `SEM-206` (Assessment Semantics), governed by +ADR-016. The formal artifacts are +{download}`specs/formal/assessment/README.md <../../../specs/formal/assessment/README.md>` +and +{download}`specs/formal/assessment/pipeline-consistency.md <../../../specs/formal/assessment/pipeline-consistency.md>`; +this note is the working summary. + +## The SDL carries no scoring/assessment pipeline + +[ADR-073](../../decisions/adrs/adr-073-scoring-reward-language-scope.md) removed +the OCR-inherited scoring/assessment pipeline from the SDL authoring language. +The graded chain `conditions -> metrics -> evaluations -> tlos -> goals` and the +CybORG `agents.reward_calculator` label are no longer SDL surfaces: + +- `metrics`, `evaluations`, `tlos` (Training Learning Objectives), and `goals` + are removed. They expressed graded values, thresholds, and training-exercise + objective/goal trees — read by a grader, not by a participant in-horizon. +- `agents.reward_calculator` is removed. It was an unbound free-text CybORG label + with no cross-reference validator. + +There is no score aggregation, no per-condition metric-exclusivity rule, and no +scoring-chain ordering/refresh derivation in authored SDL. + +## What the SDL keeps + +- **`conditions`** are observable state and remain a first-class SDL surface. A + declared condition compiles onto a runtime `evaluation.condition.*` address + (`evaluation.condition..` once bound); an unbound or ambiguous + binding is reported at compilation as `evaluation.condition-ref-unbound` / + `evaluation.condition-ref-ambiguous`. +- **`objectives`** are participant intent and remain first-class. An objective's + `success` references **only** `conditions` — observable state, in-horizon and + reproducible — never a graded score. Workflow predicates likewise reference + `conditions`. The objective-success semantics are detailed in + [objective-semantics.md](objective-semantics.md). + +## Where graded scoring now lives + +Graded scoring, cumulative reward, pass/fail evaluation, leaderboard values, and +evaluation outputs are an experiment/evaluator-plane concern, never authored SDL: + +- **experiment-core contracts** + ([ADR-055](../../decisions/adrs/adr-055-experiment-core-contract-boundary.md)): + `experiment-task-v1` metric definitions and `experiment-study-v1` analysis + plans; +- **evidence/measure contracts** + ([ADR-064](../../decisions/adrs/adr-064-experiment-evidence-and-measure-contract-boundary.md)): + `experiment-evidence-record-v1` (raw evidence) and + `experiment-derived-measure-v1` (a derived measure or evaluation output); +- **the backend Evaluator** + ([ADR-069](../../decisions/adrs/adr-069-cage-2-replication-architecture.md) + §3), which projects reward, objective, terminal-condition, and scoring facts + into ACES evaluation results, evidence records, and derived measures. + +The runtime evaluator-result and execution contracts (`EvaluationResultContract`, +`EvaluationExecutionContract`, `validate_evaluation_result()`) remain the +portable, fail-closed observation boundary for evaluated success; score fields +stay confined to score-supporting evaluator resources and experiment-derived +measures, not SDL objectives or participant outcomes. + +## Participant outcome interpretation + +The SEM-215 participant outcome-interpretation layer keeps its +`reward_signal` / `evaluation_result` interpretation layers as a governed +interpretation relation. They no longer bind to any SDL `evaluations` section — +a governed `reward_signal` interpretation is not an authored reward calculator +and adds no score/reward field to participant outcome reports. diff --git a/docs/explain/reference/glossary.md b/docs/explain/reference/glossary.md index e86716dc5..3a63b091e 100644 --- a/docs/explain/reference/glossary.md +++ b/docs/explain/reference/glossary.md @@ -90,8 +90,7 @@ published schemas, source code, and ADRs. inject state. **Evaluation plan** -: The execution-plan portion for condition bindings, scoring graph nodes, and - objectives. +: The execution-plan portion for condition bindings and objectives. **Runtime snapshot** : The typed state model used by the planner and manager to represent current diff --git a/docs/explain/reference/objective-semantics.md b/docs/explain/reference/objective-semantics.md index 9f183b1a4..5a67e7ebf 100644 --- a/docs/explain/reference/objective-semantics.md +++ b/docs/explain/reference/objective-semantics.md @@ -16,7 +16,9 @@ A declarative *objective* binds, in one place: index (bare or section-qualified; objectives, workflows, and variables are not targetable); - a **success** interpretation — `mode` (`all_of` / `any_of`) over referenced - conditions, metrics, evaluations, TLOs, and goals; + observable `conditions` only (the OCR scoring surfaces `metrics`, + `evaluations`, `tlos`, and `goals` were removed by + [ADR-073](../../decisions/adrs/adr-073-scoring-reward-language-scope.md)); - an optional **window** that constrains when the objective matters (the story/script/event/workflow/workflow-step reachability and consistency rules live with the window helper — see @@ -79,11 +81,10 @@ those constants, so a change to a success condition (or to a depended-on objective) propagates as a refresh through `objective -> depends_on -> objective`. In the analyzer's name-level IR the derived `ordering_names` / `refresh_names` -are kind-qualified (`condition.`, `metric.`, `evaluation.`, `tlo.`, -`goal.`, `objective.`, `story.`, `script.`, `event.`, -`workflow.`) so a metric and a condition with the same SDL name remain -distinguishable, mirroring the canonical `evaluation.*` addresses the compiler -builds independently. +are kind-qualified (`condition.`, `objective.`, `story.`, `script.`, +`event.`, `workflow.`) so a condition and an objective with the same SDL +name remain distinguishable, mirroring the canonical `evaluation.*` addresses the +compiler builds independently. ## Cross-stage agreement @@ -106,8 +107,8 @@ itself is unit-tested in `implementations/python/tests/test_semantics_objectives - a second objective schema beside `aces_sdl.objectives`, or a second reference resolver beside `SemanticValidator`'s named-reference index / the compiler's canonical address helpers; -- duplicating the window, assessment, dependency, or reference-resolution rules - in compiler, planner, or tests; +- duplicating the window, dependency, or reference-resolution rules in compiler, + planner, or tests; - mixing objective actor binding with participant episode lifecycle or apparatus realization, or treating objective targets as backend execution targets; - encoding evaluator query language, probe commands, credentials, or polling diff --git a/docs/explain/reference/shared-semantic-integrity.md b/docs/explain/reference/shared-semantic-integrity.md index 17a372dfa..1d5dbce17 100644 --- a/docs/explain/reference/shared-semantic-integrity.md +++ b/docs/explain/reference/shared-semantic-integrity.md @@ -232,7 +232,7 @@ so they are tracked by their own requirements, not here. | Declarative objective actor binding, target resolution, success interpretation, and dependency ordering | DSL-112, SEM-207 | authoring, validation, instantiation, compilation, planning | `implementations/python/packages/aces_sdl/objectives.py`, `implementations/python/packages/aces_sdl/semantics/objective_semantics.py`, `implementations/python/packages/aces_sdl/validator/__init__.py`, `implementations/python/packages/aces_processor/compiler.py`, `implementations/python/packages/aces_processor/models.py`, `specs/formal/objectives/README.md`, `specs/formal/objectives/declarative-objective-semantics.md`, `implementations/python/tests/test_semantics_objectives.py`, `implementations/python/tests/test_fm2_semantics.py`, `implementations/python/tests/test_sdl_validator.py` | active | | Workflow control semantics (branching, joins, calling, retry, completion, history) | DSL-113, SEM-203 | authoring, validation, compilation, planning, execution, observation | `implementations/python/packages/aces_sdl/orchestration.py`, `implementations/python/packages/aces_sdl/semantics/workflow.py`, `specs/formal/workflows/README.md`, `specs/formal/workflows/state-machine.md`, `implementations/python/tests/test_sdl_validator.py`, `implementations/python/tests/test_runtime_models.py`, `implementations/python/tests/test_sdl_models.py` | active | | Workflow compensation semantics (registration, triggering, ordering, observation) | SEM-204 | validation, compilation, execution, observation | `implementations/python/packages/aces_sdl/semantics/workflow.py`, `specs/formal/workflows/compensation.md`, `implementations/python/tests/test_sdl_validator.py`, `implementations/python/tests/test_runtime_manager.py` | partial | -| Assessment model and pipeline semantics (conditions, metrics, evaluations, TLOs, goals) | DSL-110, SEM-206 | authoring, validation, compilation, planning, execution, observation | `implementations/python/packages/aces_sdl/scoring.py`, `implementations/python/packages/aces_sdl/conditions.py`, `implementations/python/packages/aces_sdl/semantics/assessment.py`, `implementations/python/packages/aces_processor/compiler.py`, `implementations/python/packages/aces_processor/models.py`, `specs/formal/assessment/README.md`, `specs/formal/assessment/pipeline-consistency.md`, `implementations/python/tests/test_semantics_assessment.py`, `implementations/python/tests/test_sdl_models.py`, `implementations/python/tests/test_fm2_semantics.py` | active | +| Assessment semantics over observable state (objective success references `conditions`; SDL scoring pipeline removed per ADR-073) | DSL-110, SEM-206 | authoring, validation, compilation, planning, execution, observation | `implementations/python/packages/aces_sdl/conditions.py`, `implementations/python/packages/aces_sdl/semantics/assessment.py`, `implementations/python/packages/aces_sdl/semantics/objective_semantics.py`, `implementations/python/packages/aces_processor/compiler.py`, `implementations/python/packages/aces_processor/models.py`, `specs/formal/assessment/README.md`, `specs/formal/assessment/pipeline-consistency.md`, `implementations/python/tests/test_semantics_assessment.py`, `implementations/python/tests/test_sdl_models.py`, `implementations/python/tests/test_fm2_semantics.py` | active | | Runtime compiled representation and canonical addresses | RUN-302 | compilation | `implementations/python/packages/aces_processor/compiler.py`, `implementations/python/tests/test_runtime_models.py`, `implementations/python/tests/test_fm2_semantics.py` | active | | Planner dependency, ordering, refresh, and applicability semantics | RUN-303 | planning | `implementations/python/packages/aces_processor/semantics/planner.py`, `implementations/python/packages/aces_processor/planner.py`, `specs/formal/planner/README.md`, `specs/formal/planner/dependency-ordering.md`, `implementations/python/tests/test_semantics_planner.py`, `implementations/python/tests/test_runtime_planner.py` | active | | Live execution state and lifecycle (snapshots, results, history) | RUN-304, API-402 | execution, observation | `implementations/python/packages/aces_runtime/manager.py`, `implementations/python/packages/aces_runtime/result_contracts.py`, `implementations/python/packages/aces_processor/models.py`, `implementations/python/tests/test_runtime_manager.py`, `implementations/python/tests/test_runtime_models.py` | active | diff --git a/docs/explain/sdl/complex-scenarios.md b/docs/explain/sdl/complex-scenarios.md index b13ff3727..b069bd223 100644 --- a/docs/explain/sdl/complex-scenarios.md +++ b/docs/explain/sdl/complex-scenarios.md @@ -13,7 +13,9 @@ The corresponding specifications live in `examples/scenarios/*.sdl.yaml`. application, data, vendor access, and recovery paths. - Include both attack and defense experiments, not just topology. - Use the current declarative experiment surface: - scoring, entities, orchestration, agents, objectives, and variables. + conditions, entities, orchestration, agents, objectives, and variables. + (Graded scoring is not an SDL surface — it lives in the experiment/evaluator + plane per [ADR-073](../../decisions/adrs/adr-073-scoring-reward-language-scope.md).) - Prefer scenarios grounded enough to expose parser, validation, runtime, and contract limits rather than abstract toy graphs. - Surface authoring friction explicitly when the SDL makes a concept @@ -84,14 +86,15 @@ red team attempts data theft and radiology disruption? restore from immutable backups, produce an incident report - Windows tied to an exercise story with pre-surgery, live-clinic, and recovery phases -- Success based on a mix of conditional uptime metrics and manual - reporting / recovery evaluation +- Objective success expressed against observable `conditions` (e.g. service + uptime and recovery state); any graded scoring of the exercise is an + experiment/evaluator-plane concern, not authored SDL - Strong distinction between in-world telemetry and any extra experiment-side evidence capture outside the current SDL syntax ### SDL Stress Surface -- All 21 sections +- All 17 sections - Hybrid IT + clinical + vendor trust boundaries - Multiple agents with distinct initial knowledge and subnet scope - Objectives that target systems, relationships, and content diff --git a/docs/explain/sdl/index.md b/docs/explain/sdl/index.md index 86ac86565..5b241b1a4 100644 --- a/docs/explain/sdl/index.md +++ b/docs/explain/sdl/index.md @@ -136,7 +136,7 @@ accounts: ## Documentation -- [SDL Sections Reference](sections.md) — Complete reference for all 21 sections +- [SDL Sections Reference](sections.md) — Complete reference for all 17 sections - [Parser Behavior](parser.md) — Key normalization, shorthand expansion, SDL-only parsing - [Language-Service Tools](language-service.md) — Agent-facing completions, references, formatting, diagnostics, and structured edits - [Agent Guidance Profile](agent-guidance.md) — Machine-readable scope boundaries, invariants, review priorities, and safe-operating expectations diff --git a/docs/explain/sdl/language-service.md b/docs/explain/sdl/language-service.md index 9b9a7cea7..3e2cc0108 100644 --- a/docs/explain/sdl/language-service.md +++ b/docs/explain/sdl/language-service.md @@ -71,7 +71,7 @@ Completion contexts include: - top-level SDL keys - known fields for SDL sections -- reference targets such as features, conditions, metrics, TLOs, entities, +- reference targets such as features, conditions, entities, accounts, objectives, and workflow steps - generic target fields that can refer to more than one section diff --git a/docs/explain/sdl/lineage.md b/docs/explain/sdl/lineage.md index 375f15fb1..6c47ebb85 100644 --- a/docs/explain/sdl/lineage.md +++ b/docs/explain/sdl/lineage.md @@ -16,9 +16,12 @@ against precedent systems, including where those systems lead ACES, see - [Open Cyber Range SDL](https://documentation.opencyberrange.ee/docs/sdl/reference/) is the closest direct SDL precedent. ACES starts from its author-facing section surface, including logical nodes, infrastructure, features, - conditions, scoring concepts, entities, injects, events, scripts, and - stories. ACES keeps the logical scenario surface separate from backend - realization instead of treating the SDL as a deployment format. + conditions, entities, injects, events, scripts, and stories. ACES keeps the + logical scenario surface separate from backend realization instead of + treating the SDL as a deployment format, and per + [ADR-073](../../decisions/adrs/adr-073-scoring-reward-language-scope.md) it + dropped OCR's scoring concepts (metrics/evaluations/TLOs/goals) — graded + scoring/reward lives in the experiment/evaluator plane. - [Open Cybersecurity Schema Framework](https://ocsf.io/) influences the event and schema side of the architecture. Its schema, profile, extension, and attribute-dictionary model is the main precedent for portable telemetry and diff --git a/docs/explain/sdl/precedents.md b/docs/explain/sdl/precedents.md index 6388af5c4..9abc8b958 100644 --- a/docs/explain/sdl/precedents.md +++ b/docs/explain/sdl/precedents.md @@ -26,7 +26,7 @@ is not a borrowing table and carries no such column. ## Core Structure (from Open Cyber Range SDL) -The 14 base sections start from the [OCR SDL](https://github.com/Open-Cyber-Range/SDL-parser) v0.21.2 surface and are adapted into Python/Pydantic. This repository aims for coverage parity across the adopted OCR concepts while remaining its own SDL; when behavior diverges or OCR's own sources disagree, this document states repository behavior explicitly instead of making clone-level compatibility claims. The OCR SDL was developed by the Norwegian Cyber Range (CR14/NTNU). +The base sections start from the [OCR SDL](https://github.com/Open-Cyber-Range/SDL-parser) v0.21.2 surface and are adapted into Python/Pydantic (per ADR-073 the OCR scoring pipeline — metrics/evaluations/TLOs/goals — was not adopted; it lives in the experiment/evaluator plane instead). This repository aims for coverage parity across the adopted OCR concepts while remaining its own SDL; when behavior diverges or OCR's own sources disagree, this document states repository behavior explicitly instead of making clone-level compatibility claims. The OCR SDL was developed by the Norwegian Cyber Range (CR14/NTNU). | SDL Element | OCR Source | Borrowed | Changes | @@ -39,7 +39,7 @@ The 14 base sections start from the [OCR SDL](https://github.com/Open-Cyber-Rang | Feature | `Feature` | Syntax | Direct port | | Condition | `Condition` | Both | Added `timeout`, `retries`, `start_period` | | Vulnerability | `Vulnerability` | Syntax | Direct port | -| Metric/Evaluation/TLO/Goal | OCR scoring pipeline | Both | Direct port | +| Metric/Evaluation/TLO/Goal | OCR scoring pipeline | Not adopted | Removed from the SDL per ADR-073; graded scoring/reward lives in the experiment/evaluator plane (ADR-055/064/069) | | Entity | `Entity` + OCR entity surface | Both | Direct port, including OCR fact maps | | Inject/Event/Script/Story | OCR orchestration | Both | Direct port | | Source | `Source` (name + version) | Syntax | Made provider-neutral | diff --git a/docs/explain/sdl/related-work-comparison.md b/docs/explain/sdl/related-work-comparison.md index 96f6a8a3c..813222e2b 100644 --- a/docs/explain/sdl/related-work-comparison.md +++ b/docs/explain/sdl/related-work-comparison.md @@ -219,10 +219,15 @@ contract between definition and backend. Objectives and workflow graphs (branching, parallel, joins) as authored constructs. -- **ACES — yes.** Declarative objectives (actor-target-window-success) and a - workflow graph (decisions, switch/case, parallel, joins, retries, cancel and - timeout, compensation) +- **ACES — yes.** Declarative objectives (actor-target-window-success, where + success references observable `conditions`) and a workflow graph (decisions, + switch/case, parallel, joins, retries, cancel and timeout, compensation). + Unlike OCR, ACES carries **no** in-SDL scoring chain: the OCR-inherited + `metrics`/`evaluations`/`tlos`/`goals` sections were removed by + [ADR-073](../../decisions/adrs/adr-073-scoring-reward-language-scope.md), and + graded scoring/reward lives in the experiment/evaluator plane (ADR-055/064/069) ([objective-semantics.md](../reference/objective-semantics.md), + [assessment-semantics.md](../reference/assessment-semantics.md), `specs/formal/objectives/`, `specs/formal/workflows/`). - **OCR SDL — yes.** A Goals → TLOs → Evaluations → Metrics → Conditions scoring chain and Stories → Scripts → Events → Injects timelines with parallel diff --git a/docs/explain/sdl/runtime-architecture.md b/docs/explain/sdl/runtime-architecture.md index 548381eb8..e3e5c1a54 100644 --- a/docs/explain/sdl/runtime-architecture.md +++ b/docs/explain/sdl/runtime-architecture.md @@ -89,7 +89,7 @@ It separates reusable definitions from bound runtime instances: - `injects` -> first-class orchestration inject resources - `node.injects` -> optional node-scoped inject bindings layered on top of top-level inject resources - `nodes` + `infrastructure` -> deployable network/node resources -- orchestration and scoring/objective sections -> resolved runtime programs and graph nodes +- orchestration and objective sections -> resolved runtime programs and graph nodes The output is a `RuntimeModel` with canonical addresses for every runtime-owned object. @@ -142,7 +142,7 @@ objects against the current `RuntimeSnapshot`. - `ProvisioningPlan` for deployable resources and bindings - `OrchestrationPlan` for events, scripts, stories, workflows, and inject state -- `EvaluationPlan` for condition bindings, scoring graph nodes, and objectives +- `EvaluationPlan` for condition bindings and objectives Each plan is provenance-bound to: @@ -257,7 +257,7 @@ Validation is semantic, not section-only. Current checks include: - fine-grained workflow feature usage (`decision`, `retry`, `parallel` barriers, failure transitions) - workflow predicate condition refs - workflow predicate prior-step state refs and state-predicate subfeatures (`outcome-matching`, `attempt-counts`) -- scoring/objective usage +- objective usage `OrchestratorCapabilities` expose both coarse workflow support and fine-grained workflow semantics: diff --git a/docs/explain/sdl/sections.md b/docs/explain/sdl/sections.md index 4d000ca42..abb5ab4e6 100644 --- a/docs/explain/sdl/sections.md +++ b/docs/explain/sdl/sections.md @@ -18,19 +18,21 @@ Canonical `imports.source` classes are: ## Section Overview -### From Open Cyber Range SDL (14 sections) +### From Open Cyber Range SDL (10 sections) + +The OCR scoring pipeline sections (`metrics`, `evaluations`, `tlos`, `goals`) +were removed from the SDL by +[ADR-073](../../decisions/adrs/adr-073-scoring-reward-language-scope.md); graded +scoring, reward, and evaluation outputs now live in the experiment/evaluator +plane (ADR-055/064/069). `conditions` (observable state) remain. | Section | Type | Purpose | |---------|------|---------| | `nodes` | `dict[str, Node]` | VMs and network switches — the compute/network topology | | `infrastructure` | `dict[str, InfraNode]` | Deployment topology: counts, links, dependencies, IP/CIDR, ACLs | | `features` | `dict[str, Feature]` | Software (Service/Configuration/Artifact) deployed to VMs | -| `conditions` | `dict[str, Condition]` | Health checks (command+interval or library source) | +| `conditions` | `dict[str, Condition]` | Health checks (command+interval or library source) — observable state | | `vulnerabilities` | `dict[str, Vulnerability]` | CWE-classified vulnerabilities assigned to nodes/features | -| `metrics` | `dict[str, Metric]` | Scoring: Manual (human-graded) or Conditional (automated) | -| `evaluations` | `dict[str, Evaluation]` | Metric groups with pass/fail thresholds | -| `tlos` | `dict[str, TLO]` | Training Learning Objectives linked to evaluations | -| `goals` | `dict[str, Goal]` | High-level goals composed of TLOs | | `entities` | `dict[str, Entity]` | Teams, organizations, people (recursive, with exercise roles) | | `injects` | `dict[str, Inject]` | Actions between entities during exercises | | `events` | `dict[str, Event]` | Triggered actions combining conditions + injects | @@ -46,7 +48,7 @@ Canonical `imports.source` classes are: | `relationships` | `dict[str, Relationship]` | Typed edges between elements (auth, trust, federation) | STIX Relationship SRO | | `agents` | `dict[str, Agent]` | Autonomous participants (actions, knowledge, scope) | CybORG Agents | | `behavior-specifications` | `dict[str, ParticipantBehaviorSpecification]` | Versioned aggregates over participant action, observation, outcome, authority, and mode surfaces | ACES ACT-606 | -| `objectives` | `dict[str, Objective]` | Scenario-local objectives binding actors, targets, windows, and success; not EXP task records | OCR scoring + CACAO action/target/agent | +| `objectives` | `dict[str, Objective]` | Scenario-local objectives binding actors, targets, windows, and success (against observable `conditions`); not EXP task records | CACAO action/target/agent | | `workflows` | `dict[str, Workflow]` | Branching and parallel control graphs over declared objectives | CACAO workflow graph patterns; semantics tightened using Step Functions / Argo / SCXML style control-flow rules | | `variables` | `dict[str, Variable]` | Parameterization (types, defaults, substitution) | CACAO playbook_variables | @@ -1384,38 +1386,28 @@ vulnerabilities: --- -## Scoring Pipeline: Metrics, Evaluations, TLOs, Goals - -``` -Conditions → Metrics → Evaluations → TLOs → Goals -``` - -```yaml -metrics: - service-uptime: - type: CONDITIONAL - max-score: 100 - condition: web-alive - report-quality: - type: MANUAL - max-score: 50 - artifact: true - -evaluations: - overall: - metrics: [service-uptime, report-quality] - min-score: 75 # shorthand = percentage - # or: min-score: {absolute: 100} - -tlos: - web-defense: - name: Web Application Defense - evaluation: overall - -goals: - pass-exercise: - tlos: [web-defense] -``` +## Scoring: removed from the SDL + +The OCR-inherited SDL scoring pipeline +(`conditions → metrics → evaluations → TLOs → goals`) and the CybORG +`agents.reward_calculator` label were removed from the authoring language by +[ADR-073](../../decisions/adrs/adr-073-scoring-reward-language-scope.md). The +`metrics`, `evaluations`, `tlos` (Training Learning Objectives), and `goals` +sections are no longer SDL surfaces. + +`conditions` remain first-class **observable state**: an objective's `success` +is expressed against `conditions` (see [Objectives](#objectives)), and workflow +predicates reference `conditions`. When a scenario genuinely needs a graded +score, cumulative reward, pass/fail evaluation, or a leaderboard value, that +concern lives in the experiment/evaluator plane — experiment-core contracts +(`experiment-task-v1` metric definitions, `experiment-study-v1` analysis plans; +[ADR-055](../../decisions/adrs/adr-055-experiment-core-contract-boundary.md)), +the evidence/measure contracts (`experiment-evidence-record-v1`, +`experiment-derived-measure-v1`; +[ADR-064](../../decisions/adrs/adr-064-experiment-evidence-and-measure-contract-boundary.md)), +and the backend Evaluator +([ADR-069](../../decisions/adrs/adr-069-cage-2-replication-architecture.md)) — +never as authored SDL. --- @@ -1430,7 +1422,6 @@ entities: name: Blue Team role: Blue mission: Defend infrastructure - tlos: [web-defense] facts: department: SOC primary-shift: nights @@ -1636,9 +1627,14 @@ agents: operating_scope: # broader targetable scope beyond subnets - corp-net - user-net - reward_calculator: HybridImpactPwn ``` +The CybORG-inherited `agents.reward_calculator` label was removed from the SDL by +[ADR-073](../../decisions/adrs/adr-073-scoring-reward-language-scope.md): it was +an unbound free-text label that named a reward class running outside participant +perception. Reward now lives in the experiment/evaluator plane (ADR-055/064/069), +not as an authored SDL agent field. + `entity` is required and must resolve to the `entities` section; the participant's authored identity and role both come from this binding (per ADR-020). `initial_knowledge.hosts` references VM node names, `subnets` @@ -1737,7 +1733,7 @@ direct adoption of MITRE ATLAS tactics release v2026.06, pinned by `contracts/concept-authority/atlas-tactics-source-v1.json` and checked by `tools/check_atlas_tactic_vocabulary.py`. These refs classify authored attack-oriented participant tasks, goals, or activities without replacing -action contracts, SDL `goals`, experiment tasks, workflow steps, or runtime +action contracts, experiment tasks, workflow steps, or runtime history. Extensions are only allowed when `extension_policy` permits them, and extension keys must use `x-:`. @@ -1750,7 +1746,7 @@ outcome-rule runtime addresses. ## Objectives -Declarative experiment semantics that bind actors, targets, timing, and success criteria in the same SDL. Inspired by OCR's in-spec assessment model and CACAO's separation of agent, target, and workflow context. +Declarative experiment semantics that bind actors, targets, timing, and success criteria in the same SDL. Inspired by CACAO's separation of agent, target, and workflow context; objective success is expressed against observable `conditions` ([ADR-073](../../decisions/adrs/adr-073-scoring-reward-language-scope.md)). ```yaml objectives: @@ -1764,8 +1760,7 @@ objectives: - infrastructure.dmz-switch.acls.allow-dmz-https success: mode: all_of # all_of, any_of - goals: [pass-exercise] - metrics: [service-uptime] + conditions: [beacon-online] # observable state only window: stories: [exercise] scripts: [main-timeline] @@ -1776,11 +1771,11 @@ objectives: blue-reporting: entity: blue-team success: - metrics: [report-quality] + conditions: [web-alive] # observable state only depends_on: [red-initial-access] ``` -Every objective must declare exactly one actor: either `agent` or `entity`. `success` is required and must reference at least one declared `condition`, `metric`, `evaluation`, `tlo`, or `goal`. `targets` are optional, but when present they must resolve to named scenario elements. Bare target refs work when unambiguous; otherwise use a qualified ref such as `nodes.web-server`, `features.app-to-db`, or `content.mailbox.items.invoice.eml`. `window` is optional; when supplied, referenced stories/scripts/events/workflows must exist and remain internally consistent. Workflow steps use qualified refs of the form `.`. +Every objective must declare exactly one actor: either `agent` or `entity`. `success` is required and must reference at least one declared `condition` (observable state; the OCR scoring surfaces `metrics`/`evaluations`/`tlos`/`goals` were removed by [ADR-073](../../decisions/adrs/adr-073-scoring-reward-language-scope.md)). `targets` are optional, but when present they must resolve to named scenario elements. Bare target refs work when unambiguous; otherwise use a qualified ref such as `nodes.web-server`, `features.app-to-db`, or `content.mailbox.items.invoice.eml`. `window` is optional; when supplied, referenced stories/scripts/events/workflows must exist and remain internally consistent. Workflow steps use qualified refs of the form `.`. `depends_on` is an ordering relation, not just commentary. It defines a partial order over objectives: downstream objectives are not considered ready until their predecessors have been satisfied. Objective dependency cycles are rejected. @@ -1866,7 +1861,7 @@ terminates with a configured trigger. Workflow predicates may observe: -- scoring/evaluation data via `conditions`, `metrics`, `evaluations`, `tlos`, `goals`, and `objectives` +- observable state via `conditions` and objective status via `objectives` (the OCR scoring surfaces `metrics`/`evaluations`/`tlos`/`goals` were removed by [ADR-073](../../decisions/adrs/adr-073-scoring-reward-language-scope.md)) - prior step state via `steps`, where each entry names a prior executable step plus one or more expected outcomes (`succeeded`, `failed`, `exhausted`) and an optional `min-attempts` Example predicate over prior step state: @@ -1953,16 +1948,23 @@ Think of variables as parameterizing **properties of declared objects**, not the --- -## Scoring, Objectives, and Runtime Checks +## Objectives, Conditions, and Runtime Checks -The SDL carries both: +The SDL carries: -- the OCR-style scoring pipeline (`conditions → metrics → evaluations → TLOs → goals`) -- declarative objectives that bind actors, targets, windows, and success criteria +- `conditions` — observable state (health checks and library-sourced checks) +- declarative objectives that bind actors, targets, windows, and success criteria expressed against observable `conditions` - workflow graphs that branch or parallelize declared objectives without embedding runtime probe logic +The SDL carries **no** graded scoring pipeline: the OCR-inherited +`metrics`/`evaluations`/`tlos`/`goals` sections and the `agents.reward_calculator` +label were removed by +[ADR-073](../../decisions/adrs/adr-073-scoring-reward-language-scope.md). Graded +scoring, reward, leaderboard values, and evaluation outputs live in the +experiment/evaluator plane (ADR-055/064/069). + Experiment-core task, run, apparatus-context, and study records are separate contracts. They may reference SDL scenarios or scenario snapshots, but they are not SDL sections. -Backend-specific auto-validation mechanics still live outside the SDL. The runtime may use Wazuh queries, command probes, file checks, or other adapters to determine whether an SDL-declared objective or scoring condition has been satisfied, but those probe details are not the language itself. +Backend-specific auto-validation mechanics still live outside the SDL. The runtime may use Wazuh queries, command probes, file checks, or other adapters to determine whether an SDL-declared objective or observable condition has been satisfied, but those probe details are not the language itself. diff --git a/docs/explain/sdl/validation.md b/docs/explain/sdl/validation.md index 556153318..d9229f84c 100644 --- a/docs/explain/sdl/validation.md +++ b/docs/explain/sdl/validation.md @@ -22,17 +22,20 @@ becoming a validator-only interpretation of the SDL. | `verify_features` | Vulnerability references exist. Dependency references exist. **Dependency cycle detection** via topological sort. | | `verify_conditions` | (Structural: command+interval XOR source — enforced by Pydantic) | | `verify_vulnerabilities` | (Structural: CWE format — enforced by Pydantic) | -| `verify_metrics` | Conditional metrics reference existing conditions. Each condition used by at most one metric. | -| `verify_evaluations` | Referenced metrics exist. Absolute min-score doesn't exceed sum of metric max-scores. | -| `verify_tlos` | Referenced evaluations exist. | -| `verify_goals` | Referenced TLOs exist. | -| `verify_entities` | TLO, vulnerability, and event references on entities (including nested) exist. | -| `verify_injects` | from-entity and to-entities reference existing (possibly nested) entities. TLO references exist. | +| `verify_entities` | Vulnerability and event references on entities (including nested) exist. | +| `verify_injects` | from-entity and to-entities reference existing (possibly nested) entities. | | `verify_events` | Condition and inject references exist. | | `verify_scripts` | Event references exist. Event times within script start/end bounds. | | `verify_stories` | Script references exist. | | `verify_roles` | Entity references in node roles resolve to flattened entity names. | +The former scoring-pipeline passes (`verify_metrics`, `verify_evaluations`, +`verify_tlos`, `verify_goals`, and the internal `_verify_assessment_pipeline` +check) were removed with the `metrics`/`evaluations`/`tlos`/`goals` sections by +[ADR-073](../../decisions/adrs/adr-073-scoring-reward-language-scope.md). There is +no longer a "references undefined metric/evaluation/TLO/goal" validation error; +`conditions` remain the observable-state surface objective success references. + ### Extension passes | Pass | What It Checks | @@ -58,9 +61,9 @@ becoming a validator-only interpretation of the SDL. | `verify_relationship_mail_access` | A relationship with `mail_access` must target a runtime mail service. Concrete `listener_ref`, `mailbox_ref`, and `domain_ref` values resolve within that target service, while protocol, auth-mechanism, and TLS-mode fields are structurally normalized by the `RelationshipMailAccess` model. | | `verify_agents` | Entity references resolve. Starting accounts and initial-knowledge accounts exist in accounts section. Allowed subnets and initial-knowledge subnets must resolve to switch-backed infrastructure entries. Initial-knowledge hosts must resolve to VM nodes. Initial-knowledge services exist in `nodes.*.services[].name`. | | `verify_participant_behavior` | Agent action refs resolve to declared action contracts, observation-boundary refs resolve to declared boundaries, interaction refs resolve to declared actions or targetable state, and boundary view rules/transitions resolve to declared observable, hidden, or evidence refs. | -| `verify_objectives` | Objective actors resolve (`agent` or `entity`). Objective actions must be declared by the referenced agent. Targets resolve to named scenario elements, including qualified service/ACL refs and section-qualified top-level refs. Ambiguous bare refs are rejected with qualified alternatives. Success criteria resolve to declared conditions/metrics/evaluations/TLOs/goals. Optional windows resolve through one shared normalized analysis over stories/scripts/events/workflows/workflow-steps, must remain internally consistent, and fail closed on dangling or out-of-window refs. Objective dependencies must resolve and stay acyclic. | -| `verify_workflows` | Workflow `start` and every referenced step must exist. `objective`/`retry` steps must reference declared objectives. Predicate refs must resolve to declared conditions/metrics/evaluations/TLOs/goals/objectives, and step-state refs must resolve to prior executable steps whose state is guaranteed to be known before the predicate runs. Workflow graphs must be acyclic and fully reachable from `start`. Parallel joins must be explicit barriers, every explicit branch path must converge on the declared join, branch-local state remains scoped until the join, and post-join predicates may inspect only branch steps guaranteed on every path within their branch before the join. | -| `verify_participant_outcomes` | Outcome interpretation source and target refs resolve for action contracts, objectives, workflows, and evaluations. Reward-signal targets require governed assessment refs structurally, while runtime conformance grounds emitted interpretation records in action results, event evidence, and participant episode history. | +| `verify_objectives` | Objective actors resolve (`agent` or `entity`). Objective actions must be declared by the referenced agent. Targets resolve to named scenario elements, including qualified service/ACL refs and section-qualified top-level refs. Ambiguous bare refs are rejected with qualified alternatives. Success criteria resolve to declared `conditions` (observable state only, per [ADR-073](../../decisions/adrs/adr-073-scoring-reward-language-scope.md)). Optional windows resolve through one shared normalized analysis over stories/scripts/events/workflows/workflow-steps, must remain internally consistent, and fail closed on dangling or out-of-window refs. Objective dependencies must resolve and stay acyclic. | +| `verify_workflows` | Workflow `start` and every referenced step must exist. `objective`/`retry` steps must reference declared objectives. Predicate refs must resolve to declared `conditions`/`objectives` (the scoring surfaces were removed per [ADR-073](../../decisions/adrs/adr-073-scoring-reward-language-scope.md)), and step-state refs must resolve to prior executable steps whose state is guaranteed to be known before the predicate runs. Workflow graphs must be acyclic and fully reachable from `start`. Parallel joins must be explicit barriers, every explicit branch path must converge on the declared join, branch-local state remains scoped until the join, and post-join predicates may inspect only branch steps guaranteed on every path within their branch before the join. | +| `verify_participant_outcomes` | Outcome interpretation source and target refs resolve for action contracts, objectives, and workflows. The SEM-215 `reward_signal` / `evaluation_result` interpretation layers remain a governed interpretation relation but no longer bind to any SDL `evaluations` section ([ADR-073](../../decisions/adrs/adr-073-scoring-reward-language-scope.md)); runtime conformance grounds emitted interpretation records in action results, event evidence, and participant episode history. | | `verify_variables` | Checks that full-value `${var}` placeholders and embedded `${var}` tokens reference declared variables. Structural validation of variable declaration names, typed defaults, and `allowed_values` still happens in the model/schema layer. | Pydantic structural validation also enforces model-local node rules before @@ -574,7 +577,7 @@ from the SDL runtime-family registry. Registered runtime refs include: connector, and setting refs This means a relationship can reference any node, feature, condition, -vulnerability, infrastructure entry, metric, evaluation, TLO, goal, entity +vulnerability, infrastructure entry, entity (including nested), inject, event, script, story, content entry, content item, account, agent, objective, workflow, relationship, variable, named service binding, registered runtime-family object, registered runtime-family child diff --git a/examples/README.md b/examples/README.md index 866bff147..1e0b514bf 100644 --- a/examples/README.md +++ b/examples/README.md @@ -9,9 +9,9 @@ backend guarantees. | File | Best use | Current coverage | Limits | |------|----------|------------------|--------| -| [`scenarios/hospital-ransomware-surgery-day.sdl.yaml`](scenarios/hospital-ransomware-surgery-day.sdl.yaml) | Large enterprise and clinical operations scenario | Disk-backed example test; complex example checks for objectives, agents, relationships, content, stories, metrics, direct refs | Does not deploy a hospital range or prove clinical exercise adequacy | -| [`scenarios/satcom-release-poisoning.sdl.yaml`](scenarios/satcom-release-poisoning.sdl.yaml) | Supply-chain, release, tenant, and rollback scenario | Disk-backed example test; complex example checks for objectives, agents, relationships, content, stories, metrics, workflows, enum-backed variables | Does not implement a CI/CD backend or production release system | -| [`scenarios/port-authority-surge-response.sdl.yaml`](scenarios/port-authority-surge-response.sdl.yaml) | IT/OT, customs, yard operations, and recovery scenario | Disk-backed example test; complex example checks for objectives, agents, relationships, content, stories, metrics, workflows, direct refs | Does not implement OT control, safety validation, or port operations | +| [`scenarios/hospital-ransomware-surgery-day.sdl.yaml`](scenarios/hospital-ransomware-surgery-day.sdl.yaml) | Large enterprise and clinical operations scenario | Disk-backed example test; complex example checks for objectives, agents, relationships, content, stories, conditions, direct refs | Does not deploy a hospital range or prove clinical exercise adequacy | +| [`scenarios/satcom-release-poisoning.sdl.yaml`](scenarios/satcom-release-poisoning.sdl.yaml) | Supply-chain, release, tenant, and rollback scenario | Disk-backed example test; complex example checks for objectives, agents, relationships, content, stories, conditions, workflows, enum-backed variables | Does not implement a CI/CD backend or production release system | +| [`scenarios/port-authority-surge-response.sdl.yaml`](scenarios/port-authority-surge-response.sdl.yaml) | IT/OT, customs, yard operations, and recovery scenario | Disk-backed example test; complex example checks for objectives, agents, relationships, content, stories, conditions, workflows, direct refs | Does not implement OT control, safety validation, or port operations | | [`scenarios/techvault.sdl.yaml`](scenarios/techvault.sdl.yaml) | Runtime inventory and image provenance parity example | Disk-backed example test | Does not provide a deployable TechVault application or image build pipeline | | [`scenarios/enterprise-participant-evidence-loop.sdl.yaml`](scenarios/enterprise-participant-evidence-loop.sdl.yaml) | Reference scenario for a generic enterprise participant/evidence loop | Disk-backed example test; focused processor compile check for participant behaviors, action contracts, observation boundaries, Wazuh evidence, policy provenance, and boundary evidence surfaces | Does not prove a concrete coding-agent runner, APTL/libvirt realization, TechVault coverage, or broad benchmark capability | @@ -31,7 +31,7 @@ machine-readable catalog for the current non-normative authoring library. | Participant behavior | [`library/templates/participant_behavior/action-contract-observation-boundary.yaml`](library/templates/participant_behavior/action-contract-observation-boundary.yaml) | [`library/patterns/participant-behavior-contract-binding.yaml`](library/patterns/participant-behavior-contract-binding.yaml) | | Task | [`library/templates/task/single-objective-task.yaml`](library/templates/task/single-objective-task.yaml) | [`library/patterns/task-as-objective-contract.yaml`](library/patterns/task-as-objective-contract.yaml) | | Run | [`library/templates/run/timed-run-control.yaml`](library/templates/run/timed-run-control.yaml) | [`library/patterns/run-window-with-evidence.yaml`](library/patterns/run-window-with-evidence.yaml) | -| Study | [`library/templates/study/scored-study-protocol.yaml`](library/templates/study/scored-study-protocol.yaml) | [`library/patterns/study-scoring-chain.yaml`](library/patterns/study-scoring-chain.yaml) | +| Study | [`library/templates/study/observational-study-protocol.yaml`](library/templates/study/observational-study-protocol.yaml) | [`library/patterns/observable-study-conditions.yaml`](library/patterns/observable-study-conditions.yaml) | Each template has metadata plus a complete current-SDL `body`. The `tools/check_example_library.py` policy gate validates catalog shape, stable @@ -100,5 +100,7 @@ conformance tests. Task, run, and study templates use current SDL wrappers rather than first-class `tasks`, `runs`, or `studies` sections. They show how to express those concepts -with objectives, workflows, timing, scoring, and evidence-like references that -the current implementation can validate. +with objectives, workflows, timing, observable conditions, and evidence-like +references that the current implementation can validate. Graded scoring and +reward are experiment/evaluator-plane concerns (experiment-* contracts) per +ADR-073, not SDL surfaces. diff --git a/examples/library/catalog.yaml b/examples/library/catalog.yaml index c1df583a0..12fb5ea82 100644 --- a/examples/library/catalog.yaml +++ b/examples/library/catalog.yaml @@ -71,14 +71,14 @@ surfaces: - id: run-window-with-evidence path: examples/library/patterns/run-window-with-evidence.yaml study: - summary: Study-oriented authoring with metrics, evaluations, TLOs, goals, and workflow protocol. + summary: Study-oriented authoring with observable conditions, objectives, and workflow protocol. Graded scoring is an experiment/evaluator-plane concern per ADR-073. worked_examples: - - id: satcom-study-scoring + - id: satcom-study-scenario path: examples/scenarios/satcom-release-poisoning.sdl.yaml source_refs: [examples/README.md] templates: - - id: scored-study-protocol - path: examples/library/templates/study/scored-study-protocol.yaml + - id: observational-study-protocol + path: examples/library/templates/study/observational-study-protocol.yaml patterns: - - id: study-scoring-chain - path: examples/library/patterns/study-scoring-chain.yaml + - id: observable-study-conditions + path: examples/library/patterns/observable-study-conditions.yaml diff --git a/examples/library/patterns/observable-study-conditions.yaml b/examples/library/patterns/observable-study-conditions.yaml new file mode 100644 index 000000000..2df3fe6ca --- /dev/null +++ b/examples/library/patterns/observable-study-conditions.yaml @@ -0,0 +1,25 @@ +pattern: aces-library-pattern +version: 1 +id: observable-study-conditions +surface: study +requirement_refs: [AUT-806] +source_refs: + - examples/scenarios/satcom-release-poisoning.sdl.yaml + - docs/decisions/adrs/adr-073-scoring-reward-language-scope.md +summary: Represent study-style examples through observable conditions and objective success, keeping graded scoring in the experiment/evaluator plane. +intent: > + Make study-like examples reusable by connecting participant activity to + observable conditions, objectives, and workflow protocol. Graded scoring, + reward, and interpretation live in the experiment/evaluator plane + (experiment-* contracts) per ADR-073, not in the SDL. +use_when: + - A study needs a repeatable, observable definition of participant success. + - Reviewers need to see how a measured condition gates an objective and workflow. +authoring_steps: + - Define the observable condition with a real command and interval. + - Reference the condition from the objective's success.conditions list. + - Drive the objective from a workflow protocol step. + - Carry any graded scoring or reward in the experiment/evaluator plane, not the SDL. +validation: + - command: parse_sdl_file + expected: condition, objective, and workflow references resolve with no removed scoring surfaces. diff --git a/examples/library/patterns/study-scoring-chain.yaml b/examples/library/patterns/study-scoring-chain.yaml deleted file mode 100644 index 55ae66205..000000000 --- a/examples/library/patterns/study-scoring-chain.yaml +++ /dev/null @@ -1,23 +0,0 @@ -pattern: aces-library-pattern -version: 1 -id: study-scoring-chain -surface: study -requirement_refs: [AUT-806] -source_refs: - - examples/scenarios/satcom-release-poisoning.sdl.yaml - - docs/explain/reference/assessment-semantics.md -summary: Represent current study templates with a traceable scoring chain from condition to goal. -intent: > - Make study-like examples reusable by connecting participant activity to - metrics, evaluations, TLOs, goals, objectives, and workflow protocol. -use_when: - - A study needs repeatable scoring or interpretation of participant outcomes. - - Reviewers need to see how a measured condition contributes to a higher-level goal. -authoring_steps: - - Define the observable condition or manual metric. - - Bind metrics into evaluations with explicit thresholds. - - Link evaluations to TLOs and TLOs to goals. - - Reference the scoring chain from objectives and workflow protocol steps. -validation: - - command: parse_sdl_file - expected: metric, evaluation, TLO, goal, objective, and workflow references resolve. diff --git a/examples/library/templates/study/scored-study-protocol.yaml b/examples/library/templates/study/observational-study-protocol.yaml similarity index 52% rename from examples/library/templates/study/scored-study-protocol.yaml rename to examples/library/templates/study/observational-study-protocol.yaml index 43b8cc1a7..b5775f079 100644 --- a/examples/library/templates/study/scored-study-protocol.yaml +++ b/examples/library/templates/study/observational-study-protocol.yaml @@ -1,15 +1,15 @@ template: aces-library-template version: 1 -id: scored-study-protocol +id: observational-study-protocol surface: study requirement_refs: [AUT-806] source_refs: - docs/explain/sdl/sections.md - - docs/explain/reference/assessment-semantics.md -summary: Study template connecting task outcome, metric, evaluation, TLO, goal, and workflow protocol. + - docs/decisions/adrs/adr-073-scoring-reward-language-scope.md +summary: Study template connecting a task outcome to an observable condition, objective, and workflow protocol. Graded scoring lives in the experiment/evaluator plane per ADR-073. body: name: library-study-template - description: Study-oriented SDL wrapper with objectives mapped into metric, evaluation, TLO, and goal evidence. + description: Study-oriented SDL wrapper whose objective asserts observable success via a condition. Graded scoring and reward are experiment/evaluator-plane concerns per ADR-073, not SDL surfaces. nodes: study-target: type: VM @@ -20,21 +20,6 @@ body: study-task-success: command: test -f /var/lib/aces/study-task-success interval: 30 - metrics: - task-success-score: - type: conditional - condition: study-task-success - max-score: 100 - evaluations: - study-evaluation: - metrics: [task-success-score] - min-score: {percentage: 80} - tlos: - study-learning-objective: - evaluation: study-evaluation - goals: - study-goal: - tlos: [study-learning-objective] entities: participant-team: role: blue @@ -46,10 +31,7 @@ body: agent: study-participant targets: [nodes.study-target.services.https] success: - metrics: [task-success-score] - evaluations: [study-evaluation] - tlos: [study-learning-objective] - goals: [study-goal] + conditions: [study-task-success] workflows: study-protocol: start: complete-task diff --git a/examples/scenarios/enterprise-participant-evidence-loop.sdl.yaml b/examples/scenarios/enterprise-participant-evidence-loop.sdl.yaml index 5c6e7e009..07524a8ab 100644 --- a/examples/scenarios/enterprise-participant-evidence-loop.sdl.yaml +++ b/examples/scenarios/enterprise-participant-evidence-loop.sdl.yaml @@ -219,64 +219,6 @@ conditions: reachability checks from the participant host were retained as negative evaluator evidence where supported. -metrics: - participant-evidence-complete: - type: conditional - max_score: 100 - condition: participant-observation-recorded - description: > - Scores the handoff only when the participant action produced the expected - bounded observation evidence. - wazuh-evidence-complete: - type: conditional - max_score: 100 - condition: wazuh-evidence-recorded - description: > - Scores the handoff only when Wazuh evaluator evidence is retained. - policy-provenance-complete: - type: conditional - max_score: 100 - condition: policy-decision-recorded - description: > - Scores the handoff only when optional policy provenance is retained. - boundary-evidence-complete: - type: conditional - max_score: 100 - condition: boundary-checks-recorded - description: > - Scores the handoff only when negative participant-boundary evidence is - retained by a backend that supports live checks. - -evaluations: - participant-loop-evaluation: - metrics: - - participant-evidence-complete - - wazuh-evidence-complete - - policy-provenance-complete - - boundary-evidence-complete - min_score: {percentage: 100} - description: > - Demonstrates that participant-local evidence, Wazuh evaluator evidence, - policy provenance, and negative boundary checks can support the reference scenario - objective without claiming broad autonomous-agent, defensive-tool, or - model-defense capability. - -tlos: - authored-runtime-handoff: - evaluation: participant-loop-evaluation - description: > - The authored SDL participant behavior compiles into runtime-addressable - participant, action, observation, Wazuh-evidence, policy-provenance, and - boundary-evidence surfaces. - -goals: - reference-demonstration: - tlos: [authored-runtime-handoff] - description: > - Provide a reusable ACES-side reference for APTL and libvirt n=2 proof - issues while signaling richer defensive and model-defense evidence - possibilities for later work. - action-contracts: probe-customer-portal-login: semantic-version: 1.0.0 @@ -617,6 +559,9 @@ behavior-specifications: extension-policy: governed-extension objectives: + # Graded scoring/interpretation of this evidence loop now lives in the + # experiment/evaluator plane (experiment-* contracts) per ADR-073. The SDL + # objective asserts only observable success via conditions. demonstrate-handoff: agent: participant-agent actions: [probe-customer-portal-login] @@ -627,13 +572,11 @@ objectives: - content.policy-decision-log - content.boundary-check-evidence success: - metrics: - - participant-evidence-complete - - wazuh-evidence-complete - - policy-provenance-complete - - boundary-evidence-complete - evaluations: [participant-loop-evaluation] - goals: [reference-demonstration] + conditions: + - participant-observation-recorded + - wazuh-evidence-recorded + - policy-decision-recorded + - boundary-checks-recorded window: workflows: [reference-handoff] steps: [reference-handoff.probe] diff --git a/examples/scenarios/hospital-ransomware-surgery-day.sdl.yaml b/examples/scenarios/hospital-ransomware-surgery-day.sdl.yaml index 4be947be1..a87621f2e 100644 --- a/examples/scenarios/hospital-ransomware-surgery-day.sdl.yaml +++ b/examples/scenarios/hospital-ransomware-surgery-day.sdl.yaml @@ -494,60 +494,11 @@ vulnerabilities: technical: true class: CWE-732 -metrics: - blue-ehr-availability: - type: CONDITIONAL - max-score: 100 - condition: ehr-api-healthy - blue-pacs-availability: - type: CONDITIONAL - max-score: 100 - condition: pacs-archive-healthy - blue-backup-integrity: - type: CONDITIONAL - max-score: 100 - condition: backup-catalog-intact - blue-ir-report: - type: MANUAL - max-score: 50 - artifact: true - red-phi-staging: - type: CONDITIONAL - max-score: 100 - condition: phi-staging-observed - red-clinical-disruption: - type: CONDITIONAL - max-score: 100 - condition: clinical-disruption-observed - -evaluations: - blue-clinical-resilience: - metrics: [blue-ehr-availability, blue-pacs-availability, blue-backup-integrity, blue-ir-report] - min-score: 75 - red-ransom-impact: - metrics: [red-phi-staging, red-clinical-disruption] - min-score: {percentage: 75} - -tlos: - sustain-clinical-operations: - name: Sustain clinical operations - evaluation: blue-clinical-resilience - achieve-ransomware-impact: - name: Achieve ransomware impact - evaluation: red-ransom-impact - -goals: - hospital-blue-goal: - tlos: [sustain-clinical-operations] - ransom-crew-goal: - tlos: [achieve-ransomware-impact] - entities: hospital-blue: name: Hospital Blue Team role: Blue mission: Keep patient care and core services available throughout the incident - tlos: [sustain-clinical-operations] entities: soc: name: SOC @@ -750,7 +701,6 @@ agents: services: [smtp-inbound, vpn-portal, vendor-rdp] accounts: [vendor-tech] allowed_subnets: [internet-edge, vendor-segment, corp-it] - reward_calculator: InitialAccessImpact red-operator: entity: ransom-crew actions: [Pivot, Exfiltrate, Encrypt, Disable] @@ -761,7 +711,6 @@ agents: services: [ehr-https, ehr-postgres, dicom-store] accounts: [svc-sql, backup-operator] allowed_subnets: [clinical-it, backup-net, identity-net] - reward_calculator: HybridImpactPwn blue-soc-agent: entity: hospital-blue.soc actions: [Inspect, Triage, Isolate, Escalate] @@ -772,7 +721,6 @@ agents: services: [soc-ui, mail-admin-ui, vpn-portal] accounts: [helpdesk-user, vendor-tech] allowed_subnets: [security-net, corp-it, vendor-segment] - reward_calculator: HybridAvailabilityConfidentiality blue-ir-agent: entity: hospital-blue.recovery actions: [Contain, Restore, ReissueCreds, Validate] @@ -783,9 +731,11 @@ agents: services: [backup-api, backup-nfs, pacs-web-ui] accounts: [svc-sql, backup-operator] allowed_subnets: [backup-net, clinical-it, identity-net] - reward_calculator: RecoveryResilience objectives: + # This scenario intended graded red/blue comparison. Graded scoring now lives + # in the experiment/evaluator plane (experiment-* contracts) per ADR-073; + # SDL objectives assert only observable success via conditions. red-establish-foothold: agent: red-initial actions: [Phish, AbuseVPN] @@ -807,7 +757,6 @@ objectives: targets: [phi-records, ehr-to-database, ehr-db] success: conditions: [phi-staging-observed] - metrics: [red-phi-staging] depends_on: [red-establish-foothold] window: stories: [surgery-day] @@ -816,7 +765,7 @@ objectives: entity: ransom-crew targets: [pacs01, radiology-images, pacs-sync-to-backup] success: - goals: [ransom-crew-goal] + conditions: [radiology-workflow-delayed, clinical-disruption-observed] depends_on: [red-stage-phi] window: stories: [surgery-day] @@ -841,8 +790,7 @@ objectives: actions: [Contain, Restore, Validate] targets: [backup-vault, ehr-db, pacs01, backup-manages-ehr] success: - goals: [hospital-blue-goal] - metrics: [blue-backup-integrity] + conditions: [ehr-api-healthy, pacs-archive-healthy, backup-catalog-intact] depends_on: [blue-detect-and-triage, red-disrupt-radiology] window: stories: [surgery-day] diff --git a/examples/scenarios/port-authority-surge-response.sdl.yaml b/examples/scenarios/port-authority-surge-response.sdl.yaml index 6d9c1ad79..77841e019 100644 --- a/examples/scenarios/port-authority-surge-response.sdl.yaml +++ b/examples/scenarios/port-authority-surge-response.sdl.yaml @@ -311,56 +311,11 @@ vulnerabilities: technical: true class: CWE-306 -metrics: - blue-customs-integrity: - type: CONDITIONAL - max-score: 100 - condition: manifest-integrity-ok - blue-yard-safety: - type: CONDITIONAL - max-score: 100 - condition: crane-safe-mode - blue-telemetry-retention: - type: CONDITIONAL - max-score: 100 - condition: yard-telemetry-flowing - blue-ops-brief: - type: MANUAL - max-score: 50 - artifact: true - red-manifest-impact: - type: CONDITIONAL - max-score: 100 - condition: tampered-manifest-observed - -evaluations: - blue-surge-resilience: - metrics: [blue-customs-integrity, blue-yard-safety, blue-telemetry-retention, blue-ops-brief] - min-score: 75 - red-yard-impact: - metrics: [red-manifest-impact] - min-score: {absolute: 80} - -tlos: - sustain-safe-throughput: - name: Sustain safe throughput - evaluation: blue-surge-resilience - achieve-yard-disruption: - name: Achieve yard disruption - evaluation: red-yard-impact - -goals: - port-blue-goal: - tlos: [sustain-safe-throughput] - red-cartel-goal: - tlos: [achieve-yard-disruption] - entities: port-blue: name: Port Authority Blue Team role: Blue mission: Maintain customs integrity and safe yard operations during the surge - tlos: [sustain-safe-throughput] entities: it: name: Port IT @@ -550,7 +505,6 @@ agents: services: [shipping-portal-https, vendor-jump-ssh, yard-hmi-web] accounts: [vendor-tech] allowed_subnets: [public-edge, vendor-net, yard-ot, terminal-it] - reward_calculator: OTImpact blue-yard-agent: entity: port-blue.yard-ops actions: [Inspect, Isolate, Restore, Validate] @@ -561,7 +515,6 @@ agents: services: [yard-hmi-rdp, crane-opcua, historian-ui] accounts: [historian-svc] allowed_subnets: [yard-ot, safety-net, backup-net] - reward_calculator: SafetyFirst blue-soc-agent: entity: port-blue.incident-command actions: [Monitor, Triage, Coordinate, Recover] @@ -572,16 +525,17 @@ agents: services: [port-soc-ui, shipping-portal-https, tos-https] accounts: [harbor-master, customs-officer] allowed_subnets: [security-net, terminal-it, customs-link, backup-net] - reward_calculator: CoordinatedResponse objectives: + # This scenario intended graded red/blue comparison. Graded scoring now lives + # in the experiment/evaluator plane (experiment-* contracts) per ADR-073; + # SDL objectives assert only observable success via conditions. red-corrupt-manifests: agent: red-yard-agent actions: [Phish, Tamper] targets: [cargo-manifests, manifest-db, portal-to-manifest] success: conditions: [tampered-manifest-observed] - metrics: [red-manifest-impact] window: stories: [surge-day] scripts: [arrival-phase, customs-surge-phase] @@ -589,7 +543,7 @@ objectives: entity: red-cartel targets: [yard-hmi, crane-plc, hmi-manages-crane] success: - goals: [red-cartel-goal] + conditions: [tampered-manifest-observed] depends_on: [red-corrupt-manifests] window: stories: [surge-day] @@ -600,8 +554,7 @@ objectives: actions: [Monitor, Triage, Coordinate] targets: [customs-gateway, manifest-db, customs-federates-port] success: - metrics: [blue-customs-integrity] - conditions: [customs-link-healthy] + conditions: [customs-link-healthy, manifest-integrity-ok] window: stories: [surge-day] scripts: [arrival-phase, customs-surge-phase] @@ -618,7 +571,7 @@ objectives: - nodes.crane-plc.services.crane-opcua - infrastructure.yard-ot.acls.allow-yard-control success: - goals: [port-blue-goal] + conditions: [crane-safe-mode, yard-telemetry-flowing] depends_on: [blue-maintain-customs-integrity, red-degrade-yard-ops] window: stories: [surge-day] @@ -636,7 +589,6 @@ objectives: - infrastructure.yard-ot.acls.allow-yard-control success: conditions: [yard-telemetry-flowing, historian-replication-ok] - metrics: [blue-telemetry-retention] depends_on: [red-degrade-yard-ops] window: stories: [surge-day] diff --git a/examples/scenarios/satcom-release-poisoning.sdl.yaml b/examples/scenarios/satcom-release-poisoning.sdl.yaml index 64f4994fa..56e2ccfae 100644 --- a/examples/scenarios/satcom-release-poisoning.sdl.yaml +++ b/examples/scenarios/satcom-release-poisoning.sdl.yaml @@ -359,60 +359,11 @@ vulnerabilities: technical: true class: CWE-345 -metrics: - blue-release-signing: - type: CONDITIONAL - max-score: 100 - condition: release-signature-valid - blue-telemetry-availability: - type: CONDITIONAL - max-score: 100 - condition: telemetry-flowing - blue-tenant-isolation: - type: CONDITIONAL - max-score: 100 - condition: tenant-isolation-healthy - blue-rollback-report: - type: MANUAL - max-score: 50 - artifact: true - red-release-poisoning: - type: CONDITIONAL - max-score: 100 - condition: rogue-release-promoted - red-edge-compromise: - type: CONDITIONAL - max-score: 100 - condition: edge-east-poisoned - -evaluations: - blue-release-resilience: - metrics: [blue-release-signing, blue-telemetry-availability, blue-tenant-isolation, blue-rollback-report] - min-score: 75 - red-propagation: - metrics: [red-release-poisoning, red-edge-compromise] - min-score: {percentage: 75} - -tlos: - preserve-release-integrity: - name: Preserve release integrity - evaluation: blue-release-resilience - achieve-poisoned-propagation: - name: Achieve poisoned propagation - evaluation: red-propagation - -goals: - platform-blue-goal: - tlos: [preserve-release-integrity] - supply-chain-red-goal: - tlos: [achieve-poisoned-propagation] - entities: platform-blue: name: Platform Blue Team role: Blue mission: Deliver a trusted release while maintaining customer service continuity - tlos: [preserve-release-integrity] entities: release-engineering: name: Release Engineering @@ -607,7 +558,6 @@ agents: services: [forge-https, ci-api, bastion-ssh] accounts: [support-vendor] allowed_subnets: [corp-eng, build-net, federation-net, control-net] - reward_calculator: SupplyChainImpact blue-release-agent: entity: platform-blue.release-engineering actions: [Review, Promote, Revoke, Rollback] @@ -618,7 +568,6 @@ agents: services: [forge-https, registry-api, signing-api] accounts: [ci-bot, registry-bot] allowed_subnets: [corp-eng, build-net, control-net] - reward_calculator: TrustedRelease blue-sre-agent: entity: platform-blue.sre actions: [Inspect, Isolate, Restore, Validate] @@ -629,9 +578,11 @@ agents: services: [control-api, edge-east-api, edge-west-api, telemetry-kafka] accounts: [satops] allowed_subnets: [control-net, edge-net, telemetry-net, security-net] - reward_calculator: TelemetryAvailability objectives: + # This scenario intended graded red/blue comparison. Graded scoring now lives + # in the experiment/evaluator plane (experiment-* contracts) per ADR-073; + # SDL objectives assert only observable success via conditions. red-compromise-build: agent: red-build-agent actions: [Exploit, StealToken] @@ -646,7 +597,7 @@ objectives: actions: [PushArtifact] targets: [release-manifest, artifact-registry, ci-publishes-registry] success: - metrics: [red-release-poisoning] + conditions: [rogue-release-promoted] depends_on: [red-compromise-build] window: stories: [release-day] @@ -656,7 +607,7 @@ objectives: entity: red-supply-chain targets: [edge-gateway-east, control-manages-east] success: - goals: [supply-chain-red-goal] + conditions: [edge-east-poisoned] depends_on: [red-poison-release] window: stories: [release-day] @@ -668,7 +619,6 @@ objectives: targets: [artifact-registry, signing-hsm, registry-relies-signing] success: conditions: [release-signature-valid] - metrics: [blue-release-signing] window: stories: [release-day] scripts: [release-prep, canary-rollout] @@ -696,8 +646,7 @@ objectives: actions: [Inspect, Isolate, Restore, Validate] targets: [control-api, edge-gateway-east, telemetry-broker, telemetry-replicates-analytics] success: - goals: [platform-blue-goal] - metrics: [blue-telemetry-availability, blue-tenant-isolation] + conditions: [telemetry-flowing, tenant-isolation-healthy] depends_on: [blue-validate-release, red-reach-edge] window: stories: [release-day] diff --git a/implementations/python/packages/aces_backend_stubs/stubs.py b/implementations/python/packages/aces_backend_stubs/stubs.py index 3d63e9a53..1365aa895 100644 --- a/implementations/python/packages/aces_backend_stubs/stubs.py +++ b/implementations/python/packages/aces_backend_stubs/stubs.py @@ -189,7 +189,7 @@ def create_stub_manifest( ), evaluator=EvaluatorCapabilities( name="stub-evaluator", - supported_sections=frozenset({"conditions", "metrics", "evaluations", "tlos", "goals", "objectives"}), + supported_sections=frozenset({"conditions", "objectives"}), supports_scoring=True, supports_objectives=True, ), diff --git a/implementations/python/packages/aces_mcp/tools/authoring.py b/implementations/python/packages/aces_mcp/tools/authoring.py index b666e6c5d..f0e7f1f80 100644 --- a/implementations/python/packages/aces_mcp/tools/authoring.py +++ b/implementations/python/packages/aces_mcp/tools/authoring.py @@ -122,10 +122,6 @@ def sdl_validate_section( "features", "conditions", "vulnerabilities", - "metrics", - "evaluations", - "tlos", - "goals", "entities", "injects", "events", @@ -181,7 +177,7 @@ def sdl_validate_section( description=( "Generate a starter SDL scenario skeleton. Choose a complexity " "level: 'minimal' (topology + features only), 'standard' " - "(adds scoring, entities, accounts), or 'full' (all 21 sections " + "(adds objectives, entities, accounts), or 'full' (all sections " "with placeholder structure). Optionally provide a scenario name " "and description. The output is valid SDL YAML you can edit." ), @@ -278,10 +274,6 @@ def sdl_instantiate( "features", "conditions", "vulnerabilities", - "metrics", - "evaluations", - "tlos", - "goals", "entities", "injects", "events", @@ -397,35 +389,24 @@ def _section_summary(scenario: object) -> list[tuple[str, int]]: technical: true class: CWE-89 -metrics: - web-uptime: - type: CONDITIONAL - max-score: 100 - condition: web-healthy - -evaluations: - availability: - metrics: [web-uptime] - min-score: 75 - -tlos: - defend-web: - name: Defend the web application - evaluation: availability - -goals: - exercise-goal: - tlos: [defend-web] - entities: blue-team: name: Blue Team role: Blue - tlos: [defend-web] red-team: name: Red Team role: Red +# Objective success references observable state (conditions) per ADR-073. +# Graded scoring/reward, if a study needs it, lives in the experiment/evaluator +# plane (ADR-055/064/069), not in the SDL. +objectives: + keep-web-available: + description: Keep the web application available + entity: blue-team + success: + conditions: [web-healthy] + accounts: web-admin-account: username: webadmin @@ -520,36 +501,11 @@ def _section_summary(scenario: object) -> list[tuple[str, int]]: technical: true class: CWE-89 -# --- Scoring Pipeline --- -metrics: - web-uptime: - type: CONDITIONAL - max-score: 100 - condition: web-healthy - report-quality: - type: MANUAL - max-score: 50 - -evaluations: - overall: - metrics: [web-uptime, report-quality] - min-score: 75 - -tlos: - defend-web: - name: Defend the web application - evaluation: overall - -goals: - exercise-goal: - tlos: [defend-web] - # --- Teams --- entities: blue-team: name: Blue Team role: Blue - tlos: [defend-web] entities: web-ops: {name: Web Operations} red-team: @@ -630,7 +586,7 @@ def _section_summary(scenario: object) -> list[tuple[str, int]]: blue-defend: entity: blue-team success: - goals: [exercise-goal] + conditions: [web-healthy] depends_on: [red-access] # --- Workflows --- diff --git a/implementations/python/packages/aces_mcp/tools/inspection.py b/implementations/python/packages/aces_mcp/tools/inspection.py index 078793610..be1e103f6 100644 --- a/implementations/python/packages/aces_mcp/tools/inspection.py +++ b/implementations/python/packages/aces_mcp/tools/inspection.py @@ -114,10 +114,6 @@ def sdl_diagram(sdl_content: str) -> str: "features", "conditions", "vulnerabilities", - "metrics", - "evaluations", - "tlos", - "goals", "entities", "injects", "events", @@ -444,25 +440,6 @@ def _build_reference_map(scenario) -> dict[tuple[str, str], list[str]]: if feat.dependencies: refs[("features", name)] = list(feat.dependencies) - # Metrics -> conditions - for name, metric in scenario.metrics.items(): - if metric.condition: - refs[("metrics", name)] = [metric.condition] - - # Evaluations -> metrics - for name, ev in scenario.evaluations.items(): - if ev.metrics: - refs[("evaluations", name)] = list(ev.metrics) - - # TLOs -> evaluations - for name, tlo in scenario.tlos.items(): - refs[("tlos", name)] = [tlo.evaluation] - - # Goals -> TLOs - for name, goal in scenario.goals.items(): - if goal.tlos: - refs[("goals", name)] = list(goal.tlos) - # Events -> conditions, injects for name, event in scenario.events.items(): targets = [] @@ -526,10 +503,6 @@ def _build_reference_map(scenario) -> dict[tuple[str, str], list[str]]: targets.extend(obj.depends_on) if obj.success: targets.extend(obj.success.conditions) - targets.extend(obj.success.metrics) - targets.extend(obj.success.evaluations) - targets.extend(obj.success.tlos) - targets.extend(obj.success.goals) if targets: refs[("objectives", name)] = targets diff --git a/implementations/python/packages/aces_mcp/tools/operation_support.py b/implementations/python/packages/aces_mcp/tools/operation_support.py index 1c002e3ee..9355db196 100644 --- a/implementations/python/packages/aces_mcp/tools/operation_support.py +++ b/implementations/python/packages/aces_mcp/tools/operation_support.py @@ -14,10 +14,6 @@ "features", "conditions", "vulnerabilities", - "metrics", - "evaluations", - "tlos", - "goals", "entities", "injects", "events", @@ -220,10 +216,6 @@ def runtime_model_summary(model: Any) -> dict[str, Any]: }, "evaluation": { "condition_bindings": len(model.condition_bindings), - "metrics": len(model.metrics), - "evaluations": len(model.evaluations), - "tlos": len(model.tlos), - "goals": len(model.goals), "objectives": len(model.objectives), }, "participant": { @@ -286,12 +278,13 @@ def design_notes(scenario: Any, model: Any, execution_plan: Any) -> list[dict[st "No objectives are authored; range intent and success criteria may be hard to assess.", ) ) - if scenario.objectives and not (scenario.metrics or scenario.evaluations or scenario.tlos or scenario.goals): + if scenario.objectives and not any(objective.success.conditions for objective in scenario.objectives.values()): notes.append( note( "assessment", "warning", - "Objectives exist without a scoring pipeline; objective success may be under-specified.", + "Objectives exist without any observable-state (conditions) success criteria; " + "objective success may be under-specified.", ) ) if scenario.agents and not scenario.action_contracts: @@ -385,22 +378,14 @@ def claim_assessment(scenario: Any, model: Any, execution_plan: Any) -> dict[str ), ] - if scenario.metrics or scenario.evaluations or scenario.tlos or scenario.goals: - supported.append( - claim( - "assessment-pipeline", - "The scenario contains authored assessment material for scoring or outcome review.", - "Metrics/evaluations/TLOs/goals are present.", - ) - ) - else: - unsupported.append( - claim( - "assessment-result", - "The scenario supports scoring or assessment-result claims.", - "No metrics, evaluations, TLOs, or goals are authored.", - ) + unsupported.append( + claim( + "assessment-result", + "The scenario supports scoring or assessment-result claims.", + "Per ADR-073 the SDL no longer authors scoring/reward surfaces; graded scoring and " + "evaluation results live in the experiment/evaluator plane (ADR-055/064/069).", ) + ) if model.action_contracts and model.observation_boundaries and model.participant_behaviors: supported.append( diff --git a/implementations/python/packages/aces_mcp/tools/reference.py b/implementations/python/packages/aces_mcp/tools/reference.py index b0a21cf08..50affff74 100644 --- a/implementations/python/packages/aces_mcp/tools/reference.py +++ b/implementations/python/packages/aces_mcp/tools/reference.py @@ -72,11 +72,6 @@ def _read_example(name: str) -> str: "features": "Features", "conditions": "Conditions", "vulnerabilities": "Vulnerabilities", - "metrics": "Scoring Pipeline: Metrics, Evaluations, TLOs, Goals", - "evaluations": "Scoring Pipeline: Metrics, Evaluations, TLOs, Goals", - "tlos": "Scoring Pipeline: Metrics, Evaluations, TLOs, Goals", - "goals": "Scoring Pipeline: Metrics, Evaluations, TLOs, Goals", - "scoring": "Scoring Pipeline: Metrics, Evaluations, TLOs, Goals", "entities": "Entities", "injects": "Orchestration: Injects, Events, Scripts, Stories", "events": "Orchestration: Injects, Events, Scripts, Stories", @@ -118,7 +113,7 @@ def register(mcp: FastMCP) -> None: name="sdl_overview", description=( "Get a comprehensive overview of the ACES Scenario Description " - "Language (SDL). Returns what the SDL is, its 21 sections, how " + "Language (SDL). Returns what the SDL is, its 17 sections, how " "parsing/validation works, the variable system, and a complete " "minimal example. Start here if you have never seen the SDL before." ), @@ -132,11 +127,11 @@ def sdl_overview() -> str: "Get detailed documentation for a specific SDL section including " "its schema, fields, YAML examples, shorthands, and validation " "rules. Valid section names: nodes, infrastructure, features, " - "conditions, vulnerabilities, scoring (metrics+evaluations+tlos+" - "goals), entities, orchestration (injects+events+scripts+stories), " - "content, accounts, relationships, agents, objectives, workflows, " - "variables. You can also pass the individual section name like " - "'metrics' or 'events'." + "conditions, vulnerabilities, entities, orchestration " + "(injects+events+scripts+stories), content, accounts, " + "relationships, agents, objectives, workflows, variables. You can " + "also pass the individual section name like 'conditions' or " + "'events'." ), ) def sdl_section_reference(section: str) -> str: @@ -166,7 +161,7 @@ def sdl_section_reference(section: str) -> str: description=( "Get a complete, real-world annotated SDL scenario example. " "Available examples: 'hospital' (hospital ransomware exercise, " - "~750 lines, uses all 21 sections), 'satcom' (satellite supply-chain " + "~750 lines, uses all 17 sections), 'satcom' (satellite supply-chain " "exercise, ~750 lines), 'port' (port authority OT exercise, ~680 lines), " "'minimal' (a small annotated pentest-lab example to learn the basics). " "Use 'hospital' for a comprehensive reference of all SDL features." @@ -292,37 +287,24 @@ def sdl_validation_reference() -> str: command: "curl -sf http://localhost:8080/ || exit 1" interval: 15 -# --- Scoring pipeline: conditions -> metrics -> evaluations -> TLOs -> goals --- -metrics: - uptime: - type: CONDITIONAL - max-score: 100 - condition: web-alive - -evaluations: - basic-eval: - metrics: [uptime] - min-score: 75 # shorthand for {percentage: 75} - -tlos: - web-defense: - name: Defend the web application - evaluation: basic-eval - -goals: - pass: - tlos: [web-defense] - # --- Teams / people --- entities: blue-team: name: Blue Team role: Blue # Blue, Red, White, Green - tlos: [web-defense] red-team: name: Red Team role: Red +# Objective success references observable state (conditions) per ADR-073. +# Graded scoring/reward lives in the experiment/evaluator plane, not the SDL. +objectives: + keep-web-alive: + description: Keep the web application available + entity: blue-team + success: + conditions: [web-alive] + # --- Parameterization (${var} syntax, resolved at instantiation, not parse time) --- variables: lab_cidr: @@ -344,9 +326,9 @@ def sdl_validation_reference() -> str: It descends from the Open Cyber Range (OCR) SDL and extends it with 7 \ additional sections for richer experiment semantics. -## The 21 Sections +## The 17 Sections -A scenario is a YAML document with a required `name` and up to 21 optional \ +A scenario is a YAML document with a required `name` and up to 17 optional \ sections, organized into four concerns: ### Topology & Software (5 sections) @@ -358,15 +340,10 @@ def sdl_validation_reference() -> str: | `conditions` | Health checks (command+interval or library source) | | `vulnerabilities` | CWE-classified weaknesses | -### Scoring Pipeline (4 sections) -| Section | Purpose | -|---------|---------| -| `metrics` | Scoring criteria: CONDITIONAL (automated) or MANUAL | -| `evaluations` | Metric groups with pass/fail thresholds | -| `tlos` | Training Learning Objectives linked to evaluations | -| `goals` | High-level goals composed of TLOs | - -Flow: `conditions -> metrics -> evaluations -> TLOs -> goals` +Per ADR-073 the OCR scoring pipeline (`metrics` / `evaluations` / `tlos` / \ +`goals`) and `agents.reward_calculator` were removed from the SDL. Objective \ +success references observable state (`conditions`); graded scoring, reward, and \ +evaluation outputs live in the experiment/evaluator plane (ADR-055/064/069). ### Exercise Orchestration (4 sections) | Section | Purpose | diff --git a/implementations/python/packages/aces_operations/_cross_backend_corpus_ledger.py b/implementations/python/packages/aces_operations/_cross_backend_corpus_ledger.py index a5137191c..356d36343 100644 --- a/implementations/python/packages/aces_operations/_cross_backend_corpus_ledger.py +++ b/implementations/python/packages/aces_operations/_cross_backend_corpus_ledger.py @@ -30,7 +30,6 @@ "action_contracts", "observation_boundaries", "objectives", - "evaluations", "networks", "node_deployments", ) diff --git a/implementations/python/packages/aces_operations/_evidence_run_artifact.py b/implementations/python/packages/aces_operations/_evidence_run_artifact.py index 3a773bf87..c93e18985 100644 --- a/implementations/python/packages/aces_operations/_evidence_run_artifact.py +++ b/implementations/python/packages/aces_operations/_evidence_run_artifact.py @@ -169,7 +169,6 @@ def _compiled_artifact_section(model: CompiledModel) -> dict[str, Any]: "action_contracts": sorted(model.action_contracts), "observation_boundaries": sorted(model.observation_boundaries), "objectives": sorted(model.objectives), - "evaluations": sorted(model.evaluations), "networks": sorted(model.networks), "node_deployments": sorted(model.node_deployments), } @@ -535,7 +534,6 @@ def _invariant_ledger_refs(model: CompiledModel, scenario_section: Mapping[str, "participant_behaviors": sorted(model.participant_behaviors), "action_contracts": sorted(model.action_contracts), "observation_boundaries": sorted(model.observation_boundaries), - "evaluations": sorted(model.evaluations), "evidence_refs": [ "participant_action_proof", "terminal_observation", diff --git a/implementations/python/packages/aces_processor/compiler.py b/implementations/python/packages/aces_processor/compiler.py index fb32d62d8..af36b1f71 100644 --- a/implementations/python/packages/aces_processor/compiler.py +++ b/implementations/python/packages/aces_processor/compiler.py @@ -19,7 +19,6 @@ OutcomeInterpretationTargetLayer, ) from aces_sdl.scenario import InstantiatedScenario, Scenario -from aces_sdl.semantics.assessment import partition_assessment_dependencies from aces_sdl.semantics.objective_semantics import ( OBJECTIVE_WINDOW_DEPENDENCY_ROLES, partition_objective_dependencies, @@ -36,13 +35,10 @@ Diagnostic, EvaluationExecutionContract, EvaluationResultContract, - EvaluationRuntime, EventRuntime, FeatureBinding, - GoalRuntime, InjectBinding, InjectRuntime, - MetricRuntime, NetworkRuntime, NodeRuntime, ObjectiveRuntime, @@ -56,7 +52,6 @@ RuntimeTemplate, ScriptRuntime, StoryRuntime, - TLORuntime, WorkflowExecutionContract, WorkflowPredicateRuntime, WorkflowResultContract, @@ -286,22 +281,13 @@ def _workflow_address(name: str) -> str: return _address("orchestration", "workflow", name) -def _metric_address(name: str) -> str: - return _address("evaluation", "metric", name) - - def _evaluation_address(name: str) -> str: + # Address form for the experiment/evaluator-plane EVALUATION_RESULT + # interpretation layer (SEM-215). Per ADR-073 the SDL no longer authors an + # ``evaluations`` section; this address no longer resolves an SDL resource. return _address("evaluation", "evaluation", name) -def _tlo_address(name: str) -> str: - return _address("evaluation", "tlo", name) - - -def _goal_address(name: str) -> str: - return _address("evaluation", "goal", name) - - def _objective_address(name: str) -> str: return _address("evaluation", "objective", name) @@ -544,27 +530,9 @@ def _initial_knowledge_addresses( def _evaluation_contracts( resource_type: str, - spec: dict[str, Any] | None = None, ) -> tuple[EvaluationResultContract, EvaluationExecutionContract]: - payload = spec or {} - if resource_type == "metric": - max_score_raw = payload.get("max-score", payload.get("max_score")) - fixed_max_score = ( - max_score_raw if isinstance(max_score_raw, int) and not isinstance(max_score_raw, bool) else None - ) - return ( - EvaluationResultContract( - resource_type=resource_type, - supports_score=True, - fixed_max_score=fixed_max_score, - ), - EvaluationExecutionContract(resource_type=resource_type), - ) if resource_type in { "condition-binding", - "evaluation", - "tlo", - "goal", "objective", }: return ( @@ -1763,157 +1731,13 @@ def _compile_stories( return stories -def _compile_metrics( - scenario: InstantiatedScenario, - condition_bindings: dict[str, ConditionBinding], - diagnostics: list[Diagnostic], -) -> dict[str, MetricRuntime]: - metrics: dict[str, MetricRuntime] = {} - for name, metric in scenario.metrics.items(): - metric_spec = _dump(metric) - metric_address = _metric_address(name) - condition_addresses = _metric_condition_addresses(metric_spec, metric_address, condition_bindings, diagnostics) - result_contract, execution_contract = _evaluation_contracts("metric", metric_spec) - ordering_dependencies, refresh_dependencies = partition_assessment_dependencies(condition_addresses) - metrics[metric_address] = MetricRuntime( - address=metric_address, - name=name, - condition_name=metric_spec.get("condition") or "", - condition_addresses=condition_addresses, - ordering_dependencies=ordering_dependencies, - refresh_dependencies=refresh_dependencies, - spec=metric_spec, - result_contract=result_contract, - execution_contract=execution_contract, - ) - return metrics - - -def _metric_condition_addresses( - metric_spec: dict[str, Any], - metric_address: str, - condition_bindings: dict[str, ConditionBinding], - diagnostics: list[Diagnostic], -) -> tuple[str, ...]: - condition_name = metric_spec.get("condition") or "" - if not condition_name: - return () - condition_addresses, metric_diagnostics = _resolve_binding_ref( - condition_bindings, - ref_name=condition_name, - owner_address=metric_address, - domain="evaluation", - code_prefix="evaluation.condition-ref", - binding_attr="condition_name", - binding_label="condition", - ) - diagnostics.extend(metric_diagnostics) - return condition_addresses - - -def _compile_evaluations( - scenario: InstantiatedScenario, - diagnostics: list[Diagnostic], -) -> dict[str, EvaluationRuntime]: - evaluations: dict[str, EvaluationRuntime] = {} - for name, evaluation in scenario.evaluations.items(): - evaluation_address = _evaluation_address(name) - metric_addresses, evaluation_diagnostics = _resolve_named_refs( - ref_names=list(evaluation.metrics), - available_names=set(scenario.metrics), - address_builder=_metric_address, - owner_address=evaluation_address, - domain="evaluation", - code_prefix="evaluation.metric-ref", - resource_label="metric", - ) - diagnostics.extend(evaluation_diagnostics) - result_contract, execution_contract = _evaluation_contracts("evaluation") - ordering_dependencies, refresh_dependencies = partition_assessment_dependencies(metric_addresses) - evaluations[evaluation_address] = EvaluationRuntime( - address=evaluation_address, - name=name, - metric_addresses=metric_addresses, - ordering_dependencies=ordering_dependencies, - refresh_dependencies=refresh_dependencies, - spec=_dump(evaluation), - result_contract=result_contract, - execution_contract=execution_contract, - ) - return evaluations - - -def _compile_tlos( - scenario: InstantiatedScenario, - diagnostics: list[Diagnostic], -) -> dict[str, TLORuntime]: - tlos: dict[str, TLORuntime] = {} - for name, tlo in scenario.tlos.items(): - tlo_address = _tlo_address(name) - evaluation_addresses, tlo_diagnostics = _resolve_named_refs( - ref_names=[tlo.evaluation], - available_names=set(scenario.evaluations), - address_builder=_evaluation_address, - owner_address=tlo_address, - domain="evaluation", - code_prefix="evaluation.evaluation-ref", - resource_label="evaluation", - ) - diagnostics.extend(tlo_diagnostics) - result_contract, execution_contract = _evaluation_contracts("tlo") - ordering_dependencies, refresh_dependencies = partition_assessment_dependencies(evaluation_addresses) - tlos[tlo_address] = TLORuntime( - address=tlo_address, - name=name, - evaluation_address=evaluation_addresses[0] if evaluation_addresses else "", - ordering_dependencies=ordering_dependencies, - refresh_dependencies=refresh_dependencies, - spec=_dump(tlo), - result_contract=result_contract, - execution_contract=execution_contract, - ) - return tlos - - -def _compile_goals( - scenario: InstantiatedScenario, - diagnostics: list[Diagnostic], -) -> dict[str, GoalRuntime]: - goals: dict[str, GoalRuntime] = {} - for name, goal in scenario.goals.items(): - goal_address = _goal_address(name) - tlo_addresses, goal_diagnostics = _resolve_named_refs( - ref_names=list(goal.tlos), - available_names=set(scenario.tlos), - address_builder=_tlo_address, - owner_address=goal_address, - domain="evaluation", - code_prefix="evaluation.tlo-ref", - resource_label="TLO", - ) - diagnostics.extend(goal_diagnostics) - result_contract, execution_contract = _evaluation_contracts("goal") - ordering_dependencies, refresh_dependencies = partition_assessment_dependencies(tlo_addresses) - goals[goal_address] = GoalRuntime( - address=goal_address, - name=name, - tlo_addresses=tlo_addresses, - ordering_dependencies=ordering_dependencies, - refresh_dependencies=refresh_dependencies, - spec=_dump(goal), - result_contract=result_contract, - execution_contract=execution_contract, - ) - return goals - - def _objective_success_addresses( - scenario: InstantiatedScenario, condition_bindings: dict[str, ConditionBinding], objective: Any, objective_address: str, diagnostics: list[Diagnostic], ) -> list[str]: + # Per ADR-073 objective success references observable ``conditions`` only. condition_addresses, condition_diagnostics = _resolve_binding_refs( condition_bindings, ref_names=list(objective.success.conditions), @@ -1923,52 +1747,8 @@ def _objective_success_addresses( binding_attr="condition_name", binding_label="condition", ) - metric_addresses, metric_diagnostics = _resolve_named_refs( - ref_names=list(objective.success.metrics), - available_names=set(scenario.metrics), - address_builder=_metric_address, - owner_address=objective_address, - domain="evaluation", - code_prefix="evaluation.metric-ref", - resource_label="metric", - ) - evaluation_addresses, evaluation_diagnostics = _resolve_named_refs( - ref_names=list(objective.success.evaluations), - available_names=set(scenario.evaluations), - address_builder=_evaluation_address, - owner_address=objective_address, - domain="evaluation", - code_prefix="evaluation.evaluation-ref", - resource_label="evaluation", - ) - tlo_addresses, tlo_diagnostics = _resolve_named_refs( - ref_names=list(objective.success.tlos), - available_names=set(scenario.tlos), - address_builder=_tlo_address, - owner_address=objective_address, - domain="evaluation", - code_prefix="evaluation.tlo-ref", - resource_label="TLO", - ) - goal_addresses, goal_diagnostics = _resolve_named_refs( - ref_names=list(objective.success.goals), - available_names=set(scenario.goals), - address_builder=_goal_address, - owner_address=objective_address, - domain="evaluation", - code_prefix="evaluation.goal-ref", - resource_label="goal", - ) - diagnostics.extend( - [ - *condition_diagnostics, - *metric_diagnostics, - *evaluation_diagnostics, - *tlo_diagnostics, - *goal_diagnostics, - ] - ) - return [*condition_addresses, *metric_addresses, *evaluation_addresses, *tlo_addresses, *goal_addresses] + diagnostics.extend(condition_diagnostics) + return list(condition_addresses) def _objective_dependency_addresses( @@ -2060,7 +1840,6 @@ def _compile_objectives( for name, objective in scenario.objectives.items(): objective_address = _objective_address(name) success_addresses = _objective_success_addresses( - scenario, condition_bindings, objective, objective_address, @@ -2120,42 +1899,6 @@ def _compile_workflow_predicate( binding_attr="condition_name", binding_label="condition", ) - metric_addresses, metric_diagnostics = _resolve_named_refs( - ref_names=list(predicate_source.metrics), - available_names=set(scenario.metrics), - address_builder=_metric_address, - owner_address=predicate_address, - domain="orchestration", - code_prefix="orchestration.metric-ref", - resource_label="metric", - ) - evaluation_addresses, evaluation_diagnostics = _resolve_named_refs( - ref_names=list(predicate_source.evaluations), - available_names=set(scenario.evaluations), - address_builder=_evaluation_address, - owner_address=predicate_address, - domain="orchestration", - code_prefix="orchestration.evaluation-ref", - resource_label="evaluation", - ) - tlo_addresses, tlo_diagnostics = _resolve_named_refs( - ref_names=list(predicate_source.tlos), - available_names=set(scenario.tlos), - address_builder=_tlo_address, - owner_address=predicate_address, - domain="orchestration", - code_prefix="orchestration.tlo-ref", - resource_label="TLO", - ) - goal_addresses, goal_diagnostics = _resolve_named_refs( - ref_names=list(predicate_source.goals), - available_names=set(scenario.goals), - address_builder=_goal_address, - owner_address=predicate_address, - domain="orchestration", - code_prefix="orchestration.goal-ref", - resource_label="goal", - ) objective_addresses, objective_diagnostics = _resolve_named_refs( ref_names=list(predicate_source.objectives), available_names=set(scenario.objectives), @@ -2168,10 +1911,6 @@ def _compile_workflow_predicate( diagnostics.extend( [ *workflow_diagnostics, - *metric_diagnostics, - *evaluation_diagnostics, - *tlo_diagnostics, - *goal_diagnostics, *objective_diagnostics, ] ) @@ -2187,20 +1926,12 @@ def _compile_workflow_predicate( predicate_addresses = _dedupe( [ *condition_addresses, - *metric_addresses, - *evaluation_addresses, - *tlo_addresses, - *goal_addresses, *objective_addresses, ] ) return _WorkflowPredicateCompilation( predicate=WorkflowPredicateRuntime( condition_addresses=condition_addresses, - metric_addresses=tuple(metric_addresses), - evaluation_addresses=tuple(evaluation_addresses), - tlo_addresses=tuple(tlo_addresses), - goal_addresses=tuple(goal_addresses), objective_addresses=tuple(objective_addresses), step_state_predicates=step_state_predicates, ), @@ -2697,10 +2428,6 @@ def compile_runtime_model(scenario: Scenario | InstantiatedScenario) -> RuntimeM events = _compile_events(scenario, condition_bindings, injects, inject_bindings, diagnostics) scripts = _compile_scripts(scenario, diagnostics) stories = _compile_stories(scenario, diagnostics) - metrics = _compile_metrics(scenario, condition_bindings, diagnostics) - evaluations = _compile_evaluations(scenario, diagnostics) - tlos = _compile_tlos(scenario, diagnostics) - goals = _compile_goals(scenario, diagnostics) objectives = _compile_objectives(scenario, condition_bindings, diagnostics) workflows = _compile_workflows(scenario, condition_bindings, diagnostics) @@ -2732,10 +2459,6 @@ def compile_runtime_model(scenario: Scenario | InstantiatedScenario) -> RuntimeM scripts=scripts, stories=stories, workflows=workflows, - metrics=metrics, - evaluations=evaluations, - tlos=tlos, - goals=goals, objectives=objectives, diagnostics=diagnostics, realization_requirements=_compile_realization_requirements(scenario), diff --git a/implementations/python/packages/aces_processor/models.py b/implementations/python/packages/aces_processor/models.py index c0a0a97f0..cff1962f0 100644 --- a/implementations/python/packages/aces_processor/models.py +++ b/implementations/python/packages/aces_processor/models.py @@ -665,10 +665,6 @@ class WorkflowPredicateRuntime: """Resolved workflow predicate semantics.""" condition_addresses: tuple[str, ...] = () - metric_addresses: tuple[str, ...] = () - evaluation_addresses: tuple[str, ...] = () - tlo_addresses: tuple[str, ...] = () - goal_addresses: tuple[str, ...] = () objective_addresses: tuple[str, ...] = () step_state_predicates: tuple[WorkflowStepStatePredicateRuntime, ...] = () @@ -678,10 +674,6 @@ def external_addresses(self) -> tuple[str, ...]: ordered: list[str] = [] for address in ( *self.condition_addresses, - *self.metric_addresses, - *self.evaluation_addresses, - *self.tlo_addresses, - *self.goal_addresses, *self.objective_addresses, ): if address in seen: @@ -4166,59 +4158,6 @@ def iter_participant_behavior_joint_action_violations( yield from _participant_behavior_joint_action_order_violations(normalized_events) -@dataclass(frozen=True) -class MetricRuntime(ResolvedResource): - """Resolved metric node.""" - - condition_name: str = "" - condition_addresses: tuple[str, ...] = () - result_contract: "EvaluationResultContract" = field( - default_factory=lambda: EvaluationResultContract(resource_type="metric") - ) - execution_contract: "EvaluationExecutionContract" = field( - default_factory=lambda: EvaluationExecutionContract(resource_type="metric") - ) - - -@dataclass(frozen=True) -class EvaluationRuntime(ResolvedResource): - """Resolved evaluation node.""" - - metric_addresses: tuple[str, ...] = () - result_contract: "EvaluationResultContract" = field( - default_factory=lambda: EvaluationResultContract(resource_type="evaluation") - ) - execution_contract: "EvaluationExecutionContract" = field( - default_factory=lambda: EvaluationExecutionContract(resource_type="evaluation") - ) - - -@dataclass(frozen=True) -class TLORuntime(ResolvedResource): - """Resolved TLO node.""" - - evaluation_address: str = "" - result_contract: "EvaluationResultContract" = field( - default_factory=lambda: EvaluationResultContract(resource_type="tlo") - ) - execution_contract: "EvaluationExecutionContract" = field( - default_factory=lambda: EvaluationExecutionContract(resource_type="tlo") - ) - - -@dataclass(frozen=True) -class GoalRuntime(ResolvedResource): - """Resolved goal node.""" - - tlo_addresses: tuple[str, ...] = () - result_contract: "EvaluationResultContract" = field( - default_factory=lambda: EvaluationResultContract(resource_type="goal") - ) - execution_contract: "EvaluationExecutionContract" = field( - default_factory=lambda: EvaluationExecutionContract(resource_type="goal") - ) - - @dataclass(frozen=True) class ObjectiveRuntime(ResolvedResource): """Resolved objective node.""" @@ -4285,10 +4224,6 @@ class RuntimeModel: scripts: dict[str, ScriptRuntime] = field(default_factory=dict) stories: dict[str, StoryRuntime] = field(default_factory=dict) workflows: dict[str, WorkflowRuntime] = field(default_factory=dict) - metrics: dict[str, MetricRuntime] = field(default_factory=dict) - evaluations: dict[str, EvaluationRuntime] = field(default_factory=dict) - tlos: dict[str, TLORuntime] = field(default_factory=dict) - goals: dict[str, GoalRuntime] = field(default_factory=dict) objectives: dict[str, ObjectiveRuntime] = field(default_factory=dict) diagnostics: list[Diagnostic] = field(default_factory=list) # SEM-218 typed compiler emission: each authored realization concern with diff --git a/implementations/python/packages/aces_processor/planner.py b/implementations/python/packages/aces_processor/planner.py index 273a07d5b..11b3f7f7d 100644 --- a/implementations/python/packages/aces_processor/planner.py +++ b/implementations/python/packages/aces_processor/planner.py @@ -135,34 +135,6 @@ def _collect_resources(model: RuntimeModel) -> dict[str, PlannedResource]: "condition-binding", resource, ) - for address, resource in model.metrics.items(): - resources[address] = _planned_resource( - address, - RuntimeDomain.EVALUATION, - "metric", - resource, - ) - for address, resource in model.evaluations.items(): - resources[address] = _planned_resource( - address, - RuntimeDomain.EVALUATION, - "evaluation", - resource, - ) - for address, resource in model.tlos.items(): - resources[address] = _planned_resource( - address, - RuntimeDomain.EVALUATION, - "tlo", - resource, - ) - for address, resource in model.goals.items(): - resources[address] = _planned_resource( - address, - RuntimeDomain.EVALUATION, - "goal", - resource, - ) for address, resource in model.objectives.items(): resources[address] = _planned_resource( address, @@ -714,10 +686,6 @@ def _validate_manifest(model: RuntimeModel, manifest: BackendManifest) -> list[D evaluation_sections = { "conditions": bool(model.condition_bindings), - "metrics": bool(model.metrics), - "evaluations": bool(model.evaluations), - "tlos": bool(model.tlos), - "goals": bool(model.goals), "objectives": bool(model.objectives), } if any(evaluation_sections.values()): @@ -742,18 +710,6 @@ def _validate_manifest(model: RuntimeModel, manifest: BackendManifest) -> list[D message=f"Evaluator does not support '{section}'.", ) ) - scoring_in_use = bool( - model.condition_bindings or model.metrics or model.evaluations or model.tlos or model.goals - ) - if scoring_in_use and not manifest.supports_scoring: - diagnostics.append( - Diagnostic( - code="evaluator.scoring-unsupported", - domain="evaluation", - address="evaluation.scoring", - message="Evaluator does not support scoring resources.", - ) - ) if model.objectives and not manifest.supports_objectives: diagnostics.append( Diagnostic( diff --git a/implementations/python/packages/aces_reference_backend/manifest.py b/implementations/python/packages/aces_reference_backend/manifest.py index 979e440dd..2ddf3b76e 100644 --- a/implementations/python/packages/aces_reference_backend/manifest.py +++ b/implementations/python/packages/aces_reference_backend/manifest.py @@ -153,7 +153,7 @@ def _capabilities() -> BackendCapabilitySet: ), evaluator=EvaluatorCapabilities( name="reference-emulation-evaluator", - supported_sections=frozenset({"conditions", "metrics", "evaluations", "tlos", "goals", "objectives"}), + supported_sections=frozenset({"conditions", "objectives"}), supports_scoring=True, supports_objectives=True, ), diff --git a/implementations/python/packages/aces_sdl/_language_metadata.py b/implementations/python/packages/aces_sdl/_language_metadata.py index cd84c83e7..6c7fce8c8 100644 --- a/implementations/python/packages/aces_sdl/_language_metadata.py +++ b/implementations/python/packages/aces_sdl/_language_metadata.py @@ -10,10 +10,6 @@ ("infrastructure", "links"): "infrastructure", ("infrastructure", "dependencies"): "infrastructure", ("features", "dependencies"): "features", - ("metrics", "condition"): "conditions", - ("evaluations", "metrics"): "metrics", - ("tlos", "evaluation"): "evaluations", - ("goals", "tlos"): "tlos", ("events", "conditions"): "conditions", ("events", "injects"): "injects", ("scripts", "events"): "events", @@ -49,11 +45,7 @@ "features": ("type", "source", "version", "dependencies"), "conditions": ("command", "interval", "description"), "vulnerabilities": ("name", "description", "technical", "class"), - "metrics": ("type", "max_score", "condition"), - "evaluations": ("metrics", "min_score"), - "tlos": ("name", "evaluation", "description"), - "goals": ("tlos", "description"), - "entities": ("name", "role", "tlos", "facts", "entities"), + "entities": ("name", "role", "facts", "entities"), "injects": ("source", "from_entity", "to_entities"), "events": ("conditions", "injects"), "scripts": ("start_time", "end_time", "speed", "events"), diff --git a/implementations/python/packages/aces_sdl/_language_references.py b/implementations/python/packages/aces_sdl/_language_references.py index a392d8ea4..9b7187c5b 100644 --- a/implementations/python/packages/aces_sdl/_language_references.py +++ b/implementations/python/packages/aces_sdl/_language_references.py @@ -13,7 +13,7 @@ from ._language_metadata import REFERENCE_COMPLETION_TARGETS _CODE_PARSE = "sdl.parse" -_SUCCESS_REFERENCE_TARGETS = frozenset({"conditions", "metrics", "evaluations", "tlos", "goals"}) +_SUCCESS_REFERENCE_TARGETS = frozenset({"conditions"}) def find_references( diff --git a/implementations/python/packages/aces_sdl/_module_symbols.py b/implementations/python/packages/aces_sdl/_module_symbols.py index 559e02648..9c9fef841 100644 --- a/implementations/python/packages/aces_sdl/_module_symbols.py +++ b/implementations/python/packages/aces_sdl/_module_symbols.py @@ -19,10 +19,6 @@ "features", "conditions", "vulnerabilities", - "metrics", - "evaluations", - "tlos", - "goals", "entities", "injects", "events", @@ -166,10 +162,6 @@ def symbol_index( "features": section_maps.get("features", {}), "conditions": section_maps.get("conditions", {}), "vulnerabilities": section_maps.get("vulnerabilities", {}), - "metrics": section_maps.get("metrics", {}), - "evaluations": section_maps.get("evaluations", {}), - "tlos": section_maps.get("tlos", {}), - "goals": section_maps.get("goals", {}), "entities": entity_map, "injects": section_maps.get("injects", {}), "events": section_maps.get("events", {}), diff --git a/implementations/python/packages/aces_sdl/agents.py b/implementations/python/packages/aces_sdl/agents.py index 0da430922..c6c881648 100644 --- a/implementations/python/packages/aces_sdl/agents.py +++ b/implementations/python/packages/aces_sdl/agents.py @@ -58,6 +58,10 @@ class Agent(SDLModel): - ``observation_boundaries`` links to declared participant observation boundaries that define participant-specific projections of world and evidence state (SEM-208) + + Per ADR-073 the CybORG-inherited ``reward_calculator`` label was removed; + it was an unbound, unvalidated string and graded reward lives in the + experiment/evaluator plane (ADR-055/064/069). """ entity: str = "" @@ -66,7 +70,6 @@ class Agent(SDLModel): starting_accounts: list[str] = Field(default_factory=list) initial_knowledge: InitialKnowledge | None = None allowed_subnets: list[str] = Field(default_factory=list) - reward_calculator: str = "" starting_conditions: list[str] = Field(default_factory=list) authority_anchors: list[str] = Field(default_factory=list) operating_scope: list[str] = Field(default_factory=list) diff --git a/implementations/python/packages/aces_sdl/composition.py b/implementations/python/packages/aces_sdl/composition.py index 44a055967..820191f94 100644 --- a/implementations/python/packages/aces_sdl/composition.py +++ b/implementations/python/packages/aces_sdl/composition.py @@ -106,7 +106,6 @@ def _rewrite_entity(payload: dict[str, Any], symbols: dict[str, dict[str, str] | payload["vulnerabilities"] = [ _maybe_rename(name, symbols["vulnerabilities"]) for name in payload.get("vulnerabilities", []) ] - payload["tlos"] = [_maybe_rename(name, symbols["tlos"]) for name in payload.get("tlos", [])] payload["events"] = [_maybe_rename(name, symbols["events"]) for name in payload.get("events", [])] for child in payload.get("entities", {}).values(): if isinstance(child, dict): @@ -135,10 +134,6 @@ def _rewrite_workflow(payload: dict[str, Any], symbols: dict[str, dict[str, str] when = step.get("when") if isinstance(when, dict): when["conditions"] = [_maybe_rename(name, symbols["conditions"]) for name in when.get("conditions", [])] - when["metrics"] = [_maybe_rename(name, symbols["metrics"]) for name in when.get("metrics", [])] - when["evaluations"] = [_maybe_rename(name, symbols["evaluations"]) for name in when.get("evaluations", [])] - when["tlos"] = [_maybe_rename(name, symbols["tlos"]) for name in when.get("tlos", [])] - when["goals"] = [_maybe_rename(name, symbols["goals"]) for name in when.get("goals", [])] when["objectives"] = [_maybe_rename(name, symbols["objectives"]) for name in when.get("objectives", [])] @@ -181,18 +176,6 @@ def _namespace_payload( for feature in namespaced.get("features", {}).values(): if isinstance(feature, dict): _rewrite_feature(feature, symbols) - for metric in namespaced.get("metrics", {}).values(): - if isinstance(metric, dict) and metric.get("condition"): - metric["condition"] = _maybe_rename(str(metric["condition"]), symbols["conditions"]) - for evaluation in namespaced.get("evaluations", {}).values(): - if isinstance(evaluation, dict): - evaluation["metrics"] = [_maybe_rename(name, symbols["metrics"]) for name in evaluation.get("metrics", [])] - for tlo in namespaced.get("tlos", {}).values(): - if isinstance(tlo, dict) and tlo.get("evaluation"): - tlo["evaluation"] = _maybe_rename(str(tlo["evaluation"]), symbols["evaluations"]) - for goal in namespaced.get("goals", {}).values(): - if isinstance(goal, dict): - goal["tlos"] = [_maybe_rename(name, symbols["tlos"]) for name in goal.get("tlos", [])] for entity in namespaced.get("entities", {}).values(): if isinstance(entity, dict): _rewrite_entity(entity, symbols) @@ -201,7 +184,6 @@ def _namespace_payload( if inject.get("from_entity"): inject["from_entity"] = _maybe_rename(str(inject["from_entity"]), symbols["entities"]) inject["to_entities"] = [_maybe_rename(name, symbols["entities"]) for name in inject.get("to_entities", [])] - inject["tlos"] = [_maybe_rename(name, symbols["tlos"]) for name in inject.get("tlos", [])] for event in namespaced.get("events", {}).values(): if isinstance(event, dict): event["conditions"] = [_maybe_rename(name, symbols["conditions"]) for name in event.get("conditions", [])] @@ -294,14 +276,9 @@ def _namespace_payload( ] success = objective.get("success") if isinstance(success, dict): - for field_name, symbol_key in ( - ("conditions", "conditions"), - ("metrics", "metrics"), - ("evaluations", "evaluations"), - ("tlos", "tlos"), - ("goals", "goals"), - ): - success[field_name] = [_maybe_rename(name, symbols[symbol_key]) for name in success.get(field_name, [])] + success["conditions"] = [ + _maybe_rename(name, symbols["conditions"]) for name in success.get("conditions", []) + ] window = objective.get("window") if isinstance(window, dict): for field_name, symbol_key in ( diff --git a/implementations/python/packages/aces_sdl/entities.py b/implementations/python/packages/aces_sdl/entities.py index ae5795416..03adc63c0 100644 --- a/implementations/python/packages/aces_sdl/entities.py +++ b/implementations/python/packages/aces_sdl/entities.py @@ -41,7 +41,6 @@ def normalize_role(cls, v): mission: str = "" categories: list[str] = Field(default_factory=list) vulnerabilities: list[str] = Field(default_factory=list) - tlos: list[str] = Field(default_factory=list) facts: dict[str, str] = Field(default_factory=dict) events: list[str] = Field(default_factory=list) entities: dict[str, "Entity"] = Field(default_factory=dict) diff --git a/implementations/python/packages/aces_sdl/language_service.py b/implementations/python/packages/aces_sdl/language_service.py index 5e2139a90..034a0cfb7 100644 --- a/implementations/python/packages/aces_sdl/language_service.py +++ b/implementations/python/packages/aces_sdl/language_service.py @@ -199,10 +199,6 @@ def _completion_target_section(pointer: list[str]) -> str | None: if len(pointer) >= 4 and pointer[-2] == "success": success_targets = { "conditions": "conditions", - "metrics": "metrics", - "evaluations": "evaluations", - "tlos": "tlos", - "goals": "goals", } return success_targets.get(field) return None diff --git a/implementations/python/packages/aces_sdl/module_registry.py b/implementations/python/packages/aces_sdl/module_registry.py index 2346459c8..d9cdd250c 100644 --- a/implementations/python/packages/aces_sdl/module_registry.py +++ b/implementations/python/packages/aces_sdl/module_registry.py @@ -128,10 +128,6 @@ def _scenario_module_descriptor(scenario: Scenario, *, source_id: str) -> Module "features", "conditions", "vulnerabilities", - "metrics", - "evaluations", - "tlos", - "goals", "entities", "injects", "events", diff --git a/implementations/python/packages/aces_sdl/objectives.py b/implementations/python/packages/aces_sdl/objectives.py index eac0e5fe1..0e64b27cc 100644 --- a/implementations/python/packages/aces_sdl/objectives.py +++ b/implementations/python/packages/aces_sdl/objectives.py @@ -26,14 +26,16 @@ class SuccessMode(str, Enum): class ObjectiveSuccess(SDLModel): - """Declarative success criteria for an objective.""" + """Declarative success criteria for an objective. + + Per ADR-073, objective success references observable state (``conditions``) + only. The OCR-inherited scoring pipeline (``metrics`` / ``evaluations`` / + ``tlos`` / ``goals``) was removed from the SDL; graded scoring and reward + live in the experiment/evaluator plane (ADR-055/064/069). + """ mode: SuccessMode | str = SuccessMode.ALL_OF conditions: list[str] = Field(default_factory=list) - metrics: list[str] = Field(default_factory=list) - evaluations: list[str] = Field(default_factory=list) - tlos: list[str] = Field(default_factory=list) - goals: list[str] = Field(default_factory=list) @field_validator("mode", mode="before") @classmethod @@ -42,17 +44,9 @@ def normalize_mode(cls, v: str) -> SuccessMode | str: @model_validator(mode="after") def validate_non_empty(self) -> "ObjectiveSuccess": - if any( - ( - self.conditions, - self.metrics, - self.evaluations, - self.tlos, - self.goals, - ) - ): + if self.conditions: return self - raise ValueError("Objective success must reference at least one condition, metric, evaluation, TLO, or goal") + raise ValueError("Objective success must reference at least one condition") class ObjectiveWindow(SDLModel): diff --git a/implementations/python/packages/aces_sdl/orchestration.py b/implementations/python/packages/aces_sdl/orchestration.py index 431ca30b4..a5e830615 100644 --- a/implementations/python/packages/aces_sdl/orchestration.py +++ b/implementations/python/packages/aces_sdl/orchestration.py @@ -141,7 +141,6 @@ class Inject(SDLModel): source: Source | None = None from_entity: str = "" to_entities: list[str] = Field(default_factory=list) - tlos: list[str] = Field(default_factory=list) description: str = "" environment: list[str] = Field(default_factory=list) @@ -271,13 +270,14 @@ def validate_unique_outcomes(self) -> "WorkflowStepStateRef": class WorkflowPredicate(SDLModel): - """Branch predicate over runtime evaluation data and prior step state.""" + """Branch predicate over observable state, objectives, and prior step state. + + Per ADR-073 the OCR scoring references (``metrics`` / ``evaluations`` / + ``tlos`` / ``goals``) were removed; a predicate branches on observable + ``conditions``, declared ``objectives``, and prior workflow ``steps``. + """ conditions: list[str] = Field(default_factory=list) - metrics: list[str] = Field(default_factory=list) - evaluations: list[str] = Field(default_factory=list) - tlos: list[str] = Field(default_factory=list) - goals: list[str] = Field(default_factory=list) objectives: list[str] = Field(default_factory=list) steps: list[WorkflowStepStateRef] = Field(default_factory=list) @@ -286,19 +286,12 @@ def validate_non_empty(self) -> "WorkflowPredicate": if any( ( self.conditions, - self.metrics, - self.evaluations, - self.tlos, - self.goals, self.objectives, self.steps, ) ): return self - raise ValueError( - "Workflow predicate must reference at least one condition, " - "metric, evaluation, TLO, goal, objective, or step state" - ) + raise ValueError("Workflow predicate must reference at least one condition, objective, or step state") class WorkflowSwitchCase(SDLModel): diff --git a/implementations/python/packages/aces_sdl/parser.py b/implementations/python/packages/aces_sdl/parser.py index 85439e127..2a5bb81d8 100644 --- a/implementations/python/packages/aces_sdl/parser.py +++ b/implementations/python/packages/aces_sdl/parser.py @@ -28,10 +28,6 @@ "features", "conditions", "vulnerabilities", - "metrics", - "evaluations", - "tlos", - "goals", "entities", "injects", "events", @@ -186,11 +182,26 @@ def _expand_roles(roles: dict[str, Any]) -> dict[str, Any]: return result -def _expand_min_score(value: Any) -> Any: - """Expand min-score shorthand: 50 → {percentage: 50}.""" - if isinstance(value, int) or is_variable_ref(value): - return {"percentage": value} - return value +# OCR scoring sections removed from the SDL by ADR-073. Detected up front so +# authors get a migration pointer instead of a raw "extra fields" error. +_REMOVED_SCORING_SECTIONS = ("metrics", "evaluations", "tlos", "goals") + + +def _reject_removed_scoring_sections(data: dict[str, Any], *, path: Path | None) -> None: + """Raise a migration-pointing error when a removed scoring section is used.""" + present = [section for section in _REMOVED_SCORING_SECTIONS if section in data] + if not present: + return + raise SDLParseError( + "SDL scoring sections " + f"{', '.join(present)} were removed from the language by ADR-073. " + "Express objective success against observable state via " + "'objectives.*.success.conditions', and route graded scoring, reward, " + "and evaluation outputs to the experiment/evaluator plane " + "(ADR-055/064/069). The CybORG 'agents.*.reward_calculator' label was " + "removed for the same reason.", + path=path, + ) def _expand_shorthands(data: dict[str, Any]) -> dict[str, Any]: @@ -255,12 +266,6 @@ def expand_sources_scoped( if field in node_data and isinstance(node_data[field], list): node_data[field] = {name: "" for name in node_data[field]} - # Expand min_score in evaluations - if "evaluations" in data and isinstance(data["evaluations"], dict): - for eval_data in data["evaluations"].values(): - if isinstance(eval_data, dict) and "min_score" in eval_data: - eval_data["min_score"] = _expand_min_score(eval_data["min_score"]) - return data @@ -290,6 +295,7 @@ def parse_sdl( SDLValidationError: If semantic validation finds errors. """ data = _load_normalized_data(content, path=path) + _reject_removed_scoring_sections(data, path=path) module_variable_specs: dict[str, dict[str, object]] = {} module_node_variable_refs: dict[str, dict[str, str | None]] = {} if data.get("imports"): diff --git a/implementations/python/packages/aces_sdl/scenario.py b/implementations/python/packages/aces_sdl/scenario.py index ee09224fd..077e60d01 100644 --- a/implementations/python/packages/aces_sdl/scenario.py +++ b/implementations/python/packages/aces_sdl/scenario.py @@ -1,10 +1,12 @@ """Top-level Scenario model — the root of the SDL. -The Scenario combines 23 specification sections covering +The Scenario combines specification sections covering who (entities, accounts, agents), what (nodes, features, vulnerabilities, content), when (scripts, stories, events), -and declarative experiment semantics (objectives, scoring -pipeline, conditions, relationships, workflows, variables). +and declarative experiment semantics (objectives, conditions, +relationships, workflows, variables). Per ADR-073 the SDL no +longer carries the OCR scoring pipeline; graded scoring/reward +live in the experiment/evaluator plane (ADR-055/064/069). Delivery-level concerns (Docker, Terraform, cloud APIs) are outside the SDL. @@ -36,7 +38,6 @@ from .participant_outcome_semantics import OutcomeInterpretationRule from .relationships import Relationship from .runtime_forwarding_agent import RuntimeForwardingAgent -from .scoring import TLO, Evaluation, Goal, Metric from .variables import Variable from .vulnerabilities import Vulnerability @@ -135,10 +136,6 @@ class Scenario(SDLModel): features: dict[str, Feature] = Field(default_factory=dict) conditions: dict[str, Condition] = Field(default_factory=dict) vulnerabilities: dict[str, Vulnerability] = Field(default_factory=dict) - metrics: dict[str, Metric] = Field(default_factory=dict) - evaluations: dict[str, Evaluation] = Field(default_factory=dict) - tlos: dict[str, TLO] = Field(default_factory=dict) - goals: dict[str, Goal] = Field(default_factory=dict) entities: dict[str, Entity] = Field(default_factory=dict) injects: dict[str, Inject] = Field(default_factory=dict) events: dict[str, Event] = Field(default_factory=dict) diff --git a/implementations/python/packages/aces_sdl/scoring.py b/implementations/python/packages/aces_sdl/scoring.py deleted file mode 100644 index 4c24e2739..000000000 --- a/implementations/python/packages/aces_sdl/scoring.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Scoring models — Metrics, Evaluations, TLOs, and Goals. - -Implements the OCR SDL scoring pipeline: - Conditions -> Metrics -> Evaluations -> TLOs -> Goals - -Metrics are either manual (human-graded) or conditional (automated -via condition checks). Evaluations group metrics with pass/fail -thresholds. TLOs link to evaluations. Goals compose TLOs. -""" - -from enum import Enum - -from pydantic import Field, field_validator, model_validator - -from ._base import ( - SDLModel, - normalize_enum_value, - parse_bool_or_var, - parse_int_or_var, -) - - -class MetricType(str, Enum): - """How a metric is scored.""" - - MANUAL = "manual" - CONDITIONAL = "conditional" - - -class Metric(SDLModel): - """A scoring metric — either manual or conditional. - - Manual metrics may require artifact submission. Conditional - metrics reference a condition that produces the score. - """ - - name: str = "" - type: MetricType = Field(alias="type") - artifact: bool | str | None = None - - @field_validator("type", mode="before") - @classmethod - def normalize_type(cls, v: str) -> str: - return normalize_enum_value(v) - - max_score: int | str - condition: str | None = None - description: str = "" - - @field_validator("artifact", mode="before") - @classmethod - def parse_artifact(cls, v: bool | str | None) -> bool | str | None: - return parse_bool_or_var(v, field_name="artifact") - - @field_validator("max_score", mode="before") - @classmethod - def parse_max_score(cls, v: int | str) -> int | str: - return parse_int_or_var(v, minimum=1, field_name="max_score") - - @model_validator(mode="after") - def validate_type_fields(self) -> "Metric": - if self.type == MetricType.MANUAL: - if self.condition is not None: - raise ValueError("Manual metric cannot have a condition") - elif self.type == MetricType.CONDITIONAL: - if self.condition is None: - raise ValueError("Conditional metric requires a condition") - if self.artifact is not None: - raise ValueError("Conditional metric cannot have artifact flag") - return self - - -class MinScore(SDLModel): - """Pass/fail threshold — either absolute points or percentage. - - Shorthand: ``min-score: 50`` (interpreted as percentage). - Longhand: ``min-score: {absolute: 50}`` or ``{percentage: 75}``. - """ - - absolute: int | str | None = None - percentage: int | str | None = None - - @field_validator("absolute", mode="before") - @classmethod - def parse_absolute(cls, v: int | str | None) -> int | str | None: - return parse_int_or_var(v, minimum=0, field_name="absolute") - - @field_validator("percentage", mode="before") - @classmethod - def parse_percentage(cls, v: int | str | None) -> int | str | None: - return parse_int_or_var( - v, - minimum=0, - maximum=100, - field_name="percentage", - ) - - @model_validator(mode="after") - def validate_exclusive(self) -> "MinScore": - if self.absolute is not None and self.percentage is not None: - raise ValueError("MinScore cannot have both 'absolute' and 'percentage'") - if self.absolute is None and self.percentage is None: - raise ValueError("MinScore must have either 'absolute' or 'percentage'") - return self - - -class Evaluation(SDLModel): - """A group of metrics with a pass/fail threshold.""" - - name: str = "" - description: str = "" - metrics: list[str] = Field(min_length=1) - min_score: MinScore - - -class TLO(SDLModel): - """Training Learning Objective — linked to an evaluation.""" - - name: str = "" - description: str = "" - evaluation: str - - -class Goal(SDLModel): - """High-level goal composed of TLOs.""" - - name: str = "" - description: str = "" - tlos: list[str] = Field(min_length=1) diff --git a/implementations/python/packages/aces_sdl/semantics/assessment.py b/implementations/python/packages/aces_sdl/semantics/assessment.py index 799e6fd0e..9b204b604 100644 --- a/implementations/python/packages/aces_sdl/semantics/assessment.py +++ b/implementations/python/packages/aces_sdl/semantics/assessment.py @@ -1,353 +1,25 @@ -"""Pure assessment-pipeline semantic helpers (SEM-206). - -The *assessment pipeline* is the SDL scoring chain:: - - condition bindings -> metrics -> evaluations -> TLOs -> goals - -This module is the single name-level source of truth for that chain: which -cross-resource references are well-formed, how evaluation scores aggregate from -their metrics, and which dependency roles each pipeline edge carries. -``aces_sdl.validator`` enforces these rules on authored SDL (mapping the -machine-readable issues here back onto its authoring-error strings); -``aces_processor.compiler`` reuses the dependency-role decision when it maps the -chain onto canonical ``evaluation.*`` runtime addresses, and the planner then -walks those edges generically (ordering for execution order, refresh for change -propagation). Per ADR-015 this helper lives with the SDL package and has no -processor-runtime dependencies; per ADR-016 it is part of the realized artifact -set for SEM-206. - -Whether a declared condition is actually *bound* to a node — i.e. *realized* — -is a compilation-phase concern (the compiler emits ``evaluation.condition-ref`` -diagnostics for unbound/ambiguous bindings); this module deals only with the -name-level reference graph that is meaningful before binding resolution. +"""Objective-success resource-kind tag (SEM-206). + +Per ADR-073 the OCR-inherited SDL scoring chain +(``metric -> evaluation -> TLO -> goal``) was removed from the SDL. Graded +scoring, reward, and evaluation outputs live in the experiment/evaluator plane +(ADR-055/064/069), not in authored SDL. + +What remains here is the resource-kind qualifier that objective success +references carry. Objective success references observable state only, so +``CONDITION`` is the sole member; the enum is retained as the kind-qualifier +seam so a future ADR that admits another observable-state carrier adds a member +here rather than reviving the removed scoring sections. Per ADR-015 this helper +lives with the SDL package and has no processor-runtime dependencies; per +ADR-016 it is part of the realized artifact set for SEM-206. """ from __future__ import annotations -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass, field from enum import Enum class AssessmentResourceKind(str, Enum): - """Kinds of resource that participate in the assessment pipeline.""" + """Resource kinds an objective's success may reference (post ADR-073).""" CONDITION = "condition" - METRIC = "metric" - EVALUATION = "evaluation" - TLO = "tlo" - GOAL = "goal" - - -class AssessmentDependencyRole(str, Enum): - """Semantic roles an assessment-pipeline edge can carry.""" - - ORDERING = "ordering" - REFRESH = "refresh" - - -#: Every edge in the assessment pipeline is both an ordering edge (the -#: downstream resource is computed after its inputs) and a refresh edge (the -#: downstream resource must recompute when any input changes). Keeping this in -#: one place lets the compiler derive ``ordering_dependencies`` / -#: ``refresh_dependencies`` from a single semantic fact instead of restating it -#: at every resource site. -ASSESSMENT_DEPENDENCY_ROLES: tuple[AssessmentDependencyRole, ...] = ( - AssessmentDependencyRole.ORDERING, - AssessmentDependencyRole.REFRESH, -) - - -def partition_assessment_dependencies( - upstream: Sequence[str], -) -> tuple[tuple[str, ...], tuple[str, ...]]: - """Split an upstream-dependency list into (ordering, refresh) tuples. - - Both roles cover the whole upstream set today; callers should still route - through here so a future refresh-only (or ordering-only) edge changes one - place rather than every compiled-resource construction. - """ - - ordering = tuple(upstream) if AssessmentDependencyRole.ORDERING in ASSESSMENT_DEPENDENCY_ROLES else () - refresh = tuple(upstream) if AssessmentDependencyRole.REFRESH in ASSESSMENT_DEPENDENCY_ROLES else () - return ordering, refresh - - -@dataclass(frozen=True) -class AssessmentReference: - """A normalized assessment-pipeline edge from one resource to an upstream one.""" - - raw: str - source_kind: AssessmentResourceKind - source_name: str - target_kind: AssessmentResourceKind - target_name: str - dependency_roles: tuple[AssessmentDependencyRole, ...] = ASSESSMENT_DEPENDENCY_ROLES - #: Reserved for later module/import expansion; the analysis is run on - #: already-composed scenarios, so it is empty today. - namespace_path: tuple[str, ...] = () - - -@dataclass(frozen=True) -class AssessmentIssue: - """A machine-readable assessment-pipeline consistency problem. - - ``ref`` names the offending reference target (an undeclared condition / - metric / evaluation / TLO). ``observed`` / ``limit`` carry the numbers for - aggregation issues (e.g. an absolute min-score and the metric-max-score - total it exceeds). - """ - - code: str - resource_kind: AssessmentResourceKind - resource_name: str - ref: str | None = None - observed: int | None = None - limit: int | None = None - - -@dataclass(frozen=True) -class AssessmentResourceDependencies: - """Derived upstream dependencies for one assessment-pipeline resource.""" - - kind: AssessmentResourceKind - name: str - ordering_names: tuple[str, ...] = () - refresh_names: tuple[str, ...] = () - - -@dataclass(frozen=True) -class AssessmentPipelineAnalysis: - """Result of analyzing the assessment pipeline of a scenario.""" - - references: tuple[AssessmentReference, ...] = () - issues: tuple[AssessmentIssue, ...] = () - dependencies: tuple[AssessmentResourceDependencies, ...] = () - evaluation_metric_totals: Mapping[str, int | None] = field(default_factory=dict) - - @property - def has_issues(self) -> bool: - return bool(self.issues) - - def issues_of_code(self, code: str) -> tuple[AssessmentIssue, ...]: - return tuple(issue for issue in self.issues if issue.code == code) - - def dependencies_for(self, kind: AssessmentResourceKind, name: str) -> AssessmentResourceDependencies: - for dependency in self.dependencies: - if dependency.kind == kind and dependency.name == name: - return dependency - raise KeyError((kind, name)) - - -def _ordered_unique(items: list[str]) -> tuple[str, ...]: - return tuple(dict.fromkeys(items)) - - -def _never_unresolved(_value: object) -> bool: - return False - - -def analyze_assessment_pipeline( - *, - conditions_by_name: Mapping[str, object], - metrics_by_name: Mapping[str, object], - evaluations_by_name: Mapping[str, object], - tlos_by_name: Mapping[str, object], - goals_by_name: Mapping[str, object], - is_unresolved: Callable[[object], bool] | None = None, -) -> AssessmentPipelineAnalysis: - """Resolve the assessment-pipeline reference graph and derive its semantics. - - Inputs are name-keyed mappings of the SDL constructs (accessed structurally - via ``getattr``); ``is_unresolved`` (default: never) lets a caller skip - references that are still ``${var}`` placeholders. Returns the normalized - reference edges, the per-resource ordering/refresh dependency names, the - per-evaluation metric-max-score totals (``None`` when any contributing - max-score is unknown), and any consistency issues — in pipeline order: - metrics, then evaluations, then TLOs, then goals. - """ - - unresolved = is_unresolved or _never_unresolved - references: list[AssessmentReference] = [] - issues: list[AssessmentIssue] = [] - dependencies: list[AssessmentResourceDependencies] = [] - evaluation_metric_totals: dict[str, int | None] = {} - - # --- metrics -> conditions ------------------------------------------------ - scored_conditions: set[str] = set() - for metric_name, metric in metrics_by_name.items(): - condition_name = getattr(metric, "condition", None) - ordering: tuple[str, ...] = () - if condition_name and not unresolved(condition_name): - if condition_name not in conditions_by_name: - issues.append( - AssessmentIssue( - code="metric.condition-undeclared", - resource_kind=AssessmentResourceKind.METRIC, - resource_name=metric_name, - ref=condition_name, - ) - ) - else: - references.append( - AssessmentReference( - raw=condition_name, - source_kind=AssessmentResourceKind.METRIC, - source_name=metric_name, - target_kind=AssessmentResourceKind.CONDITION, - target_name=condition_name, - ) - ) - ordering = (condition_name,) - if condition_name in scored_conditions: - issues.append( - AssessmentIssue( - code="metric.condition-multiply-scored", - resource_kind=AssessmentResourceKind.CONDITION, - resource_name=condition_name, - ref=metric_name, - ) - ) - scored_conditions.add(condition_name) - dependencies.append( - AssessmentResourceDependencies( - kind=AssessmentResourceKind.METRIC, - name=metric_name, - ordering_names=ordering, - refresh_names=ordering, - ) - ) - - # --- evaluations -> metrics ---------------------------------------------- - for evaluation_name, evaluation in evaluations_by_name.items(): - resolved_metric_names: list[str] = [] - total = 0 - total_known = True - for ref_name in getattr(evaluation, "metrics", []) or []: - if unresolved(ref_name): - total_known = False - continue - if ref_name not in metrics_by_name: - issues.append( - AssessmentIssue( - code="evaluation.metric-undeclared", - resource_kind=AssessmentResourceKind.EVALUATION, - resource_name=evaluation_name, - ref=ref_name, - ) - ) - continue - references.append( - AssessmentReference( - raw=ref_name, - source_kind=AssessmentResourceKind.EVALUATION, - source_name=evaluation_name, - target_kind=AssessmentResourceKind.METRIC, - target_name=ref_name, - ) - ) - resolved_metric_names.append(ref_name) - metric_max_score = getattr(metrics_by_name[ref_name], "max_score", None) - if isinstance(metric_max_score, int) and not isinstance(metric_max_score, bool): - total += metric_max_score - else: - total_known = False - evaluation_metric_totals[evaluation_name] = total if total_known else None - upstream = _ordered_unique(resolved_metric_names) - dependencies.append( - AssessmentResourceDependencies( - kind=AssessmentResourceKind.EVALUATION, - name=evaluation_name, - ordering_names=upstream, - refresh_names=upstream, - ) - ) - min_score = getattr(evaluation, "min_score", None) - absolute = getattr(min_score, "absolute", None) if min_score is not None else None - if isinstance(absolute, int) and not isinstance(absolute, bool) and total_known and absolute > total: - issues.append( - AssessmentIssue( - code="evaluation.min-score-exceeds-metric-total", - resource_kind=AssessmentResourceKind.EVALUATION, - resource_name=evaluation_name, - observed=absolute, - limit=total, - ) - ) - - # --- TLOs -> evaluations -------------------------------------------------- - for tlo_name, tlo in tlos_by_name.items(): - evaluation_name = getattr(tlo, "evaluation", None) - ordering = () - if evaluation_name is not None and not unresolved(evaluation_name): - if evaluation_name not in evaluations_by_name: - issues.append( - AssessmentIssue( - code="tlo.evaluation-undeclared", - resource_kind=AssessmentResourceKind.TLO, - resource_name=tlo_name, - ref=evaluation_name, - ) - ) - else: - references.append( - AssessmentReference( - raw=evaluation_name, - source_kind=AssessmentResourceKind.TLO, - source_name=tlo_name, - target_kind=AssessmentResourceKind.EVALUATION, - target_name=evaluation_name, - ) - ) - ordering = (evaluation_name,) - dependencies.append( - AssessmentResourceDependencies( - kind=AssessmentResourceKind.TLO, - name=tlo_name, - ordering_names=ordering, - refresh_names=ordering, - ) - ) - - # --- goals -> TLOs -------------------------------------------------------- - for goal_name, goal in goals_by_name.items(): - resolved_tlo_names: list[str] = [] - for ref_name in getattr(goal, "tlos", []) or []: - if unresolved(ref_name): - continue - if ref_name not in tlos_by_name: - issues.append( - AssessmentIssue( - code="goal.tlo-undeclared", - resource_kind=AssessmentResourceKind.GOAL, - resource_name=goal_name, - ref=ref_name, - ) - ) - continue - references.append( - AssessmentReference( - raw=ref_name, - source_kind=AssessmentResourceKind.GOAL, - source_name=goal_name, - target_kind=AssessmentResourceKind.TLO, - target_name=ref_name, - ) - ) - resolved_tlo_names.append(ref_name) - upstream = _ordered_unique(resolved_tlo_names) - dependencies.append( - AssessmentResourceDependencies( - kind=AssessmentResourceKind.GOAL, - name=goal_name, - ordering_names=upstream, - refresh_names=upstream, - ) - ) - - return AssessmentPipelineAnalysis( - references=tuple(references), - issues=tuple(issues), - dependencies=tuple(dependencies), - evaluation_metric_totals=dict(evaluation_metric_totals), - ) diff --git a/implementations/python/packages/aces_sdl/semantics/objective_semantics.py b/implementations/python/packages/aces_sdl/semantics/objective_semantics.py index eeafc008f..5bafb77b5 100644 --- a/implementations/python/packages/aces_sdl/semantics/objective_semantics.py +++ b/implementations/python/packages/aces_sdl/semantics/objective_semantics.py @@ -2,7 +2,7 @@ :func:`analyze_objective_semantics` is the single name-level source of truth for the SDL declarative-objective construct — actor binding, target resolution, -success interpretation (over conditions/metrics/evaluations/TLOs/goals), the +success interpretation (over ``conditions`` observable state, per ADR-073), the optional window (delegated to :func:`aces_sdl.semantics.objectives.analyze_objective_window`), and the acyclic ``depends_on`` ordering relation. It returns normalized references with their dependency-role tags, the per-objective ordering/refresh @@ -140,13 +140,13 @@ def dependencies_for(self, name: str) -> ObjectiveResourceDependencies: @dataclass(frozen=True) class AssessmentResourceCatalog: - """The five assessment-pipeline section maps an objective's success may name.""" + """The observable-state section an objective's success may name (post ADR-073). + + Objective success references ``conditions`` only; the OCR scoring sections + (metrics/evaluations/tlos/goals) were removed by ADR-073. + """ conditions: Mapping[str, object] - metrics: Mapping[str, object] - evaluations: Mapping[str, object] - tlos: Mapping[str, object] - goals: Mapping[str, object] @dataclass(frozen=True) @@ -222,10 +222,6 @@ def _has_cycle(graph: Mapping[str, list[str]]) -> bool: _SUCCESS_REFERENCE_SECTIONS: tuple[tuple[str, AssessmentResourceKind, str], ...] = ( ("conditions", AssessmentResourceKind.CONDITION, "objective.success-condition-undeclared"), - ("metrics", AssessmentResourceKind.METRIC, "objective.success-metric-undeclared"), - ("evaluations", AssessmentResourceKind.EVALUATION, "objective.success-evaluation-undeclared"), - ("tlos", AssessmentResourceKind.TLO, "objective.success-tlo-undeclared"), - ("goals", AssessmentResourceKind.GOAL, "objective.success-goal-undeclared"), ) @@ -353,24 +349,18 @@ def _analyze_success( assessment_resources: AssessmentResourceCatalog, unresolved: Callable[[object], bool], ) -> tuple[list[ObjectiveReference], list[ObjectiveIssue], list[str]]: - """Resolve ``success.{conditions,metrics,evaluations,tlos,goals}``. + """Resolve ``success.conditions`` (observable state, per ADR-073). - Each success namespace contributes its own keyspace; resolved names are - kind-qualified before they enter the derived ordering/refresh tuples so a - metric and a condition with the same SDL name remain distinguishable. + Resolved names are kind-qualified before they enter the derived + ordering/refresh tuples, preserving the kind-qualifier seam even though + ``conditions`` is the only success reference kind today. """ refs: list[ObjectiveReference] = [] issues: list[ObjectiveIssue] = [] resolved: list[str] = [] success = getattr(objective, "success", None) - sections = ( - (assessment_resources.conditions, _SUCCESS_REFERENCE_SECTIONS[0]), - (assessment_resources.metrics, _SUCCESS_REFERENCE_SECTIONS[1]), - (assessment_resources.evaluations, _SUCCESS_REFERENCE_SECTIONS[2]), - (assessment_resources.tlos, _SUCCESS_REFERENCE_SECTIONS[3]), - (assessment_resources.goals, _SUCCESS_REFERENCE_SECTIONS[4]), - ) + sections = ((assessment_resources.conditions, _SUCCESS_REFERENCE_SECTIONS[0]),) for section, (attr, kind, code) in sections: for ref_name in getattr(success, attr, []) or []: if unresolved(ref_name): diff --git a/implementations/python/packages/aces_sdl/semantics/participant_outcome.py b/implementations/python/packages/aces_sdl/semantics/participant_outcome.py index 0753b3b06..f31050135 100644 --- a/implementations/python/packages/aces_sdl/semantics/participant_outcome.py +++ b/implementations/python/packages/aces_sdl/semantics/participant_outcome.py @@ -40,7 +40,6 @@ def _source_ref_issue( action_contracts: Mapping[str, object], objectives: Mapping[str, object], workflows: Mapping[str, object], - evaluations: Mapping[str, object], is_unresolved: Callable[[object], bool], ) -> ParticipantOutcomeIssue | None: layer = getattr(binding, "source_layer", None) @@ -72,14 +71,10 @@ def _source_ref_issue( ref=str(ref), layer=layer.value, ) - if layer == OutcomeInterpretationSourceLayer.EVALUATION_RESULT and ref not in evaluations: - return ParticipantOutcomeIssue( - code="participant.outcome.source-evaluation-unbound", - rule_name=rule_name, - binding_id=binding_id, - ref=str(ref), - layer=layer.value, - ) + # Per ADR-073 the SDL `evaluations` section was removed; the + # EVALUATION_RESULT interpretation layer remains a governed + # experiment/evaluator-plane concept whose ref is not bound to an SDL + # section, so no SDL cross-reference check applies to it here. return None @@ -89,7 +84,6 @@ def _target_ref_issue( binding: object, objectives: Mapping[str, object], workflows: Mapping[str, object], - evaluations: Mapping[str, object], is_unresolved: Callable[[object], bool], ) -> ParticipantOutcomeIssue | None: layer = getattr(binding, "target_layer", None) @@ -113,14 +107,8 @@ def _target_ref_issue( ref=str(ref), layer=layer.value, ) - if layer == OutcomeInterpretationTargetLayer.EVALUATION_RESULT and ref not in evaluations: - return ParticipantOutcomeIssue( - code="participant.outcome.target-evaluation-unbound", - rule_name=rule_name, - binding_id=binding_id, - ref=str(ref), - layer=layer.value, - ) + # EVALUATION_RESULT targets are an experiment/evaluator-plane concept after + # ADR-073 and carry no SDL-section cross-reference here. return None @@ -130,7 +118,6 @@ def analyze_participant_outcome_interpretations( action_contracts: Mapping[str, object], objectives: Mapping[str, object], workflows: Mapping[str, object], - evaluations: Mapping[str, object], is_unresolved: Callable[[object], bool], ) -> ParticipantOutcomeAnalysis: """Validate source/target refs declared by SEM-215 interpretation rules.""" @@ -144,7 +131,6 @@ def analyze_participant_outcome_interpretations( action_contracts=action_contracts, objectives=objectives, workflows=workflows, - evaluations=evaluations, is_unresolved=is_unresolved, ) if issue is not None: @@ -155,7 +141,6 @@ def analyze_participant_outcome_interpretations( binding=binding, objectives=objectives, workflows=workflows, - evaluations=evaluations, is_unresolved=is_unresolved, ) if issue is not None: diff --git a/implementations/python/packages/aces_sdl/validator/_content_objectives.py b/implementations/python/packages/aces_sdl/validator/_content_objectives.py index b68e615fc..b86912cbc 100644 --- a/implementations/python/packages/aces_sdl/validator/_content_objectives.py +++ b/implementations/python/packages/aces_sdl/validator/_content_objectives.py @@ -44,18 +44,6 @@ "objective.success-condition-undeclared": ( lambda i: f"Objective '{i.objective_name}' references undefined condition '{i.ref}' in success criteria" ), - "objective.success-metric-undeclared": ( - lambda i: f"Objective '{i.objective_name}' references undefined metric '{i.ref}' in success criteria" - ), - "objective.success-evaluation-undeclared": ( - lambda i: f"Objective '{i.objective_name}' references undefined evaluation '{i.ref}' in success criteria" - ), - "objective.success-tlo-undeclared": ( - lambda i: f"Objective '{i.objective_name}' references undefined TLO '{i.ref}' in success criteria" - ), - "objective.success-goal-undeclared": ( - lambda i: f"Objective '{i.objective_name}' references undefined goal '{i.ref}' in success criteria" - ), "objective.window.story-unbound": ( lambda i: f"Objective '{i.objective_name}' references undefined story '{i.ref}' in window" ), @@ -208,18 +196,12 @@ "participant.outcome.source-workflow-unbound": ( lambda i: f"Outcome interpretation rule '{i.rule_name}' source '{i.ref}' references undefined workflow" ), - "participant.outcome.source-evaluation-unbound": ( - lambda i: f"Outcome interpretation rule '{i.rule_name}' source '{i.ref}' references undefined evaluation" - ), "participant.outcome.target-objective-unbound": ( lambda i: f"Outcome interpretation rule '{i.rule_name}' target '{i.ref}' references undefined objective" ), "participant.outcome.target-workflow-unbound": ( lambda i: f"Outcome interpretation rule '{i.rule_name}' target '{i.ref}' references undefined workflow" ), - "participant.outcome.target-evaluation-unbound": ( - lambda i: f"Outcome interpretation rule '{i.rule_name}' target '{i.ref}' references undefined evaluation" - ), } @@ -402,7 +384,6 @@ def _verify_participant_outcomes(self) -> None: action_contracts=self._s.action_contracts, objectives=self._s.objectives, workflows=self._s.workflows, - evaluations=self._s.evaluations, is_unresolved=self._is_unresolved_var, ) for issue in analysis.issues: @@ -421,10 +402,6 @@ def _verify_objectives(self) -> None: entity_names=self._all_entity_names(), assessment_resources=AssessmentResourceCatalog( conditions=self._s.conditions, - metrics=self._s.metrics, - evaluations=self._s.evaluations, - tlos=self._s.tlos, - goals=self._s.goals, ), window_resources=WindowResourceCatalog( stories=self._s.stories, diff --git a/implementations/python/packages/aces_sdl/validator/_core.py b/implementations/python/packages/aces_sdl/validator/_core.py index 9b6124b73..52f7daebb 100644 --- a/implementations/python/packages/aces_sdl/validator/_core.py +++ b/implementations/python/packages/aces_sdl/validator/_core.py @@ -111,10 +111,6 @@ def _named_ref_index(self, *, targetable: bool = False) -> dict[str, set[str]]: ("conditions", True), ("vulnerabilities", True), ("infrastructure", False), - ("metrics", True), - ("evaluations", True), - ("tlos", True), - ("goals", True), ("content", True), ("accounts", True), ("agents", True), @@ -202,7 +198,7 @@ def _operating_scope_ref_index(self) -> dict[str, set[str]]: - services come from declared services on VM nodes. - content references stay open across content sections and items. - Non-spatial, non-resource elements (conditions, metrics, accounts, + Non-spatial, non-resource elements (conditions, accounts, relationships, objectives, …) are not scope boundaries even though they appear in the generic targetable index. """ @@ -321,7 +317,6 @@ def validate(self) -> None: self._verify_features() self._verify_conditions() self._verify_vulnerabilities() - self._verify_assessment_pipeline() self._verify_entities() self._verify_injects() self._verify_events() diff --git a/implementations/python/packages/aces_sdl/validator/_sections.py b/implementations/python/packages/aces_sdl/validator/_sections.py index 08478b015..c4fe671aa 100644 --- a/implementations/python/packages/aces_sdl/validator/_sections.py +++ b/implementations/python/packages/aces_sdl/validator/_sections.py @@ -9,27 +9,8 @@ from ..entities import flatten_entities from ..explicitness import classify_scenario_explicitness from ..scenario import Scenario -from ..semantics.assessment import AssessmentIssue, analyze_assessment_pipeline from ._support import _topological_sort -# Renders an assessment-pipeline issue (machine-readable code from -# ``aces_sdl.semantics.assessment``) into the authoring-error string. Keyed by -# issue code so a new code is a new line here rather than a new conditional. -_ASSESSMENT_ISSUE_RENDERERS = { - "metric.condition-undeclared": (lambda i: f"Metric '{i.resource_name}' references undefined condition '{i.ref}'"), - "metric.condition-multiply-scored": (lambda i: f"Condition '{i.resource_name}' is referenced by multiple metrics"), - "evaluation.metric-undeclared": (lambda i: f"Evaluation '{i.resource_name}' references undefined metric '{i.ref}'"), - "evaluation.min-score-exceeds-metric-total": ( - lambda i: ( - f"Evaluation '{i.resource_name}' absolute min-score " - f"({i.observed}) exceeds sum of " - f"metric max-scores ({i.limit})" - ) - ), - "tlo.evaluation-undeclared": (lambda i: f"TLO '{i.resource_name}' references undefined evaluation '{i.ref}'"), - "goal.tlo-undeclared": (lambda i: f"Goal '{i.resource_name}' references undefined TLO '{i.ref}'"), -} - class _SectionsMixin: def _verify_variables(self) -> None: @@ -120,37 +101,11 @@ def _verify_vulnerabilities(self) -> None: # CWE format validation is handled by the Pydantic field_validator. pass - def _verify_assessment_pipeline(self) -> None: - # The condition -> metric -> evaluation -> TLO -> goal scoring chain. - # Reference, aggregation, and dependency-role semantics live in - # ``aces_sdl.semantics.assessment`` (SEM-206); this pass renders the - # machine-readable issues it reports as authoring errors. - analysis = analyze_assessment_pipeline( - conditions_by_name=self._s.conditions, - metrics_by_name=self._s.metrics, - evaluations_by_name=self._s.evaluations, - tlos_by_name=self._s.tlos, - goals_by_name=self._s.goals, - is_unresolved=self._is_unresolved_var, - ) - for issue in analysis.issues: - self._err(self._format_assessment_issue(issue)) - - @staticmethod - def _format_assessment_issue(issue: AssessmentIssue) -> str: - renderer = _ASSESSMENT_ISSUE_RENDERERS.get(issue.code) - if renderer is None: - raise AssertionError(f"unhandled assessment-pipeline issue code: {issue.code}") - return renderer(issue) - def _verify_entities(self) -> None: for name, entity in flatten_entities(self._s.entities).items(): self._verify_entity_refs(name, entity) def _verify_entity_refs(self, name: str, entity: object) -> None: - self._verify_membership_refs( - entity.tlos, self._s.tlos, lambda ref: f"Entity '{name}' references undefined TLO '{ref}'" - ) self._verify_membership_refs( entity.vulnerabilities, self._s.vulnerabilities, @@ -175,9 +130,6 @@ def _verify_inject_refs(self, name: str, inject: object, flat_names: set[str]) - self._verify_membership_refs( inject.to_entities, flat_names, lambda ref: f"Inject '{name}' to_entity '{ref}' is not a defined entity" ) - self._verify_membership_refs( - inject.tlos, self._s.tlos, lambda ref: f"Inject '{name}' references undefined TLO '{ref}'" - ) def _verify_events(self) -> None: for name, event in self._s.events.items(): diff --git a/implementations/python/packages/aces_sdl/validator/_workflows_analysis.py b/implementations/python/packages/aces_sdl/validator/_workflows_analysis.py index 074023010..f59d893ad 100644 --- a/implementations/python/packages/aces_sdl/validator/_workflows_analysis.py +++ b/implementations/python/packages/aces_sdl/validator/_workflows_analysis.py @@ -25,10 +25,6 @@ def _validate_predicate_section_refs( ) -> None: predicate_sections = ( ("condition", predicate.conditions, self._s.conditions), - ("metric", predicate.metrics, self._s.metrics), - ("evaluation", predicate.evaluations, self._s.evaluations), - ("TLO", predicate.tlos, self._s.tlos), - ("goal", predicate.goals, self._s.goals), ("objective", predicate.objectives, self._s.objectives), ) for label, refs, section in predicate_sections: diff --git a/implementations/python/pyproject.toml b/implementations/python/pyproject.toml index 15fb4117a..d29b34c2a 100644 --- a/implementations/python/pyproject.toml +++ b/implementations/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "aces-sdl" -version = "0.19.0" +version = "0.19.1" description = "Backend-agnostic cyber range scenario description language and runtime." dynamic = ["readme"] requires-python = ">=3.11" diff --git a/implementations/python/src/aces/core/sdl/scoring.py b/implementations/python/src/aces/core/sdl/scoring.py deleted file mode 100644 index 78f45c3bb..000000000 --- a/implementations/python/src/aces/core/sdl/scoring.py +++ /dev/null @@ -1,5 +0,0 @@ -from aces._compat import reexport as _reexport - -_reexport(globals(), "aces_sdl.scoring") - -del _reexport diff --git a/implementations/python/tests/test_fm2_semantics.py b/implementations/python/tests/test_fm2_semantics.py index c40f4a37e..a876cb3ff 100644 --- a/implementations/python/tests/test_fm2_semantics.py +++ b/implementations/python/tests/test_fm2_semantics.py @@ -164,7 +164,7 @@ def test_validator_and_compiler_agree_on_objective_reference_errors(self): objectives: base: entity: blue - success: {metrics: [missing-metric]} + success: {conditions: [missing-condition]} depends_on: [missing-objective] """) @@ -173,13 +173,14 @@ def test_validator_and_compiler_agree_on_objective_reference_errors(self): errors = exc_info.value.errors assert any( - "Objective 'base' references undefined metric 'missing-metric' in success criteria" in e for e in errors + "Objective 'base' references undefined condition 'missing-condition' in success criteria" in e + for e in errors ) assert any("Objective 'base' depends on undefined objective 'missing-objective'" in e for e in errors) model = compile_runtime_model(parse_sdl(raw, skip_semantic_validation=True)) codes = {diag.code for diag in model.diagnostics} - assert "evaluation.metric-ref-unbound" in codes + assert "evaluation.condition-ref-unbound" in codes assert "evaluation.objective-ref-unbound" in codes def test_compiler_and_planner_agree_on_objective_dependency_ordering_and_refresh(self): @@ -221,82 +222,3 @@ def test_compiler_and_planner_agree_on_objective_dependency_ordering_and_refresh assert actions["evaluation.condition.vm.ready"] == "update" assert actions["evaluation.objective.base"] == "update" assert actions["evaluation.objective.dependent"] == "update" - - -class TestAssessmentPipelineAgreement: - def test_validator_and_compiler_agree_on_pipeline_errors(self): - raw = _scenario(""" -name: assessment-errors -metrics: - m1: {type: conditional, condition: missing-cond, max-score: 10} -evaluations: - e1: {metrics: [missing-metric], min-score: 50} -tlos: - t1: {evaluation: missing-evaluation} -goals: - g1: {tlos: [missing-tlo]} -""") - - with pytest.raises(SDLValidationError) as exc_info: - parse_sdl(raw) - - errors = exc_info.value.errors - assert any("Metric 'm1' references undefined condition 'missing-cond'" in error for error in errors) - assert any("Evaluation 'e1' references undefined metric 'missing-metric'" in error for error in errors) - assert any("TLO 't1' references undefined evaluation 'missing-evaluation'" in error for error in errors) - assert any("Goal 'g1' references undefined TLO 'missing-tlo'" in error for error in errors) - - model = compile_runtime_model(parse_sdl(raw, skip_semantic_validation=True)) - codes = {diag.code for diag in model.diagnostics} - assert "evaluation.condition-ref-unbound" in codes - assert "evaluation.metric-ref-unbound" in codes - assert "evaluation.evaluation-ref-unbound" in codes - assert "evaluation.tlo-ref-unbound" in codes - - def test_compiler_and_planner_agree_on_pipeline_refresh_semantics(self): - raw = _scenario(""" -name: assessment-refresh -nodes: - vm: - type: vm - os: linux - resources: {ram: 1 gib, cpu: 1} - conditions: {health: ops} - roles: {ops: operator} -conditions: - health: {command: /bin/true, interval: 15} -metrics: - m1: {type: conditional, condition: health, max-score: 10} -evaluations: - e1: {metrics: [m1], min-score: 50} -tlos: - t1: {evaluation: e1} -goals: - g1: {tlos: [t1]} -""") - compiled = compile_runtime_model(parse_sdl(raw)) - - metric = compiled.metrics["evaluation.metric.m1"] - assert "evaluation.condition.vm.health" in metric.ordering_dependencies - assert "evaluation.condition.vm.health" in metric.refresh_dependencies - evaluation = compiled.evaluations["evaluation.evaluation.e1"] - assert "evaluation.metric.m1" in evaluation.ordering_dependencies - assert "evaluation.metric.m1" in evaluation.refresh_dependencies - tlo = compiled.tlos["evaluation.tlo.t1"] - assert "evaluation.evaluation.e1" in tlo.ordering_dependencies - goal = compiled.goals["evaluation.goal.g1"] - assert "evaluation.tlo.t1" in goal.ordering_dependencies - assert "evaluation.tlo.t1" in goal.refresh_dependencies - - baseline = plan(compiled, create_stub_manifest()) - snapshot = _snapshot_from_plan(baseline) - - mutated = compile_runtime_model(parse_sdl(raw.replace("/bin/true", "/bin/false"))) - updated = plan(mutated, create_stub_manifest(), snapshot=snapshot) - - actions = {op.address: op.action.value for op in updated.evaluation.operations} - assert actions["evaluation.condition.vm.health"] == "update" - assert actions["evaluation.metric.m1"] == "update" - assert actions["evaluation.evaluation.e1"] == "update" - assert actions["evaluation.tlo.t1"] == "update" - assert actions["evaluation.goal.g1"] == "update" diff --git a/implementations/python/tests/test_language_service.py b/implementations/python/tests/test_language_service.py index a8fd2443f..905d5ff29 100644 --- a/implementations/python/tests/test_language_service.py +++ b/implementations/python/tests/test_language_service.py @@ -2,6 +2,7 @@ from __future__ import annotations +import pytest from aces_sdl.language_service import ( apply_structured_edit, language_completions, @@ -56,10 +57,8 @@ def test_language_completions_cover_contexts_and_filters() -> None: web: {type: VM, os: linux, resources: {ram: 2 GiB, cpu: 1}} conditions: alive: {command: "true", interval: 5} -metrics: - uptime: {type: CONDITIONAL, condition: alive} relationships: - app-to-web: {type: hosted_on, source: web, target: uptime} + app-to-web: {type: hosted_on, source: web, target: alive} workflows: flow: start: start-here @@ -77,7 +76,7 @@ def test_language_completions_cover_contexts_and_filters() -> None: any_refs = language_completions(sdl, cursor_path="/relationships/app-to-web/source") assert any(item["detail"] == "nodes.web" for item in any_refs["items"]) - assert any(item["detail"] == "metrics.uptime" for item in any_refs["items"]) + assert any(item["detail"] == "conditions.alive" for item in any_refs["items"]) workflow_refs = language_completions(sdl, cursor_path="/workflows/flow/start") assert workflow_refs["context"] == "reference:workflow_steps" @@ -244,19 +243,28 @@ def test_structured_edit_handles_root_and_list_mutations() -> None: assert "custom:" in created["content"] -def test_structured_edit_reports_invalid_edit_requests() -> None: - cases = [ - ("delete", "", None, "root pointer supports only the set operation"), - ("replace", "/description", "x", "operation must be one of"), - ("set", "description", "x", "pointer must be empty or start with '/'"), - ("append", "/name", "x", "is not a list"), - ("delete", "/missing", None, "missing path segment"), - ("delete", "/infrastructure/web/links/nope", None, "is not an integer"), - ("delete", "/infrastructure/web/links/10", None, "out of range"), - ("set", "/infrastructure/web/links/0/name", "x", "does not address a mapping or list"), - ] - - for operation, pointer, value, expected in cases: - payload = apply_structured_edit(SAMPLE_SDL, operation=operation, pointer=pointer, value=value) - assert payload["status"] == "invalid" - assert expected in payload["diagnostics"][0]["message"] +@pytest.mark.parametrize( + ("operation", "pointer", "value", "expected"), + [ + pytest.param("delete", "", None, "root pointer supports only the set operation", id="root-delete-unsupported"), + pytest.param("replace", "/description", "x", "operation must be one of", id="unknown-operation"), + pytest.param( + "set", "description", "x", "pointer must be empty or start with '/'", id="pointer-missing-leading-slash" + ), + pytest.param("append", "/name", "x", "is not a list", id="append-non-list"), + pytest.param("delete", "/missing", None, "missing path segment", id="delete-missing-segment"), + pytest.param("delete", "/infrastructure/web/links/nope", None, "is not an integer", id="index-not-integer"), + pytest.param("delete", "/infrastructure/web/links/10", None, "out of range", id="index-out-of-range"), + pytest.param( + "set", + "/infrastructure/web/links/0/name", + "x", + "does not address a mapping or list", + id="non-container-parent", + ), + ], +) +def test_structured_edit_reports_invalid_edit_requests(operation, pointer, value, expected) -> None: + payload = apply_structured_edit(SAMPLE_SDL, operation=operation, pointer=pointer, value=value) + assert payload["status"] == "invalid" + assert expected in payload["diagnostics"][0]["message"] diff --git a/implementations/python/tests/test_mcp_server.py b/implementations/python/tests/test_mcp_server.py index c58cb8159..be979d95c 100644 --- a/implementations/python/tests/test_mcp_server.py +++ b/implementations/python/tests/test_mcp_server.py @@ -89,23 +89,10 @@ def _json_call(server, tool: str, args: dict | None = None) -> dict: vulnerabilities: sqli: {name: SQL Injection, description: "SQLi in login", technical: true, class: CWE-89} -metrics: - uptime: {type: CONDITIONAL, max-score: 100, condition: alive} - -evaluations: - basic: {metrics: [uptime], min-score: 50} - -tlos: - web-defense: {name: Defend web, evaluation: basic} - -goals: - pass: {tlos: [web-defense]} - entities: blue-team: name: Blue role: Blue - tlos: [web-defense] entities: alice: {name: Alice} red-team: @@ -186,8 +173,8 @@ def test_sdl_overview_returns_content(self, server): text = _call(server, "sdl_overview") assert "SDL" in text # Both pieces of evidence must be present; an OR disjunction over - # "21" / "sections" would let either drift go undetected. - assert "21" in text + # "17" / "sections" would let either drift go undetected. + assert "17" in text assert "sections" in text.lower() assert "nodes" in text @@ -197,31 +184,24 @@ def test_sdl_section_reference_valid(self, server): assert "Switch" in text assert "VM" in text - def test_sdl_section_reference_scoring(self, server): - text = _call(server, "sdl_section_reference", {"section": "scoring"}) - assert "Metrics" in text or "metrics" in text - assert "Evaluations" in text or "evaluations" in text - def test_sdl_section_reference_invalid(self, server): text = _call(server, "sdl_section_reference", {"section": "nonexistent"}) assert "Unknown section" in text - def test_sdl_get_example_minimal(self, server): - text = _call(server, "sdl_get_example", {"name": "minimal"}) - assert "name:" in text - assert "nodes:" in text - # Pin the example identity so a regression returning a different - # SDL (still with `name:` / `nodes:`) is not silently accepted. - assert "simple-pentest-lab" in text - - def test_sdl_get_example_hospital(self, server): - text = _call(server, "sdl_get_example", {"name": "hospital"}) - assert "hospital-ransomware" in text - assert "objectives:" in text - - def test_sdl_get_example_invalid(self, server): - text = _call(server, "sdl_get_example", {"name": "nonexistent"}) - assert "Unknown example" in text + @pytest.mark.parametrize( + ("name", "expected"), + [ + # Pin the example identity so a regression returning a different + # SDL (still with `name:` / `nodes:`) is not silently accepted. + pytest.param("minimal", ["name:", "nodes:", "simple-pentest-lab"], id="minimal"), + pytest.param("hospital", ["hospital-ransomware", "objectives:"], id="hospital"), + pytest.param("nonexistent", ["Unknown example"], id="invalid"), + ], + ) + def test_sdl_get_example(self, server, name, expected): + text = _call(server, "sdl_get_example", {"name": name}) + for substring in expected: + assert substring in text def test_sdl_parser_reference(self, server): text = _call(server, "sdl_parser_reference") @@ -300,26 +280,19 @@ def test_validate_section_bad_section(self, server): ) assert "Unknown section" in text - def test_scaffold_minimal(self, server): - text = _call(server, "sdl_scaffold", {"complexity": "minimal"}) - assert "name:" in text - assert "nodes:" in text - # Should be valid SDL - validation = _call(server, "sdl_validate", {"sdl_content": text}) - assert validation.startswith("VALID") - - def test_scaffold_standard(self, server): - text = _call(server, "sdl_scaffold", {"complexity": "standard"}) - assert "entities:" in text - assert "accounts:" in text - validation = _call(server, "sdl_validate", {"sdl_content": text}) - assert validation.startswith("VALID") - - def test_scaffold_full(self, server): - text = _call(server, "sdl_scaffold", {"complexity": "full"}) - assert "workflows:" in text - assert "objectives:" in text - assert "variables:" in text + @pytest.mark.parametrize( + ("complexity", "expected"), + [ + pytest.param("minimal", ["name:", "nodes:"], id="minimal"), + pytest.param("standard", ["entities:", "accounts:"], id="standard"), + pytest.param("full", ["workflows:", "objectives:", "variables:"], id="full"), + ], + ) + def test_scaffold(self, server, complexity, expected): + text = _call(server, "sdl_scaffold", {"complexity": complexity}) + for substring in expected: + assert substring in text + # Scaffolded output should be valid SDL. validation = _call(server, "sdl_validate", {"sdl_content": text}) assert validation.startswith("VALID") @@ -754,40 +727,22 @@ def test_example_summarizes(self, server, filename): class TestServerConstruction: def test_server_has_all_tools(self): server = create_server() + # Ground truth: the tools actually registered on the FastMCP server. + # Using the real registration surface (rather than a hand-copied + # literal) means a drift between what the server exposes and what + # aces_tool_surface advertises cannot pass silently. + registered = asyncio.get_event_loop().run_until_complete(server.list_tools()) + registered_names = {tool.name for tool in registered} + assert registered_names, "server registered no tools" + + # The advertised surface from aces_tool_surface must equal the real + # registered set. aces_tool_surface documents itself outside the + # family listing, so seed it explicitly. payload = _json_call(server, "aces_tool_surface") - tool_names = {"aces_tool_surface"} + advertised = {"aces_tool_surface"} for family in payload["tool_families"].values(): - tool_names.update(family) - expected = { - "aces_tool_surface", - "aces_agent_guidance", - "aces_reference_manifests", - "sdl_overview", - "sdl_section_reference", - "sdl_get_example", - "sdl_parser_reference", - "sdl_validation_reference", - "sdl_parse", - "sdl_validate", - "sdl_validate_section", - "sdl_scaffold", - "sdl_instantiate", - "sdl_completions", - "sdl_references", - "sdl_format", - "sdl_diagnostics", - "sdl_apply_edit", - "sdl_summarize", - "sdl_list_elements", - "sdl_get_element", - "sdl_check_references", - "sdl_diagram", - "sdl_compile", - "sdl_plan", - "sdl_design_assessment", - "sdl_claims_assessment", - } - assert tool_names == expected + advertised.update(family) + assert advertised == registered_names def test_server_has_instructions(self): server = create_server() diff --git a/implementations/python/tests/test_reference_processor.py b/implementations/python/tests/test_reference_processor.py index c5c9e3177..744610d1d 100644 --- a/implementations/python/tests/test_reference_processor.py +++ b/implementations/python/tests/test_reference_processor.py @@ -50,8 +50,6 @@ roles: {ops: operator} conditions: health: {command: /bin/true, interval: 15} - metrics: - uptime: {type: conditional, max-score: 100, condition: health} entities: blue: {role: blue} objectives: @@ -83,8 +81,6 @@ roles: {ops: operator} conditions: health: {command: /bin/true, interval: 15} - metrics: - uptime: {type: conditional, max-score: 100, condition: health} entities: blue: {role: blue} objectives: diff --git a/implementations/python/tests/test_run_300_lifecycle.py b/implementations/python/tests/test_run_300_lifecycle.py index fc5c13e6c..d1f916bf7 100644 --- a/implementations/python/tests/test_run_300_lifecycle.py +++ b/implementations/python/tests/test_run_300_lifecycle.py @@ -85,8 +85,6 @@ def _raw_scenario(): roles: {{ops: operator}} conditions: health: {{command: /bin/true, interval: 15}} - metrics: - uptime: {{type: conditional, max-score: 100, condition: health}} entities: blue: {{role: blue}} objectives: diff --git a/implementations/python/tests/test_runtime_manager.py b/implementations/python/tests/test_runtime_manager.py index bd56b1ff6..79f9690d0 100644 --- a/implementations/python/tests/test_runtime_manager.py +++ b/implementations/python/tests/test_runtime_manager.py @@ -64,8 +64,6 @@ def _full_scenario(): roles: {ops: operator} conditions: health: {command: /bin/true, interval: 15} -metrics: - uptime: {type: conditional, max-score: 100, condition: health} events: kickoff: {conditions: [health]} scripts: @@ -711,11 +709,17 @@ def start(self, plan, snapshot: RuntimeSnapshot) -> ApplyResult: class InvalidEvaluatorReadyPayloadEvaluator(RecordingEvaluator): def start(self, plan, snapshot: RuntimeSnapshot) -> ApplyResult: result = super().start(plan, snapshot) - metric_address = next( - op.address for op in plan.operations if op.action != ChangeAction.DELETE and op.resource_type == "metric" - ) - self._results[metric_address]["score"] = None - self._results[metric_address]["max_score"] = None + # Post ADR-073 the observable evaluation resources are conditions + # (supports_passed); a ready result must report passed or score, so + # nulling passed with no score violates the result contract. + passed_address = next( + op.address + for op in plan.operations + if op.action != ChangeAction.DELETE and op.payload.get("result_contract", {}).get("supports_passed") + ) + self._results[passed_address]["passed"] = None + self._results[passed_address].pop("score", None) + self._results[passed_address].pop("max_score", None) return ApplyResult( success=True, snapshot=result.snapshot.with_entries( @@ -1037,117 +1041,129 @@ def test_apply_fails_on_invalid_workflow_result_contract(self): ] assert "runtime.backend-contract-invalid" in {diag.code for diag in result.diagnostics} - def test_apply_fails_on_invalid_workflow_result_schema_version(self): - calls: list[str] = [] - target = RuntimeTarget( - name="recording", - manifest=create_stub_manifest(with_participant_runtime=False), - provisioner=RecordingProvisioner(calls), - orchestrator=InvalidWorkflowSchemaVersionOrchestrator(calls), - evaluator=RecordingEvaluator(calls, "evaluator"), - ) - manager = RuntimeManager(target) - - result = manager.apply(manager.plan(_workflow_scenario())) - - assert not result.success - assert "runtime.backend-contract-invalid" in {diag.code for diag in result.diagnostics} - - def test_apply_fails_on_missing_workflow_result_fields(self): - calls: list[str] = [] - target = RuntimeTarget( - name="recording", - manifest=create_stub_manifest(with_participant_runtime=False), - provisioner=RecordingProvisioner(calls), - orchestrator=MissingWorkflowFieldsOrchestrator(calls), - evaluator=RecordingEvaluator(calls, "evaluator"), - ) - manager = RuntimeManager(target) - - result = manager.apply(manager.plan(_workflow_scenario())) - - assert not result.success - assert "runtime.backend-contract-invalid" in {diag.code for diag in result.diagnostics} - - def test_apply_fails_on_invalid_workflow_result_lifecycle(self): - calls: list[str] = [] - target = RuntimeTarget( - name="recording", - manifest=create_stub_manifest(with_participant_runtime=False), - provisioner=RecordingProvisioner(calls), - orchestrator=InvalidWorkflowLifecycleOrchestrator(calls), - evaluator=RecordingEvaluator(calls, "evaluator"), - ) - manager = RuntimeManager(target) - - result = manager.apply(manager.plan(_workflow_scenario())) - - assert not result.success - assert "runtime.backend-contract-invalid" in {diag.code for diag in result.diagnostics} - - def test_apply_fails_on_invalid_workflow_result_outcome(self): - calls: list[str] = [] - target = RuntimeTarget( - name="recording", - manifest=create_stub_manifest(with_participant_runtime=False), - provisioner=RecordingProvisioner(calls), - orchestrator=InvalidWorkflowOutcomeOrchestrator(calls), - evaluator=RecordingEvaluator(calls, "evaluator"), - ) - manager = RuntimeManager(target) - - result = manager.apply(manager.plan(_workflow_scenario())) - - assert not result.success - assert "runtime.backend-contract-invalid" in {diag.code for diag in result.diagnostics} - - def test_apply_fails_on_fixed_attempt_mismatch(self): - calls: list[str] = [] - target = RuntimeTarget( - name="recording", - manifest=create_stub_manifest(with_participant_runtime=False), - provisioner=RecordingProvisioner(calls), - orchestrator=InvalidWorkflowAttemptCountOrchestrator(calls), - evaluator=RecordingEvaluator(calls, "evaluator"), - ) - manager = RuntimeManager(target) - - result = manager.apply(manager.plan(_workflow_scenario())) - - assert not result.success - assert "runtime.backend-contract-invalid" in {diag.code for diag in result.diagnostics} - - def test_apply_fails_on_pending_step_with_outcome(self): - calls: list[str] = [] - target = RuntimeTarget( - name="recording", - manifest=create_stub_manifest(with_participant_runtime=False), - provisioner=RecordingProvisioner(calls), - orchestrator=InvalidWorkflowPendingOutcomeOrchestrator(calls), - evaluator=RecordingEvaluator(calls, "evaluator"), - ) - manager = RuntimeManager(target) - - result = manager.apply(manager.plan(_workflow_scenario())) - - assert not result.success - assert "runtime.backend-contract-invalid" in {diag.code for diag in result.diagnostics} - - def test_apply_fails_on_missing_observable_workflow_step(self): + @pytest.mark.parametrize( + ("fake_cls", "slot", "scenario_factory", "expected_code"), + [ + pytest.param( + InvalidWorkflowSchemaVersionOrchestrator, + "orchestrator", + _workflow_scenario, + "runtime.backend-contract-invalid", + id="workflow-schema-version", + ), + pytest.param( + MissingWorkflowFieldsOrchestrator, + "orchestrator", + _workflow_scenario, + "runtime.backend-contract-invalid", + id="workflow-missing-fields", + ), + pytest.param( + InvalidWorkflowLifecycleOrchestrator, + "orchestrator", + _workflow_scenario, + "runtime.backend-contract-invalid", + id="workflow-lifecycle", + ), + pytest.param( + InvalidWorkflowOutcomeOrchestrator, + "orchestrator", + _workflow_scenario, + "runtime.backend-contract-invalid", + id="workflow-outcome", + ), + pytest.param( + InvalidWorkflowAttemptCountOrchestrator, + "orchestrator", + _workflow_scenario, + "runtime.backend-contract-invalid", + id="workflow-attempt-count", + ), + pytest.param( + InvalidWorkflowPendingOutcomeOrchestrator, + "orchestrator", + _workflow_scenario, + "runtime.backend-contract-invalid", + id="workflow-pending-outcome", + ), + pytest.param( + MissingObservableWorkflowStepOrchestrator, + "orchestrator", + _workflow_scenario, + "runtime.backend-contract-invalid", + id="workflow-missing-observable-step", + ), + pytest.param( + InvalidWorkflowCallHistoryOrchestrator, + "orchestrator", + _workflow_call_scenario, + "runtime.backend-contract-invalid", + id="workflow-call-history", + ), + pytest.param( + InvalidWorkflowCompensationOrchestrator, + "orchestrator", + _workflow_scenario, + "runtime.backend-contract-invalid", + id="workflow-compensation-state", + ), + pytest.param( + InvalidEvaluatorSchemaVersionEvaluator, + "evaluator", + _full_scenario, + "runtime.backend-contract-invalid", + id="evaluation-schema-version", + ), + pytest.param( + MissingEvaluatorFieldsEvaluator, + "evaluator", + _full_scenario, + "runtime.backend-contract-invalid", + id="evaluation-missing-fields", + ), + pytest.param( + InvalidEvaluatorReadyPayloadEvaluator, + "evaluator", + _full_scenario, + "runtime.backend-contract-invalid", + id="evaluation-ready-payload", + ), + pytest.param( + MissingEvaluatorHistoryEvaluator, + "evaluator", + _full_scenario, + "runtime.backend-contract-invalid", + id="evaluation-missing-history", + ), + ], + ) + def test_apply_fails_on_invalid_backend_result_contract( + self, + fake_cls, + slot, + scenario_factory, + expected_code, + ): calls: list[str] = [] + if slot == "orchestrator": + orchestrator = fake_cls(calls) + evaluator = RecordingEvaluator(calls, "evaluator") + else: + orchestrator = RecordingOrchestrator(calls) + evaluator = fake_cls(calls, "evaluator") target = RuntimeTarget( name="recording", manifest=create_stub_manifest(with_participant_runtime=False), provisioner=RecordingProvisioner(calls), - orchestrator=MissingObservableWorkflowStepOrchestrator(calls), - evaluator=RecordingEvaluator(calls, "evaluator"), + orchestrator=orchestrator, + evaluator=evaluator, ) manager = RuntimeManager(target) - result = manager.apply(manager.plan(_workflow_scenario())) + result = manager.apply(manager.plan(scenario_factory())) assert not result.success - assert "runtime.backend-contract-invalid" in {diag.code for diag in result.diagnostics} + assert expected_code in {diag.code for diag in result.diagnostics} def test_apply_validates_against_result_contract_not_control_steps(self): calls: list[str] = [] @@ -1165,102 +1181,6 @@ def test_apply_validates_against_result_contract_not_control_steps(self): assert result.success assert manager.snapshot.for_domain(RuntimeDomain.ORCHESTRATION) - def test_apply_fails_on_invalid_workflow_call_history(self): - calls: list[str] = [] - target = RuntimeTarget( - name="recording", - manifest=create_stub_manifest(with_participant_runtime=False), - provisioner=RecordingProvisioner(calls), - orchestrator=InvalidWorkflowCallHistoryOrchestrator(calls), - evaluator=RecordingEvaluator(calls, "evaluator"), - ) - manager = RuntimeManager(target) - - result = manager.apply(manager.plan(_workflow_call_scenario())) - - assert not result.success - assert "runtime.backend-contract-invalid" in {diag.code for diag in result.diagnostics} - - def test_apply_fails_on_invalid_workflow_compensation_state(self): - calls: list[str] = [] - target = RuntimeTarget( - name="recording", - manifest=create_stub_manifest(with_participant_runtime=False), - provisioner=RecordingProvisioner(calls), - orchestrator=InvalidWorkflowCompensationOrchestrator(calls), - evaluator=RecordingEvaluator(calls, "evaluator"), - ) - manager = RuntimeManager(target) - - result = manager.apply(manager.plan(_workflow_scenario())) - - assert not result.success - assert "runtime.backend-contract-invalid" in {diag.code for diag in result.diagnostics} - - def test_apply_fails_on_invalid_evaluation_result_schema_version(self): - calls: list[str] = [] - target = RuntimeTarget( - name="recording", - manifest=create_stub_manifest(with_participant_runtime=False), - provisioner=RecordingProvisioner(calls), - orchestrator=RecordingOrchestrator(calls), - evaluator=InvalidEvaluatorSchemaVersionEvaluator(calls, "evaluator"), - ) - manager = RuntimeManager(target) - - result = manager.apply(manager.plan(_full_scenario())) - - assert not result.success - assert "runtime.backend-contract-invalid" in {diag.code for diag in result.diagnostics} - - def test_apply_fails_on_missing_evaluation_result_fields(self): - calls: list[str] = [] - target = RuntimeTarget( - name="recording", - manifest=create_stub_manifest(with_participant_runtime=False), - provisioner=RecordingProvisioner(calls), - orchestrator=RecordingOrchestrator(calls), - evaluator=MissingEvaluatorFieldsEvaluator(calls, "evaluator"), - ) - manager = RuntimeManager(target) - - result = manager.apply(manager.plan(_full_scenario())) - - assert not result.success - assert "runtime.backend-contract-invalid" in {diag.code for diag in result.diagnostics} - - def test_apply_fails_on_invalid_ready_evaluation_payload(self): - calls: list[str] = [] - target = RuntimeTarget( - name="recording", - manifest=create_stub_manifest(with_participant_runtime=False), - provisioner=RecordingProvisioner(calls), - orchestrator=RecordingOrchestrator(calls), - evaluator=InvalidEvaluatorReadyPayloadEvaluator(calls, "evaluator"), - ) - manager = RuntimeManager(target) - - result = manager.apply(manager.plan(_full_scenario())) - - assert not result.success - assert "runtime.backend-contract-invalid" in {diag.code for diag in result.diagnostics} - - def test_apply_fails_on_missing_evaluation_history(self): - calls: list[str] = [] - target = RuntimeTarget( - name="recording", - manifest=create_stub_manifest(with_participant_runtime=False), - provisioner=RecordingProvisioner(calls), - orchestrator=RecordingOrchestrator(calls), - evaluator=MissingEvaluatorHistoryEvaluator(calls, "evaluator"), - ) - manager = RuntimeManager(target) - - result = manager.apply(manager.plan(_full_scenario())) - - assert not result.success - assert "runtime.backend-contract-invalid" in {diag.code for diag in result.diagnostics} - def test_status_exposes_plain_data_workflow_results(self): target = RuntimeTarget( name="recording", @@ -1302,12 +1222,12 @@ def test_status_exposes_plain_data_evaluation_results_and_history(self): evaluation_history = status["evaluation_history"] assert isinstance(evaluation_results, dict) assert isinstance(evaluation_history, dict) - metric_payload = evaluation_results["evaluation.metric.uptime"] - assert metric_payload["state_schema_version"] == EVALUATION_STATE_SCHEMA_VERSION - assert metric_payload["resource_type"] == "metric" - assert metric_payload["status"] == "ready" - assert metric_payload["score"] == 100 - assert evaluation_history["evaluation.metric.uptime"][-1]["event_type"] == "evaluation_ready" + condition_payload = evaluation_results["evaluation.condition.vm.health"] + assert condition_payload["state_schema_version"] == EVALUATION_STATE_SCHEMA_VERSION + assert condition_payload["resource_type"] == "condition-binding" + assert condition_payload["status"] == "ready" + assert condition_payload["passed"] is True + assert evaluation_history["evaluation.condition.vm.health"][-1]["event_type"] == "evaluation_ready" json.dumps(evaluation_results) json.dumps(evaluation_history) @@ -1324,14 +1244,21 @@ def test_stub_runtime_emits_plain_data_workflow_results(self): json.dumps(manager.snapshot.orchestration_results) def test_runtime_manager_requires_explicit_manifest(self): + # RuntimeTarget.__post_init__ rejects manifest=None at construction, so + # build a valid target and null the manifest afterwards to exercise the + # RuntimeManager.__init__ guard directly (mirrors the evaluator-missing + # test's object.__setattr__ bypass below). + target = RuntimeTarget( + name="invalid", + manifest=create_stub_manifest(with_participant_runtime=False), + provisioner=RecordingProvisioner([]), + orchestrator=RecordingOrchestrator([]), + evaluator=RecordingEvaluator([], "evaluator"), + ) + object.__setattr__(target, "manifest", None) + with pytest.raises(ValueError, match="explicit manifest"): - RuntimeManager( - RuntimeTarget( # type: ignore[arg-type] - name="invalid", - manifest=None, - provisioner=RecordingProvisioner([]), - ) - ) + RuntimeManager(target) def test_apply_fails_closed_before_provisioning_when_required_service_is_missing(self): calls: list[str] = [] diff --git a/implementations/python/tests/test_runtime_models.py b/implementations/python/tests/test_runtime_models.py index 6c77e3edd..850010a43 100644 --- a/implementations/python/tests/test_runtime_models.py +++ b/implementations/python/tests/test_runtime_models.py @@ -524,12 +524,10 @@ def test_objective_windows_and_workflows_resolve_refresh_dependencies(self): roles: {ops: operator} conditions: health: {command: /bin/true, interval: 15} -metrics: - uptime: {type: conditional, max-score: 100, condition: health} objectives: initial: entity: blue - success: {conditions: [health], metrics: [uptime]} + success: {conditions: [health]} window: stories: [main] scripts: [timeline] @@ -561,7 +559,6 @@ def test_objective_windows_and_workflows_resolve_refresh_dependencies(self): objective = model.objectives["evaluation.objective.initial"] workflow = model.workflows["orchestration.workflow.flow"] - assert "evaluation.metric.uptime" in objective.success_addresses assert "evaluation.condition.vm.health" in objective.success_addresses assert objective.window_story_addresses == ("orchestration.story.main",) assert objective.window_script_addresses == ("orchestration.script.timeline",) @@ -578,7 +575,7 @@ def test_objective_windows_and_workflows_resolve_refresh_dependencies(self): ] assert objective.window_references[-1].workflow_name == "flow" assert objective.window_references[-1].step_name == "branch" - assert "evaluation.metric.uptime" in objective.ordering_dependencies + assert "evaluation.condition.vm.health" in objective.ordering_dependencies assert "orchestration.workflow.flow" in objective.refresh_dependencies assert workflow.referenced_objective_addresses == ("evaluation.objective.initial",) assert workflow.start_step == "start" @@ -599,8 +596,6 @@ def test_objective_windows_and_workflows_resolve_refresh_dependencies(self): assert "evaluation.condition.vm.health" in workflow.step_predicate_addresses["branch"] assert workflow.ordering_dependencies == () assert "evaluation.objective.initial" in workflow.refresh_dependencies - assert model.metrics["evaluation.metric.uptime"].result_contract.supports_score is True - assert model.metrics["evaluation.metric.uptime"].result_contract.fixed_max_score == 100 assert model.objectives["evaluation.objective.initial"].result_contract.supports_passed is True assert not model.diagnostics @@ -688,20 +683,11 @@ def test_missing_runtime_graph_refs_emit_partial_model_diagnostics(self): roles: {ops: operator} conditions: health: {command: /bin/true, interval: 15} -metrics: - uptime: {type: conditional, max-score: 100, condition: health} -evaluations: - overall: {metrics: [uptime, missing-metric], min-score: 50} -tlos: - defend: {evaluation: missing-evaluation} -goals: - pass: {tlos: [missing-tlo]} objectives: initial: entity: blue success: - metrics: [missing-metric] - goals: [missing-goal] + conditions: [missing-condition] window: workflows: [missing-workflow] steps: [missing-workflow.branch, badstep] @@ -717,7 +703,7 @@ def test_missing_runtime_graph_refs_emit_partial_model_diagnostics(self): steps: branch: type: decision - when: {metrics: [missing-metric], objectives: [missing-objective]} + when: {conditions: [missing-condition], objectives: [missing-objective]} then: finish else: finish finish: {type: end} @@ -729,21 +715,14 @@ def test_missing_runtime_graph_refs_emit_partial_model_diagnostics(self): codes = {diag.code for diag in model.diagnostics} assert "orchestration.event-ref-unbound" in codes assert "orchestration.script-ref-unbound" in codes - assert "evaluation.metric-ref-unbound" in codes - assert "evaluation.evaluation-ref-unbound" in codes - assert "evaluation.tlo-ref-unbound" in codes - assert "evaluation.goal-ref-unbound" in codes + assert "evaluation.condition-ref-unbound" in codes assert "evaluation.workflow-ref-unbound" in codes assert "evaluation.workflow-step-ref-workflow-unbound" in codes assert "evaluation.workflow-step-ref-invalid-format" in codes - assert "orchestration.metric-ref-unbound" in codes assert "orchestration.objective-ref-unbound" in codes assert model.scripts["orchestration.script.timeline"].event_addresses == () assert model.stories["orchestration.story.main"].script_addresses == () - assert model.evaluations["evaluation.evaluation.overall"].metric_addresses == ("evaluation.metric.uptime",) - assert model.tlos["evaluation.tlo.defend"].evaluation_address == "" - assert model.goals["evaluation.goal.pass"].tlo_addresses == () assert model.objectives["evaluation.objective.initial"].success_addresses == () assert model.objectives["evaluation.objective.initial"].window_workflow_addresses == () assert model.objectives["evaluation.objective.initial"].window_step_refs == () @@ -764,15 +743,13 @@ def test_workflow_with_retry_and_step_state_compiles(self): health: {command: /bin/true, interval: 15} entities: blue: {role: blue} -metrics: - uptime: {type: conditional, max-score: 100, condition: health} objectives: attempt: entity: blue success: {conditions: [health]} recover: entity: blue - success: {metrics: [uptime]} + success: {conditions: [health]} workflows: retry: start: attempt-loop diff --git a/implementations/python/tests/test_runtime_planner.py b/implementations/python/tests/test_runtime_planner.py index 99ace3dde..806f20430 100644 --- a/implementations/python/tests/test_runtime_planner.py +++ b/implementations/python/tests/test_runtime_planner.py @@ -252,8 +252,12 @@ def test_dependency_changes_propagate_through_evaluation_graph(self): roles: {ops: operator} conditions: health: {command: /bin/true, interval: 15} -metrics: - uptime: {type: conditional, max-score: 100, condition: health} +entities: + blue: {role: blue} +objectives: + initial: + entity: blue + success: {conditions: [health]} """) ) old_plan = plan(old_model, create_stub_manifest()) @@ -271,15 +275,22 @@ def test_dependency_changes_propagate_through_evaluation_graph(self): roles: {ops: operator} conditions: health: {command: /bin/false, interval: 15} -metrics: - uptime: {type: conditional, max-score: 100, condition: health} +entities: + blue: {role: blue} +objectives: + initial: + entity: blue + success: {conditions: [health]} """, snapshot, ) eval_actions = {op.address: op.action.value for op in new_plan.evaluation.operations} assert eval_actions["evaluation.condition.vm.health"] == "update" - assert eval_actions["evaluation.metric.uptime"] == "update" + # Objective success references the condition, so a condition change must + # propagate to the objective evaluation resource (metric->evaluation->tlo->goal + # chain removed by ADR-073). + assert eval_actions["evaluation.objective.initial"] == "update" def test_ambiguous_condition_refs_fail_closed(self): execution_plan = plan( @@ -347,8 +358,12 @@ def test_unbound_condition_and_inject_refs_invalidate_plan(self): health: {command: /bin/true, interval: 15} injects: mail: {source: inbox} -metrics: - uptime: {type: conditional, max-score: 100, condition: health} +entities: + blue: {role: blue} +objectives: + check: + entity: blue + success: {conditions: [health]} events: kickoff: {conditions: [health], injects: [mail]} """) @@ -365,23 +380,17 @@ def test_unbound_condition_and_inject_refs_invalidate_plan(self): assert "orchestration.inject-ref-unbound" in codes assert not execution_plan.is_valid - def test_workflow_condition_refs_require_orchestrator_support(self): - limited = _limited_backend_manifest( - name="limited", - provisioner=create_stub_manifest().provisioner, - orchestrator=OrchestratorCapabilities( - name="limited-orchestrator", - supported_sections=frozenset({"workflows"}), - supports_workflows=True, - supports_condition_refs=False, - supported_workflow_features=frozenset({WorkflowFeature.DECISION}), - ), - evaluator=create_stub_manifest().evaluator, - ) - - execution_plan = plan( - compile_runtime_model( - _scenario(""" + @pytest.mark.parametrize( + ("orchestrator_kwargs", "scenario_yaml", "expected_code"), + [ + pytest.param( + { + "supported_sections": frozenset({"workflows"}), + "supports_workflows": True, + "supports_condition_refs": False, + "supported_workflow_features": frozenset({WorkflowFeature.DECISION}), + }, + """ name: workflows nodes: vm: @@ -402,31 +411,17 @@ def test_workflow_condition_refs_require_orchestrator_support(self): then: finish else: finish finish: {type: end} -""") - ), - limited, - ) - - codes = {diag.code for diag in execution_plan.diagnostics} - assert "orchestrator.condition-refs-unsupported" in codes - assert not execution_plan.is_valid - - def test_workflow_feature_requires_orchestrator_support(self): - limited = _limited_backend_manifest( - name="limited", - provisioner=create_stub_manifest().provisioner, - orchestrator=OrchestratorCapabilities( - name="limited-orchestrator", - supported_sections=frozenset({"workflows"}), - supports_workflows=True, - supported_workflow_features=frozenset({WorkflowFeature.DECISION}), +""", + "orchestrator.condition-refs-unsupported", + id="condition-refs-unsupported", ), - evaluator=create_stub_manifest().evaluator, - ) - - execution_plan = plan( - compile_runtime_model( - _scenario(""" + pytest.param( + { + "supported_sections": frozenset({"workflows"}), + "supports_workflows": True, + "supported_workflow_features": frozenset({WorkflowFeature.DECISION}), + }, + """ name: workflows nodes: vm: @@ -453,32 +448,18 @@ def test_workflow_feature_requires_orchestrator_support(self): on-success: finish max-attempts: 3 finish: {type: end} -"""), - ), - limited, - ) - - codes = {diag.code for diag in execution_plan.diagnostics} - assert "orchestrator.workflow-feature-unsupported" in codes - assert not execution_plan.is_valid - - def test_step_state_predicates_require_orchestrator_support(self): - limited = _limited_backend_manifest( - name="limited", - provisioner=create_stub_manifest().provisioner, - orchestrator=OrchestratorCapabilities( - name="limited-orchestrator", - supported_sections=frozenset({"workflows"}), - supports_workflows=True, - supported_workflow_features=frozenset({WorkflowFeature.DECISION}), - supported_workflow_state_predicates=frozenset(), +""", + "orchestrator.workflow-feature-unsupported", + id="workflow-feature-unsupported", ), - evaluator=create_stub_manifest().evaluator, - ) - - execution_plan = plan( - compile_runtime_model( - _scenario(""" + pytest.param( + { + "supported_sections": frozenset({"workflows"}), + "supports_workflows": True, + "supported_workflow_features": frozenset({WorkflowFeature.DECISION}), + "supported_workflow_state_predicates": frozenset(), + }, + """ name: workflows entities: blue: {role: blue} @@ -505,32 +486,18 @@ def test_step_state_predicates_require_orchestrator_support(self): then: finish else: finish finish: {type: end} -""") - ), - limited, - ) - - codes = {diag.code for diag in execution_plan.diagnostics} - assert "orchestrator.step-state-predicate-feature-unsupported" in codes - assert not execution_plan.is_valid - - def test_attempt_count_predicates_require_specific_support(self): - limited = _limited_backend_manifest( - name="limited", - provisioner=create_stub_manifest().provisioner, - orchestrator=OrchestratorCapabilities( - name="limited-orchestrator", - supported_sections=frozenset({"workflows"}), - supports_workflows=True, - supported_workflow_features=frozenset({WorkflowFeature.DECISION}), - supported_workflow_state_predicates=frozenset({WorkflowStatePredicateFeature.OUTCOME_MATCHING}), +""", + "orchestrator.step-state-predicate-feature-unsupported", + id="step-state-predicates-unsupported", ), - evaluator=create_stub_manifest().evaluator, - ) - - execution_plan = plan( - compile_runtime_model( - _scenario(""" + pytest.param( + { + "supported_sections": frozenset({"workflows"}), + "supports_workflows": True, + "supported_workflow_features": frozenset({WorkflowFeature.DECISION}), + "supported_workflow_state_predicates": frozenset({WorkflowStatePredicateFeature.OUTCOME_MATCHING}), + }, + """ name: workflows entities: blue: {role: blue} @@ -559,31 +526,17 @@ def test_attempt_count_predicates_require_specific_support(self): then: finish else: finish finish: {type: end} -""") - ), - limited, - ) - - codes = {diag.code for diag in execution_plan.diagnostics} - assert "orchestrator.step-state-predicate-feature-unsupported" in codes - assert not execution_plan.is_valid - - def test_parallel_barrier_requires_specific_support(self): - limited = _limited_backend_manifest( - name="limited", - provisioner=create_stub_manifest().provisioner, - orchestrator=OrchestratorCapabilities( - name="limited-orchestrator", - supported_sections=frozenset({"workflows"}), - supports_workflows=True, - supported_workflow_features=frozenset({WorkflowFeature.DECISION}), +""", + "orchestrator.step-state-predicate-feature-unsupported", + id="attempt-count-predicates-unsupported", ), - evaluator=create_stub_manifest().evaluator, - ) - - execution_plan = plan( - compile_runtime_model( - _scenario(""" + pytest.param( + { + "supported_sections": frozenset({"workflows"}), + "supports_workflows": True, + "supported_workflow_features": frozenset({WorkflowFeature.DECISION}), + }, + """ name: workflows entities: blue: {role: blue} @@ -616,13 +569,27 @@ def test_parallel_barrier_requires_specific_support(self): type: join next: finish finish: {type: end} -""") +""", + "orchestrator.workflow-feature-unsupported", + id="parallel-barrier-unsupported", ), + ], + ) + def test_workflow_capability_requires_orchestrator_support(self, orchestrator_kwargs, scenario_yaml, expected_code): + limited = _limited_backend_manifest( + name="limited", + provisioner=create_stub_manifest().provisioner, + orchestrator=OrchestratorCapabilities(name="limited-orchestrator", **orchestrator_kwargs), + evaluator=create_stub_manifest().evaluator, + ) + + execution_plan = plan( + compile_runtime_model(_scenario(scenario_yaml)), limited, ) codes = {diag.code for diag in execution_plan.diagnostics} - assert "orchestrator.workflow-feature-unsupported" in codes + assert expected_code in codes assert not execution_plan.is_valid def test_workflow_condition_bindings_force_workflow_refresh(self): @@ -741,12 +708,10 @@ def test_objective_window_refs_are_refresh_only(self): roles: {ops: operator} conditions: health: {command: /bin/true, interval: 15} -metrics: - uptime: {type: conditional, max-score: 100, condition: health} objectives: initial: entity: blue - success: {metrics: [uptime]} + success: {conditions: [health]} window: workflows: [flow] steps: [flow.branch] @@ -772,7 +737,6 @@ def test_objective_window_refs_are_refresh_only(self): assert "orchestration.workflow.flow" in objective.refresh_dependencies assert execution_plan.evaluation.startup_order == [ "evaluation.condition.vm.health", - "evaluation.metric.uptime", "evaluation.objective.initial", ] @@ -788,12 +752,10 @@ def test_objective_updates_when_window_dependencies_change(self): roles: {ops: operator} conditions: health: {command: /bin/true, interval: 15} -metrics: - uptime: {type: conditional, max-score: 100, condition: health} objectives: initial: entity: blue - success: {metrics: [uptime]} + success: {conditions: [health]} window: scripts: [timeline] events: [kickoff] @@ -921,7 +883,7 @@ def test_semantic_capability_validation_catches_real_requirements(self): ), evaluator=EvaluatorCapabilities( name="limited-evaluator", - supported_sections=frozenset({"conditions", "metrics"}), + supported_sections=frozenset({"conditions"}), supports_scoring=True, supports_objectives=False, ), @@ -949,8 +911,6 @@ def test_semantic_capability_validation_catches_real_requirements(self): admin: {username: administrator, node: dc, spn: LDAP/dc.example.local} conditions: health: {command: /bin/true, interval: 15} -metrics: - uptime: {type: conditional, max-score: 100, condition: health} objectives: defend: entity: blue @@ -1396,18 +1356,10 @@ def test_dependency_ordering_across_domain_plans(self): health: {command: /bin/true, interval: 15} injects: mail: {source: inbox} -metrics: - uptime: {type: conditional, max-score: 100, condition: health} -evaluations: - overall: {metrics: [uptime], min-score: 50} -tlos: - defend: {evaluation: overall} -goals: - pass: {tlos: [defend]} objectives: initial: entity: blue - success: {metrics: [uptime], goals: [pass]} + success: {conditions: [health]} entities: blue: {role: blue} events: @@ -1447,15 +1399,11 @@ def test_dependency_ordering_across_domain_plans(self): assert orchestration_order.index("orchestration.script.timeline") < orchestration_order.index( "orchestration.story.main" ) + # Post ADR-073 the evaluation domain orders condition -> objective + # (the metric -> evaluation -> tlo -> goal chain was removed). assert evaluation_order.index("evaluation.condition.web.health") < evaluation_order.index( - "evaluation.metric.uptime" - ) - assert evaluation_order.index("evaluation.metric.uptime") < evaluation_order.index( - "evaluation.evaluation.overall" + "evaluation.objective.initial" ) - assert evaluation_order.index("evaluation.evaluation.overall") < evaluation_order.index("evaluation.tlo.defend") - assert evaluation_order.index("evaluation.tlo.defend") < evaluation_order.index("evaluation.goal.pass") - assert evaluation_order.index("evaluation.goal.pass") < evaluation_order.index("evaluation.objective.initial") def test_satcom_release_poisoning_compiles_to_valid_execution_plan(self): scenario_path = EXAMPLES_DIR / "satcom-release-poisoning.sdl.yaml" diff --git a/implementations/python/tests/test_scenarios.py b/implementations/python/tests/test_scenarios.py index e0ba75bb2..ab710eb0d 100644 --- a/implementations/python/tests/test_scenarios.py +++ b/implementations/python/tests/test_scenarios.py @@ -137,7 +137,8 @@ def test_complex_examples_have_experiment_semantics(path): assert scenario.relationships assert scenario.content assert scenario.stories - assert scenario.metrics + # Post ADR-073: objective success references observable conditions (no scoring sections). + assert any(objective.success.conditions for objective in scenario.objectives.values()) def test_complex_examples_cover_new_sdl_surfaces(): diff --git a/implementations/python/tests/test_sdl_models.py b/implementations/python/tests/test_sdl_models.py index 91697b6fb..63a09bf36 100644 --- a/implementations/python/tests/test_sdl_models.py +++ b/implementations/python/tests/test_sdl_models.py @@ -123,7 +123,6 @@ WorkflowStepType, parse_duration, ) -from aces.core.sdl.scoring import TLO, Evaluation, Goal, Metric, MetricType, MinScore from aces.core.sdl.vulnerabilities import Vulnerability # --------------------------------------------------------------------------- @@ -2085,78 +2084,6 @@ def test_invalid_cwe(self): Vulnerability(name="Test", description="Desc", **{"class": "INVALID"}) -# --------------------------------------------------------------------------- -# Scoring -# --------------------------------------------------------------------------- - - -class TestMetric: - def test_manual(self): - m = Metric(type="manual", max_score=10, artifact=True) - assert m.type == MetricType.MANUAL - - def test_conditional(self): - m = Metric(type="conditional", max_score=10, condition="cond-1") - assert m.condition == "cond-1" - - def test_manual_rejects_condition(self): - with pytest.raises(ValidationError, match="Manual.*condition"): - Metric(type="manual", max_score=10, condition="cond-1") - - def test_conditional_requires_condition(self): - with pytest.raises(ValidationError, match="requires.*condition"): - Metric(type="conditional", max_score=10) - - def test_variable_placeholders(self): - m = Metric(type="manual", max_score="${max_score}", artifact="${needs_upload}") - assert m.max_score == "${max_score}" - assert m.artifact == "${needs_upload}" - - -class TestMinScore: - def test_percentage(self): - ms = MinScore(percentage=75) - assert ms.percentage == 75 - - def test_absolute(self): - ms = MinScore(absolute=50) - assert ms.absolute == 50 - - def test_rejects_both(self): - with pytest.raises(ValidationError, match="both"): - MinScore(absolute=50, percentage=75) - - def test_rejects_neither(self): - with pytest.raises(ValidationError, match="either"): - MinScore() - - def test_placeholder_percentage(self): - ms = MinScore(percentage="${pass_percentage}") - assert ms.percentage == "${pass_percentage}" - - -class TestEvaluation: - def test_valid(self): - e = Evaluation(metrics=["m-1"], min_score=MinScore(percentage=50)) - assert len(e.metrics) == 1 - - def test_empty_metrics_rejected(self): - with pytest.raises(ValidationError, match="at least 1 item"): - Evaluation(metrics=[], min_score=MinScore(percentage=50)) - - -class TestTLO: - def test_valid(self): - t = TLO(evaluation="eval-1") - assert t.evaluation == "eval-1" - - -class TestGoal: - def test_valid(self): - g = Goal(tlos=["tlo-1"]) - assert len(g.tlos) == 1 - - # --------------------------------------------------------------------------- # Entities # --------------------------------------------------------------------------- @@ -2313,14 +2240,14 @@ def test_requires_at_least_one_reference(self): with pytest.raises(ValidationError, match="at least one condition"): ObjectiveSuccess() - def test_accepts_goal_reference(self): - success = ObjectiveSuccess(goals=["pass-exercise"]) - assert success.goals == ["pass-exercise"] + def test_accepts_condition_reference(self): + success = ObjectiveSuccess(conditions=["exercise-passed"]) + assert success.conditions == ["exercise-passed"] def test_mode_placeholder(self): success = ObjectiveSuccess( mode="${objective_mode}", - goals=["pass-exercise"], + conditions=["exercise-passed"], ) assert success.mode == "${objective_mode}" @@ -2329,14 +2256,14 @@ class TestObjective: def test_requires_exactly_one_actor_binding(self): with pytest.raises(ValidationError, match="exactly one"): Objective( - success={"goals": ["g1"]}, + success={"conditions": ["c1"]}, ) with pytest.raises(ValidationError, match="exactly one"): Objective( agent="red-agent", entity="red-team", - success={"goals": ["g1"]}, + success={"conditions": ["c1"]}, ) def test_valid_agent_objective(self): @@ -2344,7 +2271,7 @@ def test_valid_agent_objective(self): agent="red-agent", actions=["Scan"], targets=["web-server"], - success={"goals": ["initial-access"]}, + success={"conditions": ["initial-access"]}, window={ "scripts": ["main-timeline"], "events": ["attack-wave"], @@ -2354,14 +2281,14 @@ def test_valid_agent_objective(self): depends_on=["recon"], ) assert objective.agent == "red-agent" - assert objective.success.goals == ["initial-access"] + assert objective.success.conditions == ["initial-access"] assert isinstance(objective.window, ObjectiveWindow) assert objective.window.steps == ["response-flow.validate"] def test_valid_entity_objective(self): objective = Objective( entity="blue-team", - success={"metrics": ["report-quality"]}, + success={"conditions": ["report-quality"]}, ) assert objective.entity == "blue-team" @@ -2624,7 +2551,7 @@ def test_decision_step_requires_branches(self): ValidationError, match="requires 'when', 'then', and 'else'", ): - WorkflowStep(type="decision", when={"goals": ["g1"]}) + WorkflowStep(type="decision", when={"conditions": ["c1"]}) def test_parallel_step_requires_unique_branches(self): with pytest.raises(ValidationError, match="branches must be unique"): @@ -2668,7 +2595,7 @@ def test_switch_step(self): type="switch", cases=[ { - "when": {"goals": ["g1"]}, + "when": {"conditions": ["c1"]}, "next": "done", } ], @@ -2771,7 +2698,7 @@ def test_on_failure_forbidden_on_decision_step(self): ): WorkflowStep( type="decision", - when={"goals": ["g1"]}, + when={"conditions": ["c1"]}, **{"then": "a", "else": "b", "on-failure": "recover"}, ) @@ -2796,7 +2723,7 @@ def test_predicate_with_only_step_state_is_valid(self): def test_legacy_workflow_step_type_rejected(self): with pytest.raises(ValidationError, match="no longer supported"): - WorkflowStep(type="if", when={"goals": ["g1"]}, **{"then": "a", "else": "b"}) + WorkflowStep(type="if", when={"conditions": ["c1"]}, **{"then": "a", "else": "b"}) def test_predicate_empty_rejected(self): with pytest.raises(ValidationError, match="must reference at least one"): diff --git a/implementations/python/tests/test_sdl_module_registry.py b/implementations/python/tests/test_sdl_module_registry.py index 8d315acf5..e19a6b638 100644 --- a/implementations/python/tests/test_sdl_module_registry.py +++ b/implementations/python/tests/test_sdl_module_registry.py @@ -258,15 +258,20 @@ def test_module_exports_are_enforced_for_importers(tmp_path: Path): imports: - source: local:shared.yaml namespace: shared - metrics: - uptime: - type: conditional - condition: shared.health - max-score: 100 + entities: + blue: {role: blue} + objectives: + check: + entity: blue + success: + conditions: [shared.health] """, ) - with pytest.raises(SDLValidationError, match=r"Metric 'uptime' references undefined condition 'shared\.health'"): + with pytest.raises( + SDLValidationError, + match=r"Objective 'check' references undefined condition 'shared\.health' in success criteria", + ): parse_sdl_file(root) @@ -657,6 +662,7 @@ def test_oci_bundle_cache_hit_enforces_root_file_containment(tmp_path: Path): ) +@pytest.mark.integration def test_signed_oci_import_resolution_and_publish_cli(tmp_path: Path): module_path = _local_module(tmp_path / "shared.yaml") runner = CliRunner() @@ -733,6 +739,7 @@ def test_signed_oci_import_resolution_and_publish_cli(tmp_path: Path): ) +@pytest.mark.integration def test_untrusted_and_unsigned_oci_imports_fail_closed(tmp_path: Path): module_path = _local_module(tmp_path / "shared.yaml") unsigned = publish_module_to_oci_layout(module_path, output_dir=tmp_path / "dist") @@ -992,6 +999,7 @@ def _rewrite_config_field(layout_dir: Path, field: str, value: object) -> None: ) +@pytest.mark.integration def test_oci_import_rejects_tampered_config_blob(tmp_path: Path): """A registry serving config bytes that do not hash to manifest ``config.digest`` is rejected before the config JSON is trusted (issue #14, fix 1).""" @@ -1030,6 +1038,7 @@ def test_oci_import_rejects_tampered_config_blob(tmp_path: Path): parse_sdl_file(root) +@pytest.mark.integration def test_oci_import_rejects_root_file_tampering(tmp_path: Path): """A compromised registry cannot repoint ``root_file`` inside an otherwise-signed bundle: ``root_file`` is bound into the signature payload, so altering it while @@ -1079,6 +1088,7 @@ def test_oci_import_rejects_root_file_tampering(tmp_path: Path): parse_sdl_file(root) +@pytest.mark.integration def test_oci_import_rejects_non_string_root_file(tmp_path: Path): """A config declaring a non-string ``root_file`` fails closed with SDLParseError rather than flowing a bad type into the signature payload / extraction and diff --git a/implementations/python/tests/test_sdl_parser.py b/implementations/python/tests/test_sdl_parser.py index a83c6461e..36bf61e75 100644 --- a/implementations/python/tests/test_sdl_parser.py +++ b/implementations/python/tests/test_sdl_parser.py @@ -132,7 +132,7 @@ def test_non_string_top_level_keys_are_rejected_cleanly(self): ${objective_name}: agent: red-agent success: - goals: [pass-exercise] + conditions: [initial-access] """, "objectives.${objective_name}", ), @@ -168,31 +168,21 @@ def test_objectives_section_parses(self): red-agent: entity: red-team actions: [Scan, Exploit] -goals: - pass-exercise: - tlos: [web-defense] -tlos: - web-defense: - evaluation: overall -evaluations: - overall: - metrics: [service-uptime] - min-score: 75 -metrics: - service-uptime: - type: manual - max-score: 100 +conditions: + initial-access: + command: /bin/check + interval: 30 objectives: initial-access: agent: red-agent actions: [Scan] targets: [red-agent] success: - goals: [pass-exercise] + conditions: [initial-access] """ s = parse_sdl(sdl, skip_semantic_validation=True) assert s.objectives["initial-access"].agent == "red-agent" - assert s.objectives["initial-access"].success.goals == ["pass-exercise"] + assert s.objectives["initial-access"].success.conditions == ["initial-access"] assert s.advisories == [] def test_workflows_section_parses(self): @@ -201,25 +191,15 @@ def test_workflows_section_parses(self): entities: blue-team: role: Blue -metrics: - release-check: - type: manual - max-score: 100 -evaluations: - eval-1: - metrics: [release-check] - min-score: 75 -tlos: - tlo-1: - evaluation: eval-1 -goals: - pass-exercise: - tlos: [tlo-1] +conditions: + release-ready: + command: /bin/check + interval: 30 objectives: validate-release: entity: blue-team success: - goals: [pass-exercise] + conditions: [release-ready] workflows: release-response: start: validate @@ -849,52 +829,6 @@ def test_role_shorthand(self): s = parse_sdl(sdl) assert s.nodes["vm"].roles["admin"].username == "admin-user" - def test_min_score_shorthand(self): - sdl = """ -name: test -conditions: - c1: - command: /check - interval: 10 -metrics: - m1: - type: conditional - max-score: 10 - condition: c1 -evaluations: - e1: - metrics: - - m1 - min-score: 75 -""" - s = parse_sdl(sdl, skip_semantic_validation=True) - assert s.evaluations["e1"].min_score.percentage == 75 - - def test_min_score_placeholder_shorthand(self): - sdl = """ -name: test -variables: - pass_pct: - type: integer - default: 75 -conditions: - c1: - command: /check - interval: 10 -metrics: - m1: - type: conditional - max-score: 10 - condition: c1 -evaluations: - e1: - metrics: - - m1 - min-score: ${pass_pct} -""" - s = parse_sdl(sdl) - assert s.evaluations["e1"].min_score.percentage == "${pass_pct}" - def test_entity_facts_keys_preserved(self): sdl = """ name: test @@ -1007,26 +941,16 @@ def test_leaf_enum_placeholders_parse(self): username: admin node: vm password_strength: ${account_strength} -goals: - pass-exercise: - tlos: [tlo-1] -tlos: - tlo-1: - evaluation: eval-1 -evaluations: - eval-1: - metrics: [release-check] - min-score: 75 -metrics: - release-check: - type: manual - max-score: 100 +conditions: + release-ready: + command: /bin/check + interval: 30 objectives: review: entity: blue-team success: mode: ${success_mode} - goals: [pass-exercise] + conditions: [release-ready] """ s = parse_sdl(sdl) assert s.nodes["vm"].os == "${host_os}" @@ -1080,7 +1004,6 @@ def test_switch_rejects_vm_only_fields(self): "nodes.vm.type", "features.svc.type", "content.seed.type", - "metrics.m1.type", "relationships.r1.type", "variables.v1.type", ], @@ -1109,13 +1032,6 @@ def test_discriminant_enums_reject_placeholders(self, field_name): seed: type: ${content_type} target: vm -""", - "metrics.m1.type": """ -name: test -metrics: - m1: - type: ${metric_type} - max-score: 100 """, "relationships.r1.type": """ name: test @@ -1199,10 +1115,12 @@ class TestSkipSemanticValidation: def test_structural_only(self): """skip_semantic_validation=True skips cross-reference checks.""" s = parse_sdl( - "name: test\ngoals:\n g1:\n tlos:\n - missing-tlo", + "name: test\nentities:\n blue:\n role: blue\n" + "objectives:\n obj:\n entity: blue\n success:\n" + " conditions:\n - missing-condition", skip_semantic_validation=True, ) - assert "g1" in s.goals + assert "obj" in s.objectives class TestModuleImports: @@ -2280,3 +2198,80 @@ def test_database_service_variable_substitutes_on_instantiation(self): assert raw.nodes["db-host"].runtime.database_services[0].version == "${pg_version}" instantiated = instantiate_scenario(raw, parameters={"pg_version": "16.13"}) assert instantiated.nodes["db-host"].runtime.database_services[0].version == "16.13" + + +class TestADR073ScoringRemoval: + """Negative-conformance coverage for the OCR scoring removal (ADR-073, SEM-206).""" + + @pytest.mark.parametrize( + "section", + ["metrics", "evaluations", "tlos", "goals"], + ) + def test_removed_top_level_scoring_sections_rejected(self, section): + sdl = f"name: test\n{section}:\n x1: {{}}\n" + with pytest.raises(SDLParseError, match="were removed from the language by ADR-073"): + parse_sdl(sdl) + + def test_migration_message_points_to_success_conditions(self): + with pytest.raises(SDLParseError, match="objectives.\\*.success.conditions"): + parse_sdl("name: test\ngoals:\n g1:\n tlos: [t1]\n") + + def test_agent_reward_calculator_rejected(self): + sdl = """ +name: test +entities: + red-team: + role: red +agents: + red-agent: + entity: red-team + reward_calculator: some-calc +""" + with pytest.raises(SDLParseError): + parse_sdl(sdl) + + def test_entity_tlos_rejected(self): + sdl = """ +name: test +entities: + blue-team: + role: blue + tlos: [t1] +""" + with pytest.raises(SDLParseError): + parse_sdl(sdl) + + @pytest.mark.parametrize("field", ["metrics", "evaluations", "tlos", "goals"]) + def test_objective_success_removed_fields_rejected(self, field): + sdl = f""" +name: test +entities: + blue-team: + role: blue +objectives: + obj: + entity: blue-team + success: + {field}: [x1] +""" + with pytest.raises(SDLParseError): + parse_sdl(sdl) + + def test_objective_success_accepts_conditions_only(self): + sdl = """ +name: test +entities: + blue-team: + role: blue +conditions: + release-ready: + command: /bin/check + interval: 30 +objectives: + obj: + entity: blue-team + success: + conditions: [release-ready] +""" + s = parse_sdl(sdl) + assert s.objectives["obj"].success.conditions == ["release-ready"] diff --git a/implementations/python/tests/test_sdl_realworld.py b/implementations/python/tests/test_sdl_realworld.py index c2dd8ff25..2ec75e4d2 100644 --- a/implementations/python/tests/test_sdl_realworld.py +++ b/implementations/python/tests/test_sdl_realworld.py @@ -416,30 +416,8 @@ def _parse(yaml_str: str, label: str): dns-scored: {command: "dig @localhost example.com +short || exit 1", interval: 60} mysql-scored: {command: "mysqladmin ping || exit 1", interval: 60} -metrics: - http-uptime: {type: CONDITIONAL, max-score: 1000, condition: http-scored} - https-uptime: {type: CONDITIONAL, max-score: 1000, condition: https-scored} - smtp-uptime: {type: CONDITIONAL, max-score: 800, condition: smtp-scored} - ssh-uptime: {type: CONDITIONAL, max-score: 700, condition: ssh-scored} - dns-uptime: {type: CONDITIONAL, max-score: 800, condition: dns-scored} - mysql-uptime: {type: CONDITIONAL, max-score: 700, condition: mysql-scored} - -evaluations: - service-uptime: - metrics: [http-uptime, https-uptime, smtp-uptime, ssh-uptime, dns-uptime, mysql-uptime] - min-score: {absolute: 2500} - -tlos: - maintain-services: - name: "Service Availability" - evaluation: service-uptime - -goals: - defend-infrastructure: - tlos: [maintain-services] - entities: - blue-team: {name: Blue Team, role: Blue, mission: "Maintain services, patch vulnerabilities", tlos: [maintain-services]} + blue-team: {name: Blue Team, role: Blue, mission: "Maintain services, patch vulnerabilities"} red-team: {name: Red Team, role: Red, mission: "Compromise systems, steal PII"} white-team: {name: Scoring Engine, role: White, mission: "Score service availability"} """ @@ -810,36 +788,9 @@ def _parse(yaml_str: str, label: str): scada-hmi-responsive: command: /usr/bin/check-hmi-availability interval: 60 - -metrics: - maintain-scada-availability: - type: CONDITIONAL - max-score: 100 - condition: scada-hmi-responsive - execute-disruption: - type: MANUAL - max-score: 100 - artifact: true - -evaluations: - blue-resilience: - metrics: [maintain-scada-availability] - min-score: 75 - red-impact: - metrics: [execute-disruption] - min-score: {absolute: 100} - -tlos: - sustain-critical-operations: - evaluation: blue-resilience - achieve-scada-disruption: - evaluation: red-impact - -goals: - berylia-goal: - tlos: [sustain-critical-operations] - crimsonia-goal: - tlos: [achieve-scada-disruption] + scada-hmi-disrupted: + command: /usr/bin/check-hmi-disruption + interval: 60 events: disruption-wave: {} @@ -863,7 +814,7 @@ def _parse(yaml_str: str, label: str): entity: crimsonia-red targets: [hmi-controls-power, hmi-controls-water, hmi-server] success: - goals: [crimsonia-goal] + conditions: [scada-hmi-disrupted] window: stories: [main-exercise] scripts: [locked-shields-day-1] @@ -873,7 +824,7 @@ def _parse(yaml_str: str, label: str): entity: berylia-blue.ot-team targets: [hmi-server, plc-power, plc-water] success: - goals: [berylia-goal] + conditions: [scada-hmi-responsive] window: stories: [main-exercise] scripts: [locked-shields-day-1] @@ -919,7 +870,13 @@ def test_scenario_topology_integrity(label, yaml_str): @pytest.mark.parametrize("label,yaml_str", SCENARIOS, ids=[s[0] for s in SCENARIOS]) def test_scenario_stats(label, yaml_str): - """Report scenario complexity metrics.""" + """Every real-world scenario must present a non-trivial topology. + + Each computed metric is exercised by an assertion so a regression that + silently drops a scenario's hosts, networks, feature bindings, or overall + richness is caught, rather than leaving dead metric variables (mirrors the + content-presence checks in test_sdl_stress.py::test_scenario_parses_and_validates). + """ scenario = parse_sdl(textwrap.dedent(yaml_str)) nodes = len([n for n in scenario.nodes.values() if n.type.value == "vm"]) nets = len([n for n in scenario.nodes.values() if n.type.value == "switch"]) @@ -927,8 +884,18 @@ def test_scenario_stats(label, yaml_str): features = len(scenario.features) accts = len(scenario.accounts) rels = len(scenario.relationships) - # Just verify these are non-trivial scenarios - assert nodes >= 1 + + # Every study scenario is a multi-host, networked topology with service + # feature bindings, so these lower bounds hold for the whole parametrized set. + assert nodes >= 1, f"{label}: no VM nodes" + assert nets >= 1, f"{label}: no networks" + assert features >= 1, f"{label}: no feature bindings" + + # Vulnerabilities, accounts, and relationships are legitimately absent from + # some fixtures (e.g. the CCDC defense scenario models none), so they are + # folded into an overall richness floor instead of each asserted >= 1. + total_elements = nodes + nets + vulns + features + accts + rels + assert total_elements >= 10, f"{label}: scenario too sparse ({total_elements} elements)" def test_objectives_are_exercised_in_realworld_suite(): diff --git a/implementations/python/tests/test_sdl_stress.py b/implementations/python/tests/test_sdl_stress.py index dae2d460b..ed60249ea 100644 --- a/implementations/python/tests/test_sdl_stress.py +++ b/implementations/python/tests/test_sdl_stress.py @@ -107,39 +107,10 @@ def _parse(yaml_str: str, label: str): technical: true class: CWE-89 -metrics: - manual-grade: - type: MANUAL - artifact: true - max-score: 10 - auto-grade: - type: CONDITIONAL - max-score: 10 - condition: service-check - -evaluations: - eval-1: - metrics: - - manual-grade - - auto-grade - min-score: 50 - -tlos: - tlo-defense: - name: Web Defense - evaluation: eval-1 - -goals: - goal-1: - tlos: - - tlo-defense - entities: blue-team: name: Blue Team role: Blue - tlos: - - tlo-defense entities: bob: name: Blue Bob @@ -153,8 +124,6 @@ def _parse(yaml_str: str, label: str): from-entity: red-team to-entities: - blue-team - tlos: - - tlo-defense events: attack-event: @@ -454,26 +423,6 @@ def _parse(yaml_str: str, label: str): command: "test -f /home/kali/operations/exfil/loot.tar.gz" interval: 30 -metrics: - exfil-success: - type: CONDITIONAL - max-score: 100 - condition: exfil-check - -evaluations: - data-theft: - metrics: [exfil-success] - min-score: {absolute: 100} - -tlos: - exfiltration: - name: Data Exfiltration - evaluation: data-theft - -goals: - ransack-goal: - tlos: [exfiltration] - entities: attacker: {name: Ransack Operator, role: Red} """ @@ -509,26 +458,6 @@ def _parse(yaml_str: str, label: str): technical: true class: CWE-522 -metrics: - cred-dump-grade: - type: MANUAL - artifact: true - max-score: 100 - -evaluations: - cred-assessment: - metrics: [cred-dump-grade] - min-score: {absolute: 50} - -tlos: - cred-access: - name: Credential Access Techniques - evaluation: cred-assessment - -goals: - atomic-goal: - tlos: [cred-access] - entities: pentester: {name: Penetration Tester, role: Red} """ @@ -723,35 +652,6 @@ def _parse(yaml_str: str, label: str): flag-check-3: command: "/opt/ctf/check_flag.sh level3" interval: 10 - -metrics: - level-1: - type: CONDITIONAL - max-score: 100 - condition: flag-check-1 - level-2: - type: CONDITIONAL - max-score: 200 - condition: flag-check-2 - level-3: - type: CONDITIONAL - max-score: 300 - condition: flag-check-3 - -evaluations: - ctf-eval: - metrics: [level-1, level-2, level-3] - min-score: - absolute: 300 - -tlos: - ctf-skills: - name: CTF Problem Solving - evaluation: ctf-eval - -goals: - complete-ctf: - tlos: [ctf-skills] """ @@ -1286,36 +1186,9 @@ def _parse(yaml_str: str, label: str): enterprise0-compromised: command: /usr/bin/check-enterprise0-compromise interval: 60 - -metrics: - red-access-achieved: - type: CONDITIONAL - max-score: 50 - condition: enterprise0-compromised - blue-detection-report: - type: MANUAL - max-score: 50 - artifact: true - -evaluations: - red-campaign: - metrics: [red-access-achieved] - min-score: {absolute: 50} - blue-response: - metrics: [blue-detection-report] - min-score: 50 - -tlos: - establish-enterprise-foothold: - evaluation: red-campaign - detect-enterprise-compromise: - evaluation: blue-response - -goals: - red-campaign-goal: - tlos: [establish-enterprise-foothold] - blue-response-goal: - tlos: [detect-enterprise-compromise] + enterprise0-detected: + command: /usr/bin/check-enterprise0-detection + interval: 60 accounts: phished-user: @@ -1350,7 +1223,6 @@ def _parse(yaml_str: str, label: str): hosts: [user0] subnets: [user-net] allowed_subnets: [user-net, enterprise-net] - reward_calculator: HybridImpactPwn blue-agent: entity: blue-team.analyst @@ -1360,7 +1232,6 @@ def _parse(yaml_str: str, label: str): hosts: [defender, enterprise0, enterprise1, user0, user1] subnets: [user-net, enterprise-net, op-net] allowed_subnets: [user-net, enterprise-net, op-net] - reward_calculator: HybridAvailabilityConfidentiality green-agent: entity: green-team @@ -1399,7 +1270,7 @@ def _parse(yaml_str: str, label: str): actions: [DiscoverRemoteSystems, ExploitRemoteService, EternalBlue] targets: [enterprise0] success: - goals: [red-campaign-goal] + conditions: [enterprise0-compromised] window: stories: [exercise] scripts: [day-1] @@ -1410,7 +1281,7 @@ def _parse(yaml_str: str, label: str): actions: [Monitor, Analyse] targets: [enterprise0, velociraptor] success: - goals: [blue-response-goal] + conditions: [enterprise0-detected] window: stories: [exercise] scripts: [day-1] @@ -1566,25 +1437,6 @@ def _parse(yaml_str: str, label: str): command: /usr/bin/check-adfs-federation interval: 60 -metrics: - maintain-federation: - type: CONDITIONAL - max-score: 100 - condition: federation-service-up - -evaluations: - federation-health: - metrics: [maintain-federation] - min-score: 75 - -tlos: - sustain-federated-auth: - evaluation: federation-health - -goals: - blue-identity-goal: - tlos: [sustain-federated-auth] - events: federation-cutover: {} @@ -1605,7 +1457,7 @@ def _parse(yaml_str: str, label: str): entity: blue-team targets: [adfs-service, child-trusts-parent] success: - goals: [blue-identity-goal] + conditions: [federation-service-up] window: stories: [federation-exercise] scripts: [identity-day] @@ -1643,9 +1495,9 @@ def test_scenario_parses_and_validates(label, yaml_str): has_stories = bool(scenario.stories) has_entities = bool(scenario.entities) has_vulns = bool(scenario.vulnerabilities) - has_metrics = bool(scenario.metrics) + has_objectives = bool(scenario.objectives) has_content = bool(scenario.content) - assert any([has_nodes, has_features, has_stories, has_entities, has_vulns, has_metrics, has_content]), ( + assert any([has_nodes, has_features, has_stories, has_entities, has_vulns, has_objectives, has_content]), ( f"{label} parsed but has no content" ) diff --git a/implementations/python/tests/test_sdl_validator.py b/implementations/python/tests/test_sdl_validator.py index 4c129599f..939e21d81 100644 --- a/implementations/python/tests/test_sdl_validator.py +++ b/implementations/python/tests/test_sdl_validator.py @@ -397,78 +397,6 @@ def test_valid_feature_dependencies(self): assert not errors -class TestVerifyMetrics: - def test_conditional_metric_references_undefined_condition(self): - s = _make_scenario( - conditions={"c1": {"command": "/bin/check", "interval": 30}}, - metrics={ - "m1": {"type": "conditional", "max_score": 10, "condition": "missing"}, - }, - ) - errors = _validate(s) - assert any("undefined condition" in e for e in errors) - - def test_duplicate_condition_reference(self): - s = _make_scenario( - conditions={"c1": {"command": "/bin/check", "interval": 30}}, - metrics={ - "m1": {"type": "conditional", "max_score": 10, "condition": "c1"}, - "m2": {"type": "conditional", "max_score": 10, "condition": "c1"}, - }, - ) - errors = _validate(s) - assert any("multiple metrics" in e for e in errors) - - -class TestVerifyEvaluations: - def test_references_undefined_metric(self): - s = _make_scenario( - evaluations={ - "e1": {"metrics": ["missing"], "min_score": {"percentage": 50}}, - }, - ) - errors = _validate(s) - assert any("undefined metric" in e for e in errors) - - def test_absolute_min_score_exceeds_max(self): - s = _make_scenario( - conditions={"c1": {"command": "/check", "interval": 10}}, - metrics={"m1": {"type": "conditional", "max_score": 10, "condition": "c1"}}, - evaluations={ - "e1": {"metrics": ["m1"], "min_score": {"absolute": 100}}, - }, - ) - errors = _validate(s) - assert any("exceeds" in e for e in errors) - - -class TestVerifyTLOs: - def test_references_undefined_evaluation(self): - s = _make_scenario( - tlos={"t1": {"evaluation": "missing"}}, - ) - errors = _validate(s) - assert any("undefined evaluation" in e for e in errors) - - -class TestVerifyGoals: - def test_references_undefined_tlo(self): - s = _make_scenario( - goals={"g1": {"tlos": ["missing"]}}, - ) - errors = _validate(s) - assert any("undefined TLO" in e for e in errors) - - -class TestVerifyEntities: - def test_entity_references_undefined_tlo(self): - s = _make_scenario( - entities={"team": {"tlos": ["missing"]}}, - ) - errors = _validate(s) - assert any("undefined TLO" in e for e in errors) - - class TestVerifyInjects: def test_inject_references_undefined_entity(self): s = _make_scenario( @@ -527,8 +455,8 @@ def test_multiple_errors_collected(self): features={ "f1": {"type": "service", "vulnerabilities": ["missing-1"]}, "f2": {"type": "service", "vulnerabilities": ["missing-2"]}, + "f3": {"type": "service", "vulnerabilities": ["missing-3"]}, }, - goals={"g1": {"tlos": ["missing-tlo"]}}, ) errors = _validate(s) assert len(errors) >= 3 @@ -1354,20 +1282,9 @@ def _base_kwargs(self) -> dict: "actions": ["Scan", "Exploit"], }, }, - "metrics": { - "report-quality": { - "type": "manual", - "max_score": 100, - }, - }, - "evaluations": { - "overall": { - "metrics": ["report-quality"], - "min_score": {"percentage": 50}, - }, + "conditions": { + "exercise-passed": {"command": "/bin/check", "interval": 30}, }, - "tlos": {"web-defense": {"evaluation": "overall"}}, - "goals": {"pass-exercise": {"tlos": ["web-defense"]}}, "events": {"attack-wave": {}}, "scripts": { "main-timeline": { @@ -1386,7 +1303,7 @@ def test_undefined_agent(self): objectives={ "obj-1": { "agent": "ghost-agent", - "success": {"goals": ["pass-exercise"]}, + "success": {"conditions": ["exercise-passed"]}, }, }, ) @@ -1399,7 +1316,7 @@ def test_undefined_entity(self): objectives={ "obj-1": { "entity": "ghost-team", - "success": {"goals": ["pass-exercise"]}, + "success": {"conditions": ["exercise-passed"]}, }, }, ) @@ -1413,7 +1330,7 @@ def test_actions_must_be_declared_by_agent(self): "obj-1": { "agent": "red-agent", "actions": ["Persist"], - "success": {"goals": ["pass-exercise"]}, + "success": {"conditions": ["exercise-passed"]}, }, }, ) @@ -1427,7 +1344,7 @@ def test_target_must_resolve(self): "obj-1": { "agent": "red-agent", "targets": ["ghost-target"], - "success": {"goals": ["pass-exercise"]}, + "success": {"conditions": ["exercise-passed"]}, }, }, ) @@ -1443,7 +1360,7 @@ def test_target_rejects_ambiguous_bare_ref(self): "obj-1": { "agent": "red-agent", "targets": ["web"], - "success": {"goals": ["pass-exercise"]}, + "success": {"conditions": ["exercise-passed"]}, }, }, ) @@ -1459,7 +1376,7 @@ def test_targets_accept_section_qualified_refs(self): "obj-1": { "agent": "red-agent", "targets": ["nodes.web", "infrastructure.net"], - "success": {"goals": ["pass-exercise"]}, + "success": {"conditions": ["exercise-passed"]}, }, }, ) @@ -1488,7 +1405,7 @@ def test_targets_can_reference_named_services_and_acls(self): "nodes.web.services.web-https", "infrastructure.net.acls.allow-admin", ], - "success": {"goals": ["pass-exercise"]}, + "success": {"conditions": ["exercise-passed"]}, }, }, relationships={ @@ -1508,12 +1425,12 @@ def test_success_references_must_exist(self): objectives={ "obj-1": { "agent": "red-agent", - "success": {"metrics": ["ghost-metric"]}, + "success": {"conditions": ["ghost-condition"]}, }, }, ) errors = _validate(s) - assert any("undefined metric" in e for e in errors) + assert any("undefined condition" in e for e in errors) def test_window_event_must_belong_to_script(self): kwargs = self._base_kwargs() @@ -1523,7 +1440,7 @@ def test_window_event_must_belong_to_script(self): objectives={ "obj-1": { "agent": "red-agent", - "success": {"goals": ["pass-exercise"]}, + "success": {"conditions": ["exercise-passed"]}, "window": { "scripts": ["main-timeline"], "events": ["cleanup-wave"], @@ -1540,12 +1457,12 @@ def test_dependency_cycle_rejected(self): objectives={ "obj-1": { "agent": "red-agent", - "success": {"goals": ["pass-exercise"]}, + "success": {"conditions": ["exercise-passed"]}, "depends_on": ["obj-2"], }, "obj-2": { "entity": "blue", - "success": {"metrics": ["report-quality"]}, + "success": {"conditions": ["exercise-passed"]}, "depends_on": ["obj-1"], }, }, @@ -1559,7 +1476,7 @@ def test_depends_on_must_reference_defined_objective(self): objectives={ "obj-1": { "agent": "red-agent", - "success": {"goals": ["pass-exercise"]}, + "success": {"conditions": ["exercise-passed"]}, "depends_on": ["ghost-objective"], }, }, @@ -1573,7 +1490,7 @@ def test_window_steps_require_workflows(self): objectives={ "obj-1": { "agent": "red-agent", - "success": {"goals": ["pass-exercise"]}, + "success": {"conditions": ["exercise-passed"]}, "window": {"steps": ["response.validate"]}, }, }, @@ -1587,7 +1504,7 @@ def test_window_steps_must_belong_to_workflow(self): objectives={ "obj-1": { "agent": "red-agent", - "success": {"goals": ["pass-exercise"]}, + "success": {"conditions": ["exercise-passed"]}, "window": { "workflows": ["response"], "steps": ["other.validate"], @@ -1630,7 +1547,7 @@ def test_valid_objective(self): "agent": "red-agent", "actions": ["Scan"], "targets": ["web"], - "success": {"goals": ["pass-exercise"]}, + "success": {"conditions": ["exercise-passed"]}, "window": { "stories": ["exercise"], "scripts": ["main-timeline"], @@ -1639,7 +1556,7 @@ def test_valid_objective(self): }, "report": { "entity": "blue", - "success": {"metrics": ["report-quality"]}, + "success": {"conditions": ["exercise-passed"]}, "depends_on": ["recon"], }, }, @@ -1652,28 +1569,17 @@ class TestVerifyWorkflows: def _base_kwargs(self) -> dict: return { "entities": {"blue": {"role": "blue"}}, - "metrics": { - "report-quality": { - "type": "manual", - "max_score": 100, - }, - }, - "evaluations": { - "overall": { - "metrics": ["report-quality"], - "min_score": {"percentage": 50}, - }, + "conditions": { + "exercise-passed": {"command": "/bin/check", "interval": 30}, }, - "tlos": {"ops-ready": {"evaluation": "overall"}}, - "goals": {"pass-exercise": {"tlos": ["ops-ready"]}}, "objectives": { "validate-release": { "entity": "blue", - "success": {"goals": ["pass-exercise"]}, + "success": {"conditions": ["exercise-passed"]}, }, "rollback-edge": { "entity": "blue", - "success": {"metrics": ["report-quality"]}, + "success": {"conditions": ["exercise-passed"]}, }, }, } @@ -1881,7 +1787,7 @@ def test_valid_switch_and_call_workflow(self): "type": "switch", "cases": [ { - "when": {"goals": ["pass-exercise"]}, + "when": {"objectives": ["validate-release"]}, "next": "delegate", } ], @@ -2645,22 +2551,7 @@ def test_defined_variables_allow_placeholders_across_models(self): "interval": "${check_interval}", } }, - metrics={ - "m1": { - "type": "conditional", - "max_score": "${max_score}", - "condition": "check", - } - }, - evaluations={ - "e1": { - "metrics": ["m1"], - "min_score": {"percentage": "${pass_percentage}"}, - } - }, - tlos={"t1": {"evaluation": "e1"}}, - goals={"g1": {"tlos": ["t1"]}}, - entities={"blue": {"role": "blue", "tlos": ["t1"]}}, + entities={"blue": {"role": "blue"}}, events={"evt": {}}, scripts={ "timeline": { @@ -2710,7 +2601,7 @@ def test_defined_variables_allow_placeholders_across_models(self): "obj": { "agent": "a1", "targets": ["${objective_target}"], - "success": {"goals": ["g1"]}, + "success": {"conditions": ["check"]}, } }, ) @@ -2749,8 +2640,8 @@ def test_vm_without_resources_emits_advisory(self): class TestValidFullScenario: - def test_complete_ocr_scenario_validates(self): - """A complete OCR-style scenario passes validation.""" + def test_complete_scenario_validates(self): + """A complete scenario passes validation (post ADR-073, no scoring sections).""" s = Scenario( name="full-test", nodes={ @@ -2769,12 +2660,8 @@ def test_complete_ocr_scenario_validates(self): }, features={"svc": {"type": "service", "source": {"name": "apache"}}}, conditions={"check": {"command": "/bin/check", "interval": 30}}, - metrics={"m1": {"type": "conditional", "max_score": 10, "condition": "check"}}, - evaluations={"e1": {"metrics": ["m1"], "min_score": {"percentage": 50}}}, - tlos={"t1": {"evaluation": "e1"}}, - goals={"g1": {"tlos": ["t1"]}}, entities={ - "blue": {"role": "blue", "tlos": ["t1"]}, + "blue": {"role": "blue"}, }, ) errors = _validate(s) diff --git a/implementations/python/tests/test_sem_215_participant_outcome_interpretation.py b/implementations/python/tests/test_sem_215_participant_outcome_interpretation.py index e9e9ebc49..ebf3d3ca7 100644 --- a/implementations/python/tests/test_sem_215_participant_outcome_interpretation.py +++ b/implementations/python/tests/test_sem_215_participant_outcome_interpretation.py @@ -60,22 +60,13 @@ def _scenario_yaml() -> str: exfil-detected: command: "test -f /tmp/alert" interval: 10 - metrics: - exfil-score: - type: conditional - condition: exfil-detected - max-score: 10 - evaluations: - exfil-eval: - metrics: [exfil-score] - min-score: {absolute: 10} objectives: exfil-objective: agent: red-agent actions: [scan] targets: [nodes.web.services.http] success: - evaluations: [exfil-eval] + conditions: [exfil-detected] workflows: response-flow: start: verify diff --git a/implementations/python/tests/test_semantics_assessment.py b/implementations/python/tests/test_semantics_assessment.py index 02e4cad8e..07be20094 100644 --- a/implementations/python/tests/test_semantics_assessment.py +++ b/implementations/python/tests/test_semantics_assessment.py @@ -1,364 +1,54 @@ -"""Shared assessment-pipeline semantic tests (SEM-206). - -Exercises the name-level source of truth for the SDL scoring chain -``condition bindings -> metrics -> evaluations -> TLOs -> goals``: -reference resolution, score aggregation, dependency-role derivation, and -fail-closed issue reporting. +"""Assessment resource-kind tests (SEM-206, post ADR-073). + +Per ADR-073 the OCR-inherited SDL scoring chain +(``metric -> evaluation -> TLO -> goal``) was removed from the language. Graded +scoring, reward, and evaluation outputs live in the experiment/evaluator plane +(ADR-055/064/069), not in authored SDL. What remains in +``aces.core.semantics.assessment`` is the resource-kind qualifier that objective +success references carry: objective success references observable state only, so +``CONDITION`` is the sole member. + +These tests pin that reduced surface so a future revival of the removed scoring +sections is a visible, deliberate change rather than an accident. """ from __future__ import annotations -from pathlib import Path -from types import SimpleNamespace - -from aces.core.sdl.parser import parse_sdl_file -from aces.core.semantics.assessment import ( - ASSESSMENT_DEPENDENCY_ROLES, - AssessmentDependencyRole, - AssessmentPipelineAnalysis, - AssessmentResourceKind, - analyze_assessment_pipeline, - partition_assessment_dependencies, -) - - -def _metric(*, condition: object = None, max_score: object = 10) -> SimpleNamespace: - return SimpleNamespace(condition=condition, max_score=max_score) - - -def _evaluation(metrics: list[str], *, absolute: object = None, percentage: object = None) -> SimpleNamespace: - return SimpleNamespace( - metrics=list(metrics), - min_score=SimpleNamespace(absolute=absolute, percentage=percentage), - ) - - -def _tlo(evaluation: str) -> SimpleNamespace: - return SimpleNamespace(evaluation=evaluation) - - -def _goal(tlos: list[str]) -> SimpleNamespace: - return SimpleNamespace(tlos=list(tlos)) - - -def _is_var(value: object) -> bool: - return isinstance(value, str) and value.startswith("${") and value.endswith("}") - - -def _write_assessment_scenario(path: Path, *, namespace: str = "") -> None: - prefix = f"{namespace}." if namespace else "" - path.write_text( - f""" -name: {namespace or "assessment"} -version: 1.0.0 -conditions: - {prefix}health: - command: /bin/true - interval: 15 -metrics: - {prefix}health-metric: - type: conditional - condition: {prefix}health - max-score: 7 - {prefix}manual-metric: - type: manual - max-score: 5 -evaluations: - {prefix}readiness: - metrics: [{prefix}health-metric, {prefix}manual-metric] - min-score: - absolute: 10 -tlos: - {prefix}ready-tlo: - evaluation: {prefix}readiness -goals: - {prefix}ready-goal: - tlos: [{prefix}ready-tlo] -""", - encoding="utf-8", - ) - - -def _write_importing_root(path: Path, imported_name: str, *, namespace: str) -> None: - path.write_text( - f""" -name: root -imports: - - path: {imported_name} - namespace: {namespace} - version: 1.0.0 -""", - encoding="utf-8", - ) - - -def _assessment_analysis_from_file(path: Path) -> AssessmentPipelineAnalysis: - scenario = parse_sdl_file(path) - return analyze_assessment_pipeline( - conditions_by_name=scenario.conditions, - metrics_by_name=scenario.metrics, - evaluations_by_name=scenario.evaluations, - tlos_by_name=scenario.tlos, - goals_by_name=scenario.goals, - ) - - -def _assessment_reference_signature(analysis: AssessmentPipelineAnalysis) -> tuple[tuple[object, ...], ...]: - return tuple( - ( - ref.raw, - ref.source_kind, - ref.source_name, - ref.target_kind, - ref.target_name, - ref.dependency_roles, - ref.namespace_path, - ) - for ref in analysis.references - ) - - -def _strip_shared(name: str) -> str: - return name.removeprefix("shared.") - - -class TestAssessmentPipelineSemantics: - def test_well_formed_pipeline_normalizes_references_and_dependencies(self) -> None: - analysis = analyze_assessment_pipeline( - conditions_by_name={"health": object()}, - metrics_by_name={ - "m1": _metric(condition="health", max_score=10), - "m2": _metric(condition=None, max_score=5), - }, - evaluations_by_name={"e1": _evaluation(["m1", "m2"], absolute=12)}, - tlos_by_name={"t1": _tlo("e1")}, - goals_by_name={"g1": _goal(["t1"])}, - ) - - assert not analysis.has_issues - assert [ - (ref.source_kind, ref.source_name, ref.target_kind, ref.target_name) for ref in analysis.references - ] == [ - (AssessmentResourceKind.METRIC, "m1", AssessmentResourceKind.CONDITION, "health"), - (AssessmentResourceKind.EVALUATION, "e1", AssessmentResourceKind.METRIC, "m1"), - (AssessmentResourceKind.EVALUATION, "e1", AssessmentResourceKind.METRIC, "m2"), - (AssessmentResourceKind.TLO, "t1", AssessmentResourceKind.EVALUATION, "e1"), - (AssessmentResourceKind.GOAL, "g1", AssessmentResourceKind.TLO, "t1"), - ] - for ref in analysis.references: - assert ref.dependency_roles == ASSESSMENT_DEPENDENCY_ROLES - - m1_deps = analysis.dependencies_for(AssessmentResourceKind.METRIC, "m1") - assert m1_deps.ordering_names == ("health",) - assert m1_deps.refresh_names == ("health",) - assert analysis.dependencies_for(AssessmentResourceKind.METRIC, "m2").ordering_names == () - assert analysis.dependencies_for(AssessmentResourceKind.EVALUATION, "e1").ordering_names == ("m1", "m2") - assert analysis.dependencies_for(AssessmentResourceKind.TLO, "t1").ordering_names == ("e1",) - assert analysis.dependencies_for(AssessmentResourceKind.GOAL, "g1").refresh_names == ("t1",) - assert analysis.evaluation_metric_totals == {"e1": 15} - - def test_metric_with_unresolved_condition_is_skipped(self) -> None: - analysis = analyze_assessment_pipeline( - conditions_by_name={}, - metrics_by_name={"m1": _metric(condition="${cond}", max_score=10)}, - evaluations_by_name={}, - tlos_by_name={}, - goals_by_name={}, - is_unresolved=_is_var, - ) - - assert not analysis.has_issues - assert analysis.references == () - assert analysis.dependencies_for(AssessmentResourceKind.METRIC, "m1").ordering_names == () - - def test_undeclared_condition_reference_is_reported(self) -> None: - analysis = analyze_assessment_pipeline( - conditions_by_name={}, - metrics_by_name={"m1": _metric(condition="missing", max_score=10)}, - evaluations_by_name={}, - tlos_by_name={}, - goals_by_name={}, - ) - - assert [issue.code for issue in analysis.issues] == ["metric.condition-undeclared"] - issue = analysis.issues[0] - assert issue.resource_kind == AssessmentResourceKind.METRIC - assert issue.resource_name == "m1" - assert issue.ref == "missing" - assert analysis.references == () - assert analysis.dependencies_for(AssessmentResourceKind.METRIC, "m1").ordering_names == () - - def test_condition_referenced_by_multiple_metrics_is_reported_per_extra(self) -> None: - analysis = analyze_assessment_pipeline( - conditions_by_name={"c": object()}, - metrics_by_name={ - "m1": _metric(condition="c"), - "m2": _metric(condition="c"), - "m3": _metric(condition="c"), - }, - evaluations_by_name={}, - tlos_by_name={}, - goals_by_name={}, - ) - - shared = analysis.issues_of_code("metric.condition-multiply-scored") - assert len(shared) == 2 - assert all( - issue.resource_kind == AssessmentResourceKind.CONDITION and issue.resource_name == "c" for issue in shared - ) - - def test_evaluation_undeclared_metric_is_reported(self) -> None: - analysis = analyze_assessment_pipeline( - conditions_by_name={}, - metrics_by_name={}, - evaluations_by_name={"e1": _evaluation(["nope"])}, - tlos_by_name={}, - goals_by_name={}, - ) - - issues = analysis.issues_of_code("evaluation.metric-undeclared") - assert len(issues) == 1 - assert issues[0].resource_name == "e1" - assert issues[0].ref == "nope" - - def test_evaluation_absolute_min_score_over_metric_total_is_reported(self) -> None: - analysis = analyze_assessment_pipeline( - conditions_by_name={}, - metrics_by_name={"m1": _metric(condition=None, max_score=10)}, - evaluations_by_name={"e1": _evaluation(["m1"], absolute=100)}, - tlos_by_name={}, - goals_by_name={}, - ) - issue = analysis.issues_of_code("evaluation.min-score-exceeds-metric-total")[0] - assert issue.resource_name == "e1" - assert issue.observed == 100 - assert issue.limit == 10 - - ok = analyze_assessment_pipeline( - conditions_by_name={}, - metrics_by_name={"m1": _metric(condition=None, max_score=10)}, - evaluations_by_name={"e1": _evaluation(["m1"], percentage=75)}, - tlos_by_name={}, - goals_by_name={}, - ) - assert not ok.issues_of_code("evaluation.min-score-exceeds-metric-total") - - def test_evaluation_metric_total_unknown_with_var_or_nonint_max_score(self) -> None: - analysis = analyze_assessment_pipeline( - conditions_by_name={}, - metrics_by_name={"m1": _metric(condition=None, max_score="${max}")}, - evaluations_by_name={"e1": _evaluation(["m1", "${m}"], absolute=999)}, - tlos_by_name={}, - goals_by_name={}, - is_unresolved=_is_var, - ) - - assert analysis.evaluation_metric_totals == {"e1": None} - assert not analysis.issues_of_code("evaluation.min-score-exceeds-metric-total") - - def test_tlo_and_goal_undeclared_references_are_reported(self) -> None: - analysis = analyze_assessment_pipeline( - conditions_by_name={}, - metrics_by_name={}, - evaluations_by_name={}, - tlos_by_name={"t1": _tlo("missing-eval")}, - goals_by_name={"g1": _goal(["missing-tlo"])}, - ) - - codes = {issue.code for issue in analysis.issues} - assert "tlo.evaluation-undeclared" in codes - assert "goal.tlo-undeclared" in codes - - def test_issue_iteration_follows_pipeline_order(self) -> None: - analysis = analyze_assessment_pipeline( - conditions_by_name={}, - metrics_by_name={"m1": _metric(condition="missing")}, - evaluations_by_name={"e1": _evaluation(["nope"])}, - tlos_by_name={"t1": _tlo("missing-eval")}, - goals_by_name={"g1": _goal(["missing-tlo"])}, - ) - assert [issue.code for issue in analysis.issues] == [ - "metric.condition-undeclared", - "evaluation.metric-undeclared", - "tlo.evaluation-undeclared", - "goal.tlo-undeclared", - ] - - def test_composition_ready_invariant_layout_variation_preserves_normalized_references_and_aggregation( - self, tmp_path: Path - ) -> None: - flat = tmp_path / "flat.yaml" - imported = tmp_path / "assessment-module.yaml" - root = tmp_path / "root.yaml" - _write_assessment_scenario(flat, namespace="shared") - _write_assessment_scenario(imported) - _write_importing_root(root, imported.name, namespace="shared") - - flat_analysis = _assessment_analysis_from_file(flat) - imported_analysis = _assessment_analysis_from_file(root) - - assert not flat_analysis.has_issues - assert not imported_analysis.has_issues - assert _assessment_reference_signature(imported_analysis) == _assessment_reference_signature(flat_analysis) - assert ( - imported_analysis.evaluation_metric_totals - == flat_analysis.evaluation_metric_totals - == {"shared.readiness": 12} - ) - - def test_composition_ready_invariant_module_expansion_occurs_before_assessment_analysis( - self, tmp_path: Path - ) -> None: - imported = tmp_path / "assessment-module.yaml" - root = tmp_path / "root.yaml" - _write_assessment_scenario(imported) - _write_importing_root(root, imported.name, namespace="shared") - - analysis = _assessment_analysis_from_file(root) - - assert not analysis.has_issues - assert [ref.raw for ref in analysis.references] == [ - "shared.health", - "shared.health-metric", - "shared.manual-metric", - "shared.readiness", - "shared.ready-tlo", - ] - - def test_composition_ready_invariant_namespace_extends_identity_without_changing_kinds_roles_or_aggregation( - self, tmp_path: Path - ) -> None: - plain = tmp_path / "plain.yaml" - namespaced = tmp_path / "namespaced.yaml" - _write_assessment_scenario(plain) - _write_assessment_scenario(namespaced, namespace="shared") - - plain_analysis = _assessment_analysis_from_file(plain) - namespaced_analysis = _assessment_analysis_from_file(namespaced) - - assert [ - (ref.source_kind, ref.target_kind, ref.dependency_roles, ref.namespace_path) - for ref in namespaced_analysis.references - ] == [ - (ref.source_kind, ref.target_kind, ref.dependency_roles, ref.namespace_path) - for ref in plain_analysis.references - ] - assert [ - (_strip_shared(ref.source_name), _strip_shared(ref.target_name)) for ref in namespaced_analysis.references - ] == [(ref.source_name, ref.target_name) for ref in plain_analysis.references] - assert list(namespaced_analysis.evaluation_metric_totals.values()) == list( - plain_analysis.evaluation_metric_totals.values() - ) - - -class TestAssessmentDependencyPartition: - def test_partition_returns_ordering_and_refresh_copies(self) -> None: - ordering, refresh = partition_assessment_dependencies(["a", "b"]) - assert ordering == ("a", "b") - assert refresh == ("a", "b") - assert AssessmentDependencyRole.ORDERING in ASSESSMENT_DEPENDENCY_ROLES - assert AssessmentDependencyRole.REFRESH in ASSESSMENT_DEPENDENCY_ROLES - - def test_partition_handles_empty_inputs(self) -> None: - assert partition_assessment_dependencies([]) == ((), ()) - assert partition_assessment_dependencies(()) == ((), ()) +from enum import Enum + +from aces.core.semantics import assessment as compat_assessment +from aces.core.semantics.assessment import AssessmentResourceKind + + +class TestAssessmentResourceKind: + def test_condition_is_the_only_member(self) -> None: + assert [kind.name for kind in AssessmentResourceKind] == ["CONDITION"] + + def test_condition_value(self) -> None: + assert AssessmentResourceKind.CONDITION.value == "condition" + + def test_is_str_enum(self) -> None: + assert issubclass(AssessmentResourceKind, str) + assert issubclass(AssessmentResourceKind, Enum) + + def test_removed_scoring_symbols_are_gone(self) -> None: + # The OCR scoring pipeline surface (ADR-073) must not reappear. + for removed in ( + "Metric", + "Evaluation", + "TLO", + "Goal", + "analyze_assessment_pipeline", + "AssessmentReference", + "AssessmentIssue", + "AssessmentPipelineAnalysis", + "AssessmentResourceDependencies", + "AssessmentDependencyRole", + "ASSESSMENT_DEPENDENCY_ROLES", + "partition_assessment_dependencies", + ): + assert not hasattr(compat_assessment, removed) + + def test_removed_kinds_are_gone(self) -> None: + for removed in ("METRIC", "EVALUATION", "TLO", "GOAL"): + assert removed not in AssessmentResourceKind.__members__ diff --git a/implementations/python/tests/test_semantics_objectives.py b/implementations/python/tests/test_semantics_objectives.py index 5b0e82e5c..59182adb1 100644 --- a/implementations/python/tests/test_semantics_objectives.py +++ b/implementations/python/tests/test_semantics_objectives.py @@ -5,6 +5,7 @@ from pathlib import Path from types import SimpleNamespace +import pytest from hypothesis import given from hypothesis import strategies as st @@ -191,29 +192,75 @@ def test_window_analysis_reports_fail_closed_issues(self): "step-unbound", } - def test_window_invariant_story_refs_must_resolve(self) -> None: - analysis = _window_analysis(story_refs=["missing-story"]) - - assert _window_issue_codes(analysis) == {"story-unbound"} - - def test_window_invariant_script_refs_must_resolve(self) -> None: - analysis = _window_analysis(script_refs=["missing-script"]) - - assert _window_issue_codes(analysis) == {"script-unbound"} - - def test_window_invariant_event_refs_must_resolve(self) -> None: - analysis = _window_analysis(event_refs=["missing-event"]) - - assert _window_issue_codes(analysis) == {"event-unbound"} - - def test_window_invariant_steps_must_use_workflow_step_syntax(self) -> None: - analysis = _window_analysis( - workflow_refs=["flow"], - step_refs=["bad-step-ref"], - workflows_by_name={"flow": _workflow("start")}, - ) + @pytest.mark.parametrize( + ("kwargs", "expected_code"), + [ + pytest.param({"story_refs": ["missing-story"]}, "story-unbound", id="story-unbound"), + pytest.param({"script_refs": ["missing-script"]}, "script-unbound", id="script-unbound"), + pytest.param({"event_refs": ["missing-event"]}, "event-unbound", id="event-unbound"), + pytest.param( + { + "workflow_refs": ["flow"], + "step_refs": ["bad-step-ref"], + "workflows_by_name": {"flow": _workflow("start")}, + }, + "step-invalid-format", + id="step-invalid-format", + ), + pytest.param({"workflow_refs": ["missing-flow"]}, "workflow-unbound", id="workflow-unbound"), + pytest.param( + { + "workflow_refs": ["flow"], + "step_refs": ["missing-flow.start"], + "workflows_by_name": {"flow": _workflow("start")}, + }, + "step-workflow-unbound", + id="step-workflow-unbound", + ), + pytest.param( + { + "workflow_refs": ["flow"], + "step_refs": ["flow.missing"], + "workflows_by_name": {"flow": _workflow("start")}, + }, + "step-unbound", + id="step-unbound", + ), + pytest.param( + { + "workflow_refs": ["flow"], + "step_refs": ["other.done"], + "workflows_by_name": {"flow": _workflow("start"), "other": _workflow("done")}, + }, + "step-workflow-outside-window", + id="step-workflow-outside-window", + ), + pytest.param( + { + "story_refs": ["intro"], + "script_refs": ["side"], + "stories_by_name": {"intro": SimpleNamespace(scripts=["main"])}, + "scripts_by_name": {"main": SimpleNamespace(events={}), "side": SimpleNamespace(events={})}, + }, + "script-outside-window-stories", + id="script-outside-window-stories", + ), + pytest.param( + { + "script_refs": ["timeline"], + "event_refs": ["cleanup"], + "scripts_by_name": {"timeline": SimpleNamespace(events={"kickoff": 10})}, + "events_by_name": {"cleanup": SimpleNamespace()}, + }, + "event-outside-window-scripts", + id="event-outside-window-scripts", + ), + ], + ) + def test_window_invariant_reference_must_resolve(self, kwargs, expected_code) -> None: + analysis = _window_analysis(**kwargs) - assert _window_issue_codes(analysis) == {"step-invalid-format"} + assert _window_issue_codes(analysis) == {expected_code} def test_window_invariant_steps_require_workflow_window(self) -> None: analysis = _window_analysis( @@ -223,58 +270,6 @@ def test_window_invariant_steps_require_workflow_window(self) -> None: assert "step-requires-workflow-window" in _window_issue_codes(analysis) - def test_window_invariant_workflow_refs_must_resolve(self) -> None: - analysis = _window_analysis(workflow_refs=["missing-flow"]) - - assert _window_issue_codes(analysis) == {"workflow-unbound"} - - def test_window_invariant_step_workflow_must_resolve(self) -> None: - analysis = _window_analysis( - workflow_refs=["flow"], - step_refs=["missing-flow.start"], - workflows_by_name={"flow": _workflow("start")}, - ) - - assert _window_issue_codes(analysis) == {"step-workflow-unbound"} - - def test_window_invariant_step_name_must_resolve_within_workflow(self) -> None: - analysis = _window_analysis( - workflow_refs=["flow"], - step_refs=["flow.missing"], - workflows_by_name={"flow": _workflow("start")}, - ) - - assert _window_issue_codes(analysis) == {"step-unbound"} - - def test_window_invariant_step_workflow_must_be_inside_workflow_window(self) -> None: - analysis = _window_analysis( - workflow_refs=["flow"], - step_refs=["other.done"], - workflows_by_name={"flow": _workflow("start"), "other": _workflow("done")}, - ) - - assert _window_issue_codes(analysis) == {"step-workflow-outside-window"} - - def test_window_invariant_explicit_scripts_must_be_inside_story_window(self) -> None: - analysis = _window_analysis( - story_refs=["intro"], - script_refs=["side"], - stories_by_name={"intro": SimpleNamespace(scripts=["main"])}, - scripts_by_name={"main": SimpleNamespace(events={}), "side": SimpleNamespace(events={})}, - ) - - assert _window_issue_codes(analysis) == {"script-outside-window-stories"} - - def test_window_invariant_events_must_be_inside_reachable_script_window(self) -> None: - analysis = _window_analysis( - script_refs=["timeline"], - event_refs=["cleanup"], - scripts_by_name={"timeline": SimpleNamespace(events={"kickoff": 10})}, - events_by_name={"cleanup": SimpleNamespace()}, - ) - - assert _window_issue_codes(analysis) == {"event-outside-window-scripts"} - def test_composition_ready_invariant_imported_window_analysis_uses_expanded_canonical_identities( self, tmp_path: Path ) -> None: @@ -373,13 +368,9 @@ def test_workflow_step_normalization_is_stable(self, step_refs: list[str]): assert analysis.workflow_step_refs == tuple(dict.fromkeys(step_refs)) -def _success(*, conditions=None, metrics=None, evaluations=None, tlos=None, goals=None, mode="all_of"): +def _success(*, conditions=None, mode="all_of"): return SimpleNamespace( conditions=list(conditions or []), - metrics=list(metrics or []), - evaluations=list(evaluations or []), - tlos=list(tlos or []), - goals=list(goals or []), mode=mode, ) @@ -424,10 +415,6 @@ def _analyze(objectives, **overrides): section_defaults = { "conditions_by_name": {}, - "metrics_by_name": {}, - "evaluations_by_name": {}, - "tlos_by_name": {}, - "goals_by_name": {}, "stories_by_name": {}, "scripts_by_name": {}, "events_by_name": {}, @@ -440,10 +427,6 @@ def _analyze(objectives, **overrides): "entity_names": set(), "assessment_resources": AssessmentResourceCatalog( conditions=sections["conditions_by_name"], - metrics=sections["metrics_by_name"], - evaluations=sections["evaluations_by_name"], - tlos=sections["tlos_by_name"], - goals=sections["goals_by_name"], ), "window_resources": WindowResourceCatalog( stories=sections["stories_by_name"], @@ -461,20 +444,19 @@ class TestObjectiveSemantics: def test_well_formed_objectives_normalize_references_and_dependencies(self) -> None: analysis = _analyze( { - "base": _objective(entity="blue", success=_success(metrics=["m1"])), + "base": _objective(entity="blue", success=_success(conditions=["c1"])), "follow": _objective( agent="red", actions=["Scan"], targets=["nodes.web"], - success=_success(goals=["g1"]), + success=_success(conditions=["c2"]), window=_window(workflows=["flow"], steps=["flow.branch"]), depends_on=["base"], ), }, agents_by_name={"red": _agent("Scan", "Exploit")}, entity_names={"blue"}, - metrics_by_name={"m1": object()}, - goals_by_name={"g1": object()}, + conditions_by_name={"c1": object(), "c2": object()}, workflows_by_name={"flow": _workflow("start", "branch")}, targetable_name_index={"nodes.web": {"nodes.web"}}, ) @@ -487,15 +469,15 @@ def test_well_formed_objectives_normalize_references_and_dependencies(self) -> N "nodes.web" } assert {ref.canonical_name for ref in analysis.references_of_kind(ObjectiveReferenceKind.SUCCESS)} == { - "metric.m1", - "goal.g1", + "condition.c1", + "condition.c2", } success_kinds = { ref.canonical_name: ref.success_resource_kind for ref in analysis.references_of_kind(ObjectiveReferenceKind.SUCCESS) } - assert success_kinds["metric.m1"] == AssessmentResourceKind.METRIC - assert success_kinds["goal.g1"] == AssessmentResourceKind.GOAL + assert success_kinds["condition.c1"] == AssessmentResourceKind.CONDITION + assert success_kinds["condition.c2"] == AssessmentResourceKind.CONDITION assert {ref.canonical_name for ref in analysis.references_of_kind(ObjectiveReferenceKind.WINDOW)} == { "flow", "flow.branch", @@ -515,10 +497,10 @@ def test_well_formed_objectives_normalize_references_and_dependencies(self) -> N for ref in analysis.references_of_kind(kind): assert ref.dependency_roles == () - assert analysis.dependencies_for("base").ordering_names == ("metric.m1",) - assert analysis.dependencies_for("base").refresh_names == ("metric.m1",) - assert analysis.dependencies_for("follow").ordering_names == ("goal.g1", "objective.base") - assert analysis.dependencies_for("follow").refresh_names == ("goal.g1", "objective.base", "workflow.flow") + assert analysis.dependencies_for("base").ordering_names == ("condition.c1",) + assert analysis.dependencies_for("base").refresh_names == ("condition.c1",) + assert analysis.dependencies_for("follow").ordering_names == ("condition.c2", "objective.base") + assert analysis.dependencies_for("follow").refresh_names == ("condition.c2", "objective.base", "workflow.flow") assert "follow" in analysis.window_analyses def test_undeclared_actor_references_are_reported(self) -> None: @@ -564,30 +546,18 @@ def test_ambiguous_target_is_reported_with_sorted_candidates(self) -> None: assert issue.ref == "web" assert issue.candidates == ("features.web", "nodes.web") - def test_undeclared_success_references_are_reported_per_kind(self) -> None: + def test_undeclared_success_condition_is_reported(self) -> None: analysis = _analyze( { "a": _objective( entity="blue", - success=_success( - conditions=["c?"], - metrics=["m?"], - evaluations=["e?"], - tlos=["t?"], - goals=["g?"], - ), + success=_success(conditions=["c?"]), ) }, entity_names={"blue"}, ) codes = {issue.code for issue in analysis.issues} - assert { - "objective.success-condition-undeclared", - "objective.success-metric-undeclared", - "objective.success-evaluation-undeclared", - "objective.success-tlo-undeclared", - "objective.success-goal-undeclared", - } <= codes + assert "objective.success-condition-undeclared" in codes def test_window_issues_are_resurfaced_under_objective_codes(self) -> None: analysis = _analyze( @@ -633,7 +603,7 @@ def test_unresolved_variable_references_are_skipped(self) -> None: agent="${actor}", actions=["${act}"], targets=["${tgt}"], - success=_success(metrics=["${m}"]), + success=_success(conditions=["${m}"]), window=_window(stories=["${story}"]), depends_on=["${dep}"], ) diff --git a/specs/formal/assessment/README.md b/specs/formal/assessment/README.md index 3fd6b0737..018e89f87 100644 --- a/specs/formal/assessment/README.md +++ b/specs/formal/assessment/README.md @@ -1,67 +1,44 @@ -# Assessment Pipeline Semantics - -This directory holds the formal artifacts for the SDL assessment-pipeline -semantics — the scoring chain -`condition bindings -> metrics -> evaluations -> TLOs -> goals` and its -relationship to declarative objectives — under `SEM-206`. - -## Scope - -- normalized reference resolution along the scoring chain (each metric's - condition, each evaluation's metrics, each TLO's evaluation, each goal's - TLOs), and the cross-resource reference constraints they must satisfy -- score aggregation: an evaluation's metric-max-score total and the - absolute-min-score-vs-total consistency rule -- the "at most one metric per condition" exclusivity rule for conditional - metrics -- dependency-role derivation: every scoring-chain edge is both an ordering - edge (the downstream resource is computed after its inputs) and a refresh - edge (the downstream resource recomputes when any input changes), so a - single source of truth feeds the validator, the compiler's - `ordering_dependencies` / `refresh_dependencies`, and the planner's - ordering/refresh reconciliation -- fail-closed behavior for missing, ambiguous (binding-level), or - out-of-scope references -- composition-ready invariants: normalized references are independent of - source-file layout; module/import expansion must run before this analysis - -`SEM-206` does not define a scoring algorithm, publish a new contract version, -add evaluator capabilities, or define evidence/provenance semantics — those are -governed elsewhere (the runtime result/execution contracts, evaluator -capabilities, the concept-authority stack). - -## Implementation Mapping - -- shared name-level semantic source of truth: - `implementations/python/packages/aces_sdl/semantics/assessment.py` -- authoring models (closed Pydantic shape; manual/conditional metric rules; - `min-score` shorthand; `Condition` command-xor-source): - - `implementations/python/packages/aces_sdl/scoring.py` - - `implementations/python/packages/aces_sdl/conditions.py` -- semantic validation: `implementations/python/packages/aces_sdl/validator/` - (`_verify_assessment_pipeline`) -- compiled runtime addresses, contracts, and ordering/refresh derivation: - - `implementations/python/packages/aces_processor/compiler.py` - - `implementations/python/packages/aces_processor/models.py` -- planner ordering/refresh reconciliation over the compiled edges: - - `implementations/python/packages/aces_processor/planner.py` - - `implementations/python/packages/aces_processor/semantics/planner.py` -- runtime evaluator-result and execution contracts (the execution/observation - realization): `EvaluationResultContract`, `EvaluationExecutionContract`, - `EvaluationExecutionState`, `EvaluationHistoryEvent`, - `validate_evaluation_result()` in - `implementations/python/packages/aces_processor/models.py`; - evaluator-result / history-event schemas in - `implementations/python/packages/aces_contracts/contracts.py` -- implementation-facing guardrails note (preflight): - `docs/explain/reference/assessment-semantics.md` (governed by ADR-016) - -## Tests - -- `implementations/python/tests/test_semantics_assessment.py` -- `implementations/python/tests/test_fm2_semantics.py` - (`TestAssessmentPipelineAgreement`) -- `implementations/python/tests/test_sdl_validator.py` -- `implementations/python/tests/test_sdl_models.py` -- `implementations/python/tests/test_runtime_models.py` -- `implementations/python/tests/test_runtime_planner.py` +# Assessment Semantics (SDL scoring chain removed) + +[ADR-073](../../../docs/decisions/adrs/adr-073-scoring-reward-language-scope.md) +removed the OCR-inherited SDL scoring/assessment pipeline. The graded chain that +these formal artifacts once specified — +`condition bindings -> metrics -> evaluations -> TLOs -> goals` — is no longer an +SDL authoring surface: the `metrics`, `evaluations`, `tlos` (Training Learning +Objectives), and `goals` sections and the CybORG `agents.reward_calculator` +label are gone. + +## What remains in the SDL + +- `conditions` are **observable state** and stay a first-class SDL surface. They + compile onto runtime `evaluation.condition.*` addresses. +- `objectives` are participant intent and stay first-class. An objective's + `success` references **only** `conditions` (observable state), not a graded + score (see [`../objectives/`](../objectives/README.md) for the objective-success + semantics under `SEM-207`). +- Workflow predicates reference `conditions`. + +There is no condition→metric→evaluation→TLO→goal scoring chain, no score +aggregation rule, no per-condition metric-exclusivity rule, and no +scoring-chain ordering/refresh derivation in the SDL. + +## Where graded scoring now lives + +Graded scoring, cumulative reward, pass/fail evaluation, leaderboard values, and +evaluation outputs live exclusively in the experiment/evaluator plane, never as +authored SDL: + +- experiment-core contracts + ([ADR-055](../../../docs/decisions/adrs/adr-055-experiment-core-contract-boundary.md)): + `experiment-task-v1` metric definitions, `experiment-study-v1` analysis plans; +- evidence/measure contracts + ([ADR-064](../../../docs/decisions/adrs/adr-064-experiment-evidence-and-measure-contract-boundary.md)): + `experiment-evidence-record-v1`, `experiment-derived-measure-v1`; +- the backend **Evaluator** + ([ADR-069](../../../docs/decisions/adrs/adr-069-cage-2-replication-architecture.md) + §3), which projects reward, objective, terminal-condition, and scoring facts + into ACES evaluation results, evidence records, and derived measures. + +The governing requirement remains **SEM-206 (Assessment Semantics)**; +[`../../../docs/explain/reference/assessment-semantics.md`](../../../docs/explain/reference/assessment-semantics.md) +carries the implementation-facing reference for the narrowed semantics. diff --git a/specs/formal/assessment/pipeline-consistency.md b/specs/formal/assessment/pipeline-consistency.md index 14b0da47a..4df0c0fec 100644 --- a/specs/formal/assessment/pipeline-consistency.md +++ b/specs/formal/assessment/pipeline-consistency.md @@ -1,116 +1,37 @@ -# Assessment Pipeline Consistency +# Assessment Pipeline Consistency (removed) -## Reference Model - -The assessment pipeline is the scoring chain +[ADR-073](../../../docs/decisions/adrs/adr-073-scoring-reward-language-scope.md) +removed the SDL scoring/assessment pipeline. This spec formerly defined the +consistency rules for the graded chain ```text condition bindings -> metrics -> evaluations -> TLOs -> goals ``` -with declarative objectives binding actors/targets/success criteria onto any of -those resources (objective and predicate references into the chain are governed -together with objective-window semantics, not restated here). - -Every cross-resource reference along the chain is resolved into a single -normalized internal shape (`AssessmentReference`) before compiler/planner -semantics run. Each resolved reference carries: - -- raw author-facing text and the canonical source/target names -- the source and target resource kinds (`metric -> condition`, - `evaluation -> metric`, `tlo -> evaluation`, `goal -> tlo`) -- the dependency-role set the edge carries -- a namespace-extensible path slot reserved for future module/import work - -The analysis is name-level. *Which* condition addresses a metric binds to — -i.e. which VM nodes realize the condition — is resolved at compilation; an -undeclared metric/evaluation/TLO is reported here, while an unbound or -ambiguous *condition binding* is reported by the compiler against the resolved -addresses. - -## Consistency Rules - -- a conditional metric's `condition` must name a declared condition -- at most one metric may be scored by a given condition (no two conditional - metrics share a `condition`) -- each evaluation's `metrics` entries must name declared metrics -- an evaluation whose `min-score` is given as an absolute value must not exceed - the sum of its referenced metrics' `max-score` values, when every - contributing `max-score` is a known integer (a percentage `min-score`, an - unresolved `${var}` `max-score`, or a non-integer `max-score` makes the total - unknown and the check is skipped, fail-open on the *aggregate* but never on - the *references*) -- each TLO's `evaluation` must name a declared evaluation -- each goal's `tlos` entries must name declared TLOs -- `${var}` placeholders only ever substitute values; a reference that is still - an unresolved placeholder is skipped (re-checked after instantiation), never - treated as a literal name - -## Aggregation Semantics - -- `evaluation_metric_totals[e]` is the sum of the integer `max-score` values of - `e`'s resolvable referenced metrics, or `None` if any contributing - `max-score` is unknown -- the metric-result contract reports a `score` against a `fixed_max_score` - (when the metric declares an integer `max-score`); condition-binding, - evaluation, TLO, goal, and objective result contracts report `passed`. The - runtime evaluator-result/execution contracts - (`EvaluationResultContract` / `EvaluationExecutionContract` / - `validate_evaluation_result()`) are the fail-closed boundary that enforces - this — a future aggregation rule compiles into the contract or a governed - contract version, not into backend-private convention - -## Dependency and Refresh Semantics - -- every scoring-chain edge carries both an **ordering** role and a **refresh** - role: a resource is computed after its inputs, and recomputed when any input - changes. This single fact (`ASSESSMENT_DEPENDENCY_ROLES`) feeds: - - the validator (the reference checks above) - - the compiler's `ordering_dependencies` / `refresh_dependencies` for - `MetricRuntime` / `EvaluationRuntime` / `TLORuntime` / `GoalRuntime` - (derived via `partition_assessment_dependencies`) - - the planner's generic ordering/refresh reconciliation - (`reconcile_resource_actions` / `refresh_impacted_nodes`) -- consequently a change to a condition binding propagates as a refresh through - `metric -> evaluation -> TLO -> goal`; the validator, the compiler, and the - planner derive that propagation from one shared model rather than separate - per-stage logic - -## Fail-Closed Cases - -- a conditional metric referencing an undeclared condition -- two conditional metrics scoring the same condition -- an evaluation referencing an undeclared metric -- an absolute evaluation `min-score` above the known metric-max-score total -- a TLO referencing an undeclared evaluation -- a goal referencing an undeclared TLO -- (at compilation) a metric whose condition resolves to no bound node, or to - more than one — emitted as `evaluation.condition-ref-unbound` / - `evaluation.condition-ref-ambiguous` against the resolved addresses - -## Composition-Ready Invariants - -- normalized references are independent of source-file layout -- future module/import expansion must occur before this analysis runs -- namespacing may extend a reference's identity, but must not change: - - source/target resource kinds - - dependency-role semantics - - the aggregation rule - -## Implementation Mapping - -- shared semantic source of truth: - `implementations/python/packages/aces_sdl/semantics/assessment.py` -- validator checks: `implementations/python/packages/aces_sdl/validator/` -- compiled runtime references, contracts, and ordering/refresh derivation: - - `implementations/python/packages/aces_processor/compiler.py` - - `implementations/python/packages/aces_processor/models.py` -- planner reconciliation: - - `implementations/python/packages/aces_processor/planner.py` - - `implementations/python/packages/aces_processor/semantics/planner.py` -- differential and cross-stage tests: - - `implementations/python/tests/test_semantics_assessment.py` - (`test_composition_ready_invariant_layout_variation_preserves_normalized_references_and_aggregation`, - `test_composition_ready_invariant_module_expansion_occurs_before_assessment_analysis`, - `test_composition_ready_invariant_namespace_extends_identity_without_changing_kinds_roles_or_aggregation`) - - `implementations/python/tests/test_fm2_semantics.py` +— reference resolution along the chain, the "at most one metric per condition" +exclusivity rule, the `min-score` / metric-`max-score` aggregation rule, and the +scoring-chain ordering/refresh derivation. None of those surfaces +(`metrics`, `evaluations`, `tlos`, `goals`) exist in the SDL any longer, so none +of those consistency rules apply. + +## What is left + +`conditions` remain the SDL's **observable state**. A condition compiles onto a +runtime `evaluation.condition.*` address, and its consistency (a condition +binding that resolves to no bound node, or to more than one) is still enforced at +compilation as `evaluation.condition-ref-unbound` / +`evaluation.condition-ref-ambiguous` against the resolved addresses. Objective +`success` and workflow predicates reference `conditions` only; the +objective-success reference model is in +[`../objectives/declarative-objective-semantics.md`](../objectives/declarative-objective-semantics.md). + +## Where graded scoring now lives + +Graded scoring, reward, and evaluation outputs are an experiment/evaluator-plane +concern — experiment-core contracts +([ADR-055](../../../docs/decisions/adrs/adr-055-experiment-core-contract-boundary.md)), +the evidence/measure contracts +([ADR-064](../../../docs/decisions/adrs/adr-064-experiment-evidence-and-measure-contract-boundary.md)), +and the backend Evaluator +([ADR-069](../../../docs/decisions/adrs/adr-069-cage-2-replication-architecture.md) +§3) — never authored SDL. diff --git a/specs/formal/objectives/README.md b/specs/formal/objectives/README.md index bf164600e..701b0ae56 100644 --- a/specs/formal/objectives/README.md +++ b/specs/formal/objectives/README.md @@ -12,8 +12,9 @@ target-resolution, success-interpretation, and dependency-ordering semantics - target resolution through the targetable named-reference index (bare or section-qualified), fail-closed on missing/ambiguous references - success interpretation: `mode` (`all_of` / `any_of`) over referenced - conditions, metrics, evaluations, TLOs, and goals; the assessment pipeline - stays the authority for upstream ordering/refresh among those resources + `conditions` (observable state only; the OCR scoring surfaces `metrics`, + `evaluations`, `tlos`, and `goals` were removed by + [ADR-073](../../../docs/decisions/adrs/adr-073-scoring-reward-language-scope.md)) - windows: normalized story/script/event/workflow/workflow-step reference resolution; consistency between `window.stories`, `window.scripts`, `window.events`, `window.workflows`, and `window.steps`; reachability diff --git a/specs/formal/objectives/declarative-objective-semantics.md b/specs/formal/objectives/declarative-objective-semantics.md index 0f0e706d5..d8ed1702f 100644 --- a/specs/formal/objectives/declarative-objective-semantics.md +++ b/specs/formal/objectives/declarative-objective-semantics.md @@ -8,7 +8,7 @@ Objective semantics are declarative SDL meaning. They answer: - which declared actor owns the objective - which declared scenario elements the objective names as targets -- which assessment resources define success +- which observable `conditions` define success - which window bounds when the objective matters - which other objectives must precede it @@ -26,7 +26,8 @@ The implementation must build on these existing authorities: rejection, and `SDLParseError` - static validation: `SemanticValidator` and `SDLValidationError` - objective-window analysis: `aces_sdl.semantics.objectives` -- assessment pipeline semantics: `aces_sdl.semantics.assessment` +- condition resolution: the targetable named-reference index over declared + `conditions` - runtime compilation: `compile_runtime_model()` and `aces_processor.models.ObjectiveRuntime` - dependency graph semantics: `aces_processor.semantics.planner` @@ -58,14 +59,16 @@ Target resolution: Success interpretation: -- success references only declared conditions, metrics, evaluations, TLOs, and - goals -- the assessment pipeline remains the authority for upstream ordering and - refresh dependencies among those resources -- objective success mode describes interpretation of referenced success - resources; it must not encode evaluator implementation mechanics +- success references only declared `conditions` (observable state). The OCR + scoring surfaces `metrics`, `evaluations`, `tlos`, and `goals` were removed by + [ADR-073](../../../docs/decisions/adrs/adr-073-scoring-reward-language-scope.md); + objective success is no longer expressible against a graded score +- objective success mode describes interpretation of the referenced conditions; + it must not encode evaluator implementation mechanics - runtime result and execution contracts remain the portable observation boundary for evaluated success +- graded scoring, reward, and evaluation outputs are an experiment/evaluator-plane + concern (ADR-055/064/069), never authored SDL success criteria Windows: @@ -93,10 +96,9 @@ Dependency roles (which references propagate through the planner): compiles actor or target into runtime addresses lifts the role constant in lockstep - the derived per-objective `ordering_names` / `refresh_names` tuples are - kind-qualified (`condition.`, `metric.`, `evaluation.`, `tlo.`, - `goal.`, `objective.`, `story.`, `script.`, `event.`, - `workflow.`) so a metric and a condition with the same SDL name remain - distinguishable + kind-qualified (`condition.`, `objective.`, `story.`, `script.`, + `event.`, `workflow.`) so a condition and an objective with the same SDL + name remain distinguishable ## Cross-Cutting Gates @@ -151,7 +153,7 @@ Avoid: - a second objective schema beside `aces_sdl.objectives` - a second reference resolver beside `SemanticValidator`'s named-reference index or the compiler's canonical address helpers -- a second assessment pipeline or objective-dependency graph implementation +- a second objective-dependency graph implementation - mixing objective actor binding with participant episode lifecycle or apparatus realization - treating objective targets as backend execution targets diff --git a/specs/sdl/observability-and-evidence.md b/specs/sdl/observability-and-evidence.md index 62d6a0181..7ec0b64b4 100644 --- a/specs/sdl/observability-and-evidence.md +++ b/specs/sdl/observability-and-evidence.md @@ -70,8 +70,7 @@ Authored evidence requirements: - MAY reference a scenario-native observability system as a source; - MAY map to `experiment-capture-spec-v1` concepts when executable capture contracts are generated; -- MUST remain independent of participant objectives, metrics, evaluations, - TLOs, and goals; +- MUST remain independent of participant `objectives`; - MUST NOT be objective targets or implied by objective success criteria; - MUST remain distinct from `experiment-evidence-record-v1` raw evidence; and - MUST remain distinct from `experiment-derived-measure-v1` interpreted diff --git a/specs/sdl/references.md b/specs/sdl/references.md index b64a911cd..4edf92480 100644 --- a/specs/sdl/references.md +++ b/specs/sdl/references.md @@ -46,7 +46,7 @@ the `wazuh.manager` node, not a `wazuh` node with a `manager` member. 1. A reference **MUST** resolve to **exactly one** declared element of a kind the referencing field accepts. 2. A field defines its **candidate set** — the section or sections a value may - name. Some fields accept a single section (e.g. `metric.condition_ref` → + name. Some fields accept a single section (e.g. an objective's `success` → `conditions`); others accept a set of targetable sections (e.g. an objective's `target`, a relationship's `source`/`target`). The candidate set is part of each field's definition and is reflected in the edge catalog (§5). @@ -98,24 +98,18 @@ validation pass rather than one-at-a-time ([diagnostics.md](diagnostics.md)). Each row is a reference edge: a source section's field names a target. Unless noted, an unresolved (dangling) or ambiguous reference is a fatal error. -### Assessment pipeline - -| Source | Field | Target | -|--------|-------|--------| -| `metrics` | condition ref | `conditions` | -| `evaluations` | metric refs | `metrics` | -| `tlos` | evaluation refs | `evaluations` | -| `goals` | tlo refs | `tlos` | - -A condition referenced by a metric MUST be scored by exactly one metric; -an evaluation's minimum score MUST NOT exceed the sum of its metrics' maxima. +The SDL carries no graded scoring pipeline: the OCR-inherited `metrics`, +`evaluations`, `tlos`, and `goals` sections were removed with +[ADR-073](../../docs/decisions/adrs/adr-073-scoring-reward-language-scope.md), so +no reference edge targets them. Graded scoring, reward, and evaluation outputs +live in the experiment/evaluator plane (ADR-055/064/069). `conditions` remain the +observable-state target for objective success and workflow predicates. ### Narrative chain | Source | Field | Target | |--------|-------|--------| | `injects` | from/to entity | `entities` | -| `injects` | tlo refs | `tlos` | | `events` | condition refs | `conditions` | | `events` | inject refs | `injects` | | `scripts` | event refs | `events` | @@ -127,7 +121,7 @@ an evaluation's minimum score MUST NOT exceed the sum of its metrics' maxima. |--------|-------|--------| | `features` | vulnerability refs | `vulnerabilities` | | `features` | dependencies | `features` (acyclic) | -| `entities` | tlos / vulnerabilities | `tlos` / `vulnerabilities` | +| `entities` | vulnerabilities | `vulnerabilities` | | `nodes` | feature/condition/inject/vulnerability refs | `features` / `conditions` / `injects` / `vulnerabilities` | | `infrastructure` | node / link / dependency | `nodes` / switch-backed `infrastructure` | | `content` | target | `nodes` (VM) | @@ -149,11 +143,11 @@ an evaluation's minimum score MUST NOT exceed the sum of its metrics' maxima. | `objectives` | actor | `agents` or flattened `entities` | | `objectives` | action | the bound agent's `action_contracts` | | `objectives` | target | targetable elements (excl. `variables`/`objectives`/`workflows`) | -| `objectives` | success criteria | `conditions`/`metrics`/`evaluations`/`tlos`/`goals` | +| `objectives` | success criteria | `conditions` (observable state only, [ADR-073](../../docs/decisions/adrs/adr-073-scoring-reward-language-scope.md)) | | `objectives` | window | `stories`/`scripts`/`events`/`workflows` (with closure rules) | | `objectives` | depends_on | `objectives` (acyclic) | -| `outcome_interpretation_rules` | source | `action_contracts`/`objectives`/`workflows`/`evaluations` | -| `outcome_interpretation_rules` | target | `objectives`/`workflows`/`evaluations` | +| `outcome_interpretation_rules` | source | `action_contracts`/`objectives`/`workflows` | +| `outcome_interpretation_rules` | target | `objectives`/`workflows` | ### Observability and evidence authoring @@ -177,7 +171,7 @@ element is the source or channel. | `workflows` | start | own steps | | `workflows` | step successors (`on_success`/`on_failure`) | own steps | | `workflows` | compensation | other `workflows` | -| `workflows` | predicate assessment refs | `conditions`/`metrics`/`evaluations`/`tlos`/`goals` | +| `workflows` | predicate condition refs | `conditions` (observable state) | | `workflows` | predicate step refs | own steps (executable) | Parallel/join control flow MUST be closed: every branch reaches its join and no diff --git a/specs/sdl/sections.md b/specs/sdl/sections.md index 2ab000f81..fb1b65906 100644 --- a/specs/sdl/sections.md +++ b/specs/sdl/sections.md @@ -47,12 +47,8 @@ and defaults to an empty map when omitted. | `features` | optional | identifier | `vulnerabilities`; other `features` (dependencies, acyclic) | | `conditions` | optional | identifier | — | | `vulnerabilities` | optional | identifier | — | -| `metrics` | optional | identifier | `conditions` | -| `evaluations` | optional | identifier | `metrics` | -| `tlos` | optional | identifier | `evaluations` | -| `goals` | optional | identifier | `tlos` | -| `entities` | optional | identifier | `tlos`, `vulnerabilities` | -| `injects` | optional | identifier | `entities`, `tlos` | +| `entities` | optional | identifier | `vulnerabilities` | +| `injects` | optional | identifier | `entities` | | `events` | optional | identifier | `conditions`, `injects` | | `scripts` | optional | identifier | `events` | | `stories` | optional | identifier | `scripts` | @@ -62,10 +58,10 @@ and defaults to an empty map when omitted. | `agents` | optional | identifier | `entities`, `accounts`, `infrastructure`, `nodes`, `conditions`, `action_contracts`, `observation_boundaries`, targetable elements | | `action_contracts` | optional | identifier | other `action_contracts` (interactions) | | `observation_boundaries` | optional | identifier | own information refs (observable/hidden/evidence) | -| `outcome_interpretation_rules` | optional | identifier | `action_contracts`, `objectives`, `workflows`, `evaluations` | +| `outcome_interpretation_rules` | optional | identifier | `action_contracts`, `objectives`, `workflows` | | `evidence_requirements` | optional | identifier | targetable elements for source, scope, channel, trigger, and boundary refs; distinct from `objectives` and scenario-native observability systems ([observability-and-evidence.md](observability-and-evidence.md)) | -| `objectives` | optional | identifier | `agents`/`entities` (actor), `action_contracts` (action), targetable elements (target), `conditions`/`metrics`/`evaluations`/`tlos`/`goals` (success), `stories`/`scripts`/`events`/`workflows` (window), other `objectives` (depends_on, acyclic) | -| `workflows` | optional | identifier | own steps (`start`, successors), other `workflows` (compensation), assessment sections (predicates) | +| `objectives` | optional | identifier | `agents`/`entities` (actor), `action_contracts` (action), targetable elements (target), `conditions` (success — observable state only, [ADR-073](../../docs/decisions/adrs/adr-073-scoring-reward-language-scope.md)), `stories`/`scripts`/`events`/`workflows` (window), other `objectives` (depends_on, acyclic) | +| `workflows` | optional | identifier | own steps (`start`, successors), other `workflows` (compensation), `conditions` (predicates) | | `variables` | optional | identifier matching `[A-Za-z_][A-Za-z0-9_-]*` | referenced by `${…}` placeholders ([variables-and-instantiation.md](variables-and-instantiation.md)) | ## Authoring section — list-valued @@ -80,19 +76,28 @@ lives under `nodes..runtime` ([runtime-inventory.md](runtime-inventory.md)); both carry `forwarding_agent_id` identity and the same family invariants, but they occupy different positions in the document. -## Assessment and narrative chains +## Narrative chain -Two reference chains run through the catalog and are called out because their +One reference chain runs through the catalog and is called out because its ordering is normative (resolution and failure semantics in [`references.md`](references.md)): -- **Assessment pipeline:** `conditions` ← `metrics` ← `evaluations` ← `tlos` ← - `goals`. Each link names the prior section; the chain feeds objectives and - workflow predicates. - **Narrative chain:** `injects` → `events` → `scripts` → `stories`, with - `injects` naming `entities`/`tlos` and `events` naming `conditions`. Objective + `injects` naming `entities` and `events` naming `conditions`. Objective windows bind `stories`/`scripts`/`events`/`workflows`. +`conditions` are observable state: an objective's success is expressed against +`conditions`, and workflow predicates reference `conditions`. The SDL carries no +graded scoring pipeline — the OCR-inherited `metrics`, `evaluations`, `tlos` +(Training Learning Objectives), and `goals` sections were removed with +[ADR-073](../../docs/decisions/adrs/adr-073-scoring-reward-language-scope.md). +Graded scoring, reward, leaderboard values, and evaluation outputs live in the +experiment/evaluator plane +([ADR-055](../../docs/decisions/adrs/adr-055-experiment-core-contract-boundary.md), +[ADR-064](../../docs/decisions/adrs/adr-064-experiment-evidence-and-measure-contract-boundary.md), +[ADR-069](../../docs/decisions/adrs/adr-069-cage-2-replication-architecture.md)), +never as authored SDL. + ## Extending the section set A new top-level authoring section is added by: defining its model and the diff --git a/tools/policy/oversized_allowlist.yaml b/tools/policy/oversized_allowlist.yaml index 31d524b13..723237d57 100644 --- a/tools/policy/oversized_allowlist.yaml +++ b/tools/policy/oversized_allowlist.yaml @@ -13,5 +13,3 @@ files: - implementations/python/packages/aces_conformance/conformance.py - implementations/python/packages/aces_sdl/module_registry.py - implementations/python/packages/aces_mcp/tools/authoring.py - - implementations/python/packages/aces_mcp/tools/inspection.py - - implementations/python/packages/aces_sdl/orchestration.py