diff --git a/src/backtest_engine/request_dispatch.py b/src/backtest_engine/request_dispatch.py index 670f020..d2a2a3f 100644 --- a/src/backtest_engine/request_dispatch.py +++ b/src/backtest_engine/request_dispatch.py @@ -82,7 +82,26 @@ def _request_period( ) -> tuple[date, date, tuple[PinnedDataset, ...], tuple[PinnedFeatureMaterialization, ...]]: try: if lane is RequestLane.BASIC: - raise RequestProcessingError("BASIC_PERIOD_COMES_FROM_RUN", retryable=False) + return ( + date.fromisoformat(str(request["periodStart"])), + date.fromisoformat(str(request["periodEnd"])), + ( + PinnedDataset( + uuid.UUID(str(request["datasetManifestId"])), + "MARKET_BARS", + str(request["expectedDatasetHash"]), + ), + ), + tuple( + sorted( + PinnedFeatureMaterialization( + uuid.UUID(str(item["featureMaterializationId"])), + str(item["lockedResultHash"]), + ) + for item in request["featureMaterializations"] + ) + ), + ) if lane is RequestLane.CUSTOM: return ( date.fromisoformat(str(request["periodStart"])), @@ -167,15 +186,9 @@ def __call__(self, request: Mapping[str, Any], lane: RequestLane) -> None: raise RequestProcessingError("MARKET_BARS_DATASET_INVALID", retryable=False) primary = market_bars[0] if lane is RequestLane.BASIC: - start, end = run.evaluation_start, run.evaluation_end - requested_datasets: tuple[PinnedDataset, ...] = ( - PinnedDataset( - uuid.UUID(str(request["datasetManifestId"])), - "MARKET_BARS", - primary.locked_dataset_hash, - ), + start, end, requested_datasets, requested_features = _request_period( + request, lane ) - requested_features: tuple[PinnedFeatureMaterialization, ...] = stored_features period_identity: dict[str, str] = {} else: start, end, requested_datasets, requested_features = _request_period(request, lane) diff --git a/src/backtest_engine/schemas/strategy-bot/v1/official-backtest-request.schema.json b/src/backtest_engine/schemas/strategy-bot/v1/official-backtest-request.schema.json index 3ea835b..3b631b7 100644 --- a/src/backtest_engine/schemas/strategy-bot/v1/official-backtest-request.schema.json +++ b/src/backtest_engine/schemas/strategy-bot/v1/official-backtest-request.schema.json @@ -6,12 +6,21 @@ "type": "object", "required": [ "metadata", + "runId", + "lane", + "aggregateSequence", "botId", "expectedSnapshotHash", "compiledPlanChecksum", "datasetManifestId", + "expectedDatasetHash", + "periodStart", + "periodEnd", "assumptionsVersion", - "requestReason" + "executionPolicyVersion", + "requestReason", + "featureMaterializations", + "requestHash" ], "properties": { "metadata": { @@ -27,8 +36,25 @@ "expectedSnapshotHash": {"$ref": "https://contracts.idea2strategy.io/common/v1/primitives.schema.json#/$defs/sha256Prefixed"}, "compiledPlanChecksum": {"$ref": "https://contracts.idea2strategy.io/common/v1/primitives.schema.json#/$defs/sha256Prefixed"}, "datasetManifestId": {"$ref": "https://contracts.idea2strategy.io/common/v1/primitives.schema.json#/$defs/uuid"}, + "expectedDatasetHash": {"$ref": "https://contracts.idea2strategy.io/common/v1/primitives.schema.json#/$defs/sha256Prefixed"}, + "periodStart": {"type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"}, + "periodEnd": {"type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"}, "assumptionsVersion": {"$ref": "https://contracts.idea2strategy.io/common/v1/primitives.schema.json#/$defs/nonEmptyString"}, "executionPolicyVersion": {"$ref": "https://contracts.idea2strategy.io/common/v1/primitives.schema.json#/$defs/nonEmptyString"}, - "requestReason": {"const": "STRATEGY_RELEASE"} + "requestReason": {"const": "STRATEGY_RELEASE"}, + "featureMaterializations": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["featureMaterializationId", "lockedResultHash"], + "properties": { + "featureMaterializationId": {"$ref": "https://contracts.idea2strategy.io/common/v1/primitives.schema.json#/$defs/uuid"}, + "lockedResultHash": {"$ref": "https://contracts.idea2strategy.io/common/v1/primitives.schema.json#/$defs/sha256Prefixed"} + }, + "additionalProperties": false + } + }, + "requestHash": {"$ref": "https://contracts.idea2strategy.io/common/v1/primitives.schema.json#/$defs/sha256Prefixed"} } } diff --git a/tests/d_integration_stack.py b/tests/d_integration_stack.py index f9e35b7..864825a 100644 --- a/tests/d_integration_stack.py +++ b/tests/d_integration_stack.py @@ -216,7 +216,14 @@ def build_stack( main, dead = queues market_data = write_market_data(root, closes) plan = compiled_plan() - accepted_request = dict(request) if request is not None else official_backtest_request(plan=plan) + accepted_request = ( + dict(request) + if request is not None + else official_backtest_request( + plan=plan, + expected_dataset_hash=f"sha256:{market_data.manifest['dataset_hash']}", + ) + ) plan_source = StaticCompiledPlanSource( dict(plans) if plans is not None else {plan["planChecksum"]: plan} diff --git a/tests/d_reproducibility_testkit.py b/tests/d_reproducibility_testkit.py index 089687a..f85b191 100644 --- a/tests/d_reproducibility_testkit.py +++ b/tests/d_reproducibility_testkit.py @@ -82,6 +82,9 @@ #: Correlation id of B's published request fixture. CORRELATION_ID = "00000000-0000-4000-8000-000000000202" MESSAGE_ID = "00000000-0000-4000-8000-000000000213" +RUN_ID = "76a6a20c-0651-5748-8187-6bf0ae155194" +FEATURE_MATERIALIZATION_ID = "00000000-0000-4000-8000-000000000204" +FEATURE_RESULT_HASH = "sha256:" + "6" * 64 DATASET_ID = "00000000-0000-4000-8000-0000000000d2" STORAGE_OBJECT_ID = "00000000-0000-4000-8000-0000000000d3" @@ -316,18 +319,25 @@ def official_backtest_request( *, plan: Mapping[str, Any] | None = None, occurred_at: str = "2024-01-03T01:01:00Z", + expected_dataset_hash: str | None = None, ) -> dict[str, Any]: """B's ``OFFICIAL_BACKTEST_REQUESTED``, re-addressed to the seeded bot. - Only ``botId`` and ``occurredAt`` differ from B's published fixture, and both - have to: ``backtest.runs.bot_id`` is a foreign key into ``bot.bots`` and the - execution policy is selected by the ET release quarter of ``occurredAt``. The - ``idempotencyKey`` is then B's own canonical material for *this* message -- - the same function B's ``StrategyBotContractFixtures`` uses -- because an - unchanged key copied from a different message would be a forgery, not a - fixture. + The identifiers and execution window address the seeded 2024-Q1 stack rather + than B's published 2026-Q3 example. The ``idempotencyKey`` is B's own + canonical material for *this* message -- the same function B's + ``StrategyBotContractFixtures`` uses -- because an unchanged key copied from + a different message would be a forgery, not a fixture. """ document = plan if plan is not None else compiled_plan() + if expected_dataset_hash is None: + parquet_bytes = market_bars_parquet() + manifest = dataset_manifest( + hashlib.sha256(parquet_bytes).hexdigest(), + row_count=BAR_COUNT, + coverage_end=FIRST_BAR_START + BAR * BAR_COUNT, + ) + expected_dataset_hash = f"sha256:{manifest['dataset_hash']}" request: dict[str, Any] = { "metadata": { "contractVersion": STRATEGY_BOT_CONTRACT_VERSION, @@ -337,14 +347,27 @@ def official_backtest_request( "correlationId": CORRELATION_ID, "idempotencyKey": "", }, + "runId": RUN_ID, + "lane": "BASIC", + "aggregateSequence": 1, "botId": str(BOT_ID), "expectedSnapshotHash": document["executionSnapshot"]["immutableStrategyVersion"][ "snapshotHash" ], "compiledPlanChecksum": document["planChecksum"], "datasetManifestId": str(DATASET_MANIFEST_ID), + "expectedDatasetHash": expected_dataset_hash, + "periodStart": E2E_EXECUTION_POLICY.period_start.astimezone(ET).date().isoformat(), + "periodEnd": E2E_EXECUTION_POLICY.period_end.astimezone(ET).date().isoformat(), "assumptionsVersion": "accounting:1.0.0", + "executionPolicyVersion": E2E_EXECUTION_POLICY.version, "requestReason": "STRATEGY_RELEASE", + "featureMaterializations": [ + { + "featureMaterializationId": FEATURE_MATERIALIZATION_ID, + "lockedResultHash": FEATURE_RESULT_HASH, + } + ], } request["metadata"]["idempotencyKey"] = compute_message_idempotency_key( contract_version=STRATEGY_BOT_CONTRACT_VERSION, @@ -353,6 +376,9 @@ def official_backtest_request( snapshot_hash=request["expectedSnapshotHash"], operation_key=official_backtest_operation_key(request), ) + request["requestHash"] = "sha256:" + hashlib.sha256( + json.dumps(request, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() return request diff --git a/tests/fixtures/contracts/strategy-bot/v1/official-backtest-request.valid.json b/tests/fixtures/contracts/strategy-bot/v1/official-backtest-request.valid.json index 65567b8..1b754da 100644 --- a/tests/fixtures/contracts/strategy-bot/v1/official-backtest-request.valid.json +++ b/tests/fixtures/contracts/strategy-bot/v1/official-backtest-request.valid.json @@ -7,10 +7,24 @@ "correlationId": "00000000-0000-4000-8000-000000000202", "idempotencyKey": "sha256:c6dd5229151352a530ff8312f050258107370cf26ea943c68473bf81936f6c1e" }, + "runId": "00000000-0000-4000-8000-000000000214", + "lane": "BASIC", + "aggregateSequence": 1, "botId": "00000000-0000-4000-8000-000000000201", "expectedSnapshotHash": "sha256:1111111111111111111111111111111111111111111111111111111111111111", "compiledPlanChecksum": "sha256:88d61198d46dce161c2a929702a7fd1cee5c9b044c470d2590b96f3825fcacb3", "datasetManifestId": "00000000-0000-4000-8000-000000000203", + "expectedDatasetHash": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "periodStart": "2025-01-01", + "periodEnd": "2025-12-31", "assumptionsVersion": "accounting:1.0.0", - "requestReason": "STRATEGY_RELEASE" + "executionPolicyVersion": "backtest-policy:1.0.0", + "requestReason": "STRATEGY_RELEASE", + "featureMaterializations": [ + { + "featureMaterializationId": "00000000-0000-4000-8000-000000000215", + "lockedResultHash": "sha256:5555555555555555555555555555555555555555555555555555555555555555" + } + ], + "requestHash": "sha256:4df5ec8056b0857c8c841fc3a9e4f4d75c90196aef1e7ece7716445284523f33" } diff --git a/tests/test_backtest_api.py b/tests/test_backtest_api.py index cf47303..7d38043 100644 --- a/tests/test_backtest_api.py +++ b/tests/test_backtest_api.py @@ -79,11 +79,11 @@ OTHER_TOKEN = "other-owner-token" WORKER_TOKEN = "worker-token" -# B's `metadata.idempotencyKey` from the published fixture, and the run id `uuid5` -# derives from it. Pinned so a change to the derivation is a visible test failure and -# not a silently re-addressed run. +# B's published idempotency key and provider-registered run id. Both are pinned so +# the consumer cannot silently re-address an already-created run. B_IDEMPOTENCY_KEY = "sha256:c6dd5229151352a530ff8312f050258107370cf26ea943c68473bf81936f6c1e" -EXPECTED_RUN_ID = "f876f259-4158-5a9a-8973-db21764024dc" +EXPECTED_RUN_ID = "00000000-0000-4000-8000-000000000214" +DERIVED_RUN_ID = "f876f259-4158-5a9a-8973-db21764024dc" SNAPSHOT_HASH = "sha256:" + "1" * 64 RESULT_HASH = "sha256:" + "a" * 64 @@ -296,7 +296,7 @@ def complete_run(self, request: dict[str, Any]) -> None: #: loosening the catalog's "no substitution" rule. POLICY_2026Q3 = replace( D17_EXECUTION_POLICY_FIXTURE, - version="official-backtest-policy-v2", + version="backtest-policy:1.0.0", release_quarter="2026-Q3", period_start=et_quarter_start(2026, 3), period_end=et_quarter_start(2026, 4), @@ -537,7 +537,7 @@ def test_the_run_id_is_derived_from_bs_idempotency_key_not_invented( official_request: dict[str, Any], ) -> None: """Pinned literal: a random id would make redelivery create a second run.""" - assert str(run_id_for(B_IDEMPOTENCY_KEY)) == EXPECTED_RUN_ID + assert str(run_id_for(B_IDEMPOTENCY_KEY)) == DERIVED_RUN_ID assert official_request["metadata"]["idempotencyKey"] == B_IDEMPOTENCY_KEY diff --git a/tests/test_contracts.py b/tests/test_contracts.py index eb5ed3a..063eb18 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -58,6 +58,18 @@ B_BOT_ID = "00000000-0000-4000-8000-000000000201" B_DATASET_MANIFEST_ID = "00000000-0000-4000-8000-000000000203" +OFFICIAL_REQUEST_REQUIRED_RUNTIME_FIELDS = ( + "runId", + "lane", + "aggregateSequence", + "expectedDatasetHash", + "periodStart", + "periodEnd", + "executionPolicyVersion", + "featureMaterializations", + "requestHash", +) + RUN_ID = "77777777-7777-4777-8777-777777777777" OWNER_ACCOUNT_ID = "66666666-6666-4666-8666-666666666666" INPUT_BUNDLE_FINGERPRINT = "sha256:" + "e" * 64 @@ -260,6 +272,30 @@ def test_consumer_accepts_bs_official_backtest_request_verbatim( assert accepted["requestReason"] == "STRATEGY_RELEASE" +@pytest.mark.parametrize("field", OFFICIAL_REQUEST_REQUIRED_RUNTIME_FIELDS) +def test_vendored_official_request_requires_every_provider_runtime_field( + field: str, +) -> None: + request = _load( + STRATEGY_BOT_FIXTURES / "official-backtest-request.valid.json" + ) + assert field in request, f"vendored provider fixture is missing {field}" + + request.pop(field) + + with pytest.raises(ContractValidationError, match=field): + validate_official_backtest_request(request) + + +def test_runnable_reproducibility_request_keeps_the_provider_runtime_shape() -> None: + from d_reproducibility_testkit import official_backtest_request + + request = official_backtest_request() + + assert validate_official_backtest_request(request) == request + assert set(OFFICIAL_REQUEST_REQUIRED_RUNTIME_FIELDS) <= request.keys() + + def test_request_whose_dataset_manifest_was_swapped_fails_its_idempotency_key( official_request: dict[str, Any], ) -> None: diff --git a/tests/test_lifecycle.py b/tests/test_lifecycle.py index ef84fc5..1555374 100644 --- a/tests/test_lifecycle.py +++ b/tests/test_lifecycle.py @@ -51,14 +51,15 @@ MANIFEST_ID = UUID("00000000-0000-4000-8000-000000000203") B_IDEMPOTENCY_KEY = "sha256:c6dd5229151352a530ff8312f050258107370cf26ea943c68473bf81936f6c1e" -EXPECTED_RUN_ID = UUID("f876f259-4158-5a9a-8973-db21764024dc") +EXPECTED_RUN_ID = UUID("00000000-0000-4000-8000-000000000214") +DERIVED_RUN_ID = UUID("f876f259-4158-5a9a-8973-db21764024dc") SNAPSHOT_HASH = "sha256:" + "1" * 64 RESULT_HASH = "sha256:" + "a" * 64 DATASET_HASH = "d9f6310297b7eb858570086d7292a709261eecc7bf92fc9a03745c46f514161c" POLICY_2026Q3 = replace( D17_EXECUTION_POLICY_FIXTURE, - version="official-backtest-policy-v2", + version="backtest-policy:1.0.0", release_quarter="2026-Q3", period_start=et_quarter_start(2026, 3), period_end=et_quarter_start(2026, 4), @@ -110,7 +111,7 @@ def _event(service: BacktestLifecycleService, status: str, **detail: Any) -> dic correlation_id="00000000-0000-4000-8000-000000000202", message_id=str(uuid4()), expected_snapshot_hash=SNAPSHOT_HASH, - execution_policy_version="official-backtest-policy-v2", + execution_policy_version=POLICY_2026Q3.version, **detail, ) @@ -126,7 +127,7 @@ def test_run_id_namespace_is_pinned() -> None: def test_run_id_is_a_pinned_function_of_bs_idempotency_key() -> None: - assert run_id_for(B_IDEMPOTENCY_KEY) == EXPECTED_RUN_ID + assert run_id_for(B_IDEMPOTENCY_KEY) == DERIVED_RUN_ID # A second pinned literal: a constant-returning implementation fails here. assert str(run_id_for("sha256:" + "0" * 64)) == "2b97cf3a-1700-5b1a-bbab-5e02f181c281" diff --git a/tests/test_request_dispatch.py b/tests/test_request_dispatch.py index 24b0dd2..ddaa005 100644 --- a/tests/test_request_dispatch.py +++ b/tests/test_request_dispatch.py @@ -48,7 +48,7 @@ def publish(self, lane: RequestLane, job: dict[str, Any]) -> None: def projection(request: dict[str, Any], lane: RequestLane) -> QueuedRunProjection: period = ( - ("2024-01-01", "2024-12-31") + (request["periodStart"], request["periodEnd"]) if lane is RequestLane.BASIC else (request["periodStart"], request["periodEnd"]) if lane is RequestLane.CUSTOM @@ -81,11 +81,7 @@ def projection(request: dict[str, Any], lane: RequestLane) -> QueuedRunProjectio { "datasetManifestId": request["datasetManifestId"], "purposeCode": "MARKET_BARS", - "expectedDatasetHash": ( - "sha256:" + "4" * 64 - if lane is RequestLane.BASIC - else request["expectedDatasetHash"] - ), + "expectedDatasetHash": request["expectedDatasetHash"], }, ) ) @@ -98,6 +94,8 @@ def projection(request: dict[str, Any], lane: RequestLane) -> QueuedRunProjectio for item in ( request["periods"][0]["featureMaterializations"] if lane is RequestLane.COMPETITION + else request["featureMaterializations"] + if lane is RequestLane.BASIC else () ) ), @@ -238,23 +236,17 @@ def test_competition_refuses_feature_pins_that_differ_from_the_provider_bundle() publisher(request, RequestLane.COMPETITION) -def test_basic_job_uses_provider_pinned_features_not_unpinned_message_fields() -> None: +def test_basic_request_refuses_feature_pins_that_differ_from_the_provider_bundle() -> None: request = basic_request() feature = PinnedFeatureMaterialization(uuid.uuid4(), "sha256:" + "9" * 64) run = replace( projection(request, RequestLane.BASIC), feature_materializations=(feature,), ) - queue = Queue() - - BacktestRequestJobPublisher(Source(run), queue)(request, RequestLane.BASIC) + publisher = BacktestRequestJobPublisher(Source(run), Queue()) - assert queue.jobs[0][1]["featureMaterializations"] == [ - { - "featureMaterializationId": str(feature.feature_materialization_id), - "lockedResultHash": feature.locked_result_hash, - } - ] + with pytest.raises(RequestProcessingError, match="RUN_IDENTITY_MISMATCH"): + publisher(request, RequestLane.BASIC) def test_basic_request_is_dispatched_through_the_same_pinned_two_stage_boundary() -> None: @@ -278,16 +270,16 @@ def test_basic_request_is_dispatched_through_the_same_pinned_two_stage_boundary( "executionPolicyVersion": request["executionPolicyVersion"], "compiledPlanChecksum": request["compiledPlanChecksum"], "datasetManifestId": request["datasetManifestId"], - "expectedDatasetHash": "sha256:" + "4" * 64, + "expectedDatasetHash": request["expectedDatasetHash"], "expectedSnapshotHash": request["expectedSnapshotHash"], "datasets": [ { "datasetManifestId": request["datasetManifestId"], "purposeCode": "MARKET_BARS", - "expectedDatasetHash": "sha256:" + "4" * 64, + "expectedDatasetHash": request["expectedDatasetHash"], } ], - "featureMaterializations": [], + "featureMaterializations": request["featureMaterializations"], }, ) ] diff --git a/tests/test_run_outcome_detail.py b/tests/test_run_outcome_detail.py index bb6fbf7..fa44bfe 100644 --- a/tests/test_run_outcome_detail.py +++ b/tests/test_run_outcome_detail.py @@ -60,7 +60,7 @@ POLICY_2026Q3 = replace( D17_EXECUTION_POLICY_FIXTURE, - version="official-backtest-policy-v2", + version="backtest-policy:1.0.0", release_quarter="2026-Q3", period_start=et_quarter_start(2026, 3), period_end=et_quarter_start(2026, 4), @@ -99,7 +99,7 @@ def _event(service: BacktestLifecycleService, status: str, **detail: Any) -> dic correlation_id="00000000-0000-4000-8000-000000000202", message_id=str(uuid4()), expected_snapshot_hash=SNAPSHOT_HASH, - execution_policy_version="official-backtest-policy-v2", + execution_policy_version=POLICY_2026Q3.version, **detail, )