diff --git a/contracts/schema-publication/entries/runtime-snapshot-v1.json b/contracts/schema-publication/entries/runtime-snapshot-v1.json index 9eceff0b..e4265dcc 100644 --- a/contracts/schema-publication/entries/runtime-snapshot-v1.json +++ b/contracts/schema-publication/entries/runtime-snapshot-v1.json @@ -2,9 +2,9 @@ "contract_id": "runtime-snapshot-v1", "schema_path": "contracts/schemas/snapshots/runtime-snapshot-v1.json", "stability": "draft", - "content_hash": "4388410c8e7536fef3f1251b7649b7d2cefef3764bf9da31d221430c92935879", + "content_hash": "90ca4bb76c8ff2e46ba7facec0d5199bdd682f379a63686e03b4cbf8b261304d", "last_change": { - "summary": "Added typed guest-observed operating-system identity bound to operation, envelope, configuration, observer, and sequence for issue #1077.", - "content_hash": "4388410c8e7536fef3f1251b7649b7d2cefef3764bf9da31d221430c92935879" + "summary": "Published participant episode closure records for issue #1092.", + "content_hash": "90ca4bb76c8ff2e46ba7facec0d5199bdd682f379a63686e03b4cbf8b261304d" } } diff --git a/contracts/schemas/snapshots/runtime-snapshot-v1.json b/contracts/schemas/snapshots/runtime-snapshot-v1.json index 60b12ed3..0b2775e7 100644 --- a/contracts/schemas/snapshots/runtime-snapshot-v1.json +++ b/contracts/schemas/snapshots/runtime-snapshot-v1.json @@ -11558,6 +11558,17 @@ "title": "Participant Crossing History", "type": "object" }, + "participant_episode_closure_records": { + "additionalProperties": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + "title": "Participant Episode Closure Records", + "type": "object" + }, "participant_episode_history": { "additionalProperties": { "items": { diff --git a/docs/decisions/issue-1092-local-control-plane-durability-preflight.md b/docs/decisions/issue-1092-local-control-plane-durability-preflight.md new file mode 100644 index 00000000..c5f6735e --- /dev/null +++ b/docs/decisions/issue-1092-local-control-plane-durability-preflight.md @@ -0,0 +1,287 @@ +# Issue 1092 Local Control-Plane Durability Preflight + +Date: 2026-08-11 + +Issue: #1092. Requirement: API-404. + +## Decision + +`LocalControlPlaneStore` remains the single-host reference persistence owner, +but its JSON read-modify-replace files are replaced by one SQLite database. +The database uses WAL journaling, full synchronous commits, an explicit busy +timeout, unique indexed idempotency keys, and transactions that commit the +snapshot and terminal operation record as one unit. Participant transitions +continue to commit their snapshot, operation record, and audit event as one +unit. + +This closes four incumbent gaps without changing the public snapshot or +operation contracts: + +- two processes can no longer overwrite each other's operation updates; +- an idempotency lookup and claim is one atomic database operation; +- a failed participant transition cannot expose a partial snapshot, record, or + audit append; and +- operation and idempotency lookup no longer reparses a growing JSON array. + +The operation claim is durable before a backend call starts. On success or a +handled failure, the resulting snapshot and terminal operation record are +committed atomically. If the process exits after the claim but before that +terminal commit, startup converts the orphaned `ACCEPTED` or `RUNNING` record +to `FAILED` with the stable +`runtime.control-plane.operation-interrupted` diagnostic. That diagnostic +states that backend effects may be indeterminate. The runtime never replays +such a record automatically, and its retained idempotency key prevents a +client retry from blindly invoking the backend again. + +Stored JSON payloads retain canonical serialization and a SHA-256 integrity +digest. Reads verify that digest before contract reconstruction, and store +startup runs SQLite's database integrity check. These checks detect accidental +corruption; they are not a substitute for filesystem access control or an +authenticated external ledger. + +On POSIX, the store creates or tightens its owned directory to `0700` and its +main SQLite database to `0600`. SQLite creates its main database, WAL, +shared-memory, and rollback-journal files inside that owned directory. The +application validates SQLite-managed paths through descriptor-free metadata +inspection and never independently opens or closes them. Existing main +databases are tightened with +path-based, no-follow `chmod` before SQLite opens them; a new database is +created by SQLite and validated before schema work. Directory paths retain +descriptor identity checks. SQLite-managed paths fail closed when stable +metadata identifies a symlink/reparse point, wrong filesystem type, foreign +owner, non-private POSIX mode, a hard-linked SQLite alias, or main-database +identity replacement. The store pins the database identity established at +initialization and requires that same object on every later connection; an +operator restore by pathname therefore requires a runtime restart rather than +silently switching the live cache to another database. Windows +reparse/type checks remain enforced, while deployment ACLs are the authority +for permissions that POSIX mode bits cannot express. + +The durable snapshot codec enumerates every `RuntimeSnapshot` dataclass field +in both directions and fails its verification guard when the contract and codec +drift. This includes participant episode-closure records; a successful local +commit and reload must preserve them rather than silently restoring the field's +empty default. + +Existing `snapshot.json`, `operations.json`, `audit.jsonl`, and +`control-transition-state.json` files are imported once. Source files remain +untouched, and a timestamped, fsynced backup is created before the database +transaction commits. A failed import may leave an additional backup but cannot +silently discard the legacy source. + +## WAL And Backup Durability Admission + +The durability claim depends on two admission results, not merely on requesting +them. SQLite's +[`journal_mode` PRAGMA](https://sqlite.org/pragma.html#pragma_journal_mode) +returns the mode that the connection actually entered, and a request can leave +the prior mode in place. Store initialization therefore requires the exact +`wal` result before it creates schema objects, records a schema version, or +starts legacy migration. Any other result fails construction with the legacy +sources untouched. This keeps every initialized store inside the WAL topology +assumed by the transaction and cross-process tests. + +The prior backup sequence copied each legacy file and then synchronized only +the backup and parent directories. Directory synchronization persists names; +it does not establish that the copied file data reached stable storage. +Following SQLite's distinction between flushing file content and the directory +entry that names it in its +[atomic-commit protocol](https://sqlite.org/atomiccommit.html#_flushing_changes_to_mass_storage), +the store now synchronizes every copied regular backup file before the backup +directory and store directory. A file-sync failure aborts and rolls back the +migration transaction. The untouched source remains authoritative, and a +subsequent startup can retry even when the failed attempt left an incomplete, +timestamped backup directory. + +The existing-surface audit covered initialization order, transaction rollback, +legacy-source retention, backup publication, the store's directory-identity +boundary, SQLite's WAL response contract, and the repository's existing OCI +directory-publication helper. Three alternatives were rejected: retaining +best-effort directory sync would silently convert real I/O failures into +success; trusting a `journal_mode=WAL` invocation without its returned value +would admit a different journal topology; and deleting a failed backup would +add destructive recovery work without strengthening the retained legacy +source. The chosen boundary admits unsupported directory sync only on platforms +without that facility, or for the narrowly established `EINVAL`, `ENOTSUP`, +and `EOPNOTSUPP` results. Open or sync failures such as `EIO` propagate. Regular +backup-file synchronization is never downgraded because without it the store +cannot claim that the backup content is durable. + +This does not make a filesystem stronger than its documented guarantees. On a +platform without directory synchronization, the backup content is flushed but +crash persistence of its name remains a deployment property. The local-store +boundary still excludes filesystems whose locking or durability behavior is +weaker than SQLite requires. Deterministic tests enforce WAL admission before +schema/migration, file-before-directory sync ordering, rollback and restart +after backup-file `EIO`, the narrow portability errno set, and propagation of +all other directory open/sync failures. + +## Boundary + +This is a local durability mechanism, not a distributed control-plane claim. +SQLite serializes writers on one shared filesystem. Multi-host execution, +leader election, durable work queues, remote replication, disaster recovery, +and cryptographic audit-log authenticity remain outside this issue. Callers +must not place the database on a filesystem whose locking or durability +semantics are weaker than SQLite requires. + +SQLite serialization alone does not make `RuntimeControlPlane`'s cached +snapshot a cross-process compare-and-swap. Until that protocol exists, a local +store permits exactly one live runtime owner. Startup takes a non-blocking, +process-scoped filesystem lease and fails fast if another owner exists; +inherited use after `fork()` also fails. Deployments must therefore run one +ASGI worker and disable development reload for a local control-plane store. +`WEB_CONCURRENCY` or `UVICORN_WORKERS` values other than `1` are rejected at +construction, while the lease catches multiworker launchers that do not expose +their count through either variable. +Independent `LocalControlPlaneStore` maintenance/read instances remain valid, +but they do not grant another process authority to execute target mutations. +`RuntimeControlPlane` is a context manager and also exposes `close()`; callers +must release the first owner before constructing an intentional in-process +restart against the same local store. + +On POSIX the lease also holds an advisory lock on the private store directory. +That stable guard remains locked if the human-readable owner file is unlinked or +replaced, while each admitted call still verifies that the path names the +original locked file. Main databases and owner files with multiple hard links +are rejected so two store directories cannot acquire independent owner paths +for one SQLite object. Windows retains its native byte-range owner-file lock and +path-identity validation; the deployment ACL remains responsible for preventing +same-owner replacement of that file. + +Public runtime calls take lifecycle admission before reading cached state, +calling a backend, or touching the store. `close()` first stops new admission, +waits for every admitted call (including a blocked backend effect and its +terminal commit), and only then releases the runtime-owner lease. Reads remain +concurrent with an active mutation; the lifecycle counter is distinct from the +mutation lock. Closing from inside an active call fails rather than deadlocking +or releasing authority underneath that call. A call already admitted before +shutdown may re-enter another lifecycle-guarded runtime surface; shutdown blocks +only new outermost calls, so it cannot interrupt an admitted composite action. + +The lease and store directory are opened with `O_NOFOLLOW` where the platform +exposes it and are rejected unless pre-open, descriptor, and post-open metadata +identify the same owned filesystem object. No application-owned descriptor is +opened for the main database or its sidecars. The main path is checked before +and after `sqlite3.connect`, uses URI `mode=rw` for every existing database so a +concurrent disappearance cannot recreate it, and is checked again after close. +Only the initial absent path uses URI `mode=rwc`; SQLite creates it before mode, +owner, type, and identity validation and before any schema work. URI paths are +absolute and percent-encode filename delimiters. The store directory is an +owned, non-reparse directory tightened to private POSIX permissions; defending +a Windows path against an attacker who can continuously replace directory +entries requires a native handle-relative ACL boundary outside this local +reference store. + +## WAL Sidecar Lock Remediation + +The cross-process regression exposed a process-ending `SIGBUS` in SQLite's +`walIndexReadHdr` path. The operating-system report identified a 32 KiB mapped +file whose page-in failed past end-of-file. The incumbent hardening helper was +opening, applying `fchmod`, and closing each `-wal`, `-shm`, and `-journal` +path, including while a WAL connection was live. + +That is not a harmless permission check on POSIX. Closing any independent file +descriptor for a file cancels all advisory locks that the process holds on +that file, including locks acquired through SQLite's own descriptor. SQLite's +VFS works around this rule for descriptors it owns, but it cannot account for +an application descriptor. Another process can then treat the WAL shared-memory +file as unlocked and truncate it while the first process still has the WAL +index mapped. SQLite documents this failure mode in +[How To Corrupt An SQLite Database File, section 2.2](https://sqlite.org/howtocorrupt.html#posix_close_bug), +and its [WAL documentation](https://sqlite.org/wal.html) makes the `-wal` and +`-shm` files part of SQLite's own coordination protocol. + +The existing-surface audit covered the owned `0700` directory, the `0600` main +database, every application-owned database and sidecar descriptor, connection +lifetime, thread and process writer tests, and SQLite's WAL/VFS ownership. No +schema, operation contract, store topology, or public API change is needed. +The gap is confined to the filesystem boundary bypassing SQLite's lock owner. + +Three alternatives were considered: + +1. Remove only the live sidecar check. This leaves raw main-database closes able + to cancel another same-process connection's POSIX locks. +2. Serialize every store connection behind a process-global path lock while + retaining application database descriptors. This reduces read concurrency, + adds alias and fork-safe registry state, and still duplicates VFS ownership. +3. Keep descriptor identity on the owned directory, make every SQLite-managed + path application-descriptor-free, and compare the main database's identity + around SQLite's own connection. This removes the lock-canceling operation + while retaining type, owner, mode, no-recreation, and same-file checks. It is + the chosen design. + +Sidecars therefore fail closed when stable metadata shows a symlink/reparse +point, wrong type, foreign owner, or non-private POSIX mode. Their normal +creation or deletion is ephemeral, so disappearance during validation is not +an error. A concurrent unlink may surface as link count zero after pathname +lookup; that state is treated as disappearance, while multiple links remain a +hard failure. The application never raw-opens, closes, or `fchmod`s any +SQLite-managed path. The main database remains fail-closed through metadata +identity comparisons and URI open mode under the descriptor-verified `0700` +directory. + +The in-memory store gains the same atomic idempotency-claim behavior under a +re-entrant lock so reference semantics do not depend on which store is +selected. `RuntimeControlPlane` persists a claim before caching it locally; +another process that already owns the key wins, and a different request +fingerprint still fails closed. + +The JSON operation and snapshot schemas do not change. The 3.x +`ControlPlaneStore` structural contract remains source-compatible with custom +Python adapters written before crash-atomic terminal commits were added. When +an adapter does not implement the complete optional set +`claim_record(record)`, +`commit_terminal_operation(snapshot, record)` and +`reconcile_interrupted_records(records)`, one compatibility seam emits a +deprecation warning and preserves a lookup-then-save idempotency claim, the +former ordered `save_snapshot` then `save_record` commit, and per-record +startup recovery. Those fallbacks are explicitly not atomic across custom +store instances and will be removed in version 4; partial atomic +implementations also use the coherent legacy mode rather than mixing commit +semantics. If either ordered terminal write raises, the runtime reloads the +adapter's durable snapshot and operation records so both caches reflect the +actual compatibility boundary. The same reconciliation runs when a built-in +atomic method reports an error after its transaction may already have committed. +If either durable reload fails, the runtime is poisoned and rejects every new +call until it is closed and restarted; it never continues from an unknown cache. +Canonical value-free snapshot projection is applied before an idempotent +terminal retry is compared, so deliberate credential redaction does not turn an +exact retry into a false mismatch. The built-in in-memory and local SQLite +stores implement the complete atomic set and never enter the fallback. + +## Verification + +Acceptance requires regression coverage for: + +1. concurrent writers using independent store instances without lost records; +2. exactly one winner for a shared idempotency key; +3. rollback after an injected write failure; +4. restart persistence and indexed lookup; +5. legacy import with retained source, file-before-directory synchronization, + rollback after backup-sync failure, and verified restartable backup; +6. payload corruption and semantically invalid durable state; and +7. unchanged participant expected-head conflict behavior; +8. injected exits before, during, and after the atomic terminal transaction, + followed by restart and same-key retry without another backend call; +9. deterministic recovery of orphaned `ACCEPTED` and `RUNNING` records; and +10. rejection of a second runtime owner and post-fork inherited use; and +11. exhaustive snapshot-field coverage and round-trip preservation, including + participant episode-closure records; and +12. blocked-backend close/reacquisition ordering and closed-state rejection on + every public runtime surface; and +13. private store/database/sidecar modes plus rejection of unsafe existing + directory and SQLite paths; and +14. absence of application-owned descriptors for every SQLite-managed path, + encoded create-versus-existing URI modes, repeated multiprocess writes, + and the original cross-process API regression; and +15. exact WAL admission before schema or legacy migration plus fail-closed + directory synchronization outside the narrow portability boundary; and +16. database identity replacement between calls, hard-linked aliases, + owner-file replacement, admitted-call re-entry during close, post-commit + cache reconciliation/poisoning, and canonical value-free retry comparison. + +The repository policy, API-404 requirement trace, unit/integration suites, and +full verification remain release gates. Issue #1093 separately owns event-loop +offload and bounded execution. Recovery here is deliberately conservative: it +records an indeterminate non-success outcome, not background-job resumption. diff --git a/docs/explain/reference/shared-semantic-integrity.md b/docs/explain/reference/shared-semantic-integrity.md index 9dd69a70..84753ae0 100644 --- a/docs/explain/reference/shared-semantic-integrity.md +++ b/docs/explain/reference/shared-semantic-integrity.md @@ -241,7 +241,7 @@ so they are tracked by their own requirements, not here. | Planner dependency, ordering, refresh, and applicability semantics | RUN-303 | planning | `implementations/python/packages/raes_processor/semantics/planner.py`, `implementations/python/packages/raes_processor/planner/__init__.py`, `specs/formal/planner/README.md`, `specs/formal/planner/dependency-ordering.md`, `implementations/python/tests/test_semantics_planner.py`, `implementations/python/tests/test_runtime_planner.py` | active | | Live execution state and lifecycle (snapshots, results, history) | RUN-304, API-402 | execution, observation | `implementations/python/packages/raes_runtime/manager.py`, `implementations/python/packages/raes_runtime/result_contracts.py`, `implementations/python/packages/raes_processor/models/`, `implementations/python/tests/test_runtime_manager.py`, `implementations/python/tests/test_runtime_models.py` | active | | Runtime result and evaluator-result contracts | ASR-503, API-402 | execution, observation | `implementations/python/packages/raes_runtime/result_contracts.py`, `specs/formal/runtime-contracts/README.md`, `specs/formal/runtime-contracts/workflow-results.md`, `specs/formal/runtime-contracts/evaluator-results.md`, `implementations/python/tests/test_runtime_contracts.py`, `implementations/python/tests/test_run_311_participant_episode_lifecycle.py` | active | -| Control-plane semantics (auth, durable state, idempotency, audit) | API-403, API-404 | execution, observation | `implementations/python/packages/raes_runtime/control_plane_api/__init__.py`, `implementations/python/packages/raes_runtime/control_plane_security.py`, `implementations/python/packages/raes_runtime/control_plane_store.py`, `implementations/python/tests/test_runtime_control_plane.py`, `implementations/python/tests/test_runtime_control_plane_api.py` | active | +| Control-plane semantics (auth, durable state, idempotency, audit) | API-403, API-404 | execution, observation | `implementations/python/packages/raes_runtime/control_plane_api/__init__.py`, `implementations/python/packages/raes_runtime/control_plane_security.py`, `implementations/python/packages/raes_runtime/control_plane_store.py`, `implementations/python/packages/raes_runtime/control_plane_store_local.py`, `implementations/python/packages/raes_runtime/control_plane_store_paths.py`, `docs/decisions/issue-1092-local-control-plane-durability-preflight.md`, `implementations/python/tests/test_runtime_control_plane.py`, `implementations/python/tests/test_runtime_control_plane_api.py`, `implementations/python/tests/test_issue_1092_control_plane_crash_consistency.py` | active | | Backend and processor identity, capability, and compatibility manifests | API-401, API-412 | planning, execution | `implementations/python/packages/raes_processor/manifest.py`, `implementations/python/packages/raes_processor/capabilities.py`, `implementations/python/packages/raes_contracts/apparatus.py`, `implementations/python/packages/raes_contracts/manifest_authority.py`, `implementations/python/tests/test_backend_manifest.py`, `implementations/python/tests/test_processor_manifest.py` | active | | Concept authority, controlled vocabularies, reference models, and semantic profiles (meta-layer) | GOV-920 | authoring, validation, compilation, planning, execution | `specs/concept-authority/concept-authority.md`, `specs/concept-authority/semantic-profiles.md`, `implementations/python/packages/raes_contracts/semantic_profiles.py`, `implementations/python/packages/raes_contracts/controlled_vocabularies.py`, `implementations/python/packages/raes_contracts/reference_models.py`, `docs/explain/reference/shared-concept-model.md`, `implementations/python/tests/test_concept_authority.py`, `implementations/python/tests/test_semantic_profiles.py` | active | | Participant episode lifecycle boundaries and authored episode structure (initialization, reset, completion, timeout, truncation, interruption) | RUN-311, SEM-222, DSL-120, ACT-623 | authoring, validation, execution, observation | `docs/decisions/adrs/adr-013-participant-episode-lifecycle-boundaries.md`, `docs/decisions/adrs/adr-054-participant-runtime-observable-lifecycle.md`, `specs/formal/participant-episode-model/README.md`, `docs/decisions/issue-122-sem-222-episode-budget-model-preflight.md`, `implementations/python/packages/raes_contracts/participant_episode_closure.py`, `implementations/python/packages/raes_runtime/participant_result_contracts.py`, `implementations/python/tests/test_run_311_participant_episode_lifecycle.py`, `implementations/python/tests/test_sem_222_episode_termination_semantics.py`, `implementations/python/tests/test_sem_222_episode_termination_oracle.py` | partial | diff --git a/docs/explain/sdl/runtime-architecture.md b/docs/explain/sdl/runtime-architecture.md index 050a5ed7..f5a0f6a9 100644 --- a/docs/explain/sdl/runtime-architecture.md +++ b/docs/explain/sdl/runtime-architecture.md @@ -484,6 +484,49 @@ header identity must pass an explicit `ControlPlaneSecurityConfig`, set `trust_proxy_identity_headers=True`, and only trust those headers behind an authenticated proxy that strips caller-supplied identity headers. +The local control-plane store persists snapshots, operations, idempotency +claims, and audit events in a single SQLite WAL database. Full synchronous +transactions and a unique idempotency index make concurrent same-host writers +and participant transition commits atomic. A backend claim is stored before +execution, and its resulting snapshot and terminal operation record commit in +one transaction. Startup marks an orphaned non-terminal record `FAILED` with +an explicit indeterminate-outcome diagnostic and never replays it; retaining +the idempotency claim prevents a retry from blindly repeating backend effects. +On first use, legacy JSON state is imported without deleting its source and is +copied to a timestamped backup. Payload digests and SQLite integrity checks +detect accidental durable-state corruption. Owned POSIX store directories are +created or migrated to `0700` and the main SQLite database to `0600`. SQLite +alone owns descriptors for the main database, WAL, shared-memory, and rollback +journal. OpenRÆ uses descriptor-free metadata inspection and path-based mode +tightening for type, owner, private-mode, and main-database same-file checks, +so an independent `close()` cannot cancel SQLite's POSIX locks. Existing opens +use URI `mode=rw`; only first creation uses +`mode=rwc`, preventing a disappeared database from being silently recreated. +The initialized database identity remains pinned for the store lifetime, and +hard-linked aliases are rejected. Unsafe symlink, reparse, type, owner, mode, or +identity changes fail closed. If a commit reports an error after its outcome is +uncertain, the runtime reloads both durable caches; a failed reload poisons the +runtime until restart rather than allowing another mutation from stale state. + +Built-in stores use the complete crash-atomic commit capability. Existing 3.x +custom `ControlPlaneStore` adapters remain accepted when they implement the +pre-atomic structural contract: a centralized compatibility seam warns once per +runtime and uses lookup-then-save claims, ordered snapshot/record writes, and +per-record recovery. This mode preserves legacy behavior but has documented +claim and terminal-write race/crash windows and is scheduled for removal in +version 4; custom adapters should implement `claim_record` and both atomic +terminal/recovery methods before upgrading. + +The local runtime is deliberately single-process. Its cached snapshot does not +yet implement cross-process compare-and-swap, so a non-blocking filesystem +lease admits one `RuntimeControlPlane` owner and rejects another, including +inherited post-fork use. POSIX runtimes also hold a store-directory guard so +unlinking or replacing the owner-file path cannot admit a second owner. Close +drains admitted composite calls, including their nested guarded work, before it +releases authority. Run one ASGI worker with reload disabled. This is a +single-host reference boundary, not a distributed queue, replication, or +multi-host availability claim. + ## Current Scope The current runtime scope includes: diff --git a/docs/requirements/API-404/requirement.md b/docs/requirements/API-404/requirement.md index d8b74f5f..d34ba5fd 100644 --- a/docs/requirements/API-404/requirement.md +++ b/docs/requirements/API-404/requirement.md @@ -6,7 +6,7 @@ type: FUNCTIONAL priority: MUST wave: 1 created_at: 2026-04-03T05:55:58.825305Z -updated_at: 2026-04-05T06:33:22.964497Z +updated_at: 2026-08-12T00:00:00.000000Z --- # API-404 — Secure, Durable, And Idempotent Control-Plane Semantics @@ -22,7 +22,20 @@ Requirement inventory phase. Status audit deferred until the full canonical grap ## Traceability - IMPLEMENTS → GITHUB_ISSUE `8` (API-404: Secure, Durable, And Idempotent Control-Plane Semantics) +- IMPLEMENTS → GITHUB_ISSUE `1092` (Make the local control plane crash-consistent and explicitly single-process) - IMPLEMENTS → SPEC `contracts/schemas/control-plane/operation-receipt-v1.json` (Operation receipt JSON Schema — submission acknowledgment contract) - IMPLEMENTS → SPEC `contracts/schemas/control-plane/operation-status-v1.json` (Operation status JSON Schema — durable operation state contract) +- IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_runtime/control_plane_store.py` (Atomic terminal commits and interrupted-operation reconciliation contract) +- IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_runtime/control_plane_recovery.py` (Conservative startup recovery policy for interrupted operations) +- IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_runtime/control_plane_durability.py` (Commit-outcome cache publication, reconciliation, and poison-on-unknown behavior) +- IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_runtime/control_plane_lifecycle.py` (Draining close, nested-call admission, and durability-poison boundary) +- IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_runtime/control_plane_store_lease.py` (Secure single-process local runtime ownership) +- IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_runtime/control_plane_store_snapshots.py` (Compatibility-preserving portable snapshot serialization split) +- IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_runtime/control_plane_store_local.py` (Required WAL admission, pinned database identity, durable legacy backup copies, and atomic transactions) +- IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_runtime/control_plane_store_legacy.py` (Complexity-bounded legacy JSON import readers) +- IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_runtime/control_plane_store_paths.py` (Descriptor-verified private directories, fail-closed durability synchronization, and metadata-only SQLite path validation) +- IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_runtime/control_plane_store_compatibility.py` (Deprecated 3.x custom-store fallback and optional atomic capability adapter) +- DOCUMENTS → DOCUMENTATION `docs/decisions/issue-1092-local-control-plane-durability-preflight.md` (Crash recovery and supported process topology) - TESTS → TEST `implementations/python/tests/test_runtime_control_plane.py` (Core control-plane unit tests) - TESTS → TEST `implementations/python/tests/test_runtime_control_plane_api.py` (HTTP/JSON control-plane API tests — auth, idempotency, durability, audit) +- TESTS → TEST `implementations/python/tests/test_issue_1092_control_plane_crash_consistency.py` (Atomic and legacy terminal commit, WAL admission, backup file/directory synchronization, interrupted-operation recovery, descriptor-free SQLite paths, URI no-recreation, multiprocess stress, retry, and runtime-owner tests) diff --git a/docs/requirements/SEM-222/requirement.md b/docs/requirements/SEM-222/requirement.md index 7082064e..4c01ba70 100644 --- a/docs/requirements/SEM-222/requirement.md +++ b/docs/requirements/SEM-222/requirement.md @@ -27,4 +27,6 @@ Primary-source refresh shows that episode lifecycle meaning must be explicit if - IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_runtime/participant_result_contracts.py` (SEM-222 runtime episode-closure validation diagnostics seam (EBM-10 enforcement point)) - IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_runtime/result_contracts.py` (SEM-222 public re-export of participant_episode_closure_contract_diagnostics) - IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_contracts/runtime_state.py` (SEM-222 RuntimeSnapshot participant_episode_closure_records carrier (canonical closure-validation wiring)) +- IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_contracts/contracts/realization_plans.py` (Published runtime-snapshot-v1 participant episode closure-record carrier) +- IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_conformance/conformance/snapshot_semantics.py` (Published runtime snapshot closure-record preservation and semantic validation) - IMPLEMENTS → CODE_FILE `implementations/python/packages/raes_contracts/_snapshot_updates.py` (RuntimeSnapshot update-builder module split out to admit the SEM-222 closure-records field under the source-size cap) diff --git a/implementations/python/packages/raes_conformance/conformance/snapshot_semantics.py b/implementations/python/packages/raes_conformance/conformance/snapshot_semantics.py index 5afb7c8a..6fb3d8c6 100644 --- a/implementations/python/packages/raes_conformance/conformance/snapshot_semantics.py +++ b/implementations/python/packages/raes_conformance/conformance/snapshot_semantics.py @@ -16,6 +16,7 @@ ) from raes_contracts.participant_concurrency import iter_participant_concurrency_snapshot_violations from raes_contracts.participant_episode import iter_participant_episode_snapshot_violations +from raes_contracts.participant_episode_closure import iter_participant_episode_closure_violations from raes_contracts.participant_information_state_history import ( iter_participant_information_state_snapshot_violations, ) @@ -76,6 +77,10 @@ def _snapshot_from_envelope(payload: dict[str, Any]) -> RuntimeSnapshot: participant_address: [event.model_dump(mode="json") for event in history] for participant_address, history in validated.participant_episode_history.items() }, + participant_episode_closure_records={ + participant_address: [dict(record) for record in records] + for participant_address, records in validated.participant_episode_closure_records.items() + }, participant_behavior_history={ participant_address: [event.model_dump(mode="json") for event in history] for participant_address, history in validated.participant_behavior_history.items() @@ -170,6 +175,18 @@ def _participant_episode_snapshot_diagnostics( ] +def _participant_episode_closure_snapshot_diagnostics( + snapshot: RuntimeSnapshot, +) -> list[Diagnostic]: + return [ + _diagnostic(_SEMANTIC_INVALID_DIAGNOSTIC_CODE, address, message) + for address, message in iter_participant_episode_closure_violations( + snapshot.participant_episode_closure_records, + snapshot.participant_episode_history, + ) + ] + + def _participant_behavior_snapshot_references( snapshot: RuntimeSnapshot, ) -> tuple[ @@ -461,6 +478,7 @@ def _runtime_snapshot_semantic_diagnostics( *workflow_result_contract_diagnostics(snapshot), *evaluation_result_contract_diagnostics(snapshot), *_participant_episode_snapshot_diagnostics(snapshot), + *_participant_episode_closure_snapshot_diagnostics(snapshot), *_participant_behavior_snapshot_diagnostics(snapshot), *_shared_state_snapshot_diagnostics(snapshot), *_participant_concurrency_snapshot_diagnostics(snapshot), diff --git a/implementations/python/packages/raes_contracts/contracts/realization_plans.py b/implementations/python/packages/raes_contracts/contracts/realization_plans.py index 5ba40cf6..ec08dcf6 100644 --- a/implementations/python/packages/raes_contracts/contracts/realization_plans.py +++ b/implementations/python/packages/raes_contracts/contracts/realization_plans.py @@ -368,6 +368,7 @@ class RuntimeSnapshotEnvelopeModel(ContractModel): proposition_truth_results: dict[str, PropositionTruthResultModel] = Field(default_factory=dict) participant_episode_results: dict[str, ParticipantEpisodeStateModel] = Field(default_factory=dict) participant_episode_history: dict[str, list[ParticipantEpisodeHistoryEventModel]] = Field(default_factory=dict) + participant_episode_closure_records: dict[str, list[dict[str, Any]]] = Field(default_factory=dict) participant_behavior_history: dict[str, list[ParticipantBehaviorHistoryEventModel]] = Field(default_factory=dict) participant_control_history: dict[str, list[ParticipantControlOccurrenceModel]] = Field(default_factory=dict) participant_crossing_history: dict[str, list[ParticipantCrossingOccurrenceModel]] = Field(default_factory=dict) diff --git a/implementations/python/packages/raes_runtime/control_plane.py b/implementations/python/packages/raes_runtime/control_plane.py index 496a0180..8f9de341 100644 --- a/implementations/python/packages/raes_runtime/control_plane.py +++ b/implementations/python/packages/raes_runtime/control_plane.py @@ -33,17 +33,21 @@ from raes_processor.models import ParticipantBehaviorSpecificationRuntime from .backend_calls import _call_backend_diagnostics +from .control_plane_durability import RuntimeDurabilityMixin from .control_plane_execution import ( OperationExecutionRequest, _utc_now, execute_operation, ) +from .control_plane_lifecycle import RuntimeLifecycleMixin, runtime_owned +from .control_plane_recovery import reconcile_interrupted_operations from .control_plane_store import ( AuditEvent, ControlPlaneOperationRecord, ControlPlaneStore, InMemoryControlPlaneStore, ) +from .control_plane_store_compatibility import adapt_control_plane_store from .control_plane_submission import _submitted_plan_diagnostics from .control_plane_workflow_control import WorkflowControlMixin from .operational_apparatus import operational_apparatus_summary @@ -99,7 +103,13 @@ def _require_final_sink_flow_control_configuration( ) -class RuntimeControlPlane(WorkflowControlMixin, ParticipantControlMixin, ParticipantRetrievalMixin): +class RuntimeControlPlane( + RuntimeLifecycleMixin, + RuntimeDurabilityMixin, + WorkflowControlMixin, + ParticipantControlMixin, + ParticipantRetrievalMixin, +): """Reference control plane for async runtime submission and observation.""" def __init__( @@ -113,42 +123,61 @@ def __init__( information_state_context_resolver: ParticipantInformationStateContextResolver | None = None, enforce_final_sink_flow_control: bool = True, ) -> None: + self._initialize_runtime_lifecycle() _require_crossing_policy_configuration(target, crossing_policy_resolver) _require_final_sink_flow_control_configuration(crossing_policy_resolver, enforce_final_sink_flow_control) self._target = target self._enforce_final_sink_flow_control = enforce_final_sink_flow_control self._store = store or InMemoryControlPlaneStore(initial_snapshot) - self._snapshot = initial_snapshot if initial_snapshot is not None else self._store.load_snapshot() - self._operations: dict[str, ControlPlaneOperationRecord] = self._store.load_records() - self._behavior_specifications = dict(behavior_specifications or {}) - self._crossing_policy_resolver = crossing_policy_resolver - self._information_state_context_resolver = information_state_context_resolver - self._participant_control_lock = RLock() - self._trusted_provisioning_plan_lock = RLock() - self._trusted_provisioning_plan_digests: set[str] = set() - require_participant_information_state_snapshot( - self._snapshot, - information_state_context_resolver, - ) - if self._snapshot.participant_crossing_history: - if crossing_policy_resolver is None: - raise ValueError("persisted participant crossing history requires a policy resolver") - validate_persisted_crossing_history(self._snapshot, crossing_policy_resolver) + try: + self._store_commits = adapt_control_plane_store(self._store) + acquire_runtime_lease = getattr(self._store, "acquire_runtime_lease", None) + if callable(acquire_runtime_lease): + self._runtime_lease = acquire_runtime_lease() + self._snapshot = initial_snapshot if initial_snapshot is not None else self._store.load_snapshot() + self._operations = self._store.load_records() + self._operations = reconcile_interrupted_operations(self._store_commits, self._operations) + self._behavior_specifications = dict(behavior_specifications or {}) + self._crossing_policy_resolver = crossing_policy_resolver + self._information_state_context_resolver = information_state_context_resolver + self._operation_lock = RLock() + self._participant_control_lock = self._operation_lock + self._trusted_provisioning_plan_lock = RLock() + self._trusted_provisioning_plan_digests: set[str] = set() + require_participant_information_state_snapshot( + self._snapshot, + information_state_context_resolver, + ) + if self._snapshot.participant_crossing_history: + if crossing_policy_resolver is None: + raise ValueError("persisted participant crossing history requires a policy resolver") + validate_persisted_crossing_history(self._snapshot, crossing_policy_resolver) + except BaseException: + self.close() + raise @property + @runtime_owned def snapshot(self) -> RuntimeSnapshot: + self._assert_runtime_owner() return self._snapshot @property + @runtime_owned def target_name(self) -> str: + self._assert_runtime_owner() return self._target.name + @runtime_owned def audit_log(self) -> list[AuditEvent]: + self._assert_runtime_owner() return self._store.read_audit() + @runtime_owned def operational_apparatus_summary(self) -> dict[str, object]: """Return a compact operational view over existing control-plane carriers.""" + self._assert_runtime_owner() audit_events = self._store.read_audit() operation_records = list(self._operations.values()) return operational_apparatus_summary( @@ -158,6 +187,7 @@ def operational_apparatus_summary(self) -> dict[str, object]: audit_events=audit_events, ) + @runtime_owned def register_planner_produced_provisioning_plan(self, plan: ProvisioningPlan) -> str: """Trust one exact planner artifact for later HTTP relay submission. @@ -166,18 +196,22 @@ def register_planner_produced_provisioning_plan(self, plan: ProvisioningPlan) -> registered artifact, but cannot mint or widen its realization policy. """ + self._assert_runtime_owner() digest = provisioning_plan_digest(plan) with self._trusted_provisioning_plan_lock: self._trusted_provisioning_plan_digests.add(digest) return digest + @runtime_owned def is_planner_authorized_provisioning_plan(self, plan: ProvisioningPlan) -> bool: """Return whether the exact published plan was registered in-process.""" + self._assert_runtime_owner() digest = provisioning_plan_digest(plan) with self._trusted_provisioning_plan_lock: return digest in self._trusted_provisioning_plan_digests + @runtime_owned def submit_provisioning( self, plan: ProvisioningPlan, @@ -186,6 +220,7 @@ def submit_provisioning( idempotency_key: str = "", request_fingerprint: str = "", ) -> OperationReceipt: + self._assert_runtime_owner() diagnostics = _submitted_plan_diagnostics( plan, RuntimeDomain.PROVISIONING, @@ -218,6 +253,7 @@ def submit_provisioning( ), ) + @runtime_owned def submit_orchestration( self, plan: OrchestrationPlan, @@ -226,6 +262,7 @@ def submit_orchestration( idempotency_key: str = "", request_fingerprint: str = "", ) -> OperationReceipt: + self._assert_runtime_owner() if self._target.orchestrator is None: return self._reject_submission( domain=RuntimeDomain.ORCHESTRATION, @@ -253,6 +290,7 @@ def submit_orchestration( ), ) + @runtime_owned def submit_evaluation( self, plan: EvaluationPlan, @@ -261,6 +299,7 @@ def submit_evaluation( idempotency_key: str = "", request_fingerprint: str = "", ) -> OperationReceipt: + self._assert_runtime_owner() if self._target.evaluator is None: return self._reject_submission( domain=RuntimeDomain.EVALUATION, @@ -288,13 +327,18 @@ def submit_evaluation( ), ) + @runtime_owned def get_operation(self, operation_id: str) -> OperationStatus | None: + self._assert_runtime_owner() record = self._operations.get(operation_id) return None if record is None else record.status + @runtime_owned def get_snapshot(self) -> RuntimeSnapshotEnvelope: + self._assert_runtime_owner() return RuntimeSnapshotEnvelope(snapshot=self._snapshot) + @runtime_owned def record_audit( self, *, @@ -306,6 +350,7 @@ def record_audit( operation_id: str = "", details: dict[str, object] | None = None, ) -> None: + self._assert_runtime_owner() self._store.append_audit( AuditEvent( timestamp=_utc_now(), @@ -365,7 +410,7 @@ def _reject_diagnostics( updated_at=submitted_at, diagnostics=list(diagnostics), ) - self._persist_record( + persisted = self._claim_record( ControlPlaneOperationRecord( receipt=receipt, status=status, @@ -373,7 +418,7 @@ def _reject_diagnostics( request_fingerprint=request_fingerprint, ) ) - return receipt + return persisted.receipt def _idempotent_receipt( self, @@ -381,6 +426,7 @@ def _idempotent_receipt( idempotency_key: str, request_fingerprint: str, ) -> OperationReceipt | None: + self._assert_runtime_owner() if not idempotency_key: return None record = self._store.find_by_idempotency(idempotency_key) @@ -391,6 +437,14 @@ def _idempotent_receipt( self._operations[record.receipt.operation_id] = record return record.receipt - def _persist_record(self, record: ControlPlaneOperationRecord) -> None: - self._operations[record.receipt.operation_id] = record - self._store.save_record(record) + def _claim_record(self, record: ControlPlaneOperationRecord) -> ControlPlaneOperationRecord: + self._assert_runtime_owner() + persisted = self._store_commits.claim_record(record) + if ( + persisted.request_fingerprint + and record.request_fingerprint + and persisted.request_fingerprint != record.request_fingerprint + ): + raise ValueError("Idempotency-Key was reused with a different request body.") + self._operations[persisted.receipt.operation_id] = persisted + return persisted diff --git a/implementations/python/packages/raes_runtime/control_plane_durability.py b/implementations/python/packages/raes_runtime/control_plane_durability.py new file mode 100644 index 00000000..2b0d2fd5 --- /dev/null +++ b/implementations/python/packages/raes_runtime/control_plane_durability.py @@ -0,0 +1,101 @@ +"""Durable commit and cache-reconciliation helpers for the control plane.""" + +from __future__ import annotations + +from raes_contracts.runtime_state import RuntimeSnapshot + +from .control_plane_store import AuditEvent, ControlPlaneOperationRecord, ControlPlaneStore +from .control_plane_store_compatibility import ControlPlaneStoreCommitAdapter + + +class RuntimeDurabilityMixin: + """Keep live caches aligned with durable state after every store outcome.""" + + _store: ControlPlaneStore + _store_commits: ControlPlaneStoreCommitAdapter + _snapshot: RuntimeSnapshot + _operations: dict[str, ControlPlaneOperationRecord] + + def _commit_terminal_operation( + self, + snapshot: RuntimeSnapshot, + record: ControlPlaneOperationRecord, + ) -> None: + self._assert_runtime_owner() + try: + self._store_commits.commit_terminal_operation(snapshot, record) + except BaseException as exc: + self._resynchronize_after_store_error(exc) + raise + self._publish_committed_state(snapshot, record) + + def _commit_control_transition( + self, + *, + participant_address: str, + expected_head: str | None, + snapshot: RuntimeSnapshot, + record: ControlPlaneOperationRecord, + audit_event: AuditEvent, + ) -> None: + self._assert_runtime_owner() + try: + self._store.commit_control_transition( + participant_address=participant_address, + expected_head=expected_head, + snapshot=snapshot, + record=record, + audit_event=audit_event, + ) + except BaseException as exc: + self._resynchronize_after_store_error(exc) + raise + self._publish_committed_state(snapshot, record) + + def _commit_participant_transition( + self, + *, + expected_history_heads: dict[str, str | None], + snapshot: RuntimeSnapshot, + record: ControlPlaneOperationRecord, + audit_event: AuditEvent, + ) -> None: + self._assert_runtime_owner() + try: + self._store.commit_participant_transition( + expected_history_heads=expected_history_heads, + snapshot=snapshot, + record=record, + audit_event=audit_event, + ) + except BaseException as exc: + self._resynchronize_after_store_error(exc) + raise + self._publish_committed_state(snapshot, record) + + def _publish_committed_state( + self, + snapshot: RuntimeSnapshot, + record: ControlPlaneOperationRecord, + ) -> None: + self._snapshot = snapshot + self._operations[record.receipt.operation_id] = record + + def _resynchronize_after_store_error(self, error: BaseException) -> None: + """Refresh both caches after a store call with an uncertain outcome.""" + + try: + snapshot = self._store.load_snapshot() + operations = self._store.load_records() + except Exception as reconciliation_error: + self._poison_runtime_durability() + error.add_note( + "Durable state could not be reconciled after the store error; " + f"this runtime is poisoned until restart ({reconciliation_error!r})." + ) + return + self._snapshot = snapshot + self._operations = operations + + +__all__ = ("RuntimeDurabilityMixin",) diff --git a/implementations/python/packages/raes_runtime/control_plane_execution.py b/implementations/python/packages/raes_runtime/control_plane_execution.py index c05582f5..3ba0cd58 100644 --- a/implementations/python/packages/raes_runtime/control_plane_execution.py +++ b/implementations/python/packages/raes_runtime/control_plane_execution.py @@ -102,7 +102,7 @@ def _execute_participant_action_locked( submitted_at=submitted_at, accepted=True, ) - control_plane._persist_record( + claimed = control_plane._claim_record( ControlPlaneOperationRecord( receipt=receipt, status=status, @@ -110,6 +110,8 @@ def _execute_participant_action_locked( request_fingerprint=request_fingerprint, ) ) + if claimed.receipt.operation_id != operation_id: + return claimed.receipt result = _call_backend_apply( method, request, @@ -122,8 +124,6 @@ def _execute_participant_action_locked( None, ), ) - control_plane._snapshot = result.snapshot - control_plane._store.save_snapshot(control_plane._snapshot) final_state = OperationState.SUCCEEDED if result.success else OperationState.FAILED final_status = OperationStatus( operation_id=operation_id, @@ -134,13 +134,14 @@ def _execute_participant_action_locked( diagnostics=[*status.diagnostics, *result.diagnostics], changed_addresses=list(result.changed_addresses), ) - control_plane._persist_record( + control_plane._commit_terminal_operation( + result.snapshot, ControlPlaneOperationRecord( receipt=receipt, status=final_status, idempotency_key=idempotency_key, request_fingerprint=request_fingerprint, - ) + ), ) return receipt @@ -164,7 +165,7 @@ def persist_succeeded_operation( updated_at=request.submitted_at, changed_addresses=list(request.changed_addresses or []), ) - control_plane._persist_record( + claimed = control_plane._claim_record( ControlPlaneOperationRecord( receipt=receipt, status=status, @@ -172,6 +173,8 @@ def persist_succeeded_operation( request_fingerprint=request.request_fingerprint, ) ) + if claimed.receipt.operation_id != request.operation_id: + return claimed.receipt return receipt @@ -200,6 +203,14 @@ class OperationExecutionRequest: def execute_operation( control_plane: object, request: OperationExecutionRequest, +) -> OperationReceipt: + with control_plane._operation_lock: + return _execute_operation_locked(control_plane, request) + + +def _execute_operation_locked( + control_plane: object, + request: OperationExecutionRequest, ) -> OperationReceipt: existing = control_plane._idempotent_receipt( idempotency_key=request.idempotency_key, @@ -225,7 +236,7 @@ def execute_operation( accepted=True, diagnostics=list(request.diagnostics), ) - control_plane._persist_record( + claimed = control_plane._claim_record( ControlPlaneOperationRecord( receipt=receipt, status=status, @@ -233,6 +244,8 @@ def execute_operation( request_fingerprint=request.request_fingerprint, ) ) + if claimed.receipt.operation_id != operation_id: + return claimed.receipt result = _call_backend_apply( request.method, request.plan, @@ -254,8 +267,6 @@ def execute_operation( None, ), ) - control_plane._snapshot = result.snapshot - control_plane._store.save_snapshot(control_plane._snapshot) final_state = OperationState.SUCCEEDED if result.success else OperationState.FAILED final_status = OperationStatus( operation_id=operation_id, @@ -266,12 +277,13 @@ def execute_operation( diagnostics=[*status.diagnostics, *result.diagnostics], changed_addresses=list(result.changed_addresses), ) - control_plane._persist_record( + control_plane._commit_terminal_operation( + result.snapshot, ControlPlaneOperationRecord( receipt=receipt, status=final_status, idempotency_key=request.idempotency_key, request_fingerprint=request.request_fingerprint, - ) + ), ) return receipt diff --git a/implementations/python/packages/raes_runtime/control_plane_lifecycle.py b/implementations/python/packages/raes_runtime/control_plane_lifecycle.py new file mode 100644 index 00000000..e3e68243 --- /dev/null +++ b/implementations/python/packages/raes_runtime/control_plane_lifecycle.py @@ -0,0 +1,139 @@ +"""Lifecycle admission for public runtime-control-plane calls.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator +from contextlib import contextmanager, suppress +from functools import wraps +from threading import Condition, RLock, local +from typing import Concatenate, ParamSpec, Self, TypeVar + +_P = ParamSpec("_P") +_R = TypeVar("_R") + + +def runtime_owned( + method: Callable[Concatenate[object, _P], _R], +) -> Callable[Concatenate[object, _P], _R]: + """Keep one public call admitted until its reads or effects are complete.""" + + @wraps(method) + def guarded(control_plane: object, *args: _P.args, **kwargs: _P.kwargs) -> _R: + runtime_call = getattr(control_plane, "_runtime_call", None) + if not callable(runtime_call): + return method(control_plane, *args, **kwargs) + with runtime_call(): + return method(control_plane, *args, **kwargs) + + guarded.__runtime_owned__ = True + return guarded + + +class RuntimeLifecycleMixin: + """Drain admitted calls before releasing process-scoped authority.""" + + _lifecycle_condition: Condition + _lifecycle_local: local + _active_runtime_calls: int + _closing: bool + _closed: bool + _durability_poisoned: bool + _runtime_lease: object | None + + def _initialize_runtime_lifecycle(self) -> None: + self._lifecycle_condition = Condition(RLock()) + self._lifecycle_local = local() + self._active_runtime_calls = 0 + self._closing = False + self._closed = False + self._durability_poisoned = False + self._runtime_lease = None + + @runtime_owned + def __enter__(self) -> Self: + self._assert_runtime_owner() + return self + + def __exit__(self, _exc_type: object, _exc: object, _traceback: object) -> None: + self.close() + + def __del__(self) -> None: + with suppress(Exception): + self.close() + + def close(self) -> None: + """Release this control plane's process-scoped runtime authority.""" + + condition = getattr(self, "_lifecycle_condition", None) + if condition is None: + self._close_runtime_lease() + return + with condition: + if self._closed: + return + if getattr(self._lifecycle_local, "depth", 0): + raise RuntimeError("cannot close a runtime control plane from one of its active calls") + if self._closing: + condition.wait_for(lambda: self._closed) + return + self._closing = True + try: + condition.wait_for(lambda: self._active_runtime_calls == 0) + except BaseException: + self._closing = False + condition.notify_all() + raise + try: + self._close_runtime_lease() + finally: + self._closed = True + self._closing = False + condition.notify_all() + + def _close_runtime_lease(self) -> None: + lease = getattr(self, "_runtime_lease", None) + close = getattr(lease, "close", None) + try: + if callable(close): + close() + finally: + self._runtime_lease = None + + @contextmanager + def _runtime_call(self) -> Iterator[None]: + condition = self._lifecycle_condition + with condition: + depth = getattr(self._lifecycle_local, "depth", 0) + if self._durability_poisoned: + raise RuntimeError("runtime control plane requires restart after a durability reconciliation failure") + if depth == 0 and (self._closed or self._closing): + raise RuntimeError("runtime control plane is closed") + lease = self._runtime_lease + assert_owner = getattr(lease, "assert_owner", None) + if callable(assert_owner): + assert_owner() + self._active_runtime_calls += 1 + self._lifecycle_local.depth = depth + 1 + try: + yield + finally: + with condition: + self._lifecycle_local.depth -= 1 + self._active_runtime_calls -= 1 + if self._active_runtime_calls == 0: + condition.notify_all() + + def _poison_runtime_durability(self) -> None: + with self._lifecycle_condition: + self._durability_poisoned = True + + def _assert_runtime_owner(self) -> None: + if self._closed: + raise RuntimeError("runtime control plane is closed") + lease = self._runtime_lease + assert_owner = getattr(lease, "assert_owner", None) + if callable(assert_owner): + assert_owner() + + +__all__ = ("RuntimeLifecycleMixin", "runtime_owned") diff --git a/implementations/python/packages/raes_runtime/control_plane_recovery.py b/implementations/python/packages/raes_runtime/control_plane_recovery.py new file mode 100644 index 00000000..83a5285c --- /dev/null +++ b/implementations/python/packages/raes_runtime/control_plane_recovery.py @@ -0,0 +1,62 @@ +"""Crash-recovery policy for runtime control-plane startup.""" + +from __future__ import annotations + +from dataclasses import replace +from datetime import UTC, datetime + +from raes_contracts.diagnostics import Diagnostic +from raes_contracts.runtime_state import OperationState + +from .control_plane_store import ( + INTERRUPTED_OPERATION_DIAGNOSTIC_CODE, + ControlPlaneOperationRecord, +) +from .control_plane_store_compatibility import ControlPlaneStoreCommitAdapter + + +def _utc_now() -> str: + return datetime.now(UTC).isoformat().replace("+00:00", "Z") + + +def reconcile_interrupted_operations( + store_commits: ControlPlaneStoreCommitAdapter, + operations: dict[str, ControlPlaneOperationRecord], +) -> dict[str, ControlPlaneOperationRecord]: + """Seal orphaned non-terminal records without replaying backend effects.""" + + recovered_at = _utc_now() + interrupted = tuple( + replace( + record, + status=replace( + record.status, + state=OperationState.FAILED, + updated_at=recovered_at, + diagnostics=[ + *record.status.diagnostics, + Diagnostic( + code=INTERRUPTED_OPERATION_DIAGNOSTIC_CODE, + domain="runtime", + address=f"runtime.control-plane.{record.receipt.domain.value}", + message=( + "Operation was interrupted before its terminal durable commit; " + "backend effects may be indeterminate and were not replayed." + ), + ), + ], + ), + ) + for record in operations.values() + if record.status.state in {OperationState.ACCEPTED, OperationState.RUNNING} + ) + if not interrupted: + return operations + store_commits.reconcile_interrupted_records(interrupted) + return { + **operations, + **{record.receipt.operation_id: record for record in interrupted}, + } + + +__all__ = ("reconcile_interrupted_operations",) diff --git a/implementations/python/packages/raes_runtime/control_plane_store.py b/implementations/python/packages/raes_runtime/control_plane_store.py index f5e7a1e5..d880814b 100644 --- a/implementations/python/packages/raes_runtime/control_plane_store.py +++ b/implementations/python/packages/raes_runtime/control_plane_store.py @@ -4,36 +4,39 @@ import hashlib import json -import os from dataclasses import dataclass, field from enum import Enum +from threading import RLock from typing import TYPE_CHECKING, Any, Protocol -from raes_contracts.account_credentials import ( - account_placement_has_credential_bindings, - value_free_account_placement_payload, -) -from raes_contracts.artifact_requirements import ArtifactSatisfactionDisclosureModel -from raes_contracts.contracts import RealizationEnvelopeIdentityModel -from raes_contracts.contracts.time_model import TimeRuntimeStateModel from raes_contracts.participant_autonomous_state import require_participant_autonomous_runtime_snapshot -from raes_contracts.planning import RuntimeDomain from raes_contracts.runtime_state import ( - ExplicitnessClass, - ExplicitnessProvenance, OperationReceipt, + OperationState, OperationStatus, - RealizationProvenanceEntry, RuntimeSnapshot, - RuntimeSnapshotEnvelope, - SnapshotEntry, ) -from .control_plane_store_observations import realization_observation_from_payload +from .control_plane_store_snapshots import _snapshot_from_payload as _decode_snapshot_payload +from .control_plane_store_snapshots import _snapshot_payload as _encode_snapshot_payload if TYPE_CHECKING: from .control_plane_store_local import LocalControlPlaneStore +_IDEMPOTENCY_KEY_CONFLICT = "idempotency key already belongs to another operation" + + +def _snapshot_payload(snapshot: RuntimeSnapshot) -> dict[str, Any]: + """Retain the pre-split private codec import for compatible callers.""" + + return _encode_snapshot_payload(snapshot) + + +def _snapshot_from_payload(payload: dict[str, Any]) -> RuntimeSnapshot: + """Retain the pre-split private codec import for compatible callers.""" + + return _decode_snapshot_payload(payload) + class ParticipantCrossingHistoryPresence(str, Enum): """Source-level API-423 history presence before snapshot defaults apply.""" @@ -85,8 +88,88 @@ class ControlPlaneOperationRecord: result_history_heads: dict[str, str | None] = field(default_factory=dict) +INTERRUPTED_OPERATION_DIAGNOSTIC_CODE = "runtime.control-plane.operation-interrupted" +_NON_TERMINAL_OPERATION_STATES = {OperationState.ACCEPTED, OperationState.RUNNING} + + +def _require_operation_record_identity(record: ControlPlaneOperationRecord) -> None: + if record.receipt.operation_id != record.status.operation_id: + raise ValueError("operation receipt and status identities do not match") + if record.receipt.domain != record.status.domain: + raise ValueError("operation receipt and status domains do not match") + if record.receipt.submitted_at != record.status.submitted_at: + raise ValueError("operation receipt and status submission times do not match") + + +def _require_same_operation_identity( + existing: ControlPlaneOperationRecord, + replacement: ControlPlaneOperationRecord, +) -> None: + _require_operation_record_identity(existing) + _require_operation_record_identity(replacement) + if existing.receipt != replacement.receipt: + raise ValueError("operation receipt is immutable after its durable claim") + if ( + existing.status.schema_version, + existing.status.operation_id, + existing.status.domain, + existing.status.submitted_at, + existing.idempotency_key, + existing.request_fingerprint, + ) != ( + replacement.status.schema_version, + replacement.status.operation_id, + replacement.status.domain, + replacement.status.submitted_at, + replacement.idempotency_key, + replacement.request_fingerprint, + ): + raise ValueError("operation identity is immutable after its durable claim") + + +def _require_terminal_operation_transition( + existing: ControlPlaneOperationRecord | None, + replacement: ControlPlaneOperationRecord, +) -> bool: + """Validate a terminal transition and return whether it changes the record.""" + + _require_operation_record_identity(replacement) + if replacement.status.state in _NON_TERMINAL_OPERATION_STATES: + raise ValueError("terminal operation commit requires a terminal status") + if existing is None: + return True + _require_same_operation_identity(existing, replacement) + if existing.status.state in _NON_TERMINAL_OPERATION_STATES: + return True + if existing != replacement: + raise ValueError("a terminal operation record cannot be rewritten") + return False + + +def _require_interrupted_operation_transition( + existing: ControlPlaneOperationRecord | None, + replacement: ControlPlaneOperationRecord, +) -> bool: + """Validate conservative startup recovery for one interrupted operation.""" + + if existing is None: + raise ValueError("interrupted operation no longer exists in durable state") + _require_same_operation_identity(existing, replacement) + if replacement.status.state != OperationState.FAILED: + raise ValueError("interrupted operation recovery must persist a failed status") + if not any( + diagnostic.code == INTERRUPTED_OPERATION_DIAGNOSTIC_CODE for diagnostic in replacement.status.diagnostics + ): + raise ValueError("interrupted operation recovery requires its stable diagnostic") + if existing.status.state in _NON_TERMINAL_OPERATION_STATES: + return True + if existing != replacement: + raise ValueError("a terminal operation record cannot be rewritten during recovery") + return False + + class ControlPlaneStore(Protocol): - """Durable persistence for control-plane state.""" + """Legacy-compatible durable persistence for control-plane state.""" def load_snapshot(self) -> RuntimeSnapshot: ... @@ -125,6 +208,23 @@ def commit_participant_transition( ) -> None: ... +class AtomicControlPlaneStore(ControlPlaneStore, Protocol): + """Optional crash-atomic terminal commit and recovery capabilities.""" + + def claim_record(self, record: ControlPlaneOperationRecord) -> ControlPlaneOperationRecord: ... + + def commit_terminal_operation( + self, + snapshot: RuntimeSnapshot, + record: ControlPlaneOperationRecord, + ) -> None: ... + + def reconcile_interrupted_records( + self, + records: tuple[ControlPlaneOperationRecord, ...], + ) -> None: ... + + def _control_history_head(snapshot: RuntimeSnapshot, participant_address: str) -> str | None: events = snapshot.participant_control_history.get(participant_address, ()) if not events: @@ -173,261 +273,99 @@ def _require_expected_history_heads( raise ValueError("expected participant history head does not match durable state") -def _snapshot_payload(snapshot: RuntimeSnapshot) -> dict[str, Any]: - require_participant_autonomous_runtime_snapshot(snapshot) - payload = { - "schema_version": RuntimeSnapshotEnvelope().schema_version, - "entries": { - address: { - "address": entry.address, - "domain": entry.domain.value, - "resource_type": entry.resource_type, - "payload": dict(entry.payload), - "ordering_dependencies": list(entry.ordering_dependencies), - "refresh_dependencies": list(entry.refresh_dependencies), - "status": entry.status, - } - for address, entry in snapshot.entries.items() - }, - "orchestration_results": dict(snapshot.orchestration_results), - "orchestration_history": {address: list(events) for address, events in snapshot.orchestration_history.items()}, - "evaluation_results": dict(snapshot.evaluation_results), - "evaluation_history": {address: list(events) for address, events in snapshot.evaluation_history.items()}, - "proposition_truth_results": dict(snapshot.proposition_truth_results), - "participant_episode_results": dict(snapshot.participant_episode_results), - "participant_episode_history": { - participant_address: list(events) - for participant_address, events in snapshot.participant_episode_history.items() - }, - "participant_behavior_history": { - participant_address: list(events) - for participant_address, events in snapshot.participant_behavior_history.items() - }, - "participant_control_history": { - participant_address: list(events) - for participant_address, events in snapshot.participant_control_history.items() - }, - "participant_crossing_history": { - participant_address: list(events) - for participant_address, events in snapshot.participant_crossing_history.items() - }, - "information_state_history": { - participant_address: list(records) - for participant_address, records in snapshot.information_state_history.items() - }, - "participant_autonomous_execution_states": dict(snapshot.participant_autonomous_execution_states), - "participant_execution_services": dict(snapshot.participant_execution_services), - "participant_resource_budget_states": dict(snapshot.participant_resource_budget_states), - "participant_resource_pool_states": dict(snapshot.participant_resource_pool_states), - "participant_resource_budget_events": dict(snapshot.participant_resource_budget_events), - "shared_state_records": dict(snapshot.shared_state_records), - "shared_state_history": { - state_address: list(records) for state_address, records in snapshot.shared_state_history.items() - }, - "joint_action_records": dict(snapshot.joint_action_records), - "time_management_contexts": dict(snapshot.time_management_contexts), - "time_model_state": ( - snapshot.time_model_state.model_dump(mode="json") if snapshot.time_model_state is not None else None - ), - "realization_provenance": [ - { - "address": entry.address, - "field_path": entry.field_path, - "domain": entry.domain, - "requirement_kind": entry.requirement_kind, - "explicitness": entry.explicitness.value, - "provenance": entry.provenance.value, - "governing_scope": entry.governing_scope, - "artifact_satisfaction": ( - entry.artifact_satisfaction.model_dump(mode="json") - if entry.artifact_satisfaction is not None - else None - ), - } - for entry in snapshot.realization_provenance - ], - "realization_observations": [ - { - "address": entry.address, - "field_path": entry.field_path, - "domain": entry.domain, - "requirement_kind": entry.requirement_kind, - "verification_scope": entry.verification_scope.value, - "observation_strength": entry.observation_strength.value, - **( - { - "observed_value": entry.observed_value, - "operating_system": ( - { - "family": entry.operating_system.family, - "distribution": entry.operating_system.distribution, - "version": entry.operating_system.version, - } - if entry.operating_system is not None - else None - ), - "operation_id": entry.operation_id, - "envelope_digest": entry.envelope_digest, - "configuration_digest": entry.configuration_digest, - "observer_version": entry.observer_version, - "sequence": entry.sequence, - "binding_verified": entry.binding_verified, - } - if entry.requirement_kind in {"compute-substrate", "operating-system"} - else {} - ), - } - for entry in snapshot.realization_observations - ], - "realization_envelope": ( - snapshot.realization_envelope.model_dump(mode="json") if snapshot.realization_envelope is not None else None - ), - "metadata": dict(snapshot.metadata), - } - for entry in payload["entries"].values(): - if entry["resource_type"] != "account-placement": - continue - entry_payload = entry["payload"] - if account_placement_has_credential_bindings(entry_payload): - entry["payload"] = value_free_account_placement_payload(entry_payload) - return payload - - -def _snapshot_from_payload(payload: dict[str, Any]) -> RuntimeSnapshot: - entries_payload = payload.get("entries", {}) - entries = { - address: SnapshotEntry( - address=str(entry.get("address", address)), - domain=RuntimeDomain(str(entry.get("domain", "provisioning"))), - resource_type=str(entry.get("resource_type", "")), - payload=dict(entry.get("payload", {})), - ordering_dependencies=tuple(entry.get("ordering_dependencies", ())), - refresh_dependencies=tuple(entry.get("refresh_dependencies", ())), - status=str(entry.get("status", "ready")), - ) - for address, entry in entries_payload.items() - if isinstance(entry, dict) - } - snapshot = RuntimeSnapshot( - entries=entries, - orchestration_results=dict(payload.get("orchestration_results", {})), - orchestration_history={ - address: list(events) for address, events in payload.get("orchestration_history", {}).items() - }, - evaluation_results=dict(payload.get("evaluation_results", {})), - evaluation_history={address: list(events) for address, events in payload.get("evaluation_history", {}).items()}, - proposition_truth_results=dict(payload.get("proposition_truth_results", {})), - participant_episode_results=dict(payload.get("participant_episode_results", {})), - participant_episode_history={ - participant_address: list(events) - for participant_address, events in payload.get("participant_episode_history", {}).items() - }, - participant_behavior_history={ - participant_address: list(events) - for participant_address, events in payload.get("participant_behavior_history", {}).items() - }, - participant_control_history={ - participant_address: list(events) - for participant_address, events in payload.get("participant_control_history", {}).items() - }, - participant_crossing_history={ - participant_address: list(events) - for participant_address, events in payload.get("participant_crossing_history", {}).items() - }, - information_state_history={ - participant_address: list(records) - for participant_address, records in payload.get("information_state_history", {}).items() - }, - participant_autonomous_execution_states=dict(payload.get("participant_autonomous_execution_states", {})), - participant_execution_services=dict(payload.get("participant_execution_services", {})), - participant_resource_budget_states=dict(payload.get("participant_resource_budget_states", {})), - participant_resource_pool_states=dict(payload.get("participant_resource_pool_states", {})), - participant_resource_budget_events=dict(payload.get("participant_resource_budget_events", {})), - shared_state_records=dict(payload.get("shared_state_records", {})), - shared_state_history={ - state_address: list(records) for state_address, records in payload.get("shared_state_history", {}).items() - }, - joint_action_records=dict(payload.get("joint_action_records", {})), - time_management_contexts=dict(payload.get("time_management_contexts", {})), - time_model_state=( - TimeRuntimeStateModel.model_validate(payload["time_model_state"]) - if payload.get("time_model_state") is not None - else None - ), - realization_provenance=tuple( - RealizationProvenanceEntry( - address=str(item.get("address", "")), - field_path=str(item.get("field_path", "")), - domain=str(item.get("domain", "")), - requirement_kind=str(item.get("requirement_kind", "")), - explicitness=ExplicitnessClass(str(item.get("explicitness", ExplicitnessClass.EXACT.value))), - provenance=ExplicitnessProvenance( - str(item.get("provenance", ExplicitnessProvenance.AUTHOR_DECLARED.value)) - ), - governing_scope=(str(item["governing_scope"]) if item.get("governing_scope") is not None else None), - artifact_satisfaction=( - ArtifactSatisfactionDisclosureModel.model_validate(item["artifact_satisfaction"]) - if item.get("artifact_satisfaction") is not None - else None - ), - ) - for item in payload.get("realization_provenance", []) - if isinstance(item, dict) - ), - realization_observations=tuple( - realization_observation_from_payload(item) - for item in payload.get("realization_observations", []) - if isinstance(item, dict) - ), - realization_envelope=( - RealizationEnvelopeIdentityModel.model_validate(payload["realization_envelope"]) - if payload.get("realization_envelope") is not None - else None - ), - metadata=dict(payload.get("metadata", {})), - ) - require_participant_autonomous_runtime_snapshot(snapshot) - return snapshot - - class InMemoryControlPlaneStore: """Simple in-memory store.""" def __init__(self, snapshot: RuntimeSnapshot | None = None) -> None: + self._lock = RLock() self._snapshot = snapshot if snapshot is not None else RuntimeSnapshot() self._records: dict[str, ControlPlaneOperationRecord] = {} self._idempotency: dict[str, str] = {} self._audit: list[AuditEvent] = [] def load_snapshot(self) -> RuntimeSnapshot: - return self._snapshot + with self._lock: + return self._snapshot def save_snapshot(self, snapshot: RuntimeSnapshot) -> None: require_participant_autonomous_runtime_snapshot(snapshot) - self._snapshot = snapshot + with self._lock: + self._snapshot = snapshot def load_records(self) -> dict[str, ControlPlaneOperationRecord]: - return dict(self._records) + with self._lock: + return dict(self._records) def save_record(self, record: ControlPlaneOperationRecord) -> None: - self._records[record.receipt.operation_id] = record - if record.idempotency_key: - self._idempotency[record.idempotency_key] = record.receipt.operation_id + with self._lock: + self._save_record(record) + + def claim_record(self, record: ControlPlaneOperationRecord) -> ControlPlaneOperationRecord: + with self._lock: + if record.idempotency_key: + existing = self.find_by_idempotency(record.idempotency_key) + if existing is not None: + return existing + self._save_record(record) + return record + + def commit_terminal_operation( + self, + snapshot: RuntimeSnapshot, + record: ControlPlaneOperationRecord, + ) -> None: + """Atomically publish a snapshot with its terminal operation record.""" + + require_participant_autonomous_runtime_snapshot(snapshot) + with self._lock: + existing = self._records.get(record.receipt.operation_id) + changed = _require_terminal_operation_transition(existing, record) + if not changed: + if self._snapshot != snapshot: + raise ValueError("terminal operation retry does not match the durable snapshot") + return + records = {**self._records, record.receipt.operation_id: record} + idempotency = dict(self._idempotency) + if record.idempotency_key: + existing_operation_id = idempotency.get(record.idempotency_key) + if existing_operation_id is not None and existing_operation_id != record.receipt.operation_id: + raise ValueError(_IDEMPOTENCY_KEY_CONFLICT) + idempotency[record.idempotency_key] = record.receipt.operation_id + self._snapshot = snapshot + self._records = records + self._idempotency = idempotency + + def reconcile_interrupted_records( + self, + records: tuple[ControlPlaneOperationRecord, ...], + ) -> None: + """Atomically replace orphaned non-terminal records during startup.""" + + with self._lock: + staged = dict(self._records) + for record in records: + existing = staged.get(record.receipt.operation_id) + if _require_interrupted_operation_transition(existing, record): + staged[record.receipt.operation_id] = record + self._records = staged def find_by_idempotency( self, key: str, ) -> ControlPlaneOperationRecord | None: - operation_id = self._idempotency.get(key) - if operation_id is None: - return None - return self._records.get(operation_id) + with self._lock: + operation_id = self._idempotency.get(key) + if operation_id is None: + return None + return self._records.get(operation_id) def append_audit(self, event: AuditEvent) -> None: - self._audit.append(event) + with self._lock: + self._audit.append(event) def read_audit(self) -> list[AuditEvent]: - return list(self._audit) + with self._lock: + return list(self._audit) def commit_control_transition( self, @@ -438,15 +376,16 @@ def commit_control_transition( record: ControlPlaneOperationRecord, audit_event: AuditEvent, ) -> None: - _require_expected_control_head(self._snapshot, participant_address, expected_head) - self.commit_participant_transition( - expected_history_heads={ - f"participant_control_history:{participant_address}": expected_head, - }, - snapshot=snapshot, - record=record, - audit_event=audit_event, - ) + with self._lock: + _require_expected_control_head(self._snapshot, participant_address, expected_head) + self.commit_participant_transition( + expected_history_heads={ + f"participant_control_history:{participant_address}": expected_head, + }, + snapshot=snapshot, + record=record, + audit_event=audit_event, + ) def commit_participant_transition( self, @@ -456,16 +395,28 @@ def commit_participant_transition( record: ControlPlaneOperationRecord, audit_event: AuditEvent, ) -> None: - _require_expected_history_heads(self._snapshot, expected_history_heads) - require_participant_autonomous_runtime_snapshot(snapshot) - records = {**self._records, record.receipt.operation_id: record} - idempotency = dict(self._idempotency) + with self._lock: + _require_expected_history_heads(self._snapshot, expected_history_heads) + require_participant_autonomous_runtime_snapshot(snapshot) + records = {**self._records, record.receipt.operation_id: record} + idempotency = dict(self._idempotency) + if record.idempotency_key: + existing_operation_id = idempotency.get(record.idempotency_key) + if existing_operation_id is not None and existing_operation_id != record.receipt.operation_id: + raise ValueError(_IDEMPOTENCY_KEY_CONFLICT) + idempotency[record.idempotency_key] = record.receipt.operation_id + self._snapshot = snapshot + self._records = records + self._idempotency = idempotency + self._audit = [*self._audit, audit_event] + + def _save_record(self, record: ControlPlaneOperationRecord) -> None: if record.idempotency_key: - idempotency[record.idempotency_key] = record.receipt.operation_id - self._snapshot = snapshot - self._records = records - self._idempotency = idempotency - self._audit = [*self._audit, audit_event] + existing_operation_id = self._idempotency.get(record.idempotency_key) + if existing_operation_id is not None and existing_operation_id != record.receipt.operation_id: + raise ValueError(_IDEMPOTENCY_KEY_CONFLICT) + self._idempotency[record.idempotency_key] = record.receipt.operation_id + self._records[record.receipt.operation_id] = record def __getattr__(name: str) -> object: @@ -479,12 +430,13 @@ def __getattr__(name: str) -> object: __all__ = ( + "AtomicControlPlaneStore", "AuditEvent", "ControlPlaneOperationRecord", "ControlPlaneStore", + "INTERRUPTED_OPERATION_DIAGNOSTIC_CODE", "InMemoryControlPlaneStore", "LocalControlPlaneStore", "ParticipantCrossingHistoryPresence", - "os", "participant_crossing_history_presence", ) diff --git a/implementations/python/packages/raes_runtime/control_plane_store_compatibility.py b/implementations/python/packages/raes_runtime/control_plane_store_compatibility.py new file mode 100644 index 00000000..24f6ba9b --- /dev/null +++ b/implementations/python/packages/raes_runtime/control_plane_store_compatibility.py @@ -0,0 +1,99 @@ +"""Compatibility seam for optional crash-atomic store capabilities.""" + +from __future__ import annotations + +import warnings +from typing import cast + +from raes_contracts.runtime_state import RuntimeSnapshot + +from .control_plane_store import AtomicControlPlaneStore, ControlPlaneOperationRecord, ControlPlaneStore + +_LEGACY_STORE_METHODS = ( + "load_snapshot", + "save_snapshot", + "load_records", + "save_record", + "find_by_idempotency", + "append_audit", + "read_audit", + "commit_control_transition", + "commit_participant_transition", +) +_ATOMIC_STORE_METHODS = ( + "claim_record", + "commit_terminal_operation", + "reconcile_interrupted_records", +) + + +class LegacyControlPlaneStoreWarning(DeprecationWarning): + """A custom store is using the non-crash-atomic 3.x compatibility path.""" + + +class ControlPlaneStoreCommitAdapter: + """Centralize complete atomic capability use or ordered legacy fallback.""" + + def __init__(self, store: ControlPlaneStore, *, crash_atomic: bool) -> None: + self._store = store + self.crash_atomic = crash_atomic + + def commit_terminal_operation( + self, + snapshot: RuntimeSnapshot, + record: ControlPlaneOperationRecord, + ) -> None: + if self.crash_atomic: + cast(AtomicControlPlaneStore, self._store).commit_terminal_operation(snapshot, record) + return + self._store.save_snapshot(snapshot) + self._store.save_record(record) + + def claim_record(self, record: ControlPlaneOperationRecord) -> ControlPlaneOperationRecord: + if self.crash_atomic: + return cast(AtomicControlPlaneStore, self._store).claim_record(record) + if record.idempotency_key: + existing = self._store.find_by_idempotency(record.idempotency_key) + if existing is not None: + return existing + self._store.save_record(record) + return record + + def reconcile_interrupted_records( + self, + records: tuple[ControlPlaneOperationRecord, ...], + ) -> None: + if self.crash_atomic: + cast(AtomicControlPlaneStore, self._store).reconcile_interrupted_records(records) + return + for record in records: + self._store.save_record(record) + + +def adapt_control_plane_store(store: object) -> ControlPlaneStoreCommitAdapter: + """Validate the legacy contract and select one stable commit mode.""" + + missing_legacy = [name for name in _LEGACY_STORE_METHODS if not callable(getattr(store, name, None))] + if missing_legacy: + capabilities = ", ".join(missing_legacy) + raise TypeError(f"control-plane store is missing required capabilities: {capabilities}") + + missing_atomic = [name for name in _ATOMIC_STORE_METHODS if not callable(getattr(store, name, None))] + crash_atomic = not missing_atomic + if missing_atomic: + capabilities = ", ".join(missing_atomic) + warnings.warn( + "custom control-plane store is using the deprecated non-crash-atomic 3.x " + "compatibility path because it lacks a complete atomic capability set " + f"({capabilities}); implement all atomic methods before version 4", + LegacyControlPlaneStoreWarning, + stacklevel=3, + ) + return ControlPlaneStoreCommitAdapter(cast(ControlPlaneStore, store), crash_atomic=crash_atomic) + + +__all__ = ( + "ControlPlaneStoreCommitAdapter", + "LegacyControlPlaneStoreWarning", + "adapt_control_plane_store", +) diff --git a/implementations/python/packages/raes_runtime/control_plane_store_lease.py b/implementations/python/packages/raes_runtime/control_plane_store_lease.py new file mode 100644 index 00000000..baa3290c --- /dev/null +++ b/implementations/python/packages/raes_runtime/control_plane_store_lease.py @@ -0,0 +1,238 @@ +"""Secure, exclusive runtime ownership for the local control-plane store.""" + +from __future__ import annotations + +import os +import stat +from contextlib import suppress +from pathlib import Path + +_WORKER_COUNT_ENVIRONMENTS = ("WEB_CONCURRENCY", "UVICORN_WORKERS") + + +def _is_windows() -> bool: + return os.name == "nt" + + +def _runtime_owner_is_reparse_point(metadata: os.stat_result) -> bool: + attributes = getattr(metadata, "st_file_attributes", 0) + reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) + return bool(attributes & reparse_flag) + + +def _require_safe_runtime_owner_metadata(metadata: os.stat_result, path: Path) -> None: + if stat.S_ISLNK(metadata.st_mode) or _runtime_owner_is_reparse_point(metadata): + raise RuntimeError(f"runtime-owner lock path must not be a symlink or reparse point: {path}") + if not stat.S_ISREG(metadata.st_mode): + raise RuntimeError(f"runtime-owner lock path must be a regular file: {path}") + get_effective_uid = getattr(os, "geteuid", None) + if callable(get_effective_uid) and metadata.st_uid != get_effective_uid(): + raise RuntimeError(f"runtime-owner lock path must be owned by the current user: {path}") + if getattr(metadata, "st_nlink", 1) != 1: + raise RuntimeError(f"runtime-owner lock path must not have hard links: {path}") + + +def _existing_runtime_owner_metadata(path: Path) -> os.stat_result | None: + try: + metadata = path.lstat() + except FileNotFoundError: + return None + _require_safe_runtime_owner_metadata(metadata, path) + return metadata + + +def require_single_worker_configuration() -> None: + for variable in _WORKER_COUNT_ENVIRONMENTS: + configured = os.environ.get(variable) + if configured is None or not configured.strip(): + continue + try: + worker_count = int(configured) + except ValueError as exc: + raise RuntimeError(f"{variable} must be 1 for a local control-plane store") from exc + if worker_count != 1: + raise RuntimeError( + f"{variable}={worker_count} is unsupported for a local control-plane store; " + "use exactly one worker with reload disabled" + ) + + +def _lock_runtime_owner(descriptor: int) -> None: + if _is_windows(): + import msvcrt + + if os.fstat(descriptor).st_size == 0: + os.write(descriptor, b"0") + os.lseek(descriptor, 0, os.SEEK_SET) + msvcrt.locking(descriptor, msvcrt.LK_NBLCK, 1) + return + import fcntl + + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + + +def _unlock_runtime_owner(descriptor: int) -> None: + if _is_windows(): + import msvcrt + + os.lseek(descriptor, 0, os.SEEK_SET) + msvcrt.locking(descriptor, msvcrt.LK_UNLCK, 1) + return + import fcntl + + fcntl.flock(descriptor, fcntl.LOCK_UN) + + +def _acquire_store_directory_guard(path: Path) -> int | None: + """Hold a POSIX lock that survives lock-file unlink or replacement.""" + + if _is_windows(): + return None + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path.parent, flags) + except OSError as exc: + raise RuntimeError(f"could not securely open runtime-owner store directory: {path.parent}") from exc + try: + metadata = os.fstat(descriptor) + if not stat.S_ISDIR(metadata.st_mode): + raise RuntimeError(f"runtime-owner store path must be a directory: {path.parent}") + _lock_runtime_owner(descriptor) + except OSError as exc: + os.close(descriptor) + raise RuntimeError( + "local control-plane store already has a runtime owner; use exactly one worker with reload disabled" + ) from exc + except BaseException: + os.close(descriptor) + raise + return descriptor + + +def _release_store_directory_guard(descriptor: int | None, *, owner_process: bool) -> None: + if descriptor is None: + return + try: + if owner_process: + _unlock_runtime_owner(descriptor) + finally: + os.close(descriptor) + + +def _open_runtime_owner_file(path: Path) -> tuple[int, os.stat_result]: + _existing_runtime_owner_metadata(path) + flags = os.O_CREAT | os.O_RDWR + flags |= getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOINHERIT", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags, 0o600) + except OSError as exc: + raise RuntimeError(f"could not securely open runtime-owner lock path: {path}") from exc + try: + opened_metadata = _validate_open_runtime_owner_file(descriptor, path) + _claim_runtime_owner_file(descriptor) + except BaseException: + os.close(descriptor) + raise + return descriptor, opened_metadata + + +def _validate_open_runtime_owner_file(descriptor: int, path: Path) -> os.stat_result: + opened_metadata = os.fstat(descriptor) + _require_safe_runtime_owner_metadata(opened_metadata, path) + current_metadata = path.lstat() + _require_safe_runtime_owner_metadata(current_metadata, path) + if not os.path.samestat(opened_metadata, current_metadata): + raise RuntimeError(f"runtime-owner lock path changed while it was opened: {path}") + return opened_metadata + + +def _claim_runtime_owner_file(descriptor: int) -> None: + if not _is_windows(): + os.fchmod(descriptor, 0o600) + try: + _lock_runtime_owner(descriptor) + except OSError as exc: + raise RuntimeError( + "local control-plane store already has a runtime owner; use exactly one worker with reload disabled" + ) from exc + owner = f"{os.getpid()}\n".encode("ascii") + os.ftruncate(descriptor, 0) + os.write(descriptor, owner) + os.fsync(descriptor) + + +class RuntimeOwnerLease: + """Exclusive, process-bound authority to drive one local runtime target.""" + + def __init__( + self, + descriptor: int, + *, + path: Path | None = None, + identity: os.stat_result | None = None, + directory_descriptor: int | None = None, + ) -> None: + self._descriptor = descriptor + self._path = path + self._identity = identity + self._directory_descriptor = directory_descriptor + self._owner_pid = os.getpid() + self._closed = False + + @classmethod + def acquire(cls, path: Path) -> RuntimeOwnerLease: + directory_descriptor = _acquire_store_directory_guard(path) + try: + descriptor, opened_metadata = _open_runtime_owner_file(path) + except BaseException: + _release_store_directory_guard(directory_descriptor, owner_process=True) + raise + return cls( + descriptor, + path=path, + identity=opened_metadata, + directory_descriptor=directory_descriptor, + ) + + @property + def closed(self) -> bool: + return self._closed + + def assert_owner(self) -> None: + if self._closed: + raise RuntimeError("local control-plane runtime-owner lease is closed") + if os.getpid() != self._owner_pid: + raise RuntimeError( + "local control-plane runtime-owner lease cannot be used after fork; " + "construct one runtime in a single worker with reload disabled" + ) + if self._path is None or self._identity is None: + return + current_metadata = _existing_runtime_owner_metadata(self._path) + if current_metadata is None or not os.path.samestat(self._identity, current_metadata): + raise RuntimeError(f"runtime-owner lock path changed while the lease was active: {self._path}") + + def close(self) -> None: + if self._closed: + return + descriptor = self._descriptor + directory_descriptor = self._directory_descriptor + self._closed = True + owner_process = os.getpid() == self._owner_pid + try: + if owner_process: + _unlock_runtime_owner(descriptor) + finally: + try: + os.close(descriptor) + finally: + _release_store_directory_guard(directory_descriptor, owner_process=owner_process) + + def __del__(self) -> None: + with suppress(OSError): + self.close() + + +__all__ = ("RuntimeOwnerLease", "require_single_worker_configuration") diff --git a/implementations/python/packages/raes_runtime/control_plane_store_legacy.py b/implementations/python/packages/raes_runtime/control_plane_store_legacy.py new file mode 100644 index 00000000..bc3afdfa --- /dev/null +++ b/implementations/python/packages/raes_runtime/control_plane_store_legacy.py @@ -0,0 +1,83 @@ +"""Legacy JSON import readers for the local control-plane store.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from raes_contracts.runtime_state import RuntimeSnapshot + +from .control_plane_store import AuditEvent, ControlPlaneOperationRecord +from .control_plane_store_paths import _participant_transition_count, _read_json_object +from .control_plane_store_records import _audit_event_from_payload, _record_from_payload +from .control_plane_store_snapshots import _snapshot_from_payload + + +def _read_legacy_state( + *, + snapshot_path: Path, + operations_path: Path, + audit_path: Path, + control_state_path: Path, +) -> tuple[RuntimeSnapshot, dict[str, ControlPlaneOperationRecord], list[AuditEvent]]: + control_state = _read_control_state(control_state_path) + return ( + _read_snapshot(snapshot_path, control_state), + _read_records(operations_path, control_state), + _read_audits(audit_path, control_state), + ) + + +def _read_control_state(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + return _read_json_object(path) + + +def _read_snapshot(path: Path, control_state: dict[str, Any]) -> RuntimeSnapshot: + snapshot = RuntimeSnapshot() + if path.exists(): + snapshot = _snapshot_from_payload(_read_json_object(path)) + if control_state: + committed = _snapshot_from_payload(dict(control_state.get("snapshot", {}))) + if _participant_transition_count(committed) > _participant_transition_count(snapshot): + snapshot = committed + return snapshot + + +def _read_records( + path: Path, + control_state: dict[str, Any], +) -> dict[str, ControlPlaneOperationRecord]: + records = { + operation_id: _record_from_payload(payload) + for operation_id, payload in dict(control_state.get("records", {})).items() + if isinstance(payload, dict) + } + if path.exists(): + records.update( + { + operation_id: _record_from_payload(payload) + for operation_id, payload in _read_json_object(path).items() + if isinstance(payload, dict) + } + ) + return records + + +def _read_audits(path: Path, control_state: dict[str, Any]) -> list[AuditEvent]: + audits = [ + _audit_event_from_payload(payload) for payload in control_state.get("audit", []) if isinstance(payload, dict) + ] + if path.exists(): + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + event = _audit_event_from_payload(json.loads(line)) + if event not in audits: + audits.append(event) + return audits + + +__all__ = ("_read_legacy_state",) diff --git a/implementations/python/packages/raes_runtime/control_plane_store_local.py b/implementations/python/packages/raes_runtime/control_plane_store_local.py index 08ddd800..b7e5cf2d 100644 --- a/implementations/python/packages/raes_runtime/control_plane_store_local.py +++ b/implementations/python/packages/raes_runtime/control_plane_store_local.py @@ -1,11 +1,15 @@ -"""Filesystem-backed runtime control-plane persistence.""" +"""SQLite-backed runtime control-plane persistence.""" from __future__ import annotations +import hashlib import json -import tempfile -from contextlib import suppress +import os +import sqlite3 +from collections.abc import Iterator +from contextlib import contextmanager from dataclasses import asdict +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -17,129 +21,171 @@ ControlPlaneOperationRecord, _require_expected_control_head, _require_expected_history_heads, - _snapshot_from_payload, - _snapshot_payload, - os, + _require_interrupted_operation_transition, + _require_terminal_operation_transition, +) +from .control_plane_store_lease import RuntimeOwnerLease, require_single_worker_configuration +from .control_plane_store_legacy import _read_legacy_state +from .control_plane_store_paths import ( + _copy_regular_file_durably, + _fsync_directory, + _require_same_file, + _secure_database_file, + _secure_store_directory, + _validate_sqlite_sidecars, +) +from .control_plane_store_paths import ( + _participant_transition_count as _count_participant_transitions, ) from .control_plane_store_records import ( _audit_event_from_payload, _record_from_payload, _record_payload, ) +from .control_plane_store_snapshots import _snapshot_from_payload, _snapshot_payload + +_DATABASE_NAME = "control-plane.sqlite3" +_SNAPSHOT_KEY = "runtime-snapshot" +_SCHEMA_VERSION = "1" +_BUSY_TIMEOUT_MILLISECONDS = 10_000 +_RUNTIME_OWNER_LOCK_NAME = "runtime-owner.lock" +_OPERATION_RECORD_KIND = "operation record" +_INSERT_AUDIT_EVENT = "INSERT INTO audit_events(payload, digest) VALUES (?, ?)" + + +def _participant_transition_count(snapshot: RuntimeSnapshot) -> int: + """Retain the pre-split private helper for compatible test and tool imports.""" + + return _count_participant_transitions(snapshot) class LocalControlPlaneStore: - """Filesystem-backed control-plane durability.""" + """Transactional single-host control-plane durability. + + SQLite WAL transactions serialize writers across processes, keep operation + and audit lookup indexed, and make participant transition commits atomic. + Legacy JSON files are imported once and retained with a timestamped backup. + """ def __init__(self, base_dir: Path) -> None: self._base_dir = base_dir - self._base_dir.mkdir(parents=True, exist_ok=True) + _secure_store_directory(self._base_dir) + self._database_path = self._base_dir / _DATABASE_NAME + self._runtime_owner_path = self._base_dir / _RUNTIME_OWNER_LOCK_NAME + self._active_runtime_lease: RuntimeOwnerLease | None = None self._snapshot_path = self._base_dir / "snapshot.json" self._operations_path = self._base_dir / "operations.json" self._audit_path = self._base_dir / "audit.jsonl" self._control_state_path = self._base_dir / "control-transition-state.json" + self._database_identity: os.stat_result | None = None + database_existed = _secure_database_file(self._database_path, allow_missing=True) is not None + _validate_sqlite_sidecars(self._database_path) + self._initialize_database(database_existed=database_existed) + database_identity = _secure_database_file(self._database_path, allow_missing=False) + assert database_identity is not None + self._database_identity = database_identity - @staticmethod - def _atomic_write(path: Path, content: str) -> None: - """Write content atomically via a temporary file and os.replace.""" + def acquire_runtime_lease(self) -> RuntimeOwnerLease: + """Fail fast unless this process is the store's sole runtime owner.""" - fd, tmp = tempfile.mkstemp(dir=path.parent, suffix=".tmp") - try: - with os.fdopen(fd, "w", encoding="utf-8") as handle: - handle.write(content) - os.replace(tmp, path) - except BaseException: - with suppress(OSError): - os.unlink(tmp) - raise + require_single_worker_configuration() + active = self._active_runtime_lease + if active is not None and not active.closed: + raise RuntimeError( + "local control-plane store already has a runtime owner; use exactly one worker with reload disabled" + ) + lease = RuntimeOwnerLease.acquire(self._runtime_owner_path) + self._active_runtime_lease = lease + return lease def load_snapshot(self) -> RuntimeSnapshot: - legacy_snapshot = RuntimeSnapshot() - if self._snapshot_path.exists(): - payload = json.loads(self._snapshot_path.read_text(encoding="utf-8")) - legacy_snapshot = _snapshot_from_payload(payload) - control_state = self._load_control_state() - if control_state is None: - return legacy_snapshot - committed_snapshot = _snapshot_from_payload(dict(control_state.get("snapshot", {}))) - legacy_count = _participant_transition_count(legacy_snapshot) - committed_count = _participant_transition_count(committed_snapshot) - return committed_snapshot if committed_count > legacy_count else legacy_snapshot + with self._connection() as connection: + return self._load_snapshot(connection) def save_snapshot(self, snapshot: RuntimeSnapshot) -> None: - content = json.dumps(_snapshot_payload(snapshot), indent=2, sort_keys=True) + "\n" - self._atomic_write(self._snapshot_path, content) + require_participant_autonomous_runtime_snapshot(snapshot) + with self._connection() as connection, _transaction(connection): + self._upsert_snapshot(connection, snapshot) def load_records(self) -> dict[str, ControlPlaneOperationRecord]: + with self._connection() as connection: + rows = connection.execute( + "SELECT operation_id, payload, digest FROM operations ORDER BY operation_id" + ).fetchall() records: dict[str, ControlPlaneOperationRecord] = {} - control_state = self._load_control_state() - if control_state is not None: - records.update( - { - operation_id: _record_from_payload(record_payload) - for operation_id, record_payload in dict(control_state.get("records", {})).items() - if isinstance(record_payload, dict) - } - ) - if not self._operations_path.exists(): - return records - payload = json.loads(self._operations_path.read_text(encoding="utf-8")) - records.update( - { - operation_id: _record_from_payload(record_payload) - for operation_id, record_payload in payload.items() - if isinstance(record_payload, dict) - } - ) + for operation_id, payload, digest in rows: + record = _record_from_payload(_decode_payload(payload, digest, kind=_OPERATION_RECORD_KIND)) + if record.receipt.operation_id != operation_id: + raise ValueError("operation record identity does not match its durable key") + records[operation_id] = record return records def save_record(self, record: ControlPlaneOperationRecord) -> None: - records = self.load_records() - records[record.receipt.operation_id] = record - payload = { - operation_id: _record_payload(operation_record) for operation_id, operation_record in records.items() - } - content = json.dumps(payload, indent=2, sort_keys=True) + "\n" - self._atomic_write(self._operations_path, content) - - def find_by_idempotency( + with self._connection() as connection, _transaction(connection): + self._upsert_record(connection, record) + + def claim_record(self, record: ControlPlaneOperationRecord) -> ControlPlaneOperationRecord: + """Atomically claim an idempotency key or return its existing record.""" + + with self._connection() as connection, _transaction(connection): + if record.idempotency_key: + existing = self._find_by_idempotency(connection, record.idempotency_key) + if existing is not None: + return existing + self._upsert_record(connection, record) + return record + + def commit_terminal_operation( self, - key: str, - ) -> ControlPlaneOperationRecord | None: - for record in self.load_records().values(): - if record.idempotency_key == key: - return record - return None + snapshot: RuntimeSnapshot, + record: ControlPlaneOperationRecord, + ) -> None: + """Atomically publish a snapshot with its terminal operation record.""" + + require_participant_autonomous_runtime_snapshot(snapshot) + with self._connection() as connection, _transaction(connection): + existing = self._load_record(connection, record.receipt.operation_id) + changed = _require_terminal_operation_transition(existing, record) + if not changed: + canonical_snapshot = _snapshot_from_payload(_snapshot_payload(snapshot)) + if self._load_snapshot(connection) != canonical_snapshot: + raise ValueError("terminal operation retry does not match the durable snapshot") + return + self._upsert_snapshot(connection, snapshot) + self._upsert_record(connection, record) + + def reconcile_interrupted_records( + self, + records: tuple[ControlPlaneOperationRecord, ...], + ) -> None: + """Atomically replace orphaned non-terminal records during startup.""" + + with self._connection() as connection, _transaction(connection): + for record in records: + existing = self._load_record(connection, record.receipt.operation_id) + if _require_interrupted_operation_transition(existing, record): + self._upsert_record(connection, record) + + def find_by_idempotency(self, key: str) -> ControlPlaneOperationRecord | None: + if not key: + return None + with self._connection() as connection: + return self._find_by_idempotency(connection, key) def append_audit(self, event: AuditEvent) -> None: - self._audit_path.parent.mkdir(parents=True, exist_ok=True) - with self._audit_path.open("a", encoding="utf-8") as handle: - handle.write(json.dumps(asdict(event), sort_keys=True) + "\n") + payload, digest = _encode_payload(asdict(event)) + with self._connection() as connection, _transaction(connection): + connection.execute( + _INSERT_AUDIT_EVENT, + (payload, digest), + ) def read_audit(self) -> list[AuditEvent]: - events: list[AuditEvent] = [] - control_state = self._load_control_state() - if control_state is not None: - events.extend( - _audit_event_from_payload(payload) - for payload in control_state.get("audit", []) - if isinstance(payload, dict) - ) - if not self._audit_path.exists(): - return events - for line in self._audit_path.read_text(encoding="utf-8").splitlines(): - if not line.strip(): - continue - event = _audit_event_from_payload(json.loads(line)) - if event not in events: - events.append(event) - return events - - def _load_control_state(self) -> dict[str, Any] | None: - if not self._control_state_path.exists(): - return None - payload = json.loads(self._control_state_path.read_text(encoding="utf-8")) - return payload if isinstance(payload, dict) else None + with self._connection() as connection: + rows = connection.execute("SELECT payload, digest FROM audit_events ORDER BY sequence").fetchall() + return [ + _audit_event_from_payload(_decode_payload(payload, digest, kind="audit event")) for payload, digest in rows + ] def commit_control_transition( self, @@ -150,16 +196,17 @@ def commit_control_transition( record: ControlPlaneOperationRecord, audit_event: AuditEvent, ) -> None: - current_snapshot = self.load_snapshot() - _require_expected_control_head(current_snapshot, participant_address, expected_head) - self.commit_participant_transition( - expected_history_heads={ - f"participant_control_history:{participant_address}": expected_head, - }, - snapshot=snapshot, - record=record, - audit_event=audit_event, - ) + require_participant_autonomous_runtime_snapshot(snapshot) + with self._connection() as connection, _transaction(connection): + current_snapshot = self._load_snapshot(connection) + _require_expected_control_head(current_snapshot, participant_address, expected_head) + self._upsert_snapshot(connection, snapshot) + self._upsert_record(connection, record) + payload, digest = _encode_payload(asdict(audit_event)) + connection.execute( + _INSERT_AUDIT_EVENT, + (payload, digest), + ) def commit_participant_transition( self, @@ -169,33 +216,275 @@ def commit_participant_transition( record: ControlPlaneOperationRecord, audit_event: AuditEvent, ) -> None: - current_snapshot = self.load_snapshot() - _require_expected_history_heads(current_snapshot, expected_history_heads) require_participant_autonomous_runtime_snapshot(snapshot) - records = self.load_records() - records[record.receipt.operation_id] = record - audits = [*self.read_audit(), audit_event] - payload = { - "snapshot": _snapshot_payload(snapshot), - "records": { - operation_id: _record_payload(operation_record) for operation_id, operation_record in records.items() - }, - "audit": [asdict(event) for event in audits], - } - content = json.dumps(payload, indent=2, sort_keys=True) + "\n" - self._atomic_write(self._control_state_path, content) + with self._connection() as connection, _transaction(connection): + current_snapshot = self._load_snapshot(connection) + _require_expected_history_heads(current_snapshot, expected_history_heads) + self._upsert_snapshot(connection, snapshot) + self._upsert_record(connection, record) + payload, digest = _encode_payload(asdict(audit_event)) + connection.execute( + _INSERT_AUDIT_EVENT, + (payload, digest), + ) + def _connect(self, *, allow_create: bool = False) -> tuple[sqlite3.Connection, os.stat_result]: + before = _secure_database_file(self._database_path, allow_missing=allow_create) + expected_identity = self._database_identity + if expected_identity is not None and before is not None: + _require_same_file(expected_identity, before, self._database_path, "the store was active") + _validate_sqlite_sidecars(self._database_path) + database_mode = "rwc" if before is None and allow_create else "rw" + database_uri = f"{self._database_path.absolute().as_uri()}?mode={database_mode}" + connection = sqlite3.connect( + database_uri, + timeout=_BUSY_TIMEOUT_MILLISECONDS / 1000, + isolation_level=None, + uri=True, + ) + try: + after = _secure_database_file(self._database_path, allow_missing=False) + assert after is not None + if before is not None: + _require_same_file(before, after, self._database_path, "SQLite opened it") + if expected_identity is not None: + _require_same_file(expected_identity, after, self._database_path, "SQLite opened it") + connection.execute(f"PRAGMA busy_timeout={_BUSY_TIMEOUT_MILLISECONDS}") + connection.execute("PRAGMA foreign_keys=ON") + connection.execute("PRAGMA synchronous=FULL") + except BaseException: + connection.close() + raise + return connection, after -def _participant_transition_count(snapshot: RuntimeSnapshot) -> int: - return sum( - len(events) - for history in ( - snapshot.participant_control_history, - snapshot.participant_crossing_history, - snapshot.information_state_history, + @contextmanager + def _connection(self, *, allow_create: bool = False) -> Iterator[sqlite3.Connection]: + connection, connected_metadata = self._connect(allow_create=allow_create) + try: + yield connection + finally: + connection.close() + closed_metadata = _secure_database_file(self._database_path, allow_missing=False) + assert closed_metadata is not None + _require_same_file(connected_metadata, closed_metadata, self._database_path, "SQLite was connected") + if self._database_identity is not None: + _require_same_file( + self._database_identity, + closed_metadata, + self._database_path, + "the store was active", + ) + _validate_sqlite_sidecars(self._database_path) + + def _initialize_database(self, *, database_existed: bool) -> None: + with self._connection(allow_create=not database_existed) as connection: + if connection.execute("PRAGMA journal_mode=WAL").fetchone() != ("wal",): + raise RuntimeError("local control-plane database did not enter required SQLite WAL journal mode") + _validate_sqlite_sidecars(self._database_path) + connection.executescript( + """ + CREATE TABLE IF NOT EXISTS metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS state ( + key TEXT PRIMARY KEY, + payload TEXT NOT NULL, + digest TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS operations ( + operation_id TEXT PRIMARY KEY, + idempotency_key TEXT NOT NULL, + request_fingerprint TEXT NOT NULL, + payload TEXT NOT NULL, + digest TEXT NOT NULL + ); + CREATE UNIQUE INDEX IF NOT EXISTS operations_idempotency_key + ON operations(idempotency_key) WHERE idempotency_key != ''; + CREATE TABLE IF NOT EXISTS audit_events ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + payload TEXT NOT NULL, + digest TEXT NOT NULL + ); + """ + ) + with _transaction(connection): + connection.execute( + "INSERT OR IGNORE INTO metadata(key, value) VALUES ('schema-version', ?)", + (_SCHEMA_VERSION,), + ) + schema_version = connection.execute("SELECT value FROM metadata WHERE key='schema-version'").fetchone() + if schema_version is None or schema_version[0] != _SCHEMA_VERSION: + raise ValueError("unsupported local control-plane database schema") + self._migrate_legacy_json(connection) + quick_check = connection.execute("PRAGMA quick_check").fetchone() + if quick_check is None or quick_check[0] != "ok": + raise ValueError("local control-plane database failed its integrity check") + if not database_existed: + _fsync_directory(self._base_dir) + + def _migrate_legacy_json(self, connection: sqlite3.Connection) -> None: + completed = connection.execute("SELECT value FROM metadata WHERE key='legacy-json-migration'").fetchone() + if completed is not None: + return + legacy_paths = self._existing_legacy_paths() + if not legacy_paths: + connection.execute("INSERT INTO metadata(key, value) VALUES ('legacy-json-migration', 'not-present')") + return + + snapshot, records, audits = _read_legacy_state( + snapshot_path=self._snapshot_path, + operations_path=self._operations_path, + audit_path=self._audit_path, + control_state_path=self._control_state_path, + ) + backup_dir = self._backup_legacy_files(legacy_paths) + self._upsert_snapshot(connection, snapshot) + for record in records.values(): + self._upsert_record(connection, record) + for event in audits: + payload, digest = _encode_payload(asdict(event)) + connection.execute( + _INSERT_AUDIT_EVENT, + (payload, digest), + ) + stored_record_count = connection.execute("SELECT COUNT(*) FROM operations").fetchone()[0] + stored_audit_count = connection.execute("SELECT COUNT(*) FROM audit_events").fetchone()[0] + if stored_record_count != len(records) or stored_audit_count != len(audits): + raise ValueError("legacy control-plane migration verification failed") + connection.execute( + "INSERT INTO metadata(key, value) VALUES ('legacy-json-migration', ?)", + (backup_dir.name,), + ) + + def _existing_legacy_paths(self) -> list[Path]: + return [ + path + for path in ( + self._snapshot_path, + self._operations_path, + self._audit_path, + self._control_state_path, + ) + if path.exists() + ] + + def _backup_legacy_files(self, paths: list[Path]) -> Path: + timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%S.%fZ") + backup_dir = self._base_dir / f"legacy-json-backup-{timestamp}" + backup_dir.mkdir(mode=0o700) + for path in paths: + _copy_regular_file_durably(path, backup_dir / path.name) + _fsync_directory(backup_dir) + _fsync_directory(self._base_dir) + return backup_dir + + @staticmethod + def _load_snapshot(connection: sqlite3.Connection) -> RuntimeSnapshot: + row = connection.execute( + "SELECT payload, digest FROM state WHERE key=?", + (_SNAPSHOT_KEY,), + ).fetchone() + if row is None: + return RuntimeSnapshot() + return _snapshot_from_payload(_decode_payload(row[0], row[1], kind="runtime snapshot")) + + @staticmethod + def _load_record( + connection: sqlite3.Connection, + operation_id: str, + ) -> ControlPlaneOperationRecord | None: + row = connection.execute( + "SELECT payload, digest FROM operations WHERE operation_id=?", + (operation_id,), + ).fetchone() + if row is None: + return None + record = _record_from_payload(_decode_payload(row[0], row[1], kind=_OPERATION_RECORD_KIND)) + if record.receipt.operation_id != operation_id: + raise ValueError("operation record identity does not match its durable key") + return record + + @staticmethod + def _upsert_snapshot(connection: sqlite3.Connection, snapshot: RuntimeSnapshot) -> None: + payload, digest = _encode_payload(_snapshot_payload(snapshot)) + connection.execute( + """ + INSERT INTO state(key, payload, digest) VALUES (?, ?, ?) + ON CONFLICT(key) DO UPDATE SET payload=excluded.payload, digest=excluded.digest + """, + (_SNAPSHOT_KEY, payload, digest), ) - for events in history.values() - ) + + @staticmethod + def _upsert_record(connection: sqlite3.Connection, record: ControlPlaneOperationRecord) -> None: + if record.idempotency_key: + conflict = connection.execute( + "SELECT operation_id FROM operations WHERE idempotency_key=?", + (record.idempotency_key,), + ).fetchone() + if conflict is not None and conflict[0] != record.receipt.operation_id: + raise ValueError("idempotency key already belongs to another operation") + payload, digest = _encode_payload(_record_payload(record)) + connection.execute( + """ + INSERT INTO operations( + operation_id, idempotency_key, request_fingerprint, payload, digest + ) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(operation_id) DO UPDATE SET + idempotency_key=excluded.idempotency_key, + request_fingerprint=excluded.request_fingerprint, + payload=excluded.payload, + digest=excluded.digest + """, + ( + record.receipt.operation_id, + record.idempotency_key, + record.request_fingerprint, + payload, + digest, + ), + ) + + @staticmethod + def _find_by_idempotency( + connection: sqlite3.Connection, + key: str, + ) -> ControlPlaneOperationRecord | None: + row = connection.execute( + "SELECT payload, digest FROM operations WHERE idempotency_key=?", + (key,), + ).fetchone() + if row is None: + return None + return _record_from_payload(_decode_payload(row[0], row[1], kind=_OPERATION_RECORD_KIND)) + + +@contextmanager +def _transaction(connection: sqlite3.Connection) -> Iterator[None]: + connection.execute("BEGIN IMMEDIATE") + try: + yield + except BaseException: + connection.rollback() + raise + else: + connection.commit() + + +def _encode_payload(payload: dict[str, Any]) -> tuple[str, str]: + content = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return content, hashlib.sha256(content.encode("utf-8")).hexdigest() + + +def _decode_payload(content: str, expected_digest: str, *, kind: str) -> dict[str, Any]: + actual_digest = hashlib.sha256(content.encode("utf-8")).hexdigest() + if actual_digest != expected_digest: + raise ValueError(f"{kind} failed its durable integrity check") + payload = json.loads(content) + if not isinstance(payload, dict): + raise ValueError(f"{kind} payload must be an object") + return payload __all__ = ("LocalControlPlaneStore",) diff --git a/implementations/python/packages/raes_runtime/control_plane_store_paths.py b/implementations/python/packages/raes_runtime/control_plane_store_paths.py new file mode 100644 index 00000000..c31f2999 --- /dev/null +++ b/implementations/python/packages/raes_runtime/control_plane_store_paths.py @@ -0,0 +1,195 @@ +"""Private filesystem boundary for the local control-plane store.""" + +from __future__ import annotations + +import errno +import json +import os +import shutil +import stat +from pathlib import Path +from typing import Any + +from raes_contracts.runtime_state import RuntimeSnapshot + +_PRIVATE_DIRECTORY_MODE = 0o700 +_PRIVATE_FILE_MODE = 0o600 +_DATABASE_FILE_KIND = "database file" +_SQLITE_SIDECAR_SUFFIXES = ("-wal", "-shm", "-journal") +_DIRECTORY_FSYNC_SUPPORTED = os.name != "nt" +_UNSUPPORTED_DIRECTORY_FSYNC_ERRNOS = frozenset( + { + errno.EINVAL, + getattr(errno, "ENOTSUP", errno.EINVAL), + getattr(errno, "EOPNOTSUPP", errno.EINVAL), + } +) + + +def _store_path_is_reparse_point(metadata: os.stat_result) -> bool: + attributes = getattr(metadata, "st_file_attributes", 0) + reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) + return bool(attributes & reparse_flag) + + +def _require_safe_link_count(metadata: os.stat_result, path: Path, *, kind: str) -> None: + link_count = getattr(metadata, "st_nlink", 1) + unsafe = (kind == _DATABASE_FILE_KIND and link_count != 1) or (kind == "SQLite sidecar" and link_count > 1) + if unsafe: + raise RuntimeError(f"local control-plane {kind} must not have hard links: {path}") + + +def _require_safe_store_path_metadata( + metadata: os.stat_result, + path: Path, + *, + kind: str, +) -> None: + if stat.S_ISLNK(metadata.st_mode) or _store_path_is_reparse_point(metadata): + raise RuntimeError(f"local control-plane {kind} must not be a symlink or reparse point: {path}") + expected = stat.S_ISDIR(metadata.st_mode) if kind == "directory" else stat.S_ISREG(metadata.st_mode) + if not expected: + raise RuntimeError(f"local control-plane {kind} has the wrong filesystem type: {path}") + get_effective_uid = getattr(os, "geteuid", None) + if callable(get_effective_uid) and metadata.st_uid != get_effective_uid(): + raise RuntimeError(f"local control-plane {kind} must be owned by the current user: {path}") + _require_safe_link_count(metadata, path, kind=kind) + + +def _secure_store_directory(path: Path) -> None: + try: + metadata = path.lstat() + except FileNotFoundError: + path.mkdir(mode=_PRIVATE_DIRECTORY_MODE, parents=True, exist_ok=True) + metadata = path.lstat() + _require_safe_store_path_metadata(metadata, path, kind="directory") + if os.name == "nt": + return + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + opened_metadata = os.fstat(descriptor) + _require_safe_store_path_metadata(opened_metadata, path, kind="directory") + current_metadata = path.lstat() + _require_safe_store_path_metadata(current_metadata, path, kind="directory") + if not os.path.samestat(opened_metadata, current_metadata): + raise RuntimeError(f"local control-plane directory changed while it was opened: {path}") + os.fchmod(descriptor, _PRIVATE_DIRECTORY_MODE) + finally: + os.close(descriptor) + + +def _secure_database_file(path: Path, *, allow_missing: bool) -> os.stat_result | None: + try: + before = path.lstat() + except FileNotFoundError as exc: + if allow_missing: + return None + raise RuntimeError(f"local control-plane database file is missing: {path}") from exc + _require_safe_store_path_metadata(before, path, kind=_DATABASE_FILE_KIND) + if os.name != "nt" and stat.S_IMODE(before.st_mode) != _PRIVATE_FILE_MODE: + try: + os.chmod(path, _PRIVATE_FILE_MODE, follow_symlinks=False) + except (NotImplementedError, OSError) as exc: + raise RuntimeError(f"could not secure local control-plane database file: {path}") from exc + try: + after = path.lstat() + except FileNotFoundError as exc: + raise RuntimeError(f"local control-plane database file disappeared while it was secured: {path}") from exc + _require_safe_store_path_metadata(after, path, kind=_DATABASE_FILE_KIND) + if not os.path.samestat(before, after): + raise RuntimeError(f"local control-plane database file changed while it was secured: {path}") + if os.name != "nt" and stat.S_IMODE(after.st_mode) != _PRIVATE_FILE_MODE: + raise RuntimeError(f"local control-plane database file must use private permissions 0600: {path}") + return after + + +def _require_same_file( + before: os.stat_result, + after: os.stat_result, + path: Path, + activity: str, +) -> None: + if not os.path.samestat(before, after): + raise RuntimeError(f"local control-plane database file changed while {activity}: {path}") + + +def _validate_sqlite_sidecar(path: Path) -> bool: + """Validate an ephemeral SQLite file without disturbing its POSIX locks.""" + + try: + metadata = path.lstat() + except FileNotFoundError: + return False + _require_safe_store_path_metadata(metadata, path, kind="SQLite sidecar") + if getattr(metadata, "st_nlink", 1) == 0: + return False + if os.name != "nt" and stat.S_IMODE(metadata.st_mode) != _PRIVATE_FILE_MODE: + raise RuntimeError(f"local control-plane SQLite sidecar must use private permissions 0600: {path}") + return True + + +def _validate_sqlite_sidecars(database_path: Path) -> None: + for suffix in _SQLITE_SIDECAR_SUFFIXES: + _validate_sqlite_sidecar(Path(f"{database_path}{suffix}")) + + +def _read_json_object(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"legacy control-plane file must contain an object: {path.name}") + return payload + + +def _fsync_directory(path: Path) -> None: + """Persist directory entries, failing closed on real I/O failures.""" + + if not _DIRECTORY_FSYNC_SUPPORTED: + return + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as exc: + if exc.errno in _UNSUPPORTED_DIRECTORY_FSYNC_ERRNOS: + return + raise RuntimeError(f"could not durably synchronize local control-plane directory: {path}") from exc + try: + os.fsync(descriptor) + except OSError as exc: + if exc.errno in _UNSUPPORTED_DIRECTORY_FSYNC_ERRNOS: + return + raise RuntimeError(f"could not durably synchronize local control-plane directory: {path}") from exc + finally: + os.close(descriptor) + + +def _fsync_regular_file(path: Path) -> None: + """Persist a newly copied regular file before publishing its directory entry.""" + + try: + with path.open("rb") as stream: + os.fsync(stream.fileno()) + except OSError as exc: + raise RuntimeError(f"could not durably synchronize local control-plane file: {path}") from exc + + +def _copy_regular_file_durably(source: Path, destination: Path) -> None: + shutil.copy2(source, destination) + _fsync_regular_file(destination) + + +def _participant_transition_count(snapshot: RuntimeSnapshot) -> int: + return sum( + len(events) + for history in ( + snapshot.participant_control_history, + snapshot.participant_crossing_history, + snapshot.information_state_history, + ) + for events in history.values() + ) + + +__all__ = () diff --git a/implementations/python/packages/raes_runtime/control_plane_store_snapshots.py b/implementations/python/packages/raes_runtime/control_plane_store_snapshots.py new file mode 100644 index 00000000..38f8fe18 --- /dev/null +++ b/implementations/python/packages/raes_runtime/control_plane_store_snapshots.py @@ -0,0 +1,286 @@ +"""Portable runtime-snapshot serialization for control-plane stores.""" + +from __future__ import annotations + +from dataclasses import fields +from typing import Any + +from raes_contracts.account_credentials import ( + account_placement_has_credential_bindings, + value_free_account_placement_payload, +) +from raes_contracts.artifact_requirements import ArtifactSatisfactionDisclosureModel +from raes_contracts.contracts import RealizationEnvelopeIdentityModel +from raes_contracts.contracts.time_model import TimeRuntimeStateModel +from raes_contracts.participant_autonomous_state import require_participant_autonomous_runtime_snapshot +from raes_contracts.planning import RuntimeDomain +from raes_contracts.runtime_state import ( + ExplicitnessClass, + ExplicitnessProvenance, + RealizationProvenanceEntry, + RuntimeSnapshot, + RuntimeSnapshotEnvelope, + SnapshotEntry, +) + +from .control_plane_store_observations import realization_observation_from_payload + + +def _require_complete_runtime_snapshot_fields(values: dict[str, Any]) -> None: + """Keep the hand-written durable codec exhaustive as the snapshot evolves.""" + + expected = {field.name for field in fields(RuntimeSnapshot)} + actual = set(values) + if actual != expected: + missing = ", ".join(sorted(expected - actual)) or "none" + unexpected = ", ".join(sorted(actual - expected)) or "none" + raise RuntimeError(f"runtime snapshot durable codec field mismatch: missing={missing}; unexpected={unexpected}") + + +def _realization_provenance_payload(snapshot: RuntimeSnapshot) -> list[dict[str, Any]]: + return [ + { + "address": entry.address, + "field_path": entry.field_path, + "domain": entry.domain, + "requirement_kind": entry.requirement_kind, + "explicitness": entry.explicitness.value, + "provenance": entry.provenance.value, + "governing_scope": entry.governing_scope, + "artifact_satisfaction": ( + entry.artifact_satisfaction.model_dump(mode="json") if entry.artifact_satisfaction is not None else None + ), + } + for entry in snapshot.realization_provenance + ] + + +def _realization_observations_payload(snapshot: RuntimeSnapshot) -> list[dict[str, Any]]: + return [ + { + "address": entry.address, + "field_path": entry.field_path, + "domain": entry.domain, + "requirement_kind": entry.requirement_kind, + "verification_scope": entry.verification_scope.value, + "observation_strength": entry.observation_strength.value, + **( + { + "observed_value": entry.observed_value, + "operating_system": ( + { + "family": entry.operating_system.family, + "distribution": entry.operating_system.distribution, + "version": entry.operating_system.version, + } + if entry.operating_system is not None + else None + ), + "operation_id": entry.operation_id, + "envelope_digest": entry.envelope_digest, + "configuration_digest": entry.configuration_digest, + "observer_version": entry.observer_version, + "sequence": entry.sequence, + "binding_verified": entry.binding_verified, + } + if entry.requirement_kind in {"compute-substrate", "operating-system"} + else {} + ), + } + for entry in snapshot.realization_observations + ] + + +def _snapshot_payload(snapshot: RuntimeSnapshot) -> dict[str, Any]: + require_participant_autonomous_runtime_snapshot(snapshot) + snapshot_fields: dict[str, Any] = { + "entries": { + address: { + "address": entry.address, + "domain": entry.domain.value, + "resource_type": entry.resource_type, + "payload": dict(entry.payload), + "ordering_dependencies": list(entry.ordering_dependencies), + "refresh_dependencies": list(entry.refresh_dependencies), + "status": entry.status, + } + for address, entry in snapshot.entries.items() + }, + "orchestration_results": dict(snapshot.orchestration_results), + "orchestration_history": {address: list(events) for address, events in snapshot.orchestration_history.items()}, + "evaluation_results": dict(snapshot.evaluation_results), + "evaluation_history": {address: list(events) for address, events in snapshot.evaluation_history.items()}, + "proposition_truth_results": dict(snapshot.proposition_truth_results), + "participant_episode_results": dict(snapshot.participant_episode_results), + "participant_episode_history": { + participant_address: list(events) + for participant_address, events in snapshot.participant_episode_history.items() + }, + "participant_episode_closure_records": { + participant_address: list(records) + for participant_address, records in snapshot.participant_episode_closure_records.items() + }, + "participant_behavior_history": { + participant_address: list(events) + for participant_address, events in snapshot.participant_behavior_history.items() + }, + "participant_control_history": { + participant_address: list(events) + for participant_address, events in snapshot.participant_control_history.items() + }, + "participant_crossing_history": { + participant_address: list(events) + for participant_address, events in snapshot.participant_crossing_history.items() + }, + "information_state_history": { + participant_address: list(records) + for participant_address, records in snapshot.information_state_history.items() + }, + "participant_autonomous_execution_states": dict(snapshot.participant_autonomous_execution_states), + "participant_execution_services": dict(snapshot.participant_execution_services), + "participant_resource_budget_states": dict(snapshot.participant_resource_budget_states), + "participant_resource_pool_states": dict(snapshot.participant_resource_pool_states), + "participant_resource_budget_events": dict(snapshot.participant_resource_budget_events), + "shared_state_records": dict(snapshot.shared_state_records), + "shared_state_history": { + state_address: list(records) for state_address, records in snapshot.shared_state_history.items() + }, + "joint_action_records": dict(snapshot.joint_action_records), + "time_management_contexts": dict(snapshot.time_management_contexts), + "time_model_state": ( + snapshot.time_model_state.model_dump(mode="json") if snapshot.time_model_state is not None else None + ), + "realization_provenance": _realization_provenance_payload(snapshot), + "realization_observations": _realization_observations_payload(snapshot), + "realization_envelope": ( + snapshot.realization_envelope.model_dump(mode="json") if snapshot.realization_envelope is not None else None + ), + "metadata": dict(snapshot.metadata), + } + _require_complete_runtime_snapshot_fields(snapshot_fields) + payload = { + "schema_version": RuntimeSnapshotEnvelope().schema_version, + **snapshot_fields, + } + for entry in payload["entries"].values(): + if entry["resource_type"] != "account-placement": + continue + entry_payload = entry["payload"] + if account_placement_has_credential_bindings(entry_payload): + entry["payload"] = value_free_account_placement_payload(entry_payload) + return payload + + +def _snapshot_entries_from_payload(payload: dict[str, Any]) -> dict[str, SnapshotEntry]: + entries_payload = payload.get("entries", {}) + return { + address: SnapshotEntry( + address=str(entry.get("address", address)), + domain=RuntimeDomain(str(entry.get("domain", "provisioning"))), + resource_type=str(entry.get("resource_type", "")), + payload=dict(entry.get("payload", {})), + ordering_dependencies=tuple(entry.get("ordering_dependencies", ())), + refresh_dependencies=tuple(entry.get("refresh_dependencies", ())), + status=str(entry.get("status", "ready")), + ) + for address, entry in entries_payload.items() + if isinstance(entry, dict) + } + + +def _realization_provenance_from_payload(payload: dict[str, Any]) -> tuple[RealizationProvenanceEntry, ...]: + return tuple( + RealizationProvenanceEntry( + address=str(item.get("address", "")), + field_path=str(item.get("field_path", "")), + domain=str(item.get("domain", "")), + requirement_kind=str(item.get("requirement_kind", "")), + explicitness=ExplicitnessClass(str(item.get("explicitness", ExplicitnessClass.EXACT.value))), + provenance=ExplicitnessProvenance( + str(item.get("provenance", ExplicitnessProvenance.AUTHOR_DECLARED.value)) + ), + governing_scope=(str(item["governing_scope"]) if item.get("governing_scope") is not None else None), + artifact_satisfaction=( + ArtifactSatisfactionDisclosureModel.model_validate(item["artifact_satisfaction"]) + if item.get("artifact_satisfaction") is not None + else None + ), + ) + for item in payload.get("realization_provenance", []) + if isinstance(item, dict) + ) + + +def _snapshot_from_payload(payload: dict[str, Any]) -> RuntimeSnapshot: + snapshot_fields: dict[str, Any] = { + "entries": _snapshot_entries_from_payload(payload), + "orchestration_results": dict(payload.get("orchestration_results", {})), + "orchestration_history": { + address: list(events) for address, events in payload.get("orchestration_history", {}).items() + }, + "evaluation_results": dict(payload.get("evaluation_results", {})), + "evaluation_history": { + address: list(events) for address, events in payload.get("evaluation_history", {}).items() + }, + "proposition_truth_results": dict(payload.get("proposition_truth_results", {})), + "participant_episode_results": dict(payload.get("participant_episode_results", {})), + "participant_episode_history": { + participant_address: list(events) + for participant_address, events in payload.get("participant_episode_history", {}).items() + }, + "participant_episode_closure_records": { + participant_address: list(records) + for participant_address, records in payload.get("participant_episode_closure_records", {}).items() + }, + "participant_behavior_history": { + participant_address: list(events) + for participant_address, events in payload.get("participant_behavior_history", {}).items() + }, + "participant_control_history": { + participant_address: list(events) + for participant_address, events in payload.get("participant_control_history", {}).items() + }, + "participant_crossing_history": { + participant_address: list(events) + for participant_address, events in payload.get("participant_crossing_history", {}).items() + }, + "information_state_history": { + participant_address: list(records) + for participant_address, records in payload.get("information_state_history", {}).items() + }, + "participant_autonomous_execution_states": dict(payload.get("participant_autonomous_execution_states", {})), + "participant_execution_services": dict(payload.get("participant_execution_services", {})), + "participant_resource_budget_states": dict(payload.get("participant_resource_budget_states", {})), + "participant_resource_pool_states": dict(payload.get("participant_resource_pool_states", {})), + "participant_resource_budget_events": dict(payload.get("participant_resource_budget_events", {})), + "shared_state_records": dict(payload.get("shared_state_records", {})), + "shared_state_history": { + state_address: list(records) for state_address, records in payload.get("shared_state_history", {}).items() + }, + "joint_action_records": dict(payload.get("joint_action_records", {})), + "time_management_contexts": dict(payload.get("time_management_contexts", {})), + "time_model_state": ( + TimeRuntimeStateModel.model_validate(payload["time_model_state"]) + if payload.get("time_model_state") is not None + else None + ), + "realization_provenance": _realization_provenance_from_payload(payload), + "realization_observations": tuple( + realization_observation_from_payload(item) + for item in payload.get("realization_observations", []) + if isinstance(item, dict) + ), + "realization_envelope": ( + RealizationEnvelopeIdentityModel.model_validate(payload["realization_envelope"]) + if payload.get("realization_envelope") is not None + else None + ), + "metadata": dict(payload.get("metadata", {})), + } + _require_complete_runtime_snapshot_fields(snapshot_fields) + snapshot = RuntimeSnapshot(**snapshot_fields) + require_participant_autonomous_runtime_snapshot(snapshot) + return snapshot + + +__all__ = ("_snapshot_from_payload", "_snapshot_payload") diff --git a/implementations/python/packages/raes_runtime/control_plane_workflow_control.py b/implementations/python/packages/raes_runtime/control_plane_workflow_control.py index 1cdc1cd8..b4c430a8 100644 --- a/implementations/python/packages/raes_runtime/control_plane_workflow_control.py +++ b/implementations/python/packages/raes_runtime/control_plane_workflow_control.py @@ -30,6 +30,7 @@ _utc_now, persist_succeeded_operation, ) +from .control_plane_lifecycle import runtime_owned from .control_plane_store import ControlPlaneOperationRecord from .control_plane_timeouts import _reconciliation_clock, workflow_timeout_update from .control_plane_workflows import maybe_apply_compensation @@ -45,6 +46,7 @@ class WorkflowControlMixin: """Workflow cancellation and timeout reconciliation operations.""" + @runtime_owned def cancel_workflow( self, workflow_address: str, @@ -53,6 +55,24 @@ def cancel_workflow( reason: str = "cancelled by operator", idempotency_key: str = "", request_fingerprint: str = "", + ) -> OperationReceipt: + with self._operation_lock: + return self._cancel_workflow_locked( + workflow_address, + run_id=run_id, + reason=reason, + idempotency_key=idempotency_key, + request_fingerprint=request_fingerprint, + ) + + def _cancel_workflow_locked( + self, + workflow_address: str, + *, + run_id: str | None, + reason: str, + idempotency_key: str, + request_fingerprint: str, ) -> OperationReceipt: existing = self._idempotent_receipt( idempotency_key=idempotency_key, @@ -159,7 +179,7 @@ def _cancel_active_workflow( history=history, submitted_at=submitted_at, ) - self._snapshot = self._snapshot.with_entries( + next_snapshot = self._snapshot.with_entries( dict(self._snapshot.entries), orchestration_results={ **self._snapshot.orchestration_results, @@ -170,7 +190,6 @@ def _cancel_active_workflow( workflow_address: history, }, ) - self._store.save_snapshot(self._snapshot) receipt = OperationReceipt( operation_id=operation_id, domain=RuntimeDomain.ORCHESTRATION, @@ -185,22 +204,38 @@ def _cancel_active_workflow( updated_at=submitted_at, changed_addresses=[workflow_address], ) - self._persist_record( + self._commit_terminal_operation( + next_snapshot, ControlPlaneOperationRecord( receipt=receipt, status=status, idempotency_key=idempotency_key, request_fingerprint=request_fingerprint, - ) + ), ) return receipt + @runtime_owned def reconcile_workflow_timeouts( self, *, now: str | None = None, idempotency_key: str = "", request_fingerprint: str = "", + ) -> OperationReceipt: + with self._operation_lock: + return self._reconcile_workflow_timeouts_locked( + now=now, + idempotency_key=idempotency_key, + request_fingerprint=request_fingerprint, + ) + + def _reconcile_workflow_timeouts_locked( + self, + *, + now: str | None, + idempotency_key: str, + request_fingerprint: str, ) -> OperationReceipt: existing = self._idempotent_receipt( idempotency_key=idempotency_key, @@ -231,12 +266,11 @@ def reconcile_workflow_timeouts( orchestration_history[workflow_address] = timed_out[1] changed.append(workflow_address) operation_id = str(uuid4()) - self._snapshot = self._snapshot.with_entries( + next_snapshot = self._snapshot.with_entries( dict(self._snapshot.entries), orchestration_results=orchestration_results, orchestration_history=orchestration_history, ) - self._store.save_snapshot(self._snapshot) receipt = OperationReceipt( operation_id=operation_id, domain=RuntimeDomain.ORCHESTRATION, @@ -251,12 +285,13 @@ def reconcile_workflow_timeouts( updated_at=submitted_at, changed_addresses=changed, ) - self._persist_record( + self._commit_terminal_operation( + next_snapshot, ControlPlaneOperationRecord( receipt=receipt, status=status, idempotency_key=idempotency_key, request_fingerprint=request_fingerprint, - ) + ), ) return receipt diff --git a/implementations/python/packages/raes_runtime/participant_control.py b/implementations/python/packages/raes_runtime/participant_control.py index b7fd6abd..391ca151 100644 --- a/implementations/python/packages/raes_runtime/participant_control.py +++ b/implementations/python/packages/raes_runtime/participant_control.py @@ -28,6 +28,7 @@ from raes_processor.models import ParticipantBehaviorRuntime from .control_plane_execution import execute_participant_action +from .control_plane_lifecycle import runtime_owned from .participant_control_diagnostics import ( _NO_PARTICIPANT_RUNTIME_MESSAGE, _participant_binding_address, @@ -221,6 +222,7 @@ class ParticipantControlMixin( ): """Participant runtime methods for the shared runtime control plane.""" + @runtime_owned def control_participant_execution( self, request: ParticipantExecutionControlRequestModel, @@ -255,6 +257,7 @@ def control_participant_execution( request_fingerprint=request_fingerprint, ) + @runtime_owned def participant_execution_state( self, execution_scope_ref: str, @@ -267,6 +270,7 @@ def participant_execution_state( raise ValueError("participant runtime does not expose execution-service readback") return method(execution_scope_ref, self._snapshot) + @runtime_owned def initialize_participant_episode( self, participant_address: str, @@ -295,6 +299,7 @@ def initialize_participant_episode( request_fingerprint=request_fingerprint, ) + @runtime_owned def reset_participant_episode( self, participant_address: str, @@ -325,6 +330,7 @@ def reset_participant_episode( request_fingerprint=request_fingerprint, ) + @runtime_owned def restart_participant_episode( self, participant_address: str, @@ -355,6 +361,7 @@ def restart_participant_episode( request_fingerprint=request_fingerprint, ) + @runtime_owned def terminate_participant_episode( self, participant_address: str, @@ -385,6 +392,7 @@ def terminate_participant_episode( request_fingerprint=request_fingerprint, ) + @runtime_owned def admit_participant_action( self, participant_behavior: ParticipantBehaviorRuntime, @@ -424,6 +432,7 @@ def admit_participant_action( ) return submit_bound_participant_action(self, participant_behavior, request, options) + @runtime_owned def admit_participant_decision_surface_selection( self, participant_behavior: ParticipantBehaviorRuntime, diff --git a/implementations/python/packages/raes_runtime/participant_control_mediation.py b/implementations/python/packages/raes_runtime/participant_control_mediation.py index dbf24108..d891cf8c 100644 --- a/implementations/python/packages/raes_runtime/participant_control_mediation.py +++ b/implementations/python/packages/raes_runtime/participant_control_mediation.py @@ -95,15 +95,13 @@ def record_participant_control( identity, bound, ) - control_plane._store.commit_control_transition( + control_plane._commit_control_transition( participant_address=participant_address, expected_head=prepared.expected_control_head, snapshot=prepared.next_snapshot, record=prepared.record, audit_event=prepared.audit_event, ) - control_plane._snapshot = prepared.next_snapshot - control_plane._operations[prepared.record.receipt.operation_id] = prepared.record return prepared.record.receipt diff --git a/implementations/python/packages/raes_runtime/participant_crossing_boundary.py b/implementations/python/packages/raes_runtime/participant_crossing_boundary.py index a9a9dd30..4ab9724c 100644 --- a/implementations/python/packages/raes_runtime/participant_crossing_boundary.py +++ b/implementations/python/packages/raes_runtime/participant_crossing_boundary.py @@ -19,6 +19,7 @@ from raes_processor.models import ParticipantBehaviorRuntime from .control_plane_execution import apply_authorized_participant_action +from .control_plane_lifecycle import runtime_owned from .control_plane_security import ControlPlaneIdentity from .participant_control_intents import ParticipantControlIntent, ParticipantControlIntentBase from .participant_control_mediation import ( @@ -60,6 +61,7 @@ class ParticipantCrossingControlIngressMixin: """Own one RUN-319 decision and RUN-310 transition under one state cut.""" + @runtime_owned def record_participant_control( self, participant_address: str, @@ -180,14 +182,12 @@ def _record_governed_participant_control( ) if sink_decision is not None: audit = apply_flow_sink_details(audit, sink_decision) - self._store.commit_participant_transition( + self._commit_participant_transition( expected_history_heads=crossing.expected_history_heads, snapshot=next_snapshot, record=record, audit_event=audit, ) - self._snapshot = next_snapshot - self._operations[record.receipt.operation_id] = record return record.receipt @@ -251,14 +251,12 @@ def execute_action_ingress_crossing( ) if sink_decision is not None: authorization_audit = apply_flow_sink_details(authorization_audit, sink_decision) - control_plane._store.commit_participant_transition( + control_plane._commit_participant_transition( expected_history_heads=crossing.expected_history_heads, snapshot=crossing.next_snapshot, record=authorization_record, audit_event=authorization_audit, ) - control_plane._snapshot = crossing.next_snapshot - control_plane._operations[authorization_record.receipt.operation_id] = authorization_record result = apply_authorized_participant_action( method=execution.method, @@ -288,14 +286,12 @@ def execute_action_ingress_crossing( ) if sink_decision is not None: audit = apply_flow_sink_details(audit, sink_decision) - control_plane._store.commit_participant_transition( + control_plane._commit_participant_transition( expected_history_heads=crossing.record.result_history_heads, snapshot=next_snapshot, record=record, audit_event=audit, ) - control_plane._snapshot = next_snapshot - control_plane._operations[record.receipt.operation_id] = record return record.receipt diff --git a/implementations/python/packages/raes_runtime/participant_crossing_mediation.py b/implementations/python/packages/raes_runtime/participant_crossing_mediation.py index 04786116..560b1069 100644 --- a/implementations/python/packages/raes_runtime/participant_crossing_mediation.py +++ b/implementations/python/packages/raes_runtime/participant_crossing_mediation.py @@ -352,14 +352,12 @@ def commit_prepared_crossing( if prepared.existing_receipt is not None: return prepared.existing_receipt - control_plane._store.commit_participant_transition( + control_plane._commit_participant_transition( expected_history_heads=prepared.expected_history_heads, snapshot=prepared.next_snapshot, record=prepared.record, audit_event=prepared.audit_event, ) - control_plane._snapshot = prepared.next_snapshot - control_plane._operations[prepared.record.receipt.operation_id] = prepared.record return prepared.record.receipt diff --git a/implementations/python/packages/raes_runtime/participant_decision_surface_control_v2.py b/implementations/python/packages/raes_runtime/participant_decision_surface_control_v2.py index ab7efb3d..4e785832 100644 --- a/implementations/python/packages/raes_runtime/participant_decision_surface_control_v2.py +++ b/implementations/python/packages/raes_runtime/participant_decision_surface_control_v2.py @@ -18,6 +18,7 @@ validate_participant_decision_surface_v2_anchor, ) +from .control_plane_lifecycle import runtime_owned from .participant_control_diagnostics import ( _NO_PARTICIPANT_RUNTIME_MESSAGE, _participant_binding_address, @@ -28,6 +29,7 @@ class ParticipantDecisionSurfaceV2ControlMixin: """Exact-cut decision-surface operations mixed into the control plane.""" + @runtime_owned def admit_participant_decision_surface_selection_v2( self, participant_behavior: ParticipantBehaviorRuntime, diff --git a/implementations/python/packages/raes_runtime/participant_flow_sink.py b/implementations/python/packages/raes_runtime/participant_flow_sink.py index fbba1c49..54fa4166 100644 --- a/implementations/python/packages/raes_runtime/participant_flow_sink.py +++ b/implementations/python/packages/raes_runtime/participant_flow_sink.py @@ -263,14 +263,12 @@ def commit_flow_sink_denial( reason="flow-sink-denied", ) audit = apply_flow_sink_details(audit, decision) - control_plane._store.commit_participant_transition( + control_plane._commit_participant_transition( expected_history_heads=crossing.expected_history_heads, snapshot=crossing.next_snapshot, record=record, audit_event=audit, ) - control_plane._snapshot = crossing.next_snapshot - control_plane._operations[record.receipt.operation_id] = record return record.receipt diff --git a/implementations/python/packages/raes_runtime/participant_retrieval.py b/implementations/python/packages/raes_runtime/participant_retrieval.py index ef7f6836..941155fc 100644 --- a/implementations/python/packages/raes_runtime/participant_retrieval.py +++ b/implementations/python/packages/raes_runtime/participant_retrieval.py @@ -18,6 +18,7 @@ from raes_contracts.planning import RuntimeDomain from raes_contracts.runtime_state import OperationState, RuntimeSnapshot +from .control_plane_lifecycle import runtime_owned from .control_plane_security import ControlPlaneIdentity, ParticipantAudienceSubjectBinding from .control_plane_store import ControlPlaneOperationRecord from .participant_crossing_egress import ParticipantViewSerialization, serialize_participant_view @@ -87,6 +88,7 @@ class ParticipantRetrievalMixin: _snapshot: RuntimeSnapshot _operations: dict[str, ControlPlaneOperationRecord] + @runtime_owned def deliver_participant_directed_view( self, participant_address: str, @@ -127,6 +129,7 @@ def deliver_participant_directed_view( ), ) + @runtime_owned def get_participant_status_view( self, participant_address: str, @@ -179,6 +182,7 @@ def get_participant_status_view( serialization.with_crossing_evidence(crossing_evidence), ) + @runtime_owned def get_participant_history_view( self, participant_address: str, @@ -242,6 +246,7 @@ def get_participant_history_view( serialization.with_crossing_evidence(crossing_evidence), ) + @runtime_owned def get_participant_context_view( self, participant_address: str, diff --git a/implementations/python/tests/test_dsl_437_snapshot_durability_conformance.py b/implementations/python/tests/test_dsl_437_snapshot_durability_conformance.py index 0400a5c4..0ea1fa95 100644 --- a/implementations/python/tests/test_dsl_437_snapshot_durability_conformance.py +++ b/implementations/python/tests/test_dsl_437_snapshot_durability_conformance.py @@ -3,6 +3,10 @@ from __future__ import annotations import json +import sqlite3 +from collections.abc import Callable +from contextlib import closing +from hashlib import sha256 from pathlib import Path import pytest @@ -17,6 +21,31 @@ STATE_ADDRESS = f"{POLICY_ADDRESS}.state.{PARTICIPANT_ADDRESS}" +def _rewrite_durable_snapshot( + store_path: Path, + mutate: Callable[[dict[str, object]], None], + *, + update_digest: bool = True, +) -> None: + with closing(sqlite3.connect(store_path / "control-plane.sqlite3")) as connection, connection: + row = connection.execute("SELECT payload FROM state WHERE key='runtime-snapshot'").fetchone() + assert row is not None + payload = json.loads(row[0]) + mutate(payload) + content = json.dumps(payload, sort_keys=True, separators=(",", ":")) + if update_digest: + digest = sha256(content.encode("utf-8")).hexdigest() + connection.execute( + "UPDATE state SET payload=?, digest=? WHERE key='runtime-snapshot'", + (content, digest), + ) + else: + connection.execute( + "UPDATE state SET payload=? WHERE key='runtime-snapshot'", + (content,), + ) + + def _state(**updates: object) -> dict[str, object]: state: dict[str, object] = { "policy_address": POLICY_ADDRESS, @@ -170,31 +199,61 @@ def test_control_plane_stores_revalidate_mutated_autonomous_state(tmp_path: Path def test_local_control_plane_store_rejects_invalid_durable_state(tmp_path: Path) -> None: - store = LocalControlPlaneStore(tmp_path / "control-plane") + store_path = tmp_path / "control-plane" + store = LocalControlPlaneStore(store_path) store.save_snapshot(_snapshot()) - snapshot_path = tmp_path / "control-plane" / "snapshot.json" - payload = json.loads(snapshot_path.read_text(encoding="utf-8")) - payload["participant_autonomous_execution_states"][STATE_ADDRESS]["attempted_actions"] = 2 - snapshot_path.write_text(json.dumps(payload), encoding="utf-8") + + def mutate(payload: dict[str, object]) -> None: + states = payload["participant_autonomous_execution_states"] + assert isinstance(states, dict) + state = states[STATE_ADDRESS] + assert isinstance(state, dict) + state["attempted_actions"] = 2 + + _rewrite_durable_snapshot(store_path, mutate) with pytest.raises(ValueError, match="attempted_actions must equal"): store.load_snapshot() def test_local_control_plane_store_rejects_durable_clock_segment_mismatch(tmp_path: Path) -> None: - store = LocalControlPlaneStore(tmp_path / "control-plane") + store_path = tmp_path / "control-plane" + store = LocalControlPlaneStore(store_path) store.save_snapshot(_snapshot()) - snapshot_path = tmp_path / "control-plane" / "snapshot.json" - payload = json.loads(snapshot_path.read_text(encoding="utf-8")) - clock = payload["time_model_state"]["clocks"]["time.clock.scenario-clock"] - clock["coordinate"]["segment"] = 1 - clock["history"][-1]["resulting"]["segment"] = 1 - snapshot_path.write_text(json.dumps(payload), encoding="utf-8") + + def mutate(payload: dict[str, object]) -> None: + time_model = payload["time_model_state"] + assert isinstance(time_model, dict) + clocks = time_model["clocks"] + assert isinstance(clocks, dict) + clock = clocks["time.clock.scenario-clock"] + assert isinstance(clock, dict) + coordinate = clock["coordinate"] + assert isinstance(coordinate, dict) + coordinate["segment"] = 1 + history = clock["history"] + assert isinstance(history, list) + resulting = history[-1]["resulting"] + assert isinstance(resulting, dict) + resulting["segment"] = 1 + + _rewrite_durable_snapshot(store_path, mutate) with pytest.raises(ValueError, match="must match the bound shared clock segment"): store.load_snapshot() +def test_local_control_plane_store_rejects_corrupted_payload(tmp_path: Path) -> None: + store_path = tmp_path / "control-plane" + store = LocalControlPlaneStore(store_path) + store.save_snapshot(_snapshot()) + + _rewrite_durable_snapshot(store_path, lambda payload: payload.clear(), update_digest=False) + + with pytest.raises(ValueError, match="failed its durable integrity check"): + store.load_snapshot() + + def test_conformance_conversion_preserves_autonomous_state() -> None: snapshot = _snapshot_from_envelope(_envelope()) diff --git a/implementations/python/tests/test_issue_1003_final_sink_flow_enforcement.py b/implementations/python/tests/test_issue_1003_final_sink_flow_enforcement.py index 7f501769..657c12da 100644 --- a/implementations/python/tests/test_issue_1003_final_sink_flow_enforcement.py +++ b/implementations/python/tests/test_issue_1003_final_sink_flow_enforcement.py @@ -287,6 +287,7 @@ def test_local_store_restart_revalidates_and_replays_idempotently(tmp_path: Path resolver = permit_resolver() first = action_plane(resolver, store=LocalControlPlaneStore(store_path)) receipt = admit(first, idempotency_key="restart") + first.close() restarted_resolver = Sem233FlowSinkResolver() restarted_resolver.subjects = list(resolver.subjects) @@ -301,6 +302,7 @@ def test_local_store_restart_revalidates_and_replays_idempotently(tmp_path: Path assert retry.operation_id == receipt.operation_id assert len(restarted.snapshot.participant_crossing_history[PARTICIPANT]) == 2 + restarted.close() # --- Legacy path and commit-before-effect -------------------------------- diff --git a/implementations/python/tests/test_issue_1092_control_plane_crash_consistency.py b/implementations/python/tests/test_issue_1092_control_plane_crash_consistency.py new file mode 100644 index 00000000..1e6ad4ce --- /dev/null +++ b/implementations/python/tests/test_issue_1092_control_plane_crash_consistency.py @@ -0,0 +1,2645 @@ +"""API-404 crash consistency, recovery, and local runtime ownership tests.""" + +from __future__ import annotations + +import errno +import inspect +import json +import os +import sqlite3 +import stat +import sys +from contextlib import closing +from dataclasses import asdict, fields, replace +from hashlib import sha256 +from multiprocessing import get_all_start_methods, get_context +from pathlib import Path +from threading import Event, RLock, Thread +from time import monotonic, sleep +from types import SimpleNamespace +from typing import Any + +import pytest +from raes import parse_sdl +from raes_backend_stubs.stubs import StubProvisioner, create_stub_target +from raes_contracts.apparatus import RealizationObservationCapability +from raes_contracts.diagnostics import Diagnostic +from raes_contracts.planning import ProvisioningPlan, RuntimeDomain +from raes_contracts.realization_envelope import ( + BackendRealizationEnvelopeModel, + ObservationStrength, + RealizationConcern, + realization_envelope_digest, + realizer_configuration_digest, +) +from raes_contracts.realization_observation import ( + ObservedOperatingSystemIdentity, + RealizationObservation, + bind_operating_system_observations, +) +from raes_contracts.runtime_state import ( + ApplyResult, + OperationReceipt, + OperationState, + OperationStatus, + RuntimeSnapshot, + SnapshotEntry, +) +from raes_contracts.vocabulary import RealizationVerificationScope +from raes_processor.compiler import compile_runtime_model +from raes_processor.planner import plan +from raes_runtime import control_plane_store_lease as lease_module +from raes_runtime import control_plane_store_local as local_store_module +from raes_runtime import control_plane_store_paths as store_paths_module +from raes_runtime.control_plane import RuntimeControlPlane +from raes_runtime.control_plane_execution import ( + OperationExecutionRequest, + SucceededOperationRequest, + execute_operation, + execute_participant_action, + persist_succeeded_operation, +) +from raes_runtime.control_plane_store import ( + INTERRUPTED_OPERATION_DIAGNOSTIC_CODE, + AtomicControlPlaneStore, + AuditEvent, + ControlPlaneOperationRecord, + ControlPlaneStore, + InMemoryControlPlaneStore, +) +from raes_runtime.control_plane_store_compatibility import ( + LegacyControlPlaneStoreWarning, + adapt_control_plane_store, +) +from raes_runtime.control_plane_store_local import LocalControlPlaneStore +from raes_runtime.control_plane_store_snapshots import ( + _require_complete_runtime_snapshot_fields, + _snapshot_from_payload, + _snapshot_payload, +) + + +class _CountingProvisioner: + def __init__( + self, + delegate: object, + *, + realization_envelope: BackendRealizationEnvelopeModel, + interrupt_before_effect: bool = False, + ) -> None: + self._delegate = delegate + self._realization_envelope = realization_envelope + self._interrupt_before_effect = interrupt_before_effect + self.apply_count = 0 + + def validate(self, provisioning_plan: ProvisioningPlan) -> list[object]: + return self._delegate.validate(provisioning_plan) + + def apply( + self, + provisioning_plan: ProvisioningPlan, + snapshot: RuntimeSnapshot, + ) -> ApplyResult: + self.apply_count += 1 + if self._interrupt_before_effect: + raise KeyboardInterrupt("injected crash before backend effect") + result = self._delegate.apply(provisioning_plan, snapshot) + if not result.success: + return result + authority = next( + (entry for entry in provisioning_plan.realization_authority if entry.requirement_kind == "os-family"), + None, + ) + if authority is None: + return result + if provisioning_plan.operation_id is None: + raise AssertionError("fixture OS observation requires a bound operation") + observation = RealizationObservation( + address=authority.address, + field_path="guest.os-release", + concern=RealizationConcern.OPERATING_SYSTEM, + source=ObservationStrength.GUEST_OBSERVED, + value=ObservedOperatingSystemIdentity( + family="linux", + distribution="ubuntu", + version="24.04", + ), + operation_id=provisioning_plan.operation_id, + envelope_digest=self._realization_envelope.digest, + configuration_digest=self._realization_envelope.configuration.configuration_digest, + observer_version="issue-1092-fixture/v1", + sequence=0, + binding_verified=True, + ) + disclosures = bind_operating_system_observations( + plan=provisioning_plan, + observations=(observation,), + envelope=self._realization_envelope, + previous=result.snapshot.realization_observations, + ) + return replace( + result, + snapshot=result.snapshot.with_entries( + dict(result.snapshot.entries), + realization_observations=disclosures, + ), + ) + + +class _BlockingProvisioner: + def __init__(self, delegate: object) -> None: + self._delegate = delegate + self.entered = Event() + self.release = Event() + + def validate(self, provisioning_plan: ProvisioningPlan) -> list[object]: + return self._delegate.validate(provisioning_plan) + + def apply( + self, + provisioning_plan: ProvisioningPlan, + snapshot: RuntimeSnapshot, + ) -> ApplyResult: + self.entered.set() + if not self.release.wait(timeout=5): + raise TimeoutError("blocked provisioner was not released") + return self._delegate.apply(provisioning_plan, snapshot) + + +class _LegacyControlPlaneStore: + """Pre-atomic custom-store shape retained for 3.x compatibility tests.""" + + def __init__(self) -> None: + self.delegate = InMemoryControlPlaneStore() + self.write_calls: list[str] = [] + self.fail_snapshot = False + self.fail_terminal_record = False + + def load_snapshot(self) -> RuntimeSnapshot: + return self.delegate.load_snapshot() + + def save_snapshot(self, snapshot: RuntimeSnapshot) -> None: + self.write_calls.append("save_snapshot") + if self.fail_snapshot: + raise RuntimeError("injected legacy snapshot failure") + self.delegate.save_snapshot(snapshot) + + def load_records(self) -> dict[str, ControlPlaneOperationRecord]: + return self.delegate.load_records() + + def save_record(self, record: ControlPlaneOperationRecord) -> None: + self.write_calls.append("save_record") + if self.fail_terminal_record and record.status.state not in {OperationState.ACCEPTED, OperationState.RUNNING}: + raise RuntimeError("injected legacy terminal-record failure") + self.delegate.save_record(record) + + def find_by_idempotency(self, key: str) -> ControlPlaneOperationRecord | None: + return self.delegate.find_by_idempotency(key) + + def append_audit(self, event: AuditEvent) -> None: + self.delegate.append_audit(event) + + def read_audit(self) -> list[AuditEvent]: + return self.delegate.read_audit() + + def commit_control_transition( + self, + *, + participant_address: str, + expected_head: str | None, + snapshot: RuntimeSnapshot, + record: ControlPlaneOperationRecord, + audit_event: AuditEvent, + ) -> None: + self.delegate.commit_control_transition( + participant_address=participant_address, + expected_head=expected_head, + snapshot=snapshot, + record=record, + audit_event=audit_event, + ) + + def commit_participant_transition( + self, + *, + expected_history_heads: dict[str, str | None], + snapshot: RuntimeSnapshot, + record: ControlPlaneOperationRecord, + audit_event: AuditEvent, + ) -> None: + self.delegate.commit_participant_transition( + expected_history_heads=expected_history_heads, + snapshot=snapshot, + record=record, + audit_event=audit_event, + ) + + +class _PartiallyAtomicControlPlaneStore(_LegacyControlPlaneStore): + def commit_terminal_operation( + self, + _snapshot: RuntimeSnapshot, + _record: ControlPlaneOperationRecord, + ) -> None: + raise AssertionError("a partial atomic capability must not create hybrid semantics") + + +def _durability_fixture_envelope(base_target: object) -> BackendRealizationEnvelopeModel: + base_envelope = base_target.manifest.realization_envelope + if base_envelope is None: + raise AssertionError("fixture requires the governed stub realization envelope") + payload = base_envelope.model_dump(mode="json") + configuration = payload["configuration"] + configuration["operating_systems"] = [{"family": "linux", "distribution": "ubuntu", "versions": ["24.04"]}] + configuration["configuration_digest"] = realizer_configuration_digest(configuration) + operating_system = next( + claim for claim in payload["concerns"] if claim["concern"] == RealizationConcern.OPERATING_SYSTEM.value + ) + operating_system.update( + disposition="realized", + observation_strength=ObservationStrength.GUEST_OBSERVED.value, + mechanism="issue-1092-fixture-guest-os-release", + ) + payload["digest"] = realization_envelope_digest(payload) + return BackendRealizationEnvelopeModel.model_validate(payload) + + +def _target_and_plan(*, interrupt_before_effect: bool = False) -> tuple[object, ProvisioningPlan, _CountingProvisioner]: + base_target = create_stub_target() + realization_envelope = _durability_fixture_envelope(base_target) + declaration = base_target.manifest.realization_support[0] + manifest = replace( + base_target.manifest, + realization_envelope=realization_envelope, + realization_support=( + replace( + declaration, + observation_capabilities={ + **declaration.observation_capabilities, + "operating-system": RealizationObservationCapability( + verification_scope=RealizationVerificationScope.PRESENCE, + observation_strength=ObservationStrength.GUEST_OBSERVED, + ), + }, + ), + ), + ) + provisioner = _CountingProvisioner( + StubProvisioner(realization_envelope), + realization_envelope=realization_envelope, + interrupt_before_effect=interrupt_before_effect, + ) + target = replace(base_target, manifest=manifest, provisioner=provisioner) + scenario = parse_sdl( + """ +name: crash-consistency +nodes: + vm: + type: compute + os: linux + resources: {ram: 1 gib, cpu: 1} +""" + ) + provisioning_plan = plan(compile_runtime_model(scenario), target.manifest).provisioning + return target, provisioning_plan, provisioner + + +def test_durability_fixture_declares_guest_observed_os_presence_corroboration() -> None: + target, _, _ = _target_and_plan() + + capability = target.manifest.realization_support[0].observation_capabilities["operating-system"] + envelope_claim = next( + claim + for claim in target.manifest.realization_envelope.concerns + if claim.concern is RealizationConcern.OPERATING_SYSTEM + ) + assert capability == RealizationObservationCapability( + verification_scope=RealizationVerificationScope.PRESENCE, + observation_strength=ObservationStrength.GUEST_OBSERVED, + ) + assert envelope_claim.observation_strength is ObservationStrength.GUEST_OBSERVED + + +def _running_record( + operation_id: str, *, state: OperationState = OperationState.RUNNING +) -> ControlPlaneOperationRecord: + submitted_at = "2026-08-11T12:00:00Z" + return ControlPlaneOperationRecord( + receipt=OperationReceipt( + operation_id=operation_id, + domain=RuntimeDomain.PROVISIONING, + submitted_at=submitted_at, + accepted=True, + ), + status=OperationStatus( + operation_id=operation_id, + domain=RuntimeDomain.PROVISIONING, + state=state, + submitted_at=submitted_at, + updated_at=submitted_at, + ), + idempotency_key=f"key-{operation_id}", + request_fingerprint=f"fingerprint-{operation_id}", + ) + + +def _terminal_record(record: ControlPlaneOperationRecord) -> ControlPlaneOperationRecord: + return replace( + record, + status=replace( + record.status, + state=OperationState.SUCCEEDED, + updated_at="2026-08-11T12:00:01Z", + ), + ) + + +def _audit_event(action: str = "test") -> AuditEvent: + return AuditEvent( + timestamp="2026-08-11T12:00:00Z", + action=action, + identity="test-identity", + allowed=True, + target="runtime.control-plane", + ) + + +def _atomic_store(kind: str, tmp_path: Path) -> InMemoryControlPlaneStore | LocalControlPlaneStore: + if kind.startswith("memory"): + return InMemoryControlPlaneStore() + return LocalControlPlaneStore(tmp_path / f"control-plane-{kind}") + + +def _interrupted_record(record: ControlPlaneOperationRecord) -> ControlPlaneOperationRecord: + return replace( + record, + status=replace( + record.status, + state=OperationState.FAILED, + updated_at="2026-08-11T12:00:02Z", + diagnostics=[ + Diagnostic( + code=INTERRUPTED_OPERATION_DIAGNOSTIC_CODE, + domain="runtime", + address="runtime.control-plane.provisioning", + message="Backend effects may be indeterminate.", + ) + ], + ), + ) + + +def _runtime_owner_result(store_path: str, queue: Any) -> None: + try: + control_plane = RuntimeControlPlane( + create_stub_target(), + store=LocalControlPlaneStore(Path(store_path)), + ) + except RuntimeError as exc: + queue.put(str(exc)) + return + control_plane.close() + queue.put("acquired") + + +def _inherited_runtime_result(control_plane: RuntimeControlPlane, queue: Any) -> None: + try: + control_plane.get_snapshot() + except RuntimeError as exc: + queue.put(str(exc)) + return + queue.put("used") + + +def _stress_local_store_writes( + store_path: str, + process_index: int, + write_count: int, + barrier: Any, +) -> None: + for write_index in range(write_count): + barrier.wait(timeout=15) + LocalControlPlaneStore(Path(store_path)).save_record(_running_record(f"stress-{process_index}-{write_index}")) + + +def test_runtime_snapshot_durable_codec_is_exhaustive_and_round_trips_closure_records() -> None: + participant_address = "participant.behavior.alpha" + snapshot = RuntimeSnapshot( + participant_episode_closure_records={ + participant_address: [ + { + "participant_address": participant_address, + "episode_id": "episode-alpha", + "source_signal": "environment_terminal", + } + ] + }, + metadata={"generation": 1}, + ) + + payload = _snapshot_payload(snapshot) + + assert set(payload) == {"schema_version", *(field.name for field in fields(RuntimeSnapshot))} + assert _snapshot_from_payload(payload) == snapshot + + +def test_runtime_snapshot_durable_codec_rejects_missing_or_unexpected_fields() -> None: + complete = {field.name: None for field in fields(RuntimeSnapshot)} + + missing = dict(complete) + missing.pop("participant_episode_closure_records") + with pytest.raises(RuntimeError, match="missing=participant_episode_closure_records"): + _require_complete_runtime_snapshot_fields(missing) + + with pytest.raises(RuntimeError, match="unexpected=unknown"): + _require_complete_runtime_snapshot_fields({**complete, "unknown": None}) + + +@pytest.mark.parametrize("store_kind", ["memory", "local"]) +def test_terminal_commit_is_idempotent_but_rejects_snapshot_or_record_rewrite( + tmp_path: Path, + store_kind: str, +) -> None: + store = _atomic_store(store_kind, tmp_path) + terminal = _terminal_record(_running_record("terminal-idempotency")) + snapshot = RuntimeSnapshot(metadata={"generation": 1}) + + store.commit_terminal_operation(snapshot, terminal) + store.commit_terminal_operation(snapshot, terminal) + + different_snapshot = RuntimeSnapshot(metadata={"generation": 2}) + with pytest.raises(ValueError, match="does not match the durable snapshot"): + store.commit_terminal_operation(different_snapshot, terminal) + rewritten = replace(terminal, status=replace(terminal.status, updated_at="2026-08-11T12:00:03Z")) + with pytest.raises(ValueError, match="cannot be rewritten"): + store.commit_terminal_operation(snapshot, rewritten) + + +@pytest.mark.parametrize("store_kind", ["memory", "local"]) +def test_terminal_commit_rejects_nonterminal_and_immutable_identity_changes( + tmp_path: Path, + store_kind: str, +) -> None: + store = _atomic_store(store_kind, tmp_path) + running = _running_record("immutable") + empty_snapshot = RuntimeSnapshot() + + with pytest.raises(ValueError, match="requires a terminal status"): + store.commit_terminal_operation(empty_snapshot, running) + terminal_status = _terminal_record(running).status + for status, message in ( + (replace(terminal_status, operation_id="other"), "identities do not match"), + (replace(terminal_status, domain=RuntimeDomain.EVALUATION), "domains do not match"), + (replace(terminal_status, submitted_at="2026-08-11T12:00:04Z"), "submission times do not match"), + ): + invalid_record = replace(_terminal_record(running), status=status) + with pytest.raises(ValueError, match=message): + store.commit_terminal_operation(empty_snapshot, invalid_record) + + store.claim_record(running) + changed_receipt = replace( + _terminal_record(running), + receipt=replace(running.receipt, accepted=False), + ) + with pytest.raises(ValueError, match="receipt is immutable"): + store.commit_terminal_operation(empty_snapshot, changed_receipt) + changed_fingerprint = replace(_terminal_record(running), request_fingerprint="changed") + with pytest.raises(ValueError, match="operation identity is immutable"): + store.commit_terminal_operation(empty_snapshot, changed_fingerprint) + + +@pytest.mark.parametrize("store_kind", ["memory", "local"]) +def test_interrupted_reconciliation_validates_and_seals_terminal_record( + tmp_path: Path, + store_kind: str, +) -> None: + store = _atomic_store(store_kind, tmp_path) + running = _running_record("recovery-invariants") + store.save_record(running) + recovered = _interrupted_record(running) + + store.reconcile_interrupted_records((recovered,)) + store.reconcile_interrupted_records((recovered,)) + rewritten = replace(recovered, status=replace(recovered.status, updated_at="2026-08-11T12:00:05Z")) + with pytest.raises(ValueError, match="cannot be rewritten during recovery"): + store.reconcile_interrupted_records((rewritten,)) + + missing = _running_record("missing") + interrupted_missing = _interrupted_record(missing) + with pytest.raises(ValueError, match="no longer exists"): + store.reconcile_interrupted_records((interrupted_missing,)) + + +@pytest.mark.parametrize("store_kind", ["memory", "local"]) +@pytest.mark.parametrize("invalid_recovery", ["nonfailed", "missing-diagnostic"]) +def test_interrupted_reconciliation_rejects_invalid_replacement( + tmp_path: Path, + store_kind: str, + invalid_recovery: str, +) -> None: + store = _atomic_store(f"{store_kind}-{invalid_recovery}", tmp_path) + running = _running_record(f"invalid-{invalid_recovery}") + store.save_record(running) + replacement = _interrupted_record(running) + if invalid_recovery == "nonfailed": + replacement = replace(replacement, status=replace(replacement.status, state=OperationState.CANCELLED)) + message = "must persist a failed status" + else: + replacement = replace(replacement, status=replace(replacement.status, diagnostics=[])) + message = "requires its stable diagnostic" + + with pytest.raises(ValueError, match=message): + store.reconcile_interrupted_records((replacement,)) + + +@pytest.mark.parametrize("store_kind", ["memory", "local"]) +def test_terminal_commit_rolls_back_idempotency_collision( + tmp_path: Path, + store_kind: str, +) -> None: + store = _atomic_store(f"{store_kind}-collision", tmp_path) + first = replace(_running_record("first"), idempotency_key="shared") + second = replace(_terminal_record(_running_record("second")), idempotency_key="shared") + store.save_record(first) + rollback_snapshot = RuntimeSnapshot(metadata={"should": "rollback"}) + + with pytest.raises(ValueError, match="idempotency key already belongs"): + store.commit_terminal_operation(rollback_snapshot, second) + + assert store.load_snapshot() == RuntimeSnapshot() + assert set(store.load_records()) == {"first"} + + +def test_in_memory_store_idempotency_claim_and_write_collisions_fail_closed() -> None: + store = InMemoryControlPlaneStore() + first = replace(_running_record("first-claim"), idempotency_key="shared-claim") + competing = replace(_running_record("competing-claim"), idempotency_key="shared-claim") + + assert store.claim_record(first) == first + assert store.claim_record(competing) == first + + with pytest.raises(ValueError, match="idempotency key already belongs"): + store.save_record(competing) + + +def test_in_memory_participant_transition_rolls_back_idempotency_collision() -> None: + store = InMemoryControlPlaneStore() + first = replace(_running_record("first-transition"), idempotency_key="shared-transition") + competing = replace(_terminal_record(_running_record("competing-transition")), idempotency_key="shared-transition") + store.save_record(first) + rollback_snapshot = RuntimeSnapshot(metadata={"must": "rollback"}) + event = _audit_event("participant-transition") + + with pytest.raises(ValueError, match="idempotency key already belongs"): + store.commit_participant_transition( + expected_history_heads={}, + snapshot=rollback_snapshot, + record=competing, + audit_event=event, + ) + + assert store.load_snapshot() == RuntimeSnapshot() + assert store.read_audit() == [] + assert set(store.load_records()) == {first.receipt.operation_id} + + +def test_in_memory_participant_transition_accepts_record_without_idempotency_key() -> None: + store = InMemoryControlPlaneStore() + record = replace(_terminal_record(_running_record("without-idempotency")), idempotency_key="") + snapshot = RuntimeSnapshot(metadata={"committed": True}) + event = _audit_event("participant-transition-without-idempotency") + + store.commit_participant_transition( + expected_history_heads={}, + snapshot=snapshot, + record=record, + audit_event=event, + ) + + assert store.load_snapshot() == snapshot + assert store.load_records() == {record.receipt.operation_id: record} + assert store.read_audit() == [event] + + +@pytest.mark.parametrize("crash_boundary", ["before-backend", "before-terminal-transaction"]) +def test_restart_marks_interrupted_operation_failed_and_retry_does_not_repeat_backend( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + crash_boundary: str, +) -> None: + target, provisioning_plan, provisioner = _target_and_plan( + interrupt_before_effect=crash_boundary == "before-backend" + ) + store_path = tmp_path / "control-plane" + store = LocalControlPlaneStore(store_path) + control_plane = RuntimeControlPlane(target, store=store) + if crash_boundary == "before-terminal-transaction": + + def interrupt_commit(_snapshot: RuntimeSnapshot, _record: ControlPlaneOperationRecord) -> None: + raise KeyboardInterrupt("injected crash before terminal transaction") + + monkeypatch.setattr(store, "commit_terminal_operation", interrupt_commit) + + with pytest.raises(KeyboardInterrupt, match="injected crash"): + control_plane.submit_provisioning( + provisioning_plan, + idempotency_key="retry-safe", + request_fingerprint="same-request", + ) + + records = store.load_records() + assert len(records) == 1 + operation_id, interrupted = next(iter(records.items())) + assert interrupted.status.state == OperationState.RUNNING + assert store.load_snapshot() == RuntimeSnapshot() + control_plane.close() + + restarted = RuntimeControlPlane(target, store=LocalControlPlaneStore(store_path)) + recovered = restarted.get_operation(operation_id) + assert recovered is not None + assert recovered.state == OperationState.FAILED + assert any(diagnostic.code == INTERRUPTED_OPERATION_DIAGNOSTIC_CODE for diagnostic in recovered.diagnostics) + assert "indeterminate" in recovered.diagnostics[-1].message + + retry = restarted.submit_provisioning( + provisioning_plan, + idempotency_key="retry-safe", + request_fingerprint="same-request", + ) + assert retry.operation_id == operation_id + assert provisioner.apply_count == 1 + restarted.close() + + +@pytest.mark.parametrize("write_boundary", ["snapshot", "record"]) +def test_terminal_transaction_rolls_back_at_each_internal_write_boundary( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + write_boundary: str, +) -> None: + store = LocalControlPlaneStore(tmp_path / "control-plane") + running = _running_record("transaction-crash") + store.claim_record(running) + next_snapshot = RuntimeSnapshot(metadata={"generation": 2}) + method_name = f"_upsert_{write_boundary}" + real_upsert = getattr(store, method_name) + + def interrupt_after_write(connection: object, value: object) -> None: + real_upsert(connection, value) + raise KeyboardInterrupt(f"injected crash after {write_boundary} write") + + monkeypatch.setattr(store, method_name, interrupt_after_write) + terminal = _terminal_record(running) + with pytest.raises(KeyboardInterrupt, match=f"after {write_boundary} write"): + store.commit_terminal_operation(next_snapshot, terminal) + + assert store.load_snapshot() == RuntimeSnapshot() + assert store.load_records()[running.receipt.operation_id] == running + + +def test_runtime_resynchronizes_after_error_reported_after_durable_terminal_commit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + target, provisioning_plan, provisioner = _target_and_plan() + store_path = tmp_path / "control-plane" + store = LocalControlPlaneStore(store_path) + control_plane = RuntimeControlPlane(target, store=store) + real_commit = store.commit_terminal_operation + + def commit_then_error(snapshot: RuntimeSnapshot, record: ControlPlaneOperationRecord) -> None: + real_commit(snapshot, record) + raise RuntimeError("injected error after terminal commit") + + monkeypatch.setattr(store, "commit_terminal_operation", commit_then_error) + with pytest.raises(RuntimeError, match="after terminal commit"): + control_plane.submit_provisioning( + provisioning_plan, + idempotency_key="committed", + request_fingerprint="same-request", + ) + + durable_record = next(iter(store.load_records().values())) + assert durable_record.status.state == OperationState.SUCCEEDED + assert store.load_snapshot().entries + assert control_plane.snapshot == store.load_snapshot() + assert control_plane.get_operation(durable_record.receipt.operation_id) == durable_record.status + + retry = control_plane.submit_provisioning( + provisioning_plan, + idempotency_key="committed", + request_fingerprint="same-request", + ) + assert retry.operation_id == durable_record.receipt.operation_id + assert provisioner.apply_count == 1 + control_plane.close() + + restarted = RuntimeControlPlane(target, store=LocalControlPlaneStore(store_path)) + assert restarted.get_operation(durable_record.receipt.operation_id) == durable_record.status + restarted_retry = restarted.submit_provisioning( + provisioning_plan, + idempotency_key="committed", + request_fingerprint="same-request", + ) + assert restarted_retry.operation_id == durable_record.receipt.operation_id + assert provisioner.apply_count == 1 + restarted.close() + + +def test_runtime_poisoned_when_store_error_cannot_be_reconciled(monkeypatch: pytest.MonkeyPatch) -> None: + target, provisioning_plan, _ = _target_and_plan() + store = InMemoryControlPlaneStore() + control_plane = RuntimeControlPlane(target, store=store) + + def fail_commit(_snapshot: RuntimeSnapshot, _record: ControlPlaneOperationRecord) -> None: + raise RuntimeError("terminal commit failed") + + def fail_reload() -> RuntimeSnapshot: + raise OSError("durable reload failed") + + monkeypatch.setattr(store, "commit_terminal_operation", fail_commit) + monkeypatch.setattr(store, "load_snapshot", fail_reload) + + with pytest.raises(RuntimeError, match="terminal commit failed") as caught: + control_plane.submit_provisioning(provisioning_plan) + assert any("runtime is poisoned" in note for note in caught.value.__notes__) + with pytest.raises(RuntimeError, match="requires restart"): + control_plane.get_snapshot() + control_plane.close() + + +def test_startup_reconciliation_is_atomic_and_restarts_cleanly_after_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store_path = tmp_path / "control-plane" + store = LocalControlPlaneStore(store_path) + accepted = _running_record("accepted", state=OperationState.ACCEPTED) + running = _running_record("running") + store.save_record(accepted) + store.save_record(running) + real_upsert = store._upsert_record + writes = 0 + + def fail_second_recovery_write(connection: object, record: ControlPlaneOperationRecord) -> None: + nonlocal writes + writes += 1 + real_upsert(connection, record) + if writes == 2: + raise OSError("injected recovery crash") + + monkeypatch.setattr(store, "_upsert_record", fail_second_recovery_write) + target = create_stub_target() + with pytest.raises(OSError, match="injected recovery crash"): + RuntimeControlPlane(target, store=store) + + assert {record.status.state for record in store.load_records().values()} == { + OperationState.ACCEPTED, + OperationState.RUNNING, + } + + restarted = RuntimeControlPlane(create_stub_target(), store=LocalControlPlaneStore(store_path)) + recovered = restarted._operations.values() + assert {record.status.state for record in recovered} == {OperationState.FAILED} + assert all( + any(diagnostic.code == INTERRUPTED_OPERATION_DIAGNOSTIC_CODE for diagnostic in record.status.diagnostics) + for record in recovered + ) + restarted.close() + + +def test_local_store_rejects_second_runtime_owner_then_allows_clean_handoff(tmp_path: Path) -> None: + store_path = tmp_path / "control-plane" + store = LocalControlPlaneStore(store_path) + target = create_stub_target() + first = RuntimeControlPlane(target, store=store) + + with pytest.raises(RuntimeError, match="exactly one worker"): + RuntimeControlPlane(target, store=store) + competing_store = LocalControlPlaneStore(store_path) + with pytest.raises(RuntimeError, match="exactly one worker"): + RuntimeControlPlane(target, store=competing_store) + + first.close() + second = RuntimeControlPlane(target, store=LocalControlPlaneStore(store_path)) + second.close() + + +def test_local_store_rejects_empty_idempotency_lookup_and_tampered_operation_identity(tmp_path: Path) -> None: + store = LocalControlPlaneStore(tmp_path / "control-plane") + record = _running_record("durable-identity") + store.save_record(record) + assert store.find_by_idempotency("") is None + + tampered_key = "tampered-durable-key" + with store._connection() as connection, local_store_module._transaction(connection): + connection.execute( + "UPDATE operations SET operation_id=? WHERE operation_id=?", + (tampered_key, record.receipt.operation_id), + ) + + with pytest.raises(ValueError, match="identity does not match its durable key"): + store.load_records() + terminal = _terminal_record(_running_record(tampered_key)) + empty_snapshot = RuntimeSnapshot() + with pytest.raises(ValueError, match="identity does not match its durable key"): + store.commit_terminal_operation(empty_snapshot, terminal) + + +def test_local_store_rejects_unsupported_schema_and_failed_quick_check( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + schema_path = tmp_path / "unsupported-schema" + schema_store = LocalControlPlaneStore(schema_path) + with schema_store._connection() as connection, local_store_module._transaction(connection): + connection.execute("UPDATE metadata SET value='future' WHERE key='schema-version'") + with pytest.raises(ValueError, match="unsupported local control-plane database schema"): + LocalControlPlaneStore(schema_path) + + quick_check_path = tmp_path / "failed-quick-check" + LocalControlPlaneStore(quick_check_path) + real_connect = LocalControlPlaneStore._connect + + class _QuickCheckFailureConnection: + def __init__(self, connection: Any) -> None: + self._connection = connection + + def execute(self, statement: str, *args: Any) -> Any: + if statement == "PRAGMA quick_check": + return SimpleNamespace(fetchone=lambda: ("injected-corruption",)) + return self._connection.execute(statement, *args) + + def __getattr__(self, name: str) -> Any: + return getattr(self._connection, name) + + def failed_quick_check_connect( + store: LocalControlPlaneStore, + **kwargs: Any, + ) -> tuple[Any, os.stat_result]: + connection, metadata = real_connect(store, **kwargs) + return _QuickCheckFailureConnection(connection), metadata + + monkeypatch.setattr(LocalControlPlaneStore, "_connect", failed_quick_check_connect) + with pytest.raises(ValueError, match="database failed its integrity check"): + LocalControlPlaneStore(quick_check_path) + + +def test_local_store_rejects_non_wal_before_schema_or_legacy_migration( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store_path = tmp_path / "control-plane" + store_path.mkdir(mode=0o700) + legacy_path = store_path / "operations.json" + legacy_payload = "{}" + legacy_path.write_text(legacy_payload, encoding="utf-8") + real_connect = local_store_module.sqlite3.connect + + class _NonWalConnection: + def __init__(self, connection: sqlite3.Connection) -> None: + self._connection = connection + + def execute(self, statement: str, *args: Any) -> Any: + if statement == "PRAGMA journal_mode=WAL": + return SimpleNamespace(fetchone=lambda: ("delete",)) + return self._connection.execute(statement, *args) + + def executescript(self, _script: str) -> Any: + pytest.fail("schema work must not run without WAL admission") + + def __getattr__(self, name: str) -> Any: + return getattr(self._connection, name) + + def connect_without_wal(*args: Any, **kwargs: Any) -> _NonWalConnection: + return _NonWalConnection(real_connect(*args, **kwargs)) + + monkeypatch.setattr(local_store_module.sqlite3, "connect", connect_without_wal) + monkeypatch.setattr( + LocalControlPlaneStore, + "_migrate_legacy_json", + lambda *_args, **_kwargs: pytest.fail("legacy migration must not run without WAL admission"), + ) + + with pytest.raises(RuntimeError, match="did not enter required SQLite WAL journal mode"): + LocalControlPlaneStore(store_path) + + assert legacy_path.read_text(encoding="utf-8") == legacy_payload + assert list(store_path.glob("legacy-json-backup-*")) == [] + with closing(real_connect(store_path / "control-plane.sqlite3")) as connection, connection: + assert connection.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall() == [] + + +def test_local_store_rejects_non_object_durable_payload(tmp_path: Path) -> None: + store = LocalControlPlaneStore(tmp_path / "control-plane") + store.save_snapshot(RuntimeSnapshot(metadata={"stored": True})) + content = "[]" + digest = sha256(content.encode("utf-8")).hexdigest() + with store._connection() as connection, local_store_module._transaction(connection): + connection.execute( + "UPDATE state SET payload=?, digest=? WHERE key='runtime-snapshot'", + (content, digest), + ) + + with pytest.raises(ValueError, match="payload must be an object"): + store.load_snapshot() + + +def test_local_store_migrates_complete_legacy_state_and_keeps_auditable_backup( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store_path = tmp_path / "control-plane" + store_path.mkdir(mode=0o700) + file_snapshot = RuntimeSnapshot(metadata={"source": "snapshot-file"}) + committed_snapshot = RuntimeSnapshot( + participant_control_history={"participant.test": [{}]}, + metadata={"source": "control-transition-state"}, + ) + control_record = _running_record("legacy-control-record") + operations_record = _running_record("legacy-operations-record") + first_audit = _audit_event("legacy-control-audit") + second_audit = _audit_event("legacy-jsonl-audit") + (store_path / "snapshot.json").write_text( + json.dumps(_snapshot_payload(file_snapshot)), + encoding="utf-8", + ) + (store_path / "control-transition-state.json").write_text( + json.dumps( + { + "snapshot": _snapshot_payload(committed_snapshot), + "records": {control_record.receipt.operation_id: local_store_module._record_payload(control_record)}, + "audit": [asdict(first_audit)], + } + ), + encoding="utf-8", + ) + (store_path / "operations.json").write_text( + json.dumps({operations_record.receipt.operation_id: local_store_module._record_payload(operations_record)}), + encoding="utf-8", + ) + (store_path / "audit.jsonl").write_text( + "\n" + json.dumps(asdict(first_audit)) + "\n" + json.dumps(asdict(second_audit)) + "\n", + encoding="utf-8", + ) + fsync_targets: list[str] = [] + real_fsync = store_paths_module.os.fsync + + def observe_fsync(descriptor: int) -> None: + mode = os.fstat(descriptor).st_mode + fsync_targets.append("file" if stat.S_ISREG(mode) else "directory") + real_fsync(descriptor) + + monkeypatch.setattr(store_paths_module.os, "fsync", observe_fsync) + + migrated = LocalControlPlaneStore(store_path) + + assert migrated.load_snapshot().metadata == {"source": "control-transition-state"} + assert set(migrated.load_records()) == { + control_record.receipt.operation_id, + operations_record.receipt.operation_id, + } + assert migrated.read_audit() == [first_audit, second_audit] + backups = list(store_path.glob("legacy-json-backup-*")) + assert len(backups) == 1 + assert {path.name for path in backups[0].iterdir()} == { + "snapshot.json", + "control-transition-state.json", + "operations.json", + "audit.jsonl", + } + expected_fsync_targets = ["file"] * 4 + if store_paths_module._DIRECTORY_FSYNC_SUPPORTED: + expected_fsync_targets.extend(["directory"] * 3) + assert fsync_targets == expected_fsync_targets + + +def test_local_store_backup_file_fsync_failure_rolls_back_and_restarts_migration( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store_path = tmp_path / "control-plane" + store_path.mkdir(mode=0o700) + record = _running_record("legacy-fsync-restart") + legacy_path = store_path / "operations.json" + legacy_payload = json.dumps({record.receipt.operation_id: local_store_module._record_payload(record)}) + legacy_path.write_text(legacy_payload, encoding="utf-8") + real_fsync = store_paths_module.os.fsync + failed_regular_file = False + + def fail_first_regular_file(descriptor: int) -> None: + nonlocal failed_regular_file + if stat.S_ISREG(os.fstat(descriptor).st_mode) and not failed_regular_file: + failed_regular_file = True + raise OSError(errno.EIO, "injected backup fsync failure") + real_fsync(descriptor) + + monkeypatch.setattr(store_paths_module.os, "fsync", fail_first_regular_file) + + with pytest.raises(RuntimeError, match="could not durably synchronize local control-plane file") as caught: + LocalControlPlaneStore(store_path) + + assert isinstance(caught.value.__cause__, OSError) + assert caught.value.__cause__.errno == errno.EIO + assert legacy_path.read_text(encoding="utf-8") == legacy_payload + assert len(list(store_path.glob("legacy-json-backup-*"))) == 1 + with closing(sqlite3.connect(store_path / "control-plane.sqlite3")) as connection, connection: + assert connection.execute("SELECT value FROM metadata WHERE key='legacy-json-migration'").fetchone() is None + + monkeypatch.undo() + migrated = LocalControlPlaneStore(store_path) + assert migrated.load_records() == {record.receipt.operation_id: record} + assert legacy_path.read_text(encoding="utf-8") == legacy_payload + assert len(list(store_path.glob("legacy-json-backup-*"))) == 2 + + +def test_local_store_rolls_back_unverified_legacy_migration( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store_path = tmp_path / "control-plane" + store_path.mkdir(mode=0o700) + record = _running_record("legacy-unverified") + snapshot = RuntimeSnapshot(metadata={"source": "same-transition-count"}) + (store_path / "snapshot.json").write_text( + json.dumps(_snapshot_payload(snapshot)), + encoding="utf-8", + ) + (store_path / "control-transition-state.json").write_text( + json.dumps({"snapshot": _snapshot_payload(snapshot)}), + encoding="utf-8", + ) + (store_path / "operations.json").write_text( + json.dumps({record.receipt.operation_id: local_store_module._record_payload(record)}), + encoding="utf-8", + ) + monkeypatch.setattr( + LocalControlPlaneStore, + "_upsert_record", + staticmethod(lambda _connection, _record: None), + ) + + with pytest.raises(ValueError, match="legacy control-plane migration verification failed"): + LocalControlPlaneStore(store_path) + + +def test_local_store_migrates_legacy_records_without_snapshot_files(tmp_path: Path) -> None: + store_path = tmp_path / "control-plane" + store_path.mkdir(mode=0o700) + record = _running_record("legacy-record-only") + (store_path / "operations.json").write_text( + json.dumps({record.receipt.operation_id: local_store_module._record_payload(record)}), + encoding="utf-8", + ) + + migrated = LocalControlPlaneStore(store_path) + + assert migrated.load_snapshot() == RuntimeSnapshot() + assert migrated.load_records() == {record.receipt.operation_id: record} + + +def test_snapshot_serialization_preserves_account_placement_without_credentials() -> None: + address = "account-placement.test" + payload = {"account_address": "account.test", "node_address": "node.test"} + snapshot = RuntimeSnapshot( + entries={ + address: SnapshotEntry( + address=address, + domain=RuntimeDomain.PROVISIONING, + resource_type="account-placement", + payload=payload, + ) + } + ) + + assert _snapshot_payload(snapshot)["entries"][address]["payload"] == payload + + +def test_terminal_commit_retry_compares_canonical_value_free_snapshot(tmp_path: Path) -> None: + address = "provision.account.test" + snapshot = RuntimeSnapshot( + entries={ + address: SnapshotEntry( + address=address, + domain=RuntimeDomain.PROVISIONING, + resource_type="account-placement", + payload={ + "spec": { + "credential_bindings": [ + { + "credential_id": "root", + "purpose": "login", + "auth_method": "password", + "material": {"classification": "secret_fixture", "value": "secret"}, + } + ] + } + }, + ) + } + ) + store = LocalControlPlaneStore(tmp_path / "control-plane") + running = replace(_running_record("canonical-terminal-retry"), idempotency_key="canonical-retry") + terminal = _terminal_record(running) + store.claim_record(running) + + store.commit_terminal_operation(snapshot, terminal) + store.commit_terminal_operation(snapshot, terminal) + + assert store.load_records()[running.receipt.operation_id] == terminal + assert store.load_snapshot() == _snapshot_from_payload(_snapshot_payload(snapshot)) + + +def test_local_store_rejects_configured_multiworker_startup( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("WEB_CONCURRENCY", "2") + target = create_stub_target() + store = LocalControlPlaneStore(tmp_path / "control-plane") + with pytest.raises(RuntimeError, match="WEB_CONCURRENCY=2"): + RuntimeControlPlane(target, store=store) + + +def test_local_store_rejects_invalid_worker_count_configuration( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("UVICORN_WORKERS", "many") + target = create_stub_target() + store = LocalControlPlaneStore(tmp_path / "control-plane") + with pytest.raises(RuntimeError, match="UVICORN_WORKERS must be 1"): + RuntimeControlPlane(target, store=store) + + +def test_local_store_accepts_explicit_single_worker_configuration( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("WEB_CONCURRENCY", "1") + owner = RuntimeControlPlane( + create_stub_target(), + store=LocalControlPlaneStore(tmp_path / "control-plane"), + ) + assert owner.get_snapshot().snapshot == RuntimeSnapshot() + owner.close() + + +def test_local_store_migrates_directory_and_database_and_validates_private_sidecars( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + if os.name == "nt": + pytest.skip("POSIX mode bits are unavailable") + store_path = tmp_path / "control-plane" + store_path.mkdir() + store_path.chmod(0o777) + database_path = store_path / "control-plane.sqlite3" + database_path.touch(mode=0o644) + connection_observations: list[tuple[int, tuple[str, ...]]] = [] + connect = local_store_module.sqlite3.connect + + def observe_connect(*args: Any, **kwargs: Any) -> Any: + existing_sidecars = tuple( + suffix + for suffix in store_paths_module._SQLITE_SIDECAR_SUFFIXES + if Path(f"{database_path}{suffix}").exists() + ) + connection_observations.append((store_path.stat().st_mode & 0o777, existing_sidecars)) + return connect(*args, **kwargs) + + monkeypatch.setattr(local_store_module.sqlite3, "connect", observe_connect) + + store = LocalControlPlaneStore(store_path) + + assert connection_observations[0] == (0o700, ()) + assert store_path.stat().st_mode & 0o777 == 0o700 + assert database_path.stat().st_mode & 0o777 == 0o600 + with store._connection() as connection: + connection.execute("BEGIN IMMEDIATE") + connection.execute("INSERT OR REPLACE INTO metadata(key, value) VALUES ('permission-probe', 'ok')") + connection.commit() + for suffix in ("-wal", "-shm"): + sidecar = Path(f"{database_path}{suffix}") + assert sidecar.exists() + assert sidecar.stat().st_mode & 0o777 == 0o600 + + +def test_local_store_never_uses_raw_descriptors_for_sqlite_files( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store_path = tmp_path / "control-plane" + database_path = store_path / "control-plane.sqlite3" + sqlite_paths = { + database_path, + *(Path(f"{database_path}{suffix}") for suffix in store_paths_module._SQLITE_SIDECAR_SUFFIXES), + } + directory_descriptors: set[int] = set() + raw_open = store_paths_module.os.open + raw_fchmod = store_paths_module.os.fchmod + raw_path_open = Path.open + + def guarded_raw_open(path: Any, *args: Any, **kwargs: Any) -> int: + assert Path(path) not in sqlite_paths + descriptor = raw_open(path, *args, **kwargs) + directory_descriptors.add(descriptor) + return descriptor + + def guarded_fchmod(descriptor: int, mode: int) -> None: + assert descriptor in directory_descriptors + raw_fchmod(descriptor, mode) + + def guarded_path_open(path: Path, *args: Any, **kwargs: Any) -> Any: + assert path not in sqlite_paths + return raw_path_open(path, *args, **kwargs) + + monkeypatch.setattr(store_paths_module.os, "open", guarded_raw_open) + monkeypatch.setattr(store_paths_module.os, "fchmod", guarded_fchmod) + monkeypatch.setattr(Path, "open", guarded_path_open) + + store = LocalControlPlaneStore(store_path) + with store._connection() as connection: + connection.execute("BEGIN IMMEDIATE") + connection.execute("INSERT OR REPLACE INTO metadata(key, value) VALUES ('sidecar-probe', 'ok')") + connection.commit() + assert Path(f"{database_path}-wal").exists() + assert Path(f"{database_path}-shm").exists() + + +def test_local_store_uses_encoded_sqlite_uri_creation_and_existing_modes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store_path = tmp_path / "control plane?#" + connection_uris: list[tuple[str, bool]] = [] + connect = local_store_module.sqlite3.connect + + def observe_connect(database: str, *args: Any, **kwargs: Any) -> Any: + connection_uris.append((database, kwargs.get("uri") is True)) + return connect(database, *args, **kwargs) + + monkeypatch.setattr(local_store_module.sqlite3, "connect", observe_connect) + + store = LocalControlPlaneStore(store_path) + store.load_snapshot() + + assert connection_uris[0][0].endswith("control%20plane%3F%23/control-plane.sqlite3?mode=rwc") + assert connection_uris[-1][0].endswith("control%20plane%3F%23/control-plane.sqlite3?mode=rw") + assert all(uri_enabled for _, uri_enabled in connection_uris) + + +def test_local_store_does_not_recreate_existing_database_that_disappears_before_connect( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = LocalControlPlaneStore(tmp_path / "control-plane") + database_path = store._database_path + connect = local_store_module.sqlite3.connect + + def remove_then_connect(*args: Any, **kwargs: Any) -> Any: + database_path.unlink() + return connect(*args, **kwargs) + + monkeypatch.setattr(local_store_module.sqlite3, "connect", remove_then_connect) + + with pytest.raises(sqlite3.OperationalError): + store.load_snapshot() + assert not database_path.exists() + + +@pytest.mark.parametrize( + ("changed_call", "message"), + [ + (2, "database file changed while SQLite opened it"), + (3, "database file changed while SQLite was connected"), + ], +) +def test_local_store_rejects_database_identity_replacement_across_sqlite_connection( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + changed_call: int, + message: str, +) -> None: + store = LocalControlPlaneStore(tmp_path / "control-plane") + secure_database_file = local_store_module._secure_database_file + calls = 0 + + def changed_identity(*args: Any, **kwargs: Any) -> os.stat_result | None: + nonlocal calls + calls += 1 + metadata = secure_database_file(*args, **kwargs) + if calls != changed_call or metadata is None: + return metadata + values = list(metadata) + values[stat.ST_INO] += 1 + return os.stat_result(values) + + monkeypatch.setattr(local_store_module, "_secure_database_file", changed_identity) + + with pytest.raises(RuntimeError, match=message): + store.load_snapshot() + + +def test_local_store_rejects_database_replacement_between_connections(tmp_path: Path) -> None: + original_path = tmp_path / "original" + replacement_path = tmp_path / "replacement" + store = LocalControlPlaneStore(original_path) + store.save_snapshot(RuntimeSnapshot(metadata={"database": "original"})) + replacement = LocalControlPlaneStore(replacement_path) + replacement.save_snapshot(RuntimeSnapshot(metadata={"database": "replacement"})) + os.replace(replacement._database_path, store._database_path) + + with pytest.raises(RuntimeError, match="database file changed while the store was active"): + store.load_snapshot() + + +def test_local_store_rejects_hard_linked_database_alias(tmp_path: Path) -> None: + original_path = tmp_path / "original" + alias_path = tmp_path / "alias" + original = LocalControlPlaneStore(original_path) + alias_path.mkdir(mode=0o700) + try: + os.link(original._database_path, alias_path / "control-plane.sqlite3") + except OSError: + pytest.skip("hard links are unavailable") + + with pytest.raises(RuntimeError, match="database file must not have hard links"): + LocalControlPlaneStore(alias_path) + + +def test_sqlite_sidecar_validation_is_metadata_only_and_tolerates_disappearance( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + sidecar = tmp_path / "control-plane.sqlite3-shm" + sidecar.touch(mode=0o600) + + def unexpected_descriptor_operation(*_args: Any, **_kwargs: Any) -> Any: + raise AssertionError("SQLite sidecar validation must remain metadata-only") + + monkeypatch.setattr(store_paths_module.os, "open", unexpected_descriptor_operation) + monkeypatch.setattr(store_paths_module.os, "fchmod", unexpected_descriptor_operation, raising=False) + monkeypatch.setattr(store_paths_module.os, "chmod", unexpected_descriptor_operation) + monkeypatch.setattr(Path, "chmod", unexpected_descriptor_operation) + + assert store_paths_module._validate_sqlite_sidecar(sidecar) is True + disappearing = SimpleNamespace( + st_mode=stat.S_IFREG | 0o600, + st_file_attributes=0, + st_uid=getattr(os, "geteuid", lambda: 0)(), + st_nlink=0, + ) + monkeypatch.setattr(Path, "lstat", lambda _path: disappearing) + assert store_paths_module._validate_sqlite_sidecar(sidecar) is False + monkeypatch.undo() + sidecar.unlink() + assert store_paths_module._validate_sqlite_sidecar(sidecar) is False + + +def test_sqlite_sidecar_validation_rejects_unsafe_metadata( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + current_uid = getattr(os, "geteuid", lambda: 0)() + sidecar = tmp_path / "control-plane.sqlite3-shm" + cases = [ + ( + SimpleNamespace(st_mode=stat.S_IFLNK | 0o777, st_file_attributes=0, st_uid=current_uid), + "symlink or reparse point", + ), + ( + SimpleNamespace(st_mode=stat.S_IFDIR | 0o700, st_file_attributes=0, st_uid=current_uid), + "wrong filesystem type", + ), + ( + SimpleNamespace( + st_mode=stat.S_IFREG | 0o600, + st_file_attributes=0, + st_uid=current_uid, + st_nlink=2, + ), + "must not have hard links", + ), + ] + if os.name != "nt": + cases.append( + ( + SimpleNamespace(st_mode=stat.S_IFREG | 0o640, st_file_attributes=0, st_uid=current_uid), + "private permissions 0600", + ) + ) + if hasattr(os, "geteuid"): + cases.append( + ( + SimpleNamespace(st_mode=stat.S_IFREG | 0o600, st_file_attributes=0, st_uid=current_uid + 1), + "owned by the current user", + ) + ) + + for metadata, message in cases: + with monkeypatch.context() as patch: + patch.setattr(Path, "lstat", lambda _path, value=metadata: value) + with pytest.raises(RuntimeError, match=message): + store_paths_module._validate_sqlite_sidecar(sidecar) + + +def test_local_store_repeated_multiprocess_wal_lifecycle(tmp_path: Path) -> None: + store_path = tmp_path / "control-plane" + LocalControlPlaneStore(store_path) + context = get_context("spawn") + process_count = 4 + write_count = 6 + barrier = context.Barrier(process_count) + processes = [ + context.Process( + target=_stress_local_store_writes, + args=(str(store_path), process_index, write_count, barrier), + ) + for process_index in range(process_count) + ] + + for process in processes: + process.start() + for process in processes: + process.join(timeout=30) + for process in processes: + if process.is_alive(): + process.terminate() + process.join(timeout=5) + + assert [process.exitcode for process in processes] == [0] * process_count + assert set(LocalControlPlaneStore(store_path).load_records()) == { + f"stress-{process_index}-{write_index}" + for process_index in range(process_count) + for write_index in range(write_count) + } + + +def test_local_store_rejects_symlink_directory_without_touching_target(tmp_path: Path) -> None: + target = tmp_path / "target" + target.mkdir() + marker = target / "marker" + marker.write_text("unchanged", encoding="utf-8") + store_path = tmp_path / "control-plane" + try: + store_path.symlink_to(target, target_is_directory=True) + except OSError: + pytest.skip("symlink creation is unavailable") + + with pytest.raises(RuntimeError, match="directory must not be a symlink or reparse point"): + LocalControlPlaneStore(store_path) + + assert marker.read_text(encoding="utf-8") == "unchanged" + + +def test_local_store_rejects_non_directory_store_path( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store_path = tmp_path / "control-plane" + store_path.write_text("not a directory", encoding="utf-8") + before = os.stat(store_path) + + def unexpected_mutation(*_args: Any, **_kwargs: Any) -> Any: + raise AssertionError("unsafe path was opened or chmodded") + + monkeypatch.setattr(store_paths_module.os, "open", unexpected_mutation) + monkeypatch.setattr(store_paths_module.os, "fchmod", unexpected_mutation, raising=False) + + with pytest.raises(RuntimeError, match="directory has the wrong filesystem type"): + LocalControlPlaneStore(store_path) + + after = os.stat(store_path) + assert store_path.read_text(encoding="utf-8") == "not a directory" + assert (after.st_ino, after.st_mode, after.st_mtime_ns) == (before.st_ino, before.st_mode, before.st_mtime_ns) + + +def test_local_store_rejects_foreign_directory_without_open_or_chmod( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + get_effective_uid = getattr(os, "geteuid", None) + if not callable(get_effective_uid): + pytest.skip("filesystem ownership is unavailable") + store_path = tmp_path / "control-plane" + store_path.mkdir() + store_path.chmod(0o777) + before = os.stat(store_path) + lstat = Path.lstat + foreign_metadata = SimpleNamespace( + st_mode=before.st_mode, + st_file_attributes=0, + st_uid=get_effective_uid() + 1, + ) + + def fake_lstat(path: Path) -> Any: + if path == store_path: + return foreign_metadata + return lstat(path) + + def unexpected_mutation(*_args: Any, **_kwargs: Any) -> Any: + raise AssertionError("foreign path was opened or chmodded") + + monkeypatch.setattr(Path, "lstat", fake_lstat) + monkeypatch.setattr(store_paths_module.os, "open", unexpected_mutation) + monkeypatch.setattr(store_paths_module.os, "fchmod", unexpected_mutation, raising=False) + + with pytest.raises(RuntimeError, match="directory must be owned by the current user"): + LocalControlPlaneStore(store_path) + + after = os.stat(store_path) + assert (after.st_ino, after.st_mode, after.st_mtime_ns) == (before.st_ino, before.st_mode, before.st_mtime_ns) + + +def test_store_path_windows_mode_branches_do_not_call_fchmod( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + directory_path = tmp_path / "control-plane" + directory_path.mkdir() + database_path = directory_path / "control-plane.sqlite3" + database_path.touch() + fchmod_calls: list[tuple[object, ...]] = [] + + monkeypatch.setattr(store_paths_module.os, "name", "nt") + monkeypatch.setattr( + store_paths_module.os, + "fchmod", + lambda *args: fchmod_calls.append(args), + raising=False, + ) + + store_paths_module._secure_store_directory(directory_path) + assert store_paths_module._secure_database_file(database_path, allow_missing=False) is not None + assert store_paths_module._validate_sqlite_sidecar(database_path) + assert fchmod_calls == [] + + +def test_store_path_legacy_json_and_transition_count_helpers(tmp_path: Path) -> None: + legacy_path = tmp_path / "snapshot.json" + legacy_path.write_text('{"snapshot": true}', encoding="utf-8") + assert store_paths_module._read_json_object(legacy_path) == {"snapshot": True} + + legacy_path.write_text("[]", encoding="utf-8") + with pytest.raises(ValueError, match="must contain an object"): + store_paths_module._read_json_object(legacy_path) + + snapshot = RuntimeSnapshot( + participant_control_history={"participant-a": [{}, {}]}, + participant_crossing_history={"participant-a": [{}]}, + information_state_history={"participant-a": [{}, {}, {}]}, + ) + assert store_paths_module._participant_transition_count(snapshot) == 6 + + +def test_local_store_directory_fsync_skips_platforms_without_support( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(store_paths_module, "_DIRECTORY_FSYNC_SUPPORTED", False) + monkeypatch.setattr( + store_paths_module.os, + "open", + lambda *_args, **_kwargs: pytest.fail("unsupported directory fsync must not open the directory"), + ) + + store_paths_module._fsync_directory(tmp_path) + + +@pytest.mark.parametrize( + "unsupported_errno", + sorted(store_paths_module._UNSUPPORTED_DIRECTORY_FSYNC_ERRNOS), +) +@pytest.mark.parametrize("failure_stage", ["open", "fsync"]) +def test_local_store_directory_fsync_tolerates_only_known_unsupported_errnos( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + unsupported_errno: int, + failure_stage: str, +) -> None: + opened_descriptors: list[int] = [] + real_open = store_paths_module.os.open + + def unsupported_open(*_args: Any, **_kwargs: Any) -> int: + raise OSError(unsupported_errno, "unsupported directory open") + + def observe_open(*args: Any, **kwargs: Any) -> int: + descriptor = real_open(*args, **kwargs) + opened_descriptors.append(descriptor) + return descriptor + + def unsupported_fsync(_descriptor: int) -> None: + raise OSError(unsupported_errno, "unsupported directory fsync") + + monkeypatch.setattr(store_paths_module, "_DIRECTORY_FSYNC_SUPPORTED", True) + monkeypatch.setattr( + store_paths_module.os, + "open", + unsupported_open if failure_stage == "open" else observe_open, + ) + if failure_stage == "fsync": + monkeypatch.setattr(store_paths_module.os, "fsync", unsupported_fsync) + + store_paths_module._fsync_directory(tmp_path) + + if opened_descriptors: + with pytest.raises(OSError) as caught: + os.fstat(opened_descriptors[0]) + assert caught.value.errno == errno.EBADF + + +@pytest.mark.parametrize("failure_stage", ["open", "fsync"]) +def test_local_store_directory_fsync_propagates_eio( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure_stage: str, +) -> None: + opened_descriptors: list[int] = [] + real_open = store_paths_module.os.open + + def fail_open(*_args: Any, **_kwargs: Any) -> int: + raise OSError(errno.EIO, "injected directory open failure") + + def observe_open(*args: Any, **kwargs: Any) -> int: + descriptor = real_open(*args, **kwargs) + opened_descriptors.append(descriptor) + return descriptor + + def fail_fsync(_descriptor: int) -> None: + raise OSError(errno.EIO, "injected directory fsync failure") + + monkeypatch.setattr(store_paths_module, "_DIRECTORY_FSYNC_SUPPORTED", True) + monkeypatch.setattr( + store_paths_module.os, + "open", + fail_open if failure_stage == "open" else observe_open, + ) + if failure_stage == "fsync": + monkeypatch.setattr(store_paths_module.os, "fsync", fail_fsync) + + with pytest.raises(RuntimeError, match="could not durably synchronize local control-plane directory") as caught: + store_paths_module._fsync_directory(tmp_path) + + assert isinstance(caught.value.__cause__, OSError) + assert caught.value.__cause__.errno == errno.EIO + if opened_descriptors: + with pytest.raises(OSError) as closed: + os.fstat(opened_descriptors[0]) + assert closed.value.errno == errno.EBADF + + +@pytest.mark.parametrize("suffix", ["", "-wal", "-shm", "-journal"]) +def test_local_store_rejects_symlink_database_paths_without_touching_target( + tmp_path: Path, + suffix: str, +) -> None: + store_path = tmp_path / "control-plane" + store_path.mkdir(mode=0o700) + database_path = store_path / "control-plane.sqlite3" + if suffix: + database_path.touch(mode=0o600) + victim = tmp_path / f"victim{suffix or '-database'}" + victim.write_text("unchanged", encoding="utf-8") + unsafe_path = Path(f"{database_path}{suffix}") + try: + unsafe_path.symlink_to(victim) + except OSError: + pytest.skip("symlink creation is unavailable") + + path_kind = "database file" if not suffix else "SQLite sidecar" + with pytest.raises(RuntimeError, match=rf"{path_kind} must not be a symlink or reparse point"): + LocalControlPlaneStore(store_path) + + assert victim.read_text(encoding="utf-8") == "unchanged" + + +def test_local_store_path_metadata_rejects_reparse_wrong_type_and_foreign_owner(tmp_path: Path) -> None: + current_uid = getattr(os, "geteuid", lambda: 0)() + path = tmp_path / "control-plane" + cases = [ + ( + SimpleNamespace( + st_mode=stat.S_IFDIR | 0o700, + st_file_attributes=getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400), + st_uid=current_uid, + ), + "directory", + "symlink or reparse point", + ), + ( + SimpleNamespace(st_mode=stat.S_IFREG | 0o600, st_file_attributes=0, st_uid=current_uid), + "directory", + "wrong filesystem type", + ), + ] + if hasattr(os, "geteuid"): + cases.append( + ( + SimpleNamespace(st_mode=stat.S_IFREG | 0o600, st_file_attributes=0, st_uid=current_uid + 1), + "database file", + "owned by the current user", + ) + ) + for metadata, kind, message in cases: + with pytest.raises(RuntimeError, match=message): + store_paths_module._require_safe_store_path_metadata( + metadata, # type: ignore[arg-type] + path, + kind=kind, + ) + + +def test_local_store_rejects_directory_and_database_identity_changes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + directory_path = tmp_path / "directory-race" + directory_path.mkdir(mode=0o700) + with monkeypatch.context() as patch: + patch.setattr(store_paths_module.os.path, "samestat", lambda _left, _right: False) + with pytest.raises(RuntimeError, match="directory changed while it was opened"): + LocalControlPlaneStore(directory_path) + + database_path = tmp_path / "database-race" + database_path.mkdir(mode=0o700) + (database_path / "control-plane.sqlite3").touch(mode=0o600) + comparisons = 0 + + def change_database_identity(_left: object, _right: object) -> bool: + nonlocal comparisons + comparisons += 1 + return comparisons == 1 + + with monkeypatch.context() as patch: + patch.setattr(store_paths_module.os.path, "samestat", change_database_identity) + with pytest.raises(RuntimeError, match="database file changed while it was secured"): + LocalControlPlaneStore(database_path) + + +def test_secure_database_file_handles_missing_path( + tmp_path: Path, +) -> None: + database_path = tmp_path / "control-plane.sqlite3" + assert store_paths_module._secure_database_file(database_path, allow_missing=True) is None + with pytest.raises(RuntimeError, match="database file is missing"): + store_paths_module._secure_database_file(database_path, allow_missing=False) + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX path-mode tightening is unavailable") +@pytest.mark.parametrize("failure", [PermissionError("denied"), NotImplementedError("unsupported")]) +def test_secure_database_file_maps_path_chmod_failures( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure: BaseException, +) -> None: + database_path = tmp_path / "control-plane.sqlite3" + database_path.touch(mode=0o644) + monkeypatch.setattr( + store_paths_module.os, + "chmod", + lambda *_args, **_kwargs: (_ for _ in ()).throw(failure), + ) + + with pytest.raises(RuntimeError, match="could not secure local control-plane database file"): + store_paths_module._secure_database_file(database_path, allow_missing=False) + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX path-mode tightening is unavailable") +def test_secure_database_file_rejects_ineffective_path_chmod( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + database_path = tmp_path / "control-plane.sqlite3" + database_path.touch(mode=0o644) + monkeypatch.setattr(store_paths_module.os, "chmod", lambda *_args, **_kwargs: None) + + with pytest.raises(RuntimeError, match="must use private permissions 0600"): + store_paths_module._secure_database_file(database_path, allow_missing=False) + + +def test_secure_database_file_fails_when_main_database_disappears_during_identity_recheck( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + database_path = tmp_path / "control-plane.sqlite3" + database_path.touch(mode=0o600) + real_lstat = Path.lstat + calls = 0 + + def disappear_during_identity_recheck(path: Path) -> os.stat_result: + nonlocal calls + calls += 1 + if calls == 2: + path.unlink() + raise FileNotFoundError(path) + return real_lstat(path) + + monkeypatch.setattr(Path, "lstat", disappear_during_identity_recheck) + with pytest.raises(RuntimeError, match="database file disappeared while it was secured"): + store_paths_module._secure_database_file(database_path, allow_missing=False) + + +def test_local_store_fails_closed_when_database_disappears(tmp_path: Path) -> None: + store = LocalControlPlaneStore(tmp_path / "control-plane") + store._database_path.unlink() + + with pytest.raises(RuntimeError, match="database file is missing"): + store.load_snapshot() + + +def test_runtime_context_manager_closes_control_plane() -> None: + with RuntimeControlPlane(create_stub_target()) as control_plane: + assert control_plane.get_snapshot().snapshot == RuntimeSnapshot() + + with pytest.raises(RuntimeError, match="control plane is closed"): + control_plane.get_snapshot() + control_plane.close() + + +def test_close_waits_for_backend_and_keeps_lease_until_terminal_commit(tmp_path: Path) -> None: + target, provisioning_plan, _ = _target_and_plan() + provisioner = _BlockingProvisioner(target.provisioner) + target = replace(target, provisioner=provisioner) + store_path = tmp_path / "control-plane" + control_plane = RuntimeControlPlane(target, store=LocalControlPlaneStore(store_path)) + submission_errors: list[BaseException] = [] + close_errors: list[BaseException] = [] + close_completed = [Event(), Event()] + + def submit() -> None: + try: + control_plane.submit_provisioning( + provisioning_plan, + idempotency_key="blocked-close", + request_fingerprint="blocked-close-request", + ) + except BaseException as exc: + submission_errors.append(exc) + + def close(index: int) -> None: + try: + control_plane.close() + except BaseException as exc: + close_errors.append(exc) + finally: + close_completed[index].set() + + submission = Thread(target=submit) + closers = [Thread(target=close, args=(index,)) for index in range(2)] + submission.start() + try: + assert provisioner.entered.wait(timeout=2) + for closer in closers: + closer.start() + assert not any(completed.wait(timeout=0.1) for completed in close_completed) + competing_store = LocalControlPlaneStore(store_path) + with pytest.raises(RuntimeError, match="exactly one worker"): + RuntimeControlPlane(target, store=competing_store) + finally: + provisioner.release.set() + submission.join(timeout=5) + for closer in closers: + closer.join(timeout=5) + + assert not submission.is_alive() + assert all(not closer.is_alive() for closer in closers) + assert submission_errors == [] + assert close_errors == [] + assert all(completed.is_set() for completed in close_completed) + record = next(iter(LocalControlPlaneStore(store_path).load_records().values())) + assert record.status.state == OperationState.SUCCEEDED + + restarted = RuntimeControlPlane(target, store=LocalControlPlaneStore(store_path)) + restarted.close() + + +def test_close_allows_nested_work_from_an_already_admitted_call() -> None: + control_plane = RuntimeControlPlane(create_stub_target()) + outer_admitted = Event() + enter_nested = Event() + nested_completed = Event() + + def admitted_call() -> None: + with control_plane._runtime_call(): + outer_admitted.set() + assert enter_nested.wait(timeout=2) + with control_plane._runtime_call(): + nested_completed.set() + + worker = Thread(target=admitted_call) + worker.start() + assert outer_admitted.wait(timeout=2) + closer = Thread(target=control_plane.close) + closer.start() + deadline = monotonic() + 2 + while not control_plane._closing and monotonic() < deadline: + sleep(0.001) + assert control_plane._closing + + enter_nested.set() + worker.join(timeout=2) + closer.join(timeout=2) + + assert nested_completed.is_set() + assert not worker.is_alive() + assert not closer.is_alive() + assert control_plane._closed + + +@pytest.mark.parametrize("transition_kind", ["control", "participant"]) +def test_runtime_resynchronizes_after_transition_commit_reports_postcommit_error( + monkeypatch: pytest.MonkeyPatch, + transition_kind: str, +) -> None: + store = InMemoryControlPlaneStore() + control_plane = RuntimeControlPlane(create_stub_target(), store=store) + snapshot = RuntimeSnapshot(metadata={"committed": transition_kind}) + record = _terminal_record(_running_record(f"{transition_kind}-postcommit")) + event = _audit_event(f"{transition_kind}-postcommit") + method_name = f"commit_{transition_kind}_transition" + real_commit = getattr(store, method_name) + + def commit_then_error(**kwargs: object) -> None: + real_commit(**kwargs) + raise RuntimeError("postcommit transition error") + + monkeypatch.setattr(store, method_name, commit_then_error) + + def commit_transition() -> None: + if transition_kind == "control": + control_plane._commit_control_transition( + participant_address="participant.test", + expected_head=None, + snapshot=snapshot, + record=record, + audit_event=event, + ) + else: + control_plane._commit_participant_transition( + expected_history_heads={}, + snapshot=snapshot, + record=record, + audit_event=event, + ) + + with pytest.raises(RuntimeError, match="postcommit transition error"): + commit_transition() + + assert control_plane.snapshot == snapshot + assert control_plane.get_operation(record.receipt.operation_id) == record.status + control_plane.close() + + +def test_every_public_runtime_method_and_property_has_lifecycle_admission() -> None: + for name, method in inspect.getmembers(RuntimeControlPlane, predicate=inspect.isfunction): + if name.startswith("_") or name == "close": + continue + assert getattr(method, "__runtime_owned__", False), name + for name, value in inspect.getmembers(RuntimeControlPlane, lambda candidate: isinstance(candidate, property)): + if name.startswith("_"): + continue + assert value.fget is not None + assert getattr(value.fget, "__runtime_owned__", False), name + assert getattr(RuntimeControlPlane.__enter__, "__runtime_owned__", False) + + +def test_public_participant_reads_fail_after_runtime_close() -> None: + control_plane = RuntimeControlPlane(create_stub_target()) + control_plane.close() + + with pytest.raises(RuntimeError, match="control plane is closed"): + control_plane._assert_runtime_owner() + + calls = ( + lambda: control_plane.participant_execution_state("participant-execution.missing"), + lambda: control_plane.get_participant_status_view("participant.behavior.missing"), + lambda: control_plane.get_participant_history_view("participant.behavior.missing", "episode-missing"), + lambda: control_plane.get_participant_context_view( + "participant.behavior.missing", + view_ref="participant-view.missing", + ), + ) + for call in calls: + with pytest.raises(RuntimeError, match="control plane is closed"): + call() + + +def test_close_from_an_active_runtime_call_fails_without_releasing_lease() -> None: + control_plane = RuntimeControlPlane(create_stub_target()) + try: + with control_plane._runtime_call(): + with control_plane._runtime_call(): + pass + with pytest.raises(RuntimeError, match="from one of its active calls"): + control_plane.close() + assert control_plane.get_snapshot().snapshot == RuntimeSnapshot() + finally: + control_plane.close() + + +def test_interrupted_close_reopens_lifecycle_admission(monkeypatch: pytest.MonkeyPatch) -> None: + control_plane = RuntimeControlPlane(create_stub_target()) + condition = control_plane._lifecycle_condition + + with monkeypatch.context() as patch: + + def interrupt_wait(_predicate: object) -> None: + raise KeyboardInterrupt("injected close interruption") + + patch.setattr(condition, "wait_for", interrupt_wait) + with pytest.raises(KeyboardInterrupt, match="close interruption"): + control_plane.close() + + assert control_plane._closing is False + assert control_plane.get_snapshot().snapshot == RuntimeSnapshot() + control_plane.close() + + +def test_partially_initialized_runtime_can_release_lease_without_lifecycle_condition() -> None: + closed = False + + class _Lease: + def close(self) -> None: + nonlocal closed + closed = True + + control_plane = object.__new__(RuntimeControlPlane) + control_plane._runtime_lease = _Lease() + + control_plane.close() + + assert closed is True + assert control_plane._runtime_lease is None + + +def test_runtime_owner_acquisition_releases_descriptor_after_base_exception( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store_path = tmp_path / "control-plane" + store = LocalControlPlaneStore(store_path) + + with monkeypatch.context() as patch: + + def interrupt_lock(_descriptor: int) -> None: + raise KeyboardInterrupt("injected lock acquisition crash") + + patch.setattr(lease_module, "_lock_runtime_owner", interrupt_lock) + target = create_stub_target() + with pytest.raises(KeyboardInterrupt, match="lock acquisition crash"): + RuntimeControlPlane(target, store=store) + + owner = RuntimeControlPlane(create_stub_target(), store=store) + owner.close() + + +@pytest.mark.skipif(os.name == "nt", reason="directory flock guard is POSIX-specific") +def test_runtime_owner_directory_guard_maps_secure_open_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + lock_path = tmp_path / "runtime-owner.lock" + + def deny_directory_open(*_args: Any, **_kwargs: Any) -> int: + raise PermissionError("injected directory-open denial") + + monkeypatch.setattr(lease_module.os, "open", deny_directory_open) + + with pytest.raises(RuntimeError, match="could not securely open runtime-owner store directory"): + lease_module._acquire_store_directory_guard(lock_path) + + +@pytest.mark.skipif(os.name == "nt", reason="directory flock guard is POSIX-specific") +def test_runtime_owner_directory_guard_rejects_non_directory_descriptor( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + lock_path = tmp_path / "runtime-owner.lock" + opened_descriptors: list[int] = [] + real_open = lease_module.os.open + real_fstat = lease_module.os.fstat + + def observe_open(*args: Any, **kwargs: Any) -> int: + descriptor = real_open(*args, **kwargs) + opened_descriptors.append(descriptor) + return descriptor + + monkeypatch.setattr(lease_module.os, "open", observe_open) + monkeypatch.setattr( + lease_module.os, + "fstat", + lambda _descriptor: SimpleNamespace(st_mode=stat.S_IFREG | 0o600), + ) + + with pytest.raises(RuntimeError, match="store path must be a directory"): + lease_module._acquire_store_directory_guard(lock_path) + + assert len(opened_descriptors) == 1 + with pytest.raises(OSError) as closed: + real_fstat(opened_descriptors[0]) + assert closed.value.errno == errno.EBADF + + +@pytest.mark.skipif(os.name == "nt", reason="directory flock guard is POSIX-specific") +def test_runtime_owner_file_lock_failure_releases_directory_guard( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + lock_path = tmp_path / "runtime-owner.lock" + real_lock = lease_module._lock_runtime_owner + lock_calls = 0 + + def fail_file_lock(descriptor: int) -> None: + nonlocal lock_calls + lock_calls += 1 + if lock_calls == 2: + raise BlockingIOError("injected file-lock contention") + real_lock(descriptor) + + with monkeypatch.context() as patch: + patch.setattr(lease_module, "_lock_runtime_owner", fail_file_lock) + with pytest.raises(RuntimeError, match="exactly one worker"): + lease_module.RuntimeOwnerLease.acquire(lock_path) + + lease = lease_module.RuntimeOwnerLease.acquire(lock_path) + lease.close() + + +def test_closed_runtime_owner_lease_fails_closed_and_close_is_idempotent(tmp_path: Path) -> None: + store = LocalControlPlaneStore(tmp_path / "control-plane") + lease = store.acquire_runtime_lease() + lease.close() + lease.close() + + with pytest.raises(RuntimeError, match="lease is closed"): + lease.assert_owner() + + +def test_windows_runtime_owner_lock_protocol_is_directly_testable( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + locking_calls: list[tuple[int, int, int]] = [] + fake_msvcrt = SimpleNamespace( + LK_NBLCK=1, + LK_UNLCK=2, + locking=lambda descriptor, operation, size: locking_calls.append((descriptor, operation, size)), + ) + monkeypatch.setitem(sys.modules, "msvcrt", fake_msvcrt) + monkeypatch.setattr(lease_module, "_is_windows", lambda: True) + + lease = lease_module.RuntimeOwnerLease.acquire(tmp_path / "runtime-owner.lock") + descriptor = lease._descriptor + lease.assert_owner() + lease_module._lock_runtime_owner(descriptor) + lease.close() + + assert [operation for _descriptor, operation, _size in locking_calls] == [1, 1, 2] + assert all(size == 1 for _descriptor, _operation, size in locking_calls) + + +def test_runtime_owner_lease_rejects_and_closes_in_a_different_process_identity( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + lock_path = tmp_path / "runtime-owner.lock" + descriptor = os.open(lock_path, os.O_CREAT | os.O_RDWR, 0o600) + descriptor_only_lease = lease_module.RuntimeOwnerLease(descriptor) + descriptor_only_lease.assert_owner() + descriptor_only_lease.close() + + lease = lease_module.RuntimeOwnerLease.acquire(lock_path) + with monkeypatch.context() as patch: + patch.setattr(lease_module.os, "getpid", lambda: lease._owner_pid + 1) + with pytest.raises(RuntimeError, match="cannot be used after fork"): + lease.assert_owner() + lease.close() + assert lease.closed is True + + reacquired = lease_module.RuntimeOwnerLease.acquire(lock_path) + reacquired.close() + + +def test_local_store_runtime_lease_blocks_another_process(tmp_path: Path) -> None: + store_path = tmp_path / "control-plane" + owner = RuntimeControlPlane(create_stub_target(), store=LocalControlPlaneStore(store_path)) + context = get_context("spawn") + queue = context.Queue() + process = context.Process(target=_runtime_owner_result, args=(str(store_path), queue)) + process.start() + process.join(timeout=15) + if process.is_alive(): + process.terminate() + process.join(timeout=5) + try: + assert process.exitcode == 0 + assert "exactly one worker" in queue.get(timeout=2) + finally: + owner.close() + + +def test_runtime_owner_directory_guard_survives_lock_path_replacement(tmp_path: Path) -> None: + if os.name == "nt": + pytest.skip("directory flock guard is POSIX-specific") + store_path = tmp_path / "control-plane" + target = create_stub_target() + owner = RuntimeControlPlane(target, store=LocalControlPlaneStore(store_path)) + lock_path = store_path / "runtime-owner.lock" + lock_path.unlink() + competing_store = LocalControlPlaneStore(store_path) + + with pytest.raises(RuntimeError, match="exactly one worker"): + RuntimeControlPlane(target, store=competing_store) + with pytest.raises(RuntimeError, match="lock path changed while the lease was active"): + owner.get_snapshot() + + owner.close() + restarted = RuntimeControlPlane(target, store=LocalControlPlaneStore(store_path)) + restarted.close() + + +@pytest.mark.skipif("fork" not in get_all_start_methods(), reason="fork is unavailable") +def test_inherited_runtime_owner_fails_closed_after_fork(tmp_path: Path) -> None: + owner = RuntimeControlPlane( + create_stub_target(), + store=LocalControlPlaneStore(tmp_path / "control-plane"), + ) + context = get_context("fork") + queue = context.Queue() + process = context.Process(target=_inherited_runtime_result, args=(owner, queue)) + process.start() + process.join(timeout=10) + if process.is_alive(): + process.terminate() + process.join(timeout=5) + try: + assert process.exitcode == 0 + assert "cannot be used after fork" in queue.get(timeout=2) + assert owner.get_snapshot().snapshot == RuntimeSnapshot() + finally: + owner.close() + + +@pytest.mark.parametrize("store_type", [_LegacyControlPlaneStore, _PartiallyAtomicControlPlaneStore]) +def test_runtime_preserves_ordered_legacy_store_commits_with_deprecation( + store_type: type[_LegacyControlPlaneStore], +) -> None: + target, provisioning_plan, _ = _target_and_plan() + store = store_type() + + with pytest.warns(LegacyControlPlaneStoreWarning, match="before version 4"): + control_plane = RuntimeControlPlane(target, store=store) # type: ignore[arg-type] + receipt = control_plane.submit_provisioning( + provisioning_plan, + idempotency_key="legacy-compatible", + request_fingerprint="legacy-compatible-request", + ) + + assert store.write_calls == ["save_record", "save_snapshot", "save_record"] + assert store.delegate.load_snapshot() == control_plane.snapshot + assert store.delegate.load_records()[receipt.operation_id].status.state == OperationState.SUCCEEDED + + replay = control_plane.submit_provisioning( + provisioning_plan, + idempotency_key="legacy-compatible", + request_fingerprint="legacy-compatible-request", + ) + assert replay == receipt + assert store.write_calls == ["save_record", "save_snapshot", "save_record"] + with pytest.raises(ValueError, match="reused with a different request body"): + control_plane.submit_provisioning( + provisioning_plan, + idempotency_key="legacy-compatible", + request_fingerprint="different-request", + ) + assert store.write_calls == ["save_record", "save_snapshot", "save_record"] + control_plane.close() + + +def test_public_store_protocol_keeps_atomic_capabilities_optional() -> None: + assert not callable(getattr(ControlPlaneStore, "claim_record", None)) + assert callable(getattr(AtomicControlPlaneStore, "claim_record", None)) + + +def test_legacy_store_claim_fallback_returns_existing_idempotency_record() -> None: + store = _LegacyControlPlaneStore() + existing = replace( + _running_record("legacy-existing-claim"), + idempotency_key="legacy-shared-claim", + ) + competing = replace( + _running_record("legacy-competing-claim"), + idempotency_key="legacy-shared-claim", + ) + store.delegate.save_record(existing) + + with pytest.warns(LegacyControlPlaneStoreWarning, match="implement all atomic methods"): + adapter = adapt_control_plane_store(store) + + assert adapter.claim_record(competing) == existing + assert store.write_calls == [] + + +def test_runtime_legacy_store_recovers_interrupted_records_one_at_a_time() -> None: + store = _LegacyControlPlaneStore() + first = _running_record("legacy-recovery-first") + second = _running_record("legacy-recovery-second") + store.delegate.save_record(first) + store.delegate.save_record(second) + + with pytest.warns(LegacyControlPlaneStoreWarning, match="non-crash-atomic 3.x"): + control_plane = RuntimeControlPlane(create_stub_target(), store=store) # type: ignore[arg-type] + + assert store.write_calls == ["save_record", "save_record"] + for operation_id in (first.receipt.operation_id, second.receipt.operation_id): + recovered = control_plane.get_operation(operation_id) + assert recovered is not None + assert recovered.state == OperationState.FAILED + assert any(diagnostic.code == INTERRUPTED_OPERATION_DIAGNOSTIC_CODE for diagnostic in recovered.diagnostics) + control_plane.close() + + +def test_runtime_legacy_store_preserves_snapshot_first_failure_window() -> None: + target, provisioning_plan, _ = _target_and_plan() + store = _LegacyControlPlaneStore() + with pytest.warns(LegacyControlPlaneStoreWarning): + control_plane = RuntimeControlPlane(target, store=store) # type: ignore[arg-type] + store.fail_terminal_record = True + + with pytest.raises(RuntimeError, match="legacy terminal-record failure"): + control_plane.submit_provisioning(provisioning_plan) + + assert store.write_calls == ["save_record", "save_snapshot", "save_record"] + assert control_plane.snapshot.entries + assert store.delegate.load_snapshot() == control_plane.snapshot + assert next(iter(store.delegate.load_records().values())).status.state == OperationState.RUNNING + control_plane.close() + + +def test_runtime_legacy_store_resynchronizes_after_snapshot_write_failure() -> None: + target, provisioning_plan, _ = _target_and_plan() + store = _LegacyControlPlaneStore() + with pytest.warns(LegacyControlPlaneStoreWarning): + control_plane = RuntimeControlPlane(target, store=store) # type: ignore[arg-type] + store.fail_snapshot = True + + with pytest.raises(RuntimeError, match="legacy snapshot failure"): + control_plane.submit_provisioning(provisioning_plan) + + assert store.write_calls == ["save_record", "save_snapshot"] + assert control_plane.snapshot == RuntimeSnapshot() + assert store.delegate.load_snapshot() == control_plane.snapshot + assert next(iter(store.delegate.load_records().values())).status.state == OperationState.RUNNING + control_plane.close() + + +def test_runtime_rejects_store_without_the_legacy_contract() -> None: + target = create_stub_target() + missing_store = object() + with pytest.raises(TypeError, match="missing required capabilities"): + RuntimeControlPlane(target, store=missing_store) # type: ignore[arg-type] + + +def test_runtime_requires_policy_resolver_for_persisted_crossing_history() -> None: + store = InMemoryControlPlaneStore(RuntimeSnapshot(participant_crossing_history={"participant.demo": [{}]})) + target = create_stub_target() + + with pytest.raises(ValueError, match="persisted participant crossing history requires a policy resolver"): + RuntimeControlPlane(target, store=store) + + +def test_execution_helpers_return_the_durable_winner_when_an_idempotency_claim_loses() -> None: + class _LosingClaimControlPlane: + def __init__(self) -> None: + self._snapshot = RuntimeSnapshot() + self._operation_lock = RLock() + + @staticmethod + def _idempotent_receipt(**_kwargs: object) -> None: + return None + + @staticmethod + def _claim_record(record: ControlPlaneOperationRecord) -> ControlPlaneOperationRecord: + winning_receipt = replace(record.receipt, operation_id="durable-winner") + winning_status = replace(record.status, operation_id="durable-winner") + return replace(record, receipt=winning_receipt, status=winning_status) + + @staticmethod + def _commit_terminal_operation(*_args: object) -> None: + raise AssertionError("a losing claimant must not call the backend or commit") + + control_plane = _LosingClaimControlPlane() + + def unexpected_backend(*_args: object, **_kwargs: object) -> object: + raise AssertionError("a losing claimant must not call the backend") + + participant_receipt = execute_participant_action( + control_plane, + method=unexpected_backend, + request=SimpleNamespace(participant_address="participant.demo"), + address="participant.demo", + idempotency_key="participant-key", + request_fingerprint="participant-fingerprint", + ) + persisted_receipt = persist_succeeded_operation( + control_plane, + SucceededOperationRequest( + operation_id="local-persist", + domain=RuntimeDomain.EVALUATION, + submitted_at="2026-08-12T00:00:00Z", + idempotency_key="persist-key", + request_fingerprint="persist-fingerprint", + ), + ) + operation_receipt = execute_operation( + control_plane, + OperationExecutionRequest( + domain=RuntimeDomain.EVALUATION, + method=unexpected_backend, + plan=object(), + address="evaluation.demo", + diagnostics=[], + base_snapshot=None, + idempotency_key="operation-key", + request_fingerprint="operation-fingerprint", + ), + ) + + assert participant_receipt.operation_id == "durable-winner" + assert persisted_receipt.operation_id == "durable-winner" + assert operation_receipt.operation_id == "durable-winner" + + +def test_persist_succeeded_operation_returns_newly_claimed_receipt() -> None: + class _WinningClaimControlPlane: + @staticmethod + def _claim_record(record: ControlPlaneOperationRecord) -> ControlPlaneOperationRecord: + return record + + receipt = persist_succeeded_operation( + _WinningClaimControlPlane(), + SucceededOperationRequest( + operation_id="newly-claimed", + domain=RuntimeDomain.EVALUATION, + submitted_at="2026-08-12T00:00:00Z", + idempotency_key="new-key", + request_fingerprint="new-fingerprint", + ), + ) + + assert receipt.operation_id == "newly-claimed" + + +def test_runtime_rejects_losing_claim_with_different_request_fingerprint() -> None: + store = InMemoryControlPlaneStore() + control_plane = RuntimeControlPlane(create_stub_target(), store=store) + existing = replace(_running_record("fingerprint-winner"), idempotency_key="shared-fingerprint") + store.claim_record(existing) + competing = replace( + _running_record("fingerprint-loser"), + idempotency_key="shared-fingerprint", + request_fingerprint="different-fingerprint", + ) + + with pytest.raises(ValueError, match="reused with a different request body"): + control_plane._claim_record(competing) + control_plane.close() + + +def test_runtime_owner_lock_has_private_permissions(tmp_path: Path) -> None: + if os.name == "nt": + pytest.skip("POSIX mode bits are unavailable") + store_path = tmp_path / "control-plane" + owner = RuntimeControlPlane(create_stub_target(), store=LocalControlPlaneStore(store_path)) + try: + assert (store_path / "runtime-owner.lock").stat().st_mode & 0o777 == 0o600 + finally: + owner.close() + + +def test_runtime_owner_lock_rejects_symlink_without_opening_or_changing_target( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store_path = tmp_path / "control-plane" + store = LocalControlPlaneStore(store_path) + victim = tmp_path / "victim.txt" + victim.write_text("must remain unchanged", encoding="utf-8") + lock_path = store_path / "runtime-owner.lock" + try: + lock_path.symlink_to(victim) + except OSError: + pytest.skip("symlink creation is unavailable") + real_open = lease_module.os.open + opened_lock = False + + def track_open(path: object, *args: object, **kwargs: object) -> int: + nonlocal opened_lock + if os.fspath(path) == os.fspath(lock_path): + opened_lock = True + return real_open(path, *args, **kwargs) + + monkeypatch.setattr(lease_module.os, "open", track_open) + target = create_stub_target() + with pytest.raises(RuntimeError, match="must not be a symlink or reparse point"): + RuntimeControlPlane(target, store=store) + + assert opened_lock is False + assert victim.read_text(encoding="utf-8") == "must remain unchanged" + + +def test_runtime_owner_lock_rejects_hard_link_alias(tmp_path: Path) -> None: + store_path = tmp_path / "control-plane" + store = LocalControlPlaneStore(store_path) + lock_path = store_path / "runtime-owner.lock" + lock_path.touch(mode=0o600) + try: + os.link(lock_path, tmp_path / "runtime-owner-alias.lock") + except OSError: + pytest.skip("hard links are unavailable") + target = create_stub_target() + + with pytest.raises(RuntimeError, match="lock path must not have hard links"): + RuntimeControlPlane(target, store=store) + + +def test_runtime_owner_lock_rejects_post_open_identity_change_before_truncation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store_path = tmp_path / "control-plane" + store = LocalControlPlaneStore(store_path) + lock_path = store_path / "runtime-owner.lock" + lock_path.write_text("sentinel", encoding="ascii") + lock_path.chmod(0o600) + monkeypatch.setattr(lease_module.os.path, "samestat", lambda _left, _right: False) + target = create_stub_target() + + with pytest.raises(RuntimeError, match="changed while it was opened"): + RuntimeControlPlane(target, store=store) + + assert lock_path.read_text(encoding="ascii") == "sentinel" + + +def test_runtime_owner_metadata_rejects_windows_reparse_attribute(tmp_path: Path) -> None: + metadata = SimpleNamespace( + st_mode=stat.S_IFREG | 0o600, + st_file_attributes=getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400), + st_uid=getattr(os, "geteuid", lambda: 0)(), + ) + + with pytest.raises(RuntimeError, match="symlink or reparse point"): + lease_module._require_safe_runtime_owner_metadata( + metadata, # type: ignore[arg-type] + tmp_path / "runtime-owner.lock", + ) + + +def test_runtime_owner_metadata_rejects_nonregular_file(tmp_path: Path) -> None: + metadata = SimpleNamespace( + st_mode=stat.S_IFDIR | 0o700, + st_file_attributes=0, + st_uid=getattr(os, "geteuid", lambda: 0)(), + ) + + with pytest.raises(RuntimeError, match="must be a regular file"): + lease_module._require_safe_runtime_owner_metadata( + metadata, # type: ignore[arg-type] + tmp_path / "runtime-owner.lock", + ) + + +@pytest.mark.skipif(not hasattr(os, "geteuid"), reason="effective UID is unavailable") +def test_runtime_owner_metadata_rejects_foreign_owner(tmp_path: Path) -> None: + metadata = SimpleNamespace( + st_mode=stat.S_IFREG | 0o600, + st_file_attributes=0, + st_uid=os.geteuid() + 1, + ) + + with pytest.raises(RuntimeError, match="owned by the current user"): + lease_module._require_safe_runtime_owner_metadata( + metadata, # type: ignore[arg-type] + tmp_path / "runtime-owner.lock", + ) + + +def test_runtime_owner_secure_open_failure_is_fail_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store_path = tmp_path / "control-plane" + store = LocalControlPlaneStore(store_path) + lock_path = store_path / "runtime-owner.lock" + real_open = lease_module.os.open + + def deny_lock_open(path: object, *args: object, **kwargs: object) -> int: + if os.fspath(path) == os.fspath(lock_path): + raise PermissionError("injected secure-open denial") + return real_open(path, *args, **kwargs) + + monkeypatch.setattr(lease_module.os, "open", deny_lock_open) + target = create_stub_target() + with pytest.raises(RuntimeError, match="could not securely open runtime-owner lock path"): + RuntimeControlPlane(target, store=store) diff --git a/implementations/python/tests/test_issue_802_participant_control_migration.py b/implementations/python/tests/test_issue_802_participant_control_migration.py index 61fbb375..164d4ca2 100644 --- a/implementations/python/tests/test_issue_802_participant_control_migration.py +++ b/implementations/python/tests/test_issue_802_participant_control_migration.py @@ -287,6 +287,11 @@ def test_runtime_snapshot_fixture_preserves_incumbent_history_without_claim_prom RuntimeSnapshotEnvelopeModel.model_validate(after) (tmp_path / "snapshot.json").write_text(json.dumps(before), encoding="utf-8") legacy_snapshot = LocalControlPlaneStore(tmp_path).load_snapshot() + backups = list(tmp_path.glob("legacy-json-backup-*/snapshot.json")) + assert len(backups) == 1 + assert json.loads(backups[0].read_text(encoding="utf-8")) == before + assert json.loads((tmp_path / "snapshot.json").read_text(encoding="utf-8")) == before + assert (tmp_path / "control-plane.sqlite3").is_file() assert participant_crossing_history_presence(before) is ParticipantCrossingHistoryPresence.ABSENT assert participant_crossing_history_presence(after) is ParticipantCrossingHistoryPresence.PRESENT_EMPTY assert legacy_snapshot.participant_crossing_history == {} diff --git a/implementations/python/tests/test_issue_964_participant_opacity_runtime.py b/implementations/python/tests/test_issue_964_participant_opacity_runtime.py index 6c2522a4..d488d175 100644 --- a/implementations/python/tests/test_issue_964_participant_opacity_runtime.py +++ b/implementations/python/tests/test_issue_964_participant_opacity_runtime.py @@ -643,6 +643,7 @@ def test_restart_requires_the_exact_persisted_opacity_context(tmp_path) -> None: store=store, ) admit(first, idempotency_key="opacity-restart") + first.close() restarted_resolver = _OpacityResolver() restarted_resolver.subjects = list(first_resolver.subjects) @@ -654,6 +655,7 @@ def test_restart_requires_the_exact_persisted_opacity_context(tmp_path) -> None: enforce_final_sink_flow_control=False, ) assert restarted.snapshot.participant_crossing_history[CROSSING_PARTICIPANT] + restarted.close() stale = _OpacityResolver() stale.subjects = list(first_resolver.subjects) diff --git a/implementations/python/tests/test_libvirt_backend_techvault_honesty.py b/implementations/python/tests/test_libvirt_backend_techvault_honesty.py index f71737d8..a0340f78 100644 --- a/implementations/python/tests/test_libvirt_backend_techvault_honesty.py +++ b/implementations/python/tests/test_libvirt_backend_techvault_honesty.py @@ -358,8 +358,10 @@ def test_failed_techvault_admission_preserves_persisted_runtime_snapshot(tmp_pat receipt = control_plane.submit_provisioning(_plan(invalid)) status = control_plane.get_operation(receipt.operation_id) + control_plane.close() restarted = RuntimeControlPlane(create_libvirt_target(driver=driver), store=store) assert status is not None and status.state.value == "failed" assert restarted.snapshot == baseline assert driver.realize_calls == [] + restarted.close() diff --git a/implementations/python/tests/test_run_310_supervisory_lifecycle.py b/implementations/python/tests/test_run_310_supervisory_lifecycle.py index d05226f1..302a636f 100644 --- a/implementations/python/tests/test_run_310_supervisory_lifecycle.py +++ b/implementations/python/tests/test_run_310_supervisory_lifecycle.py @@ -2,6 +2,7 @@ from __future__ import annotations +import sqlite3 from dataclasses import replace from pathlib import Path @@ -623,6 +624,7 @@ def test_supervisory_control_restarts_and_replays_before_the_next_transition( identity=_identity(), idempotency_key="key-proposal", ).accepted + first.close() restarted = RuntimeControlPlane( create_stub_target(), @@ -644,6 +646,7 @@ def test_supervisory_control_restarts_and_replays_before_the_next_transition( idempotency_key="key-approval", ).accepted assert len(restarted.snapshot.participant_control_history[_PARTICIPANT]) == 2 + restarted.close() def test_controller_state_replay_is_scoped_to_one_episode() -> None: @@ -816,11 +819,16 @@ def test_failed_atomic_control_commit_exposes_no_partial_transition( payload_ref="payload:proposal-1", ) - def fail_atomic_write(path: Path, content: str) -> None: - del path, content + real_upsert = store._upsert_record + + def fail_record_upsert( + connection: sqlite3.Connection, + record: ControlPlaneOperationRecord, + ) -> None: + real_upsert(connection, record) raise OSError("commit failed") - monkeypatch.setattr(store, "_atomic_write", fail_atomic_write) + monkeypatch.setattr(store, "_upsert_record", fail_record_upsert) identity = _identity() with pytest.raises(OSError, match="commit failed"): control_plane.record_participant_control( diff --git a/implementations/python/tests/test_run_319_participant_flow_policy.py b/implementations/python/tests/test_run_319_participant_flow_policy.py index e8e3e677..0c85b680 100644 --- a/implementations/python/tests/test_run_319_participant_flow_policy.py +++ b/implementations/python/tests/test_run_319_participant_flow_policy.py @@ -438,6 +438,7 @@ def test_crossing_history_restarts_and_operation_replays_idempotently(tmp_path: store_path = tmp_path / "control-plane" first = action_plane(resolver, store=LocalControlPlaneStore(store_path)) receipt = admit(first, idempotency_key="restart-crossing") + first.close() restarted_resolver = StaticCrossingResolver() restarted_resolver.subjects = list(resolver.subjects) @@ -452,6 +453,7 @@ def test_crossing_history_restarts_and_operation_replays_idempotently(tmp_path: assert retry.operation_id == receipt.operation_id assert len(restarted.snapshot.participant_crossing_history[PARTICIPANT]) == 2 + restarted.close() class _FailingCommitStore(InMemoryControlPlaneStore): diff --git a/implementations/python/tests/test_runtime_control_plane_api.py b/implementations/python/tests/test_runtime_control_plane_api.py index 6b70ce11..b77814cf 100644 --- a/implementations/python/tests/test_runtime_control_plane_api.py +++ b/implementations/python/tests/test_runtime_control_plane_api.py @@ -2,11 +2,17 @@ from __future__ import annotations +import sqlite3 import textwrap +from concurrent.futures import ThreadPoolExecutor +from contextlib import closing +from dataclasses import replace +from multiprocessing import get_context from pathlib import Path +from threading import Barrier as ThreadBarrier +from typing import Protocol import pytest -import raes_runtime.control_plane_store as control_plane_store_module from raes import parse_sdl from raes_backend_stubs.stubs import create_stub_target from raes_contracts.contracts import ( @@ -70,6 +76,23 @@ def _participant_operation_record(operation_id: str, participant_address: str) - ) +class _BarrierLike(Protocol): + def wait(self, timeout: float | None = None) -> int: ... + + +def _save_operation_in_process(store_path: str, index: int, barrier: _BarrierLike) -> None: + record = replace( + _participant_operation_record( + f"process-operation-{index}", + f"participant.behavior.process-subject-{index}", + ), + idempotency_key=f"process-key-{index}", + request_fingerprint=f"process-fingerprint-{index}", + ) + barrier.wait(timeout=10) + LocalControlPlaneStore(Path(store_path)).save_record(record) + + def _test_security(target_name: str, *, max_request_bytes: int = 1_000_000) -> ControlPlaneSecurityConfig: return ControlPlaneSecurityConfig( max_request_bytes=max_request_bytes, @@ -423,9 +446,11 @@ def test_control_plane_api_persists_operations_and_snapshot(tmp_path: Path): headers=headers, ).json() + control_plane.close() restarted = RuntimeControlPlane(target, store=store) assert restarted.get_operation(receipt["operation_id"]) is not None assert restarted.get_snapshot().snapshot.entries + restarted.close() def test_backend_principal_cannot_rewrite_registered_realization_authority() -> None: @@ -571,45 +596,120 @@ def test_control_plane_api_rejects_invalid_content_length_header(): assert control_plane.audit_log()[-1].reason == "invalid content-length" -def test_local_control_plane_store_saves_snapshot_with_atomic_replace( +def test_local_control_plane_store_commits_snapshot_to_wal_database(tmp_path: Path): + store_path = tmp_path / "cp-store" + store = LocalControlPlaneStore(store_path) + store.save_snapshot(RuntimeSnapshot()) + + with closing(sqlite3.connect(store_path / "control-plane.sqlite3")) as connection, connection: + journal_mode = connection.execute("PRAGMA journal_mode").fetchone() + state_count = connection.execute("SELECT COUNT(*) FROM state").fetchone() + integrity = connection.execute("PRAGMA quick_check").fetchone() + + assert journal_mode == ("wal",) + assert state_count == (1,) + assert integrity == ("ok",) + + +def test_local_control_plane_store_rolls_back_snapshot_transaction_failure( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ): store = LocalControlPlaneStore(tmp_path / "cp-store") - replace_calls: list[tuple[Path, Path]] = [] - real_replace = control_plane_store_module.os.replace + real_upsert = store._upsert_snapshot - def tracked_replace(source: str, destination: str) -> None: - replace_calls.append((Path(source), Path(destination))) - real_replace(source, destination) + def fail_upsert(connection: sqlite3.Connection, snapshot: RuntimeSnapshot) -> None: + real_upsert(connection, snapshot) + raise OSError("commit failed") - monkeypatch.setattr(control_plane_store_module.os, "replace", tracked_replace) + monkeypatch.setattr(store, "_upsert_snapshot", fail_upsert) + snapshot = RuntimeSnapshot() - store.save_snapshot(RuntimeSnapshot()) + with pytest.raises(OSError, match="commit failed"): + store.save_snapshot(snapshot) - assert replace_calls - assert replace_calls[0][1] == tmp_path / "cp-store" / "snapshot.json" - assert not replace_calls[0][0].exists() - assert not list((tmp_path / "cp-store").glob("*.tmp")) + assert store.load_snapshot() == RuntimeSnapshot() -def test_local_control_plane_store_cleans_temp_file_after_atomic_replace_failure( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -): - store = LocalControlPlaneStore(tmp_path / "cp-store") +def test_local_control_plane_store_preserves_concurrent_operation_writes(tmp_path: Path) -> None: + store_path = tmp_path / "cp-store" + stores = (LocalControlPlaneStore(store_path), LocalControlPlaneStore(store_path)) + records = [ + replace( + _participant_operation_record( + f"operation-{index}", + f"participant.behavior.subject-{index}", + ), + idempotency_key=f"key-{index}", + request_fingerprint=f"fingerprint-{index}", + ) + for index in range(32) + ] + + def save(index: int) -> None: + stores[index % len(stores)].save_record(records[index]) + + with ThreadPoolExecutor(max_workers=8) as executor: + list(executor.map(save, range(len(records)))) + + assert set(LocalControlPlaneStore(store_path).load_records()) == {record.receipt.operation_id for record in records} + + +def test_local_control_plane_store_preserves_cross_process_operation_writes(tmp_path: Path) -> None: + store_path = tmp_path / "cp-store" + LocalControlPlaneStore(store_path) + context = get_context("spawn") + barrier = context.Barrier(4) + processes = [ + context.Process( + target=_save_operation_in_process, + args=(str(store_path), index, barrier), + ) + for index in range(4) + ] + + for process in processes: + process.start() + for process in processes: + process.join(timeout=15) + for process in processes: + if process.is_alive(): + process.terminate() + process.join(timeout=5) + + assert [process.exitcode for process in processes] == [0, 0, 0, 0] + assert set(LocalControlPlaneStore(store_path).load_records()) == { + f"process-operation-{index}" for index in range(4) + } + - def fail_replace(source: str, destination: str) -> None: - del source, destination - raise OSError("replace failed") +def test_local_control_plane_store_claims_idempotency_key_once_across_instances( + tmp_path: Path, +) -> None: + store_path = tmp_path / "cp-store" + stores = (LocalControlPlaneStore(store_path), LocalControlPlaneStore(store_path)) + records = tuple( + replace( + _participant_operation_record( + f"operation-{index}", + f"participant.behavior.subject-{index}", + ), + idempotency_key="shared-key", + request_fingerprint="same-request", + ) + for index in range(2) + ) + barrier = ThreadBarrier(2) - monkeypatch.setattr(control_plane_store_module.os, "replace", fail_replace) + def claim(index: int) -> ControlPlaneOperationRecord: + barrier.wait() + return stores[index].claim_record(records[index]) - with pytest.raises(OSError, match="replace failed"): - store.save_snapshot(RuntimeSnapshot()) + with ThreadPoolExecutor(max_workers=2) as executor: + claimed = list(executor.map(claim, range(2))) - assert not (tmp_path / "cp-store" / "snapshot.json").exists() - assert not list((tmp_path / "cp-store").glob("*.tmp")) + assert len({record.receipt.operation_id for record in claimed}) == 1 + assert len(LocalControlPlaneStore(store_path).load_records()) == 1 def test_control_plane_api_cancels_workflow_runs(): diff --git a/implementations/python/tests/test_sem_222_episode_termination_semantics.py b/implementations/python/tests/test_sem_222_episode_termination_semantics.py index cdf383ce..b04d9456 100644 --- a/implementations/python/tests/test_sem_222_episode_termination_semantics.py +++ b/implementations/python/tests/test_sem_222_episode_termination_semantics.py @@ -13,6 +13,9 @@ from __future__ import annotations import pytest +from jsonschema import Draft202012Validator +from raes_conformance.conformance.semantics import _semantic_diagnostics +from raes_contracts.contracts import RuntimeSnapshotEnvelopeModel, schema_bundle from raes_contracts.participant_episode import ( ParticipantEpisodeControlAction, ParticipantEpisodeStatus, @@ -24,6 +27,8 @@ ParticipantEpisodeClosureSignal, iter_participant_episode_closure_violations, ) +from raes_contracts.runtime_state import RuntimeSnapshot +from raes_runtime.control_plane_store_snapshots import _snapshot_payload _ADDRESS = "scenario/participant/agent-0" _EPISODE = "episode-0" @@ -276,3 +281,31 @@ def test_canonical_runtime_state_validation_enforces_closure_records() -> None: diagnostics = participant_runtime_state_contract_diagnostics(contradictory) assert diagnostics assert all(diag.code == "runtime.backend-contract-invalid" for diag in diagnostics) + + +def test_public_runtime_snapshot_preserves_and_validates_closure_records() -> None: + snapshot = RuntimeSnapshot( + participant_episode_results=_terminal_result(), + participant_episode_history=_terminal_history(), + participant_episode_closure_records={_ADDRESS: [_closure_payload()]}, + ) + payload = _snapshot_payload(snapshot) + + model = RuntimeSnapshotEnvelopeModel.model_validate(payload) + schema_errors = list(Draft202012Validator(schema_bundle()["runtime-snapshot-v1"]).iter_errors(payload)) + + assert model.participant_episode_closure_records == snapshot.participant_episode_closure_records + assert schema_errors == [] + assert _semantic_diagnostics("runtime-snapshot-v1", payload) == [] + + +def test_public_runtime_snapshot_rejects_invalid_closure_record_semantics() -> None: + snapshot = RuntimeSnapshot( + participant_episode_results=_terminal_result(), + participant_episode_history=_terminal_history(), + participant_episode_closure_records={_ADDRESS: [_closure_payload(sequence_number=3)]}, + ) + + diagnostics = _semantic_diagnostics("runtime-snapshot-v1", _snapshot_payload(snapshot)) + + assert any("history" in diagnostic.message for diagnostic in diagnostics)