diff --git a/.github/workflows/mutation-evidence-accountability.yml b/.github/workflows/mutation-evidence-accountability.yml new file mode 100644 index 0000000..98cf515 --- /dev/null +++ b/.github/workflows/mutation-evidence-accountability.yml @@ -0,0 +1,40 @@ +name: Mutation Evidence Accountability + +on: + pull_request: + branches: ["main"] + paths: + - "schemas/MutationEvidence*.json" + - "examples/mutation-evidence*.json" + - "fixtures/anti-patterns/mutation-evidence*.json" + - "tools/validate_mutation_evidence_accountability.py" + - ".github/workflows/mutation-evidence-accountability.yml" + push: + branches: ["main", "sourceos-mutation-evidence-v0-1"] + paths: + - "schemas/MutationEvidence*.json" + - "examples/mutation-evidence*.json" + - "fixtures/anti-patterns/mutation-evidence*.json" + - "tools/validate_mutation_evidence_accountability.py" + - ".github/workflows/mutation-evidence-accountability.yml" + +permissions: + contents: read + +jobs: + validate-mutation-evidence: + name: Validate mutation/evidence receipts + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install validator dependencies + run: python -m pip install --upgrade pip jsonschema + + - name: Validate examples and anti-pattern fixtures + run: python tools/validate_mutation_evidence_accountability.py diff --git a/docs/adr/0012-sourceos-mutation-evidence-accountability.md b/docs/adr/0012-sourceos-mutation-evidence-accountability.md new file mode 100644 index 0000000..57a820f --- /dev/null +++ b/docs/adr/0012-sourceos-mutation-evidence-accountability.md @@ -0,0 +1,70 @@ +# ADR-0012: SourceOS Mutation and Evidence Accountability + +Status: Proposed +Date: 2026-05-05 + +## Context + +A multi-artifact forensic review of macOS resource, syslog, console, Wi-Fi, and memorystatus reports exposed a recurring operating-system design failure: modern systems may enforce policy and collect telemetry while still failing to explain causality to the human operator. + +The reviewed evidence showed repeated browser disk-write pressure, browser process-family memory pressure, sandbox/TCC denials, watchdog coverage gaps, launchd/XPC service churn, Spotlight/indexing worker churn, opaque log-routing, media/codec diagnostic chatter, and weak continuity across shutdown, boot, login, and logging-service restart boundaries. + +The security conclusion is intentionally conservative: the available logs do not prove compromise, but they are not sufficient to clear compromise either. Several sensors were degraded or semantically incomplete. SourceOS must treat opaque or degraded telemetry as an evidence-quality failure, not as negative evidence. + +## Decision + +SourceOS will adopt a receipt-native accountability model for mutation, evidence routing, service lifecycle, resource pressure, media work, temporary artifacts, policy denials, and compromise assessment. + +The canonical specification home is this repository, `SourceOS-Linux/sourceos-spec`, using the existing layout: + +- `docs/adr/` for architectural decisions. +- `docs/contract-additions/` for normative contracts. +- `schemas/` for JSON schemas. +- `examples/` for valid examples. +- `fixtures/` for invalid or degraded anti-pattern examples. +- `tools/` for schema validation and CI checks. + +Implementation should fan out only after this contract is stable: + +- `SourceOS-Linux/sourceos-boot`: boot evidence topology attestation and cross-reboot session IDs. +- `SourceOS-Linux/sourceos-devtools`: validators, CLI tooling, CI gates, and synthetic anti-pattern fixtures. +- `SourceOS-Linux/sourceos-shell`: operator timeline, evidence panel, and `sourceosctl explain` flows. +- `SourceOS-Linux/BearBrowser`: browser coalition/profile/storage/write receipts. +- `SourceOS-Linux/TurtleTerm`: terminal/session/UI-observation receipts. +- `SourceOS-Linux/sourceos-syncd`: sync-cycle receipts, full-sync risk receipts, and temporary artifact budgets. + +## Required event families + +- `sourceos.observability.event.v0.1` +- `sourceos.write_accountability.v0.1` +- `sourceos.routing_receipt.v0.1` +- `sourceos.media_work.v0.1` +- `sourceos.temporary_artifact.v0.1` +- `sourceos.coalition_resource.v0.1` +- `sourceos.policy_decision.v0.1` +- `sourceos.compromise_assessment.v0.1` + +## Consequences + +Every relevant subsystem must preserve enough evidence to answer: + +1. What happened? +2. Who or what caused it? +3. Which durable or temporary object changed? +4. Which policy allowed, denied, degraded, sampled, redacted, routed, or dropped the event? +5. What resource budget was consumed? +6. What evidence was missing or degraded? +7. What downstream work or risk was created? +8. What should the user or operator do next? + +## Non-goals + +This ADR does not attempt to prove compromise in the reviewed macOS artifacts. It uses them as design evidence. It also does not require raw sensitive paths or network identifiers in default logs. SourceOS should use privacy-preserving object IDs by default and support explicit forensic expansion only when authorized. + +## Acceptance criteria + +- A base observability event schema exists and validates examples. +- Mutation/write accountability, evidence routing, media work, temporary artifact lifecycle, coalition resource, policy decision, and compromise assessment schemas exist. +- Valid examples and invalid anti-pattern fixtures are provided. +- A local validator checks valid examples pass and anti-pattern fixtures fail. +- Documentation explicitly states that lack of positive evidence is not clearance when sensors are degraded or evidence quality is insufficient. diff --git a/docs/contract-additions/browser-write-accountability.md b/docs/contract-additions/browser-write-accountability.md new file mode 100644 index 0000000..51ae197 --- /dev/null +++ b/docs/contract-additions/browser-write-accountability.md @@ -0,0 +1,60 @@ +# SourceOS Browser Write Accountability v0.1 + +## Purpose + +This contract specializes SourceOS Mutation and Evidence Accountability for browser write-pressure incidents. + +The Firefox evidence from the macOS investigation showed repeated SQLite-backed profile/storage writes, but a later Add-ons Manager screenshot showed no visible installed extensions. That correction matters: extension-driven write amplification must be downgraded unless an extension inventory actually supports it. + +## Decision + +Browser write-pressure receipts must distinguish user-installed extension storage from browser-core profile state, per-origin storage, service-worker/cache state, session restore, browser sync, downloads/cache, diagnostics, and profile repair or migration. + +A browser write-pressure incident must not blame extensions merely because the browser wrote heavily. + +## Required actor classes + +- `browser_profile_core` +- `browser_history_places` +- `browser_favicons` +- `browser_cookies_permissions` +- `browser_session_restore` +- `browser_origin_storage` +- `browser_service_worker_cache` +- `browser_download_cache` +- `browser_sync_state` +- `browser_extension_storage` +- `browser_hidden_system_addon` +- `browser_diagnostics` +- `browser_profile_repair_or_migration` +- `unknown` + +## Extension inventory states + +- `none_visible`: no user-installed extensions visible in the ordinary extension manager view. +- `installed`: visible user-installed extensions exist. +- `hidden_possible`: system, policy, hidden, or native-messaging add-ons may exist but were not fully enumerated. +- `unavailable`: the extension inventory could not be collected. +- `not_collected`: no extension inventory evidence was captured. + +## Normative rule + +If `extension_inventory_state=none_visible`, then `browser_extension_storage` must not be used as the primary actor class unless additional hidden/system/policy add-on evidence is attached. + +## Evidence-quality rule + +A complete browser-write receipt requires at minimum: + +- Browser/version/profile identifier. +- Actor class. +- Extension inventory state. +- Operation class. +- Object class and path class. +- Byte count or write-pressure budget. +- Evidence-quality status and missing fields. + +A report with only process name, stack offsets, and syscall class is partial evidence. + +## Design consequence + +SourceOS/BearBrowser must expose browser storage accountability by profile component and origin. The user should see whether writes came from core profile state, site/origin storage, service workers, downloads/cache, sync state, extensions, diagnostics, or migration/repair logic. diff --git a/docs/contract-additions/mutation-evidence-accountability.md b/docs/contract-additions/mutation-evidence-accountability.md new file mode 100644 index 0000000..9060ba1 --- /dev/null +++ b/docs/contract-additions/mutation-evidence-accountability.md @@ -0,0 +1,97 @@ +# SourceOS Mutation and Evidence Accountability v0.1 + +## Purpose + +This contract defines the minimum evidence model for mutation, resource pressure, policy denials, evidence routing, media/codec work, temporary artifacts, service lifecycle, and compromise assessment in SourceOS. + +The design motivation is a repeated failure pattern observed in opaque macOS logs: low-level mechanisms recorded fragments of truth, but no layer provided a complete causal explanation. SourceOS must make mutation and evidence accountable by default. + +## Normative principles + +1. **Receipts over loose logs.** Events that mutate state, consume resource budgets, change evidence routing, or degrade security coverage must emit structured receipts. +2. **No degraded sensor may clear an incident.** Absence of evidence from a degraded sensor is not negative evidence. +3. **Report by actor, object, operation, policy, cost, and cause.** Process name alone is not attribution. +4. **Report by coalition and service class.** Browsers, office stacks, indexers, sync agents, and model services are multi-process families. +5. **Every denial needs a classification.** A denial can be expected, expected-but-degraded, misconfigured, suspicious, impossible, or unknown. +6. **Retry and activation storms must circuit-break.** Repeated failures must be summarized with counts, first/last timestamps, backoff state, and remediation. +7. **Logs are governed artifacts.** Routed, sampled, dropped, redacted, compressed, or diverted log streams must themselves have routing receipts. +8. **Temporary artifacts are typed.** Codec scratch files, previews, thumbnails, sync staging, caches, and owned media must not collapse into one path class. +9. **Cross-reboot continuity is mandatory.** Boot sessions, shutdowns, restarts, login sessions, and logging topology changes require stable correlation IDs. +10. **Evidence quality is explicit.** Each incident must expose what is known, unknown, degraded, redacted, or insufficient. + +## Required event families + +- `sourceos.observability.event.v0.1` +- `sourceos.write_accountability.v0.1` +- `sourceos.routing_receipt.v0.1` +- `sourceos.media_work.v0.1` +- `sourceos.temporary_artifact.v0.1` +- `sourceos.coalition_resource.v0.1` +- `sourceos.policy_decision.v0.1` +- `sourceos.compromise_assessment.v0.1` + +## Minimum event envelope + +Every receipt must include: + +- `schema` +- `event_id` +- `timestamp` +- `boot_id` +- `session_id` +- `actor` +- `operation` +- `policy` +- `evidence_quality` +- `causal_parents` + +## Required evidence-quality states + +- `complete` +- `partial` +- `degraded_sensor` +- `opaque_symbolication` +- `redacted` +- `insufficient_for_clearance` + +## Required compromise-assessment states + +- `compromise_proven` +- `compromise_suspected` +- `no_positive_compromise_evidence` +- `cannot_exclude_compromise` +- `sensor_degraded` +- `evidence_inadequate` + +## Required artifact classes + +- `owned_media` +- `durable_document` +- `derived_preview` +- `codec_temp` +- `attachment_cache` +- `thumbnail_cache` +- `sync_staging` +- `failed_transcode_residue` +- `message_database_reference` +- `browser_profile_db` +- `browser_cache` +- `semantic_index` +- `opaque_vendor_state` + +## Required operator queries + +A SourceOS operator view must be able to answer: + +- Explain writes. +- Explain logs and evidence routing. +- Explain memory and coalition pressure. +- Explain sync/reindex/full-sync risks. +- Explain policy denials. +- Explain degraded security coverage. +- Explain media/temp artifacts. +- Explain why compromise can or cannot be excluded. + +## Initial repo placement + +This contract belongs in `SourceOS-Linux/sourceos-spec` under `docs/contract-additions/mutation-evidence-accountability.md`. Implementation should be fanned out only after the contract and schema fixtures are validated. diff --git a/docs/implementation/mutation-evidence-stack-integration-map.md b/docs/implementation/mutation-evidence-stack-integration-map.md new file mode 100644 index 0000000..60c1008 --- /dev/null +++ b/docs/implementation/mutation-evidence-stack-integration-map.md @@ -0,0 +1,193 @@ +# Mutation and Evidence Accountability: Stack Integration Map + +Status: Proposed +Date: 2026-05-05 + +## Purpose + +This implementation map turns the forensic lessons behind SourceOS Mutation and Evidence Accountability into concrete improvement targets for BearBrowser, TurtleTerm, SourceOS Shell, sourceos-syncd, sourceos-devtools, sourceos-boot, Exodus, FogStack, and the ontology/evidence planes. + +The critical lesson is that SourceOS must learn from opaque macOS logs by refusing process-only attribution. The stack must explain delegated mutation: human intent, UI action, app or agent actor, service delegation, database/file/temp/object mutation, evidence routing, policy decision, and evidence quality. + +## Cross-stack doctrine + +1. Every mutation requires a receipt. +2. Every delegated mutation requires an `on_behalf_of` chain. +3. Every resource-pressure incident requires actor, object, operation, policy, resource cost, causal parent, and evidence-quality fields. +4. Every degraded or blind sensor prevents security clearance. +5. Every browser write-pressure incident must distinguish core profile, origin storage, service-worker/cache, session restore, sync, extension storage, diagnostics, and profile repair. +6. Every terminal/session action that spawns, writes, downloads, extracts, or invokes agents must carry session and intent context. +7. Every sync, archive extraction, preview, index, media/codec, and diagnostic path must distinguish durable user data from temp/cache/derived state. +8. Every evidence route must say where events went, what was redacted, what was sampled, what was dropped, and what requires privilege. + +## BearBrowser improvements + +BearBrowser must become the reference implementation for browser write accountability. + +Required work: + +- Emit `BrowserWriteAccountabilityReceipt` for core profile writes, per-origin storage, service-worker/cache writes, downloads/cache writes, session restore, sync state, diagnostics, extension storage, hidden/system add-ons, and profile repair/migration. +- Add `extension_inventory_state` with values `none_visible`, `installed`, `hidden_possible`, `unavailable`, and `not_collected`. +- Prevent unsupported attribution: if no visible extensions exist, extension storage must not be the primary cause without additional evidence. +- Emit per-origin and per-storage-class counters where available. +- Capture SQLite/WAL/checkpoint counts for browser profile databases. +- Surface a user-readable `Explain browser writes` panel. +- Add fixtures mirroring Firefox-style opaque reports: SQLite profile churn, raw write bursts, service-worker cache churn, session-restore snapshots, and no-extension anti-patterns. + +Acceptance criteria: + +- A write-pressure example can identify whether the write source is profile core, origin storage, service worker, sync, extension, cache, or unknown. +- A no-visible-extension case cannot be auto-labeled as extension-caused. +- Missing database path/origin/extension fields reduce evidence quality. + +## TurtleTerm improvements + +TurtleTerm must become the reference implementation for terminal/session mutation accountability. + +Required work: + +- Emit `TerminalSessionReceipt` for interactive shells, command execution, downloads, archive extraction, file writes, chmod/chown/xattr changes, and agent invocations. +- Attach terminal session ID, working directory, command digest/redacted command class, process tree, user intent class, and output artifact class. +- Emit `ArchiveExtractionReceipt` when tools such as tar, unzip, bsdtar, ditto-equivalent, npm/pnpm extraction, package managers, or build tools expand archives. +- Emit `DiagnosticSelfNoiseReceipt` when terminal-driven diagnostics collect traces, symbolicate, export logs, or write evidence bundles. +- Mark commands that cross path boundaries: Downloads, Desktop, Documents, project workspace, app container, sync root, cache, temp, external volume, network mount, package/bundle. + +Acceptance criteria: + +- A terminal-driven extraction can explain source archive, output path class, file count, byte count, permission/xattr changes, cleanup policy, and downstream indexing/sync triggers. +- A diagnostic command can account for its own evidence writes. +- Agent-triggered terminal work carries human intent and agent identity. + +## SourceOS Shell improvements + +SourceOS Shell must provide the operator-facing explanation plane. + +Required work: + +- Add `sourceosctl explain writes`. +- Add `sourceosctl explain sync`. +- Add `sourceosctl explain browser`. +- Add `sourceosctl explain terminal`. +- Add `sourceosctl explain logs`. +- Add `sourceosctl explain compromise`. +- Add an Evidence Topology panel showing event sources, routing, sinks, privilege requirements, redaction, sampling, drops, and retention. +- Add a Mutation Graph view with actor, delegated actor, service, object, policy, resource, and evidence-quality nodes. + +Acceptance criteria: + +- An operator can distinguish no positive compromise evidence from evidence sufficient to clear compromise. +- A resource incident shows causal parents, delegated actors, evidence gaps, and next evidence required. + +## sourceos-syncd improvements + +sourceos-syncd must become the reference implementation for delegated sync accountability. + +Required work: + +- Emit `SyncCycleReceipt`, `DelegatedIOReceipt`, `FullSyncRiskReceipt`, `CloudObjectTransferReceipt`, and `SchedulerReceipt`. +- Track sync roots, object IDs, batch sizes, direction, retry/backoff, pacer state, full-sync/checkpoint events, and local DB updates. +- Distinguish local sync metadata writes from durable file content writes and cloud/fog object transfers. +- Capture logical bytes, physical bytes, clone/reflink bytes, and dirty-memory accounting bytes separately. + +Acceptance criteria: + +- A cloud/fog sync incident can explain the origin actor, requesting actor, executing service, object namespace, DB writes, object transfers, and cleanup/staging transitions. +- A full-sync or reindex risk is emitted before the expensive work starts when possible. + +## sourceos-devtools improvements + +sourceos-devtools must own the validation and fixture harness. + +Required work: + +- Wire `tools/validate_mutation_evidence_accountability.py` into CI. +- Add schema fixtures for valid and invalid cases. +- Add anti-pattern checks for process-only attribution, false clearance with blind sensors, raw write burst without object classification, cloud sync without causal parent, folder traversal without path boundary, and diagnostic writes without observer-effect accounting. +- Provide fixture generators for synthetic WAL churn, sync bursts, archive extraction, cache maintenance, and evidence routing gaps. + +Acceptance criteria: + +- Valid examples pass. +- Invalid anti-patterns fail. +- CI blocks schema regressions and unsupported security conclusions. + +## sourceos-boot improvements + +sourceos-boot must anchor cross-reboot evidence continuity. + +Required work: + +- Emit `BootEvidenceTopologyAttestation` at boot. +- Generate stable `boot_id` and `session_id` values. +- Attest logging/evidence sinks, enabled sensors, disabled sensors, privilege state, redaction profiles, and retention policies. +- Record immutable OS deployment identity, kernel build, image digest, symbolication bundle state, and measured boot references when available. + +Acceptance criteria: + +- Every event can be attached to a boot/session context. +- Missing or degraded boot evidence prevents high-confidence security clearance. + +## Exodus improvements + +Exodus must treat migration as accountable mutation, not bulk copying. + +Required work: + +- Classify durable user media separately from derived previews, codec temps, thumbnails, attachment caches, sync staging, failed transcode residue, and opaque vendor state. +- Emit archive extraction, media/codec work, temporary artifact lifecycle, and ownership/attestation receipts. +- Quarantine or exclude vendor cache/temp artifacts unless explicitly requested. + +Acceptance criteria: + +- Migration manifests explain imported, excluded, quarantined, reconstructed, and ignored artifacts. +- Owned media import does not accidentally preserve codec scratch or vendor cache as durable content. + +## FogStack and Prophet Platform improvements + +FogStack and Prophet Platform must absorb these receipts into governance and evidence-console surfaces. + +Required work: + +- Extend service manifests with write budgets, log budgets, sync budgets, temp artifact budgets, and evidence routing declarations. +- Map SourceOS receipts into the platform evidence console. +- Add policy gates that reject services with unbounded logging, opaque temp artifacts, or degraded sensor-clearance claims. + +Acceptance criteria: + +- A service manifest declares mutation and evidence behavior before deployment. +- Evidence console can show resource pressure, delegated mutation, evidence gaps, and policy decisions across local, fog, and cloud deployments. + +## Ontogenesis improvements + +Ontogenesis should host the semantic model and SHACL gates. + +Required work: + +- Model actors, delegated actors, services, objects, artifacts, policies, evidence quality, resource budgets, and compromise assessment states. +- Add SHACL gates that reject invalid clearance when sensors are degraded. +- Add ontology classes for mutation receipt, evidence pipeline receipt, execution context receipt, and service work receipt. + +Acceptance criteria: + +- Receipts can be validated semantically, not only syntactically. +- Evidence-quality limits are machine-checkable. + +## Immediate issue package + +Create implementation issues in these repos after the spec PR opens: + +1. `SourceOS-Linux/BearBrowser`: Implement browser write accountability and no-extension attribution guardrails. +2. `SourceOS-Linux/TurtleTerm`: Implement terminal/session mutation receipts and archive extraction receipts. +3. `SourceOS-Linux/sourceos-shell`: Implement `sourceosctl explain` and Evidence Topology panel. +4. `SourceOS-Linux/sourceos-syncd`: Implement delegated sync and full-sync-risk receipts. +5. `SourceOS-Linux/sourceos-devtools`: Wire schema validator and anti-pattern fixtures into CI. +6. `SourceOS-Linux/sourceos-boot`: Emit boot evidence topology attestation. +7. `SocioProphet/prophet-platform`: Integrate receipts into evidence console and FogStack manifests. +8. `ontogenesis`: Add ontology classes and SHACL gates. +9. `Exodus`: Add migration artifact taxonomy and temp/cache exclusion logic. + +## Completion definition + +This workstream is not complete until the specification, schemas, examples, validators, repo issues, and at least one implementation prototype exist. The first complete prototype should prove the path from observed mutation to operator explanation: + +`event source -> receipt -> schema validation -> evidence graph -> sourceosctl explain -> UI/evidence console`. diff --git a/examples/mutation-evidence-accountability.examples.json b/examples/mutation-evidence-accountability.examples.json new file mode 100644 index 0000000..9d685d1 --- /dev/null +++ b/examples/mutation-evidence-accountability.examples.json @@ -0,0 +1,143 @@ +{ + "examples": [ + { + "schema": "sourceos.write_accountability.v0.1", + "event_id": "evt-write-firefox-sqlite-0001", + "timestamp": "2026-05-05T19:00:00Z", + "boot_id": "boot-redacted-0001", + "session_id": "session-redacted-0001", + "actor": { + "kind": "browser_content_process", + "id": "pid:12814", + "name": "plugin-container", + "digest": null, + "coalition_id": "firefox-coalition-redacted", + "trust_zone": "user.desktop.browser" + }, + "object": { + "class": "browser_profile_db", + "id": "profile_db:firefox.default.sqlite.redacted", + "path_class": "app_state", + "redaction": "stable_pseudonym" + }, + "operation": "sqlite_wal_write", + "bytes": { + "written": 8589950000, + "dirtied": 8589950000, + "read": 0, + "write_amplification_ratio": null + }, + "db": { + "engine": "sqlite", + "wal_bytes": null, + "checkpoint_count": null, + "fsync_count": null + }, + "policy": { + "decision": "allow", + "policy_id": "browser.profile.daily.write.budget", + "classification": "allowed_but_budget_exceeded" + }, + "evidence_quality": { + "status": "partial", + "missing_fields": ["exact_database_path", "tab_or_extension_actor", "service_worker_origin"], + "confidence": "medium" + }, + "causal_parents": ["evt-network-resume-unknown", "evt-browser-active-session"] + }, + { + "schema": "sourceos.routing_receipt.v0.1", + "event_id": "evt-routing-asl-claimed-0001", + "timestamp": "2026-05-05T10:32:55Z", + "source": "message_tracer", + "routing_policy": { + "claimed_by": "sourceos.log.module.message_tracer", + "visible_in_standard_log": false, + "visible_in_evidence_store": true, + "requires_privilege": false, + "redaction_profile": "local_default", + "retention_days": 30 + }, + "counts": { + "received": 100, + "written": 100, + "sampled": 0, + "dropped": 0, + "redacted": 12, + "diverted": 100 + }, + "destinations": ["evidence_store", "operator_topology_panel"], + "evidence_quality": { + "status": "partial", + "missing_fields": [], + "confidence": "high" + }, + "explanation": "Messages claimed by this module are routed to the evidence store and may be omitted from the standard log view." + }, + { + "schema": "sourceos.media_work.v0.1", + "event_id": "evt-media-hevc-0001", + "timestamp": "2026-05-05T09:13:57Z", + "component": "messages.media.hevc", + "operation": "tile_encode", + "trigger": "unknown", + "input": { + "artifact_class": "attachment", + "content_hash": null, + "bytes": null + }, + "output": { + "artifact_class": "codec_temp", + "path_class": "tmp", + "bytes": null, + "cleanup_policy": "unknown" + }, + "codec": { + "family": "HEVC", + "mode": "tile_session", + "input_count": 1, + "processed_count": 1, + "drop_count": 0 + }, + "policy": { + "decision": "allow", + "policy_id": "media.temp.default", + "classification": "opaque_but_allowed" + }, + "evidence_quality": { + "status": "partial", + "missing_fields": ["user_object", "attachment_id", "output_bytes", "cleanup_status"], + "confidence": "medium" + } + }, + { + "schema": "sourceos.compromise_assessment.v0.1", + "assessment_id": "assess-macos-artifacts-0001", + "timestamp": "2026-05-05T20:00:00Z", + "scope": { + "system": "macos-artifact-review", + "time_window": "2026-04-28/2026-05-05" + }, + "status": ["no_positive_compromise_evidence", "cannot_exclude_compromise", "sensor_degraded", "evidence_inadequate"], + "sensor_state": [ + { + "sensor": "BlockBlock-equivalent", + "state": "degraded", + "reason": "missing full disk access / privileged coverage" + }, + { + "sensor": "browser_profile_attribution", + "state": "missing", + "reason": "no per-tab or per-extension write attribution" + } + ], + "evidence_quality": { + "status": "insufficient_for_clearance", + "missing_fields": ["network_flow_ledger", "browser_extension_audit", "tcc_history", "file_mutation_ledger"], + "confidence": "high" + }, + "conclusion": "Available artifacts do not prove compromise and cannot clear compromise due to degraded sensors and insufficient causal evidence.", + "required_next_evidence": ["endpoint security event stream", "per-file write ledger", "browser extension inventory", "network flow ledger", "TCC/permission change history"] + } + ] +} diff --git a/examples/mutation-evidence-umbrella.examples.json b/examples/mutation-evidence-umbrella.examples.json new file mode 100644 index 0000000..a6122e9 --- /dev/null +++ b/examples/mutation-evidence-umbrella.examples.json @@ -0,0 +1,308 @@ +{ + "examples": [ + { + "schema": "sourceos.mutation_evidence.umbrella.v0.1", + "primitive": "MutationReceipt", + "subtype": "browser_write", + "event_id": "evt-browser-write-no-visible-ext-0001", + "timestamp": "2026-05-05T22:30:00Z", + "boot_id": "boot-redacted-0001", + "session_id": "session-redacted-0001", + "actor_chain": [ + { + "role": "origin_actor", + "kind": "user", + "id": "user:local:501", + "name": "local-user", + "digest": null, + "coalition_id": null, + "persona_id": "persona:interactive-desktop", + "trust_zone": "user.desktop", + "attribution_status": "partial" + }, + { + "role": "execution_actor", + "kind": "browser", + "id": "process:firefox", + "name": "Firefox", + "digest": null, + "coalition_id": "coalition:firefox-redacted", + "persona_id": "persona:interactive-desktop", + "trust_zone": "user.desktop.browser", + "attribution_status": "partial" + } + ], + "object": { + "class": "browser_profile_core", + "id": "browser-profile:default:sqlite-redacted", + "path_class": "browser_profile", + "redaction": "stable_pseudonym" + }, + "operation": "sqlite_wal_write", + "resource_cost": { + "logical_bytes": null, + "physical_bytes": null, + "dirty_memory_bytes": 2147480000, + "clone_or_reflink_bytes": null, + "cpu_ms": null, + "network_bytes_in": null, + "network_bytes_out": null, + "fsync_count": null, + "wal_bytes": null, + "checkpoint_count": null, + "entries_visited": null, + "objects_processed": null + }, + "policy": { + "decision": "allow", + "policy_id": "browser.profile.write.policy", + "budget_id": "browser.profile.daily.write.budget", + "classification": "allowed_but_budget_exceeded" + }, + "evidence_quality": { + "status": "partial", + "confidence": "medium", + "missing_fields": ["exact_database_path", "origin_storage_actor", "service_worker_state", "sync_state"], + "sensor_state": "healthy" + }, + "browser": { + "browser_actor_class": "browser_profile_core", + "extension_inventory_state": "none_visible", + "extension_evidence_quality": "partial", + "primary_extension_cause_allowed": false + }, + "causal_parents": ["evt-user-active-browser-session"], + "downstream_effects": ["write_pressure_threshold_crossed"] + }, + { + "schema": "sourceos.mutation_evidence.umbrella.v0.1", + "primitive": "MutationReceipt", + "subtype": "delegated_io", + "event_id": "evt-delegated-sync-fileprovider-bird-cloudd-0001", + "timestamp": "2026-05-05T23:00:00Z", + "boot_id": "boot-redacted-0002", + "session_id": "session-redacted-0002", + "actor_chain": [ + { + "role": "origin_actor", + "kind": "file_manager", + "id": "process:finder-equivalent", + "name": "file-manager", + "digest": null, + "coalition_id": "coalition:file-manager", + "persona_id": "persona:interactive-desktop", + "trust_zone": "user.desktop.file-manager", + "attribution_status": "partial" + }, + { + "role": "requesting_actor", + "kind": "service", + "id": "service:file-provider", + "name": "file-provider-service", + "digest": null, + "coalition_id": "coalition:file-provider", + "persona_id": "persona:interactive-desktop", + "trust_zone": "user.desktop.sync", + "attribution_status": "partial" + }, + { + "role": "execution_actor", + "kind": "sync_daemon", + "id": "service:sourceos-syncd", + "name": "sourceos-syncd", + "digest": null, + "coalition_id": "coalition:syncd", + "persona_id": "persona:interactive-desktop", + "trust_zone": "user.desktop.sync", + "attribution_status": "complete" + }, + { + "role": "storage_actor", + "kind": "cloud_service", + "id": "service:fog-object-store", + "name": "fog-object-store", + "digest": null, + "coalition_id": "coalition:fog-sync", + "persona_id": "persona:interactive-desktop", + "trust_zone": "fog.sync.object-store", + "attribution_status": "complete" + } + ], + "object": { + "class": "sync_metadata_db", + "id": "sync-db:workspace-redacted", + "path_class": "sync_root", + "redaction": "stable_pseudonym" + }, + "operation": "sync_metadata_commit", + "resource_cost": { + "logical_bytes": 104857600, + "physical_bytes": 83886080, + "dirty_memory_bytes": 2147480000, + "clone_or_reflink_bytes": 0, + "cpu_ms": 1785000, + "network_bytes_in": 52428800, + "network_bytes_out": 73400320, + "fsync_count": 42, + "wal_bytes": 125829120, + "checkpoint_count": 7, + "entries_visited": 1200, + "objects_processed": 342 + }, + "policy": { + "decision": "allow", + "policy_id": "sync.delegated_io.policy", + "budget_id": "sync.workspace.default", + "classification": "delegated_mutation_with_budget_pressure" + }, + "evidence_quality": { + "status": "partial", + "confidence": "medium", + "missing_fields": ["raw_cloud_provider_record_id", "full_local_path"], + "sensor_state": "healthy" + }, + "causal_parents": ["evt-folder-sizing-0001", "evt-fileprovider-fsevent-batch-0001"], + "downstream_effects": ["sync_cycle", "cloud_object_transfer", "wal_checkpoint"] + }, + { + "schema": "sourceos.mutation_evidence.umbrella.v0.1", + "primitive": "MutationReceipt", + "subtype": "archive_extraction", + "event_id": "evt-terminal-archive-extract-0001", + "timestamp": "2026-05-05T23:10:00Z", + "boot_id": "boot-redacted-0002", + "session_id": "terminal-session-redacted-0001", + "actor_chain": [ + { + "role": "origin_actor", + "kind": "terminal", + "id": "terminal:turtleterm-session-redacted", + "name": "TurtleTerm", + "digest": null, + "coalition_id": "coalition:turtleterm", + "persona_id": "persona:interactive-developer", + "trust_zone": "user.terminal", + "attribution_status": "complete" + }, + { + "role": "execution_actor", + "kind": "process", + "id": "process:archive-tool-redacted", + "name": "archive-extractor", + "digest": null, + "coalition_id": "coalition:archive-extract", + "persona_id": "persona:interactive-developer", + "trust_zone": "user.terminal.tool", + "attribution_status": "complete" + } + ], + "object": { + "class": "archive", + "id": "archive:sha256-redacted", + "path_class": "downloads", + "redaction": "stable_pseudonym" + }, + "operation": "extract_archive", + "resource_cost": { + "logical_bytes": 734003200, + "physical_bytes": 681574400, + "dirty_memory_bytes": 681574400, + "clone_or_reflink_bytes": 0, + "cpu_ms": 8500, + "network_bytes_in": null, + "network_bytes_out": null, + "fsync_count": 12, + "wal_bytes": null, + "checkpoint_count": null, + "entries_visited": 1480, + "objects_processed": 1480 + }, + "policy": { + "decision": "allow", + "policy_id": "archive.extract.default", + "budget_id": "workspace.import.default", + "classification": "user_requested_import" + }, + "evidence_quality": { + "status": "complete", + "confidence": "high", + "missing_fields": [], + "sensor_state": "healthy" + }, + "archive": { + "permissions_changed": true, + "xattrs_changed": true, + "quarantine_state": "preserved", + "cleanup_policy": "delete_temp_on_success" + }, + "causal_parents": ["evt-terminal-command-redacted"], + "downstream_effects": ["folder_sizing_candidate", "sync_candidate", "index_candidate"] + }, + { + "schema": "sourceos.mutation_evidence.umbrella.v0.1", + "primitive": "EvidencePipelineReceipt", + "subtype": "diagnostic_self_noise", + "event_id": "evt-diagnostic-self-noise-0001", + "timestamp": "2026-05-05T23:20:00Z", + "boot_id": "boot-redacted-0002", + "session_id": "session-redacted-0002", + "actor_chain": [ + { + "role": "observer", + "kind": "diagnostic_service", + "id": "service:symbolication", + "name": "symbolication-service", + "digest": null, + "coalition_id": "coalition:diagnostics", + "persona_id": null, + "trust_zone": "system.diagnostics", + "attribution_status": "partial" + } + ], + "object": { + "class": "diagnostic_archive", + "id": "diagnostic-archive:mmap-redacted", + "path_class": "diagnostic_archive", + "redaction": "stable_pseudonym" + }, + "operation": "write_symbolication_archive", + "resource_cost": { + "logical_bytes": 67108864, + "physical_bytes": 67108864, + "dirty_memory_bytes": 67108864, + "clone_or_reflink_bytes": 0, + "cpu_ms": 4200, + "network_bytes_in": null, + "network_bytes_out": null, + "fsync_count": null, + "wal_bytes": null, + "checkpoint_count": null, + "entries_visited": null, + "objects_processed": 8 + }, + "policy": { + "decision": "allow", + "policy_id": "diagnostics.observer_effect.policy", + "budget_id": "diagnostics.default.write.budget", + "classification": "observer_effect_accounted" + }, + "evidence_quality": { + "status": "partial", + "confidence": "medium", + "missing_fields": ["observed_process_identity_for_all_samples"], + "sensor_state": "healthy" + }, + "evidence_pipeline": { + "received": 8, + "written": 8, + "sampled": 0, + "dropped": 0, + "redacted": 3, + "diverted": 0, + "clearance_allowed": false + }, + "causal_parents": ["evt-resource-pressure-report-0001"], + "downstream_effects": ["diagnostic_write_pressure_contribution"] + } + ] +} diff --git a/fixtures/anti-patterns/mutation-evidence-accountability.invalid.json b/fixtures/anti-patterns/mutation-evidence-accountability.invalid.json new file mode 100644 index 0000000..56932c3 --- /dev/null +++ b/fixtures/anti-patterns/mutation-evidence-accountability.invalid.json @@ -0,0 +1,48 @@ +{ + "anti_patterns": [ + { + "name": "routing receipt without counts", + "event": { + "schema": "sourceos.routing_receipt.v0.1", + "event_id": "bad-routing-0001", + "timestamp": "2026-05-05T10:32:55Z", + "source": "performance", + "routing_policy": { + "claimed_by": "module", + "visible_in_standard_log": false, + "visible_in_evidence_store": true, + "retention_days": 30 + }, + "destinations": ["evidence_store"], + "evidence_quality": { + "status": "partial" + } + } + }, + { + "name": "false clearance despite blind sensor", + "event": { + "schema": "sourceos.compromise_assessment.v0.1", + "assessment_id": "bad-assess-0001", + "timestamp": "2026-05-05T20:00:00Z", + "scope": { + "system": "test", + "time_window": "now" + }, + "status": ["no_positive_compromise_evidence"], + "sensor_state": [ + { + "sensor": "filesystem_monitor", + "state": "blind", + "reason": "permission denied" + } + ], + "evidence_quality": { + "status": "complete", + "confidence": "high" + }, + "conclusion": "System is clear." + } + } + ] +} diff --git a/fixtures/anti-patterns/mutation-evidence-umbrella.invalid.json b/fixtures/anti-patterns/mutation-evidence-umbrella.invalid.json new file mode 100644 index 0000000..85f84ee --- /dev/null +++ b/fixtures/anti-patterns/mutation-evidence-umbrella.invalid.json @@ -0,0 +1,224 @@ +{ + "anti_patterns": [ + { + "name": "browser write falsely attributed to extensions when no visible extensions exist", + "event": { + "schema": "sourceos.mutation_evidence.umbrella.v0.1", + "primitive": "MutationReceipt", + "subtype": "browser_write", + "event_id": "bad-browser-extension-attribution-0001", + "timestamp": "2026-05-05T22:30:00Z", + "boot_id": "boot-redacted-0001", + "session_id": "session-redacted-0001", + "actor_chain": [ + { + "role": "execution_actor", + "kind": "browser", + "id": "process:firefox", + "name": "Firefox", + "digest": null, + "coalition_id": "coalition:firefox-redacted", + "persona_id": "persona:interactive-desktop", + "trust_zone": "user.desktop.browser", + "attribution_status": "partial" + } + ], + "object": { + "class": "browser_extension_storage", + "id": "browser-extension:unknown", + "path_class": "browser_profile", + "redaction": "stable_pseudonym" + }, + "operation": "sqlite_wal_write", + "resource_cost": { + "dirty_memory_bytes": 2147480000 + }, + "policy": { + "decision": "allow", + "policy_id": "browser.profile.write.policy", + "budget_id": "browser.profile.daily.write.budget", + "classification": "extension_storage_primary" + }, + "evidence_quality": { + "status": "partial", + "confidence": "medium", + "missing_fields": ["hidden_addon_inventory", "extension_id"], + "sensor_state": "healthy" + }, + "browser": { + "browser_actor_class": "browser_extension_storage", + "extension_inventory_state": "none_visible", + "extension_evidence_quality": "partial", + "primary_extension_cause_allowed": true + }, + "causal_parents": [], + "downstream_effects": ["write_pressure_threshold_crossed"] + } + }, + { + "name": "delegated mutation without actor chain", + "event": { + "schema": "sourceos.mutation_evidence.umbrella.v0.1", + "primitive": "MutationReceipt", + "subtype": "delegated_io", + "event_id": "bad-delegated-no-chain-0001", + "timestamp": "2026-05-05T23:00:00Z", + "boot_id": "boot-redacted-0002", + "session_id": "session-redacted-0002", + "actor_chain": [ + { + "role": "execution_actor", + "kind": "sync_daemon", + "id": "service:sourceos-syncd", + "name": "sourceos-syncd", + "digest": null, + "coalition_id": "coalition:syncd", + "persona_id": "persona:interactive-desktop", + "trust_zone": "user.desktop.sync", + "attribution_status": "partial" + } + ], + "object": { + "class": "sync_metadata_db", + "id": "sync-db:workspace-redacted", + "path_class": "sync_root", + "redaction": "stable_pseudonym" + }, + "operation": "sync_metadata_commit", + "resource_cost": { + "dirty_memory_bytes": 2147480000 + }, + "policy": { + "decision": "allow", + "policy_id": "sync.delegated_io.policy", + "budget_id": "sync.workspace.default", + "classification": "delegated_mutation" + }, + "evidence_quality": { + "status": "complete", + "confidence": "high", + "missing_fields": [], + "sensor_state": "healthy" + }, + "causal_parents": [], + "downstream_effects": ["sync_cycle"] + } + }, + { + "name": "diagnostic self-noise allows clearance despite redaction and observer writes", + "event": { + "schema": "sourceos.mutation_evidence.umbrella.v0.1", + "primitive": "EvidencePipelineReceipt", + "subtype": "diagnostic_self_noise", + "event_id": "bad-diagnostic-clearance-0001", + "timestamp": "2026-05-05T23:20:00Z", + "boot_id": "boot-redacted-0002", + "session_id": "session-redacted-0002", + "actor_chain": [ + { + "role": "observer", + "kind": "diagnostic_service", + "id": "service:symbolication", + "name": "symbolication-service", + "digest": null, + "coalition_id": "coalition:diagnostics", + "persona_id": null, + "trust_zone": "system.diagnostics", + "attribution_status": "partial" + } + ], + "object": { + "class": "diagnostic_archive", + "id": "diagnostic-archive:mmap-redacted", + "path_class": "diagnostic_archive", + "redaction": "stable_pseudonym" + }, + "operation": "write_symbolication_archive", + "resource_cost": { + "logical_bytes": 67108864, + "physical_bytes": 67108864, + "dirty_memory_bytes": 67108864 + }, + "policy": { + "decision": "allow", + "policy_id": "diagnostics.observer_effect.policy", + "budget_id": "diagnostics.default.write.budget", + "classification": "observer_effect_unaccounted" + }, + "evidence_quality": { + "status": "partial", + "confidence": "medium", + "missing_fields": ["observed_process_identity_for_all_samples"], + "sensor_state": "healthy" + }, + "evidence_pipeline": { + "received": 8, + "written": 8, + "sampled": 0, + "dropped": 0, + "redacted": 3, + "diverted": 0, + "clearance_allowed": true + }, + "causal_parents": ["evt-resource-pressure-report-0001"], + "downstream_effects": ["diagnostic_write_pressure_contribution"] + } + }, + { + "name": "archive extraction without path boundary and cleanup accounting", + "event": { + "schema": "sourceos.mutation_evidence.umbrella.v0.1", + "primitive": "MutationReceipt", + "subtype": "archive_extraction", + "event_id": "bad-archive-missing-boundary-0001", + "timestamp": "2026-05-05T23:10:00Z", + "boot_id": "boot-redacted-0002", + "session_id": "terminal-session-redacted-0001", + "actor_chain": [ + { + "role": "execution_actor", + "kind": "process", + "id": "process:archive-tool-redacted", + "name": "archive-extractor", + "digest": null, + "coalition_id": "coalition:archive-extract", + "persona_id": "persona:interactive-developer", + "trust_zone": "user.terminal.tool", + "attribution_status": "partial" + } + ], + "object": { + "class": "archive", + "id": "archive:sha256-redacted", + "path_class": "unknown", + "redaction": "stable_pseudonym" + }, + "operation": "extract_archive", + "resource_cost": { + "logical_bytes": 734003200, + "physical_bytes": 681574400, + "dirty_memory_bytes": 681574400 + }, + "policy": { + "decision": "allow", + "policy_id": "archive.extract.default", + "budget_id": "workspace.import.default", + "classification": "user_requested_import" + }, + "evidence_quality": { + "status": "complete", + "confidence": "high", + "missing_fields": [], + "sensor_state": "healthy" + }, + "archive": { + "permissions_changed": null, + "xattrs_changed": null, + "quarantine_state": "unknown" + }, + "causal_parents": [], + "downstream_effects": [] + } + } + ] +} diff --git a/schemas/MutationEvidenceAccountability.schema.json b/schemas/MutationEvidenceAccountability.schema.json new file mode 100644 index 0000000..f55a502 --- /dev/null +++ b/schemas/MutationEvidenceAccountability.schema.json @@ -0,0 +1,234 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://sourceos.dev/schemas/MutationEvidenceAccountability.schema.json", + "title": "SourceOS Mutation and Evidence Accountability v0.1", + "type": "object", + "oneOf": [ + { "$ref": "#/$defs/observability_event" }, + { "$ref": "#/$defs/write_accountability" }, + { "$ref": "#/$defs/routing_receipt" }, + { "$ref": "#/$defs/media_work" }, + { "$ref": "#/$defs/temporary_artifact" }, + { "$ref": "#/$defs/coalition_resource" }, + { "$ref": "#/$defs/policy_decision" }, + { "$ref": "#/$defs/compromise_assessment" } + ], + "$defs": { + "actor": { + "type": "object", + "required": ["kind", "id", "name"], + "properties": { + "kind": { "enum": ["process", "service", "agent", "kernel", "user", "container", "browser_content_process", "indexer", "sync_agent"] }, + "id": { "type": "string" }, + "name": { "type": "string" }, + "digest": { "type": ["string", "null"] }, + "coalition_id": { "type": ["string", "null"] }, + "trust_zone": { "type": ["string", "null"] } + }, + "additionalProperties": true + }, + "object_ref": { + "type": "object", + "properties": { + "class": { "type": "string" }, + "id": { "type": "string" }, + "path_class": { "type": ["string", "null"] }, + "redaction": { "enum": ["none", "stable_pseudonym", "path_class_only", "forensic_expansion_required"] } + }, + "additionalProperties": true + }, + "policy": { + "type": "object", + "required": ["decision"], + "properties": { + "decision": { "enum": ["allow", "deny", "degrade", "sample", "drop", "route", "unknown"] }, + "policy_id": { "type": ["string", "null"] }, + "classification": { "type": ["string", "null"] } + }, + "additionalProperties": true + }, + "evidence_quality": { + "type": "object", + "required": ["status"], + "properties": { + "status": { "enum": ["complete", "partial", "degraded_sensor", "opaque_symbolication", "redacted", "insufficient_for_clearance"] }, + "missing_fields": { "type": "array", "items": { "type": "string" } }, + "confidence": { "enum": ["high", "medium", "low", "unknown"] } + }, + "additionalProperties": true + }, + "base_event_fields": { + "type": "object", + "required": ["schema", "event_id", "timestamp", "actor", "operation", "policy", "evidence_quality"], + "properties": { + "schema": { "type": "string" }, + "event_id": { "type": "string", "minLength": 8 }, + "timestamp": { "type": "string", "format": "date-time" }, + "boot_id": { "type": "string" }, + "session_id": { "type": "string" }, + "actor": { "$ref": "#/$defs/actor" }, + "operation": { "type": "string" }, + "object": { "$ref": "#/$defs/object_ref" }, + "policy": { "$ref": "#/$defs/policy" }, + "evidence_quality": { "$ref": "#/$defs/evidence_quality" }, + "causal_parents": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": true + }, + "observability_event": { + "allOf": [ + { "$ref": "#/$defs/base_event_fields" }, + { "properties": { "schema": { "const": "sourceos.observability.event.v0.1" } } } + ] + }, + "write_accountability": { + "allOf": [ + { "$ref": "#/$defs/base_event_fields" }, + { + "required": ["bytes"], + "properties": { + "schema": { "const": "sourceos.write_accountability.v0.1" }, + "operation": { "enum": ["write", "append", "sqlite_commit", "sqlite_wal_write", "sqlite_checkpoint", "atomic_rename", "fsync", "cache_write", "index_write", "unknown"] }, + "bytes": { + "type": "object", + "required": ["written"], + "properties": { + "written": { "type": "integer", "minimum": 0 }, + "dirtied": { "type": "integer", "minimum": 0 }, + "read": { "type": "integer", "minimum": 0 }, + "write_amplification_ratio": { "type": ["number", "null"], "minimum": 0 } + } + }, + "db": { + "type": "object", + "properties": { + "engine": { "enum": ["sqlite", "rocksdb", "postgres", "lmdb", "lancedb", "qdrant", "unknown"] }, + "wal_bytes": { "type": ["integer", "null"], "minimum": 0 }, + "checkpoint_count": { "type": ["integer", "null"], "minimum": 0 }, + "fsync_count": { "type": ["integer", "null"], "minimum": 0 } + }, + "additionalProperties": true + } + } + } + ] + }, + "routing_receipt": { + "type": "object", + "required": ["schema", "event_id", "timestamp", "source", "routing_policy", "counts", "destinations", "evidence_quality"], + "properties": { + "schema": { "const": "sourceos.routing_receipt.v0.1" }, + "event_id": { "type": "string" }, + "timestamp": { "type": "string", "format": "date-time" }, + "source": { "type": "string" }, + "routing_policy": { + "type": "object", + "required": ["claimed_by", "visible_in_standard_log", "visible_in_evidence_store", "retention_days"], + "properties": { + "claimed_by": { "type": "string" }, + "visible_in_standard_log": { "type": "boolean" }, + "visible_in_evidence_store": { "type": "boolean" }, + "requires_privilege": { "type": "boolean" }, + "redaction_profile": { "type": "string" }, + "retention_days": { "type": "integer", "minimum": 0 } + }, + "additionalProperties": true + }, + "counts": { + "type": "object", + "required": ["received", "written", "sampled", "dropped", "redacted", "diverted"], + "properties": { + "received": { "type": "integer", "minimum": 0 }, + "written": { "type": "integer", "minimum": 0 }, + "sampled": { "type": "integer", "minimum": 0 }, + "dropped": { "type": "integer", "minimum": 0 }, + "redacted": { "type": "integer", "minimum": 0 }, + "diverted": { "type": "integer", "minimum": 0 } + } + }, + "destinations": { "type": "array", "items": { "type": "string" } }, + "evidence_quality": { "$ref": "#/$defs/evidence_quality" } + }, + "additionalProperties": true + }, + "media_work": { + "type": "object", + "required": ["schema", "event_id", "timestamp", "component", "operation", "trigger", "input", "output", "codec", "policy", "evidence_quality"], + "properties": { + "schema": { "const": "sourceos.media_work.v0.1" }, + "event_id": { "type": "string" }, + "timestamp": { "type": "string", "format": "date-time" }, + "component": { "type": "string" }, + "operation": { "enum": ["tile_encode", "preview_generate", "attachment_transcode", "thumbnail_generate", "decode", "unknown"] }, + "trigger": { "enum": ["user_action", "message_received", "sync_replay", "cache_rebuild", "migration", "indexing", "unknown"] }, + "input": { "type": "object", "required": ["artifact_class"], "additionalProperties": true }, + "output": { "type": "object", "required": ["artifact_class", "path_class", "cleanup_policy"], "additionalProperties": true }, + "codec": { "type": "object", "required": ["family", "mode"], "additionalProperties": true }, + "policy": { "$ref": "#/$defs/policy" }, + "evidence_quality": { "$ref": "#/$defs/evidence_quality" } + }, + "additionalProperties": true + }, + "temporary_artifact": { + "type": "object", + "required": ["schema", "event_id", "timestamp", "artifact", "lifecycle", "policy", "evidence_quality"], + "properties": { + "schema": { "const": "sourceos.temporary_artifact.v0.1" }, + "event_id": { "type": "string" }, + "timestamp": { "type": "string", "format": "date-time" }, + "artifact": { "type": "object", "required": ["class", "path_class"], "additionalProperties": true }, + "lifecycle": { "type": "object", "required": ["created", "cleanup_state"], "additionalProperties": true }, + "policy": { "$ref": "#/$defs/policy" }, + "evidence_quality": { "$ref": "#/$defs/evidence_quality" } + }, + "additionalProperties": true + }, + "coalition_resource": { + "type": "object", + "required": ["schema", "event_id", "timestamp", "coalition", "processes", "resources", "evidence_quality"], + "properties": { + "schema": { "const": "sourceos.coalition_resource.v0.1" }, + "event_id": { "type": "string" }, + "timestamp": { "type": "string", "format": "date-time" }, + "coalition": { "type": "object", "required": ["id", "class"], "additionalProperties": true }, + "processes": { "type": "array", "items": { "type": "object", "required": ["pid", "name"], "additionalProperties": true } }, + "resources": { "type": "object", "additionalProperties": true }, + "evidence_quality": { "$ref": "#/$defs/evidence_quality" } + }, + "additionalProperties": true + }, + "policy_decision": { + "type": "object", + "required": ["schema", "event_id", "timestamp", "subject", "target", "decision", "classification", "evidence_quality"], + "properties": { + "schema": { "const": "sourceos.policy_decision.v0.1" }, + "event_id": { "type": "string" }, + "timestamp": { "type": "string", "format": "date-time" }, + "subject": { "$ref": "#/$defs/actor" }, + "target": { "type": "object", "required": ["kind", "id"], "additionalProperties": true }, + "decision": { "enum": ["allow", "deny", "degrade", "unknown"] }, + "classification": { "enum": ["expected_benign", "expected_degraded", "configuration_missing", "policy_violation", "possible_tamper", "impossible_state", "unknown"] }, + "retry_policy": { "type": "object", "additionalProperties": true }, + "downstream_effects": { "type": "array", "items": { "type": "string" } }, + "evidence_quality": { "$ref": "#/$defs/evidence_quality" } + }, + "additionalProperties": true + }, + "compromise_assessment": { + "type": "object", + "required": ["schema", "assessment_id", "timestamp", "scope", "status", "sensor_state", "evidence_quality", "conclusion"], + "properties": { + "schema": { "const": "sourceos.compromise_assessment.v0.1" }, + "assessment_id": { "type": "string" }, + "timestamp": { "type": "string", "format": "date-time" }, + "scope": { "type": "object", "required": ["system", "time_window"], "additionalProperties": true }, + "status": { "type": "array", "items": { "enum": ["compromise_proven", "compromise_suspected", "no_positive_compromise_evidence", "cannot_exclude_compromise", "sensor_degraded", "evidence_inadequate"] }, "minItems": 1 }, + "sensor_state": { "type": "array", "items": { "type": "object", "required": ["sensor", "state"], "additionalProperties": true } }, + "evidence_quality": { "$ref": "#/$defs/evidence_quality" }, + "conclusion": { "type": "string" }, + "required_next_evidence": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": true + } + } +} diff --git a/schemas/MutationEvidenceUmbrellaPrimitives.schema.json b/schemas/MutationEvidenceUmbrellaPrimitives.schema.json new file mode 100644 index 0000000..9e9a3b3 --- /dev/null +++ b/schemas/MutationEvidenceUmbrellaPrimitives.schema.json @@ -0,0 +1,180 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://sourceos.dev/schemas/MutationEvidenceUmbrellaPrimitives.schema.json", + "title": "SourceOS Mutation Evidence Umbrella Primitives v0.1", + "description": "Umbrella schema for SourceOS MutationReceipt, ExecutionContextReceipt, ServiceWorkReceipt, and EvidencePipelineReceipt with specialized subtypes.", + "type": "object", + "oneOf": [ + { "$ref": "#/$defs/mutation_receipt" }, + { "$ref": "#/$defs/execution_context_receipt" }, + { "$ref": "#/$defs/service_work_receipt" }, + { "$ref": "#/$defs/evidence_pipeline_receipt" } + ], + "$defs": { + "actor_ref": { + "type": "object", + "required": ["role", "kind", "id", "name"], + "properties": { + "role": { "enum": ["human", "origin_actor", "requesting_actor", "execution_actor", "storage_actor", "observer", "policy_engine", "unknown"] }, + "kind": { "enum": ["user", "process", "service", "daemon", "agent", "browser", "browser_content_process", "terminal", "file_manager", "sync_daemon", "cloud_service", "diagnostic_service", "kernel", "peripheral", "unknown"] }, + "id": { "type": "string" }, + "name": { "type": "string" }, + "digest": { "type": ["string", "null"] }, + "coalition_id": { "type": ["string", "null"] }, + "persona_id": { "type": ["string", "null"] }, + "trust_zone": { "type": ["string", "null"] }, + "attribution_status": { "enum": ["complete", "partial", "none", "unknown_origin", "delegated_unknown"] } + }, + "additionalProperties": true + }, + "object_ref": { + "type": "object", + "required": ["class", "id", "path_class"], + "properties": { + "class": { "type": "string" }, + "id": { "type": "string" }, + "path_class": { "enum": ["user_data", "app_state", "browser_profile", "origin_storage", "service_worker_cache", "cache", "tmp", "sync_root", "cloud_object", "fog_object", "archive", "diagnostic_archive", "evidence_store", "system_volume", "trash", "downloads", "external_volume", "network_mount", "package_bundle", "unknown"] }, + "redaction": { "enum": ["none", "stable_pseudonym", "path_class_only", "forensic_expansion_required"] } + }, + "additionalProperties": true + }, + "resource_cost": { + "type": "object", + "properties": { + "logical_bytes": { "type": ["integer", "null"], "minimum": 0 }, + "physical_bytes": { "type": ["integer", "null"], "minimum": 0 }, + "dirty_memory_bytes": { "type": ["integer", "null"], "minimum": 0 }, + "clone_or_reflink_bytes": { "type": ["integer", "null"], "minimum": 0 }, + "cpu_ms": { "type": ["integer", "null"], "minimum": 0 }, + "network_bytes_in": { "type": ["integer", "null"], "minimum": 0 }, + "network_bytes_out": { "type": ["integer", "null"], "minimum": 0 }, + "fsync_count": { "type": ["integer", "null"], "minimum": 0 }, + "wal_bytes": { "type": ["integer", "null"], "minimum": 0 }, + "checkpoint_count": { "type": ["integer", "null"], "minimum": 0 }, + "entries_visited": { "type": ["integer", "null"], "minimum": 0 }, + "objects_processed": { "type": ["integer", "null"], "minimum": 0 } + }, + "additionalProperties": true + }, + "policy_ref": { + "type": "object", + "required": ["decision"], + "properties": { + "decision": { "enum": ["allow", "deny", "degrade", "sample", "drop", "route", "quarantine", "unknown"] }, + "policy_id": { "type": ["string", "null"] }, + "budget_id": { "type": ["string", "null"] }, + "classification": { "type": ["string", "null"] } + }, + "additionalProperties": true + }, + "evidence_quality": { + "type": "object", + "required": ["status", "confidence"], + "properties": { + "status": { "enum": ["complete", "partial", "degraded_sensor", "opaque_symbolication", "redacted", "insufficient_for_clearance"] }, + "confidence": { "enum": ["high", "medium", "low", "unknown"] }, + "missing_fields": { "type": "array", "items": { "type": "string" } }, + "sensor_state": { "enum": ["healthy", "degraded", "blind", "missing", "not_applicable", "unknown"] } + }, + "additionalProperties": true + }, + "base_receipt": { + "type": "object", + "required": ["schema", "primitive", "subtype", "event_id", "timestamp", "actor_chain", "object", "operation", "policy", "evidence_quality"], + "properties": { + "schema": { "const": "sourceos.mutation_evidence.umbrella.v0.1" }, + "primitive": { "enum": ["MutationReceipt", "ExecutionContextReceipt", "ServiceWorkReceipt", "EvidencePipelineReceipt"] }, + "subtype": { "type": "string" }, + "event_id": { "type": "string", "minLength": 8 }, + "timestamp": { "type": "string", "format": "date-time" }, + "boot_id": { "type": ["string", "null"] }, + "session_id": { "type": ["string", "null"] }, + "actor_chain": { "type": "array", "items": { "$ref": "#/$defs/actor_ref" }, "minItems": 1 }, + "object": { "$ref": "#/$defs/object_ref" }, + "operation": { "type": "string" }, + "resource_cost": { "$ref": "#/$defs/resource_cost" }, + "policy": { "$ref": "#/$defs/policy_ref" }, + "evidence_quality": { "$ref": "#/$defs/evidence_quality" }, + "causal_parents": { "type": "array", "items": { "type": "string" } }, + "downstream_effects": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": true + }, + "mutation_receipt": { + "allOf": [ + { "$ref": "#/$defs/base_receipt" }, + { "properties": { "primitive": { "const": "MutationReceipt" }, "subtype": { "enum": ["browser_write", "delegated_io", "sync_cycle", "folder_sizing", "metadata_traversal", "archive_extraction", "diagnostic_self_noise", "write_burst", "filesystem_clone", "cache_maintenance", "media_work", "temporary_artifact"] } } } + ] + }, + "execution_context_receipt": { + "allOf": [ + { "$ref": "#/$defs/base_receipt" }, + { + "properties": { + "primitive": { "const": "ExecutionContextReceipt" }, + "subtype": { "enum": ["qos_policy", "user_activity_context", "persona_context", "path_boundary", "binary_provenance", "model_activity_state", "terminal_session"] }, + "execution_context": { + "type": "object", + "properties": { + "frontmost": { "type": ["boolean", "null"] }, + "user_active": { "type": ["boolean", "null"] }, + "effective_qos": { "type": ["string", "null"] }, + "requested_qos": { "type": ["string", "null"] }, + "qos_match": { "type": ["boolean", "null"] }, + "path_boundary_crossed": { "type": ["boolean", "null"] } + }, + "additionalProperties": true + } + } + } + ] + }, + "service_work_receipt": { + "allOf": [ + { "$ref": "#/$defs/base_receipt" }, + { + "properties": { + "primitive": { "const": "ServiceWorkReceipt" }, + "subtype": { "enum": ["scheduler_receipt", "ephemeral_service_work", "fileprovider_tracking", "cloud_object_transfer", "ui_event_route", "compositor_frame", "display_update_pressure", "ipc_serialization", "peripheral_io", "dock_channel_event"] }, + "service_work": { + "type": "object", + "properties": { + "scheduler_kind": { "type": ["string", "null"] }, + "batch_size": { "type": ["integer", "null"], "minimum": 0 }, + "max_cost": { "type": ["integer", "null"], "minimum": 0 }, + "retry_after_ms": { "type": ["integer", "null"], "minimum": 0 }, + "throttle": { "type": ["string", "null"] }, + "object_count": { "type": ["integer", "null"], "minimum": 0 } + }, + "additionalProperties": true + } + } + } + ] + }, + "evidence_pipeline_receipt": { + "allOf": [ + { "$ref": "#/$defs/base_receipt" }, + { + "properties": { + "primitive": { "const": "EvidencePipelineReceipt" }, + "subtype": { "enum": ["routing_receipt", "sink_precedence", "evidence_gap", "log_flush", "symbolication_quality", "agent_coverage_attestation", "compromise_assessment", "diagnostic_self_noise"] }, + "evidence_pipeline": { + "type": "object", + "properties": { + "received": { "type": ["integer", "null"], "minimum": 0 }, + "written": { "type": ["integer", "null"], "minimum": 0 }, + "sampled": { "type": ["integer", "null"], "minimum": 0 }, + "dropped": { "type": ["integer", "null"], "minimum": 0 }, + "redacted": { "type": ["integer", "null"], "minimum": 0 }, + "diverted": { "type": ["integer", "null"], "minimum": 0 }, + "clearance_allowed": { "type": ["boolean", "null"] } + }, + "additionalProperties": true + } + } + } + ] + } + } +} diff --git a/tools/validate_mutation_evidence_accountability.py b/tools/validate_mutation_evidence_accountability.py new file mode 100644 index 0000000..eba334f --- /dev/null +++ b/tools/validate_mutation_evidence_accountability.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Validate SourceOS Mutation and Evidence Accountability examples. + +The validator checks two schema surfaces: + +1. The initial event-family bundle in MutationEvidenceAccountability.schema.json. +2. The umbrella primitive bundle in MutationEvidenceUmbrellaPrimitives.schema.json. + +It also enforces semantic guardrails that JSON Schema alone should not carry: + +- degraded/blind/missing sensors cannot clear compromise; +- browser write pressure cannot be blamed on extensions when the visible + extension inventory is empty, unless hidden/system/policy add-on evidence + explicitly allows it; +- delegated I/O requires a meaningful multi-actor chain; +- diagnostic observer-effect writes cannot grant clearance when evidence is + redacted/partial; +- archive extraction requires path-boundary and cleanup accounting. +""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +try: + import jsonschema +except ImportError as exc: # pragma: no cover + raise SystemExit("jsonschema is required: python -m pip install jsonschema") from exc + +ROOT = Path(__file__).resolve().parents[1] +BASE_SCHEMA_PATH = ROOT / "schemas" / "MutationEvidenceAccountability.schema.json" +BASE_EXAMPLES_PATH = ROOT / "examples" / "mutation-evidence-accountability.examples.json" +BASE_ANTI_PATH = ROOT / "fixtures" / "anti-patterns" / "mutation-evidence-accountability.invalid.json" + +UMBRELLA_SCHEMA_PATH = ROOT / "schemas" / "MutationEvidenceUmbrellaPrimitives.schema.json" +UMBRELLA_EXAMPLES_PATH = ROOT / "examples" / "mutation-evidence-umbrella.examples.json" +UMBRELLA_ANTI_PATH = ROOT / "fixtures" / "anti-patterns" / "mutation-evidence-umbrella.invalid.json" + +base_validator = jsonschema.Draft202012Validator(json.loads(BASE_SCHEMA_PATH.read_text())) +umbrella_validator = jsonschema.Draft202012Validator(json.loads(UMBRELLA_SCHEMA_PATH.read_text())) + + +def _schema_errors(validator: jsonschema.Draft202012Validator, event: dict[str, Any]) -> list[str]: + return [err.message for err in sorted(validator.iter_errors(event), key=lambda e: list(e.path))] + + +def _actor_roles(event: dict[str, Any]) -> set[str]: + return {actor.get("role", "") for actor in event.get("actor_chain", [])} + + +def semantic_errors_base(event: dict[str, Any]) -> list[str]: + errors: list[str] = [] + if event.get("schema") != "sourceos.compromise_assessment.v0.1": + return errors + + sensor_states = {sensor.get("state") for sensor in event.get("sensor_state", [])} + status = set(event.get("status", [])) + evidence_status = event.get("evidence_quality", {}).get("status") + if {"blind", "degraded", "missing"} & sensor_states: + if "cannot_exclude_compromise" not in status: + errors.append("degraded/blind/missing sensors require cannot_exclude_compromise") + if evidence_status == "complete": + errors.append("degraded/blind/missing sensors cannot produce complete evidence quality") + return errors + + +def semantic_errors_umbrella(event: dict[str, Any]) -> list[str]: + errors: list[str] = [] + if event.get("schema") != "sourceos.mutation_evidence.umbrella.v0.1": + return errors + + primitive = event.get("primitive") + subtype = event.get("subtype") + evidence = event.get("evidence_quality", {}) + evidence_status = evidence.get("status") + missing_fields = set(evidence.get("missing_fields", [])) + roles = _actor_roles(event) + + if subtype == "browser_write": + browser = event.get("browser", {}) + if ( + browser.get("extension_inventory_state") == "none_visible" + and browser.get("browser_actor_class") == "browser_extension_storage" + and browser.get("primary_extension_cause_allowed") is True + ): + errors.append("browser write cannot be extension-primary when extension_inventory_state=none_visible") + + if subtype == "delegated_io": + required_roles = {"origin_actor", "requesting_actor", "execution_actor", "storage_actor"} + missing_roles = sorted(required_roles - roles) + if missing_roles: + errors.append(f"delegated_io missing required actor roles: {', '.join(missing_roles)}") + if not event.get("causal_parents"): + errors.append("delegated_io requires at least one causal parent") + + if subtype == "diagnostic_self_noise": + pipeline = event.get("evidence_pipeline", {}) + if pipeline.get("clearance_allowed") is True and ( + evidence_status != "complete" or missing_fields or pipeline.get("redacted", 0) > 0 + ): + errors.append("diagnostic self-noise with partial/redacted evidence cannot allow clearance") + + if subtype == "archive_extraction": + obj = event.get("object", {}) + archive = event.get("archive", {}) + if obj.get("path_class") == "unknown": + errors.append("archive_extraction requires a known path_class") + if "cleanup_policy" not in archive: + errors.append("archive_extraction requires cleanup_policy accounting") + if evidence_status == "complete" and (archive.get("quarantine_state") == "unknown" or not event.get("causal_parents")): + errors.append("archive_extraction cannot be complete without quarantine state and causal parent") + + if primitive == "EvidencePipelineReceipt" and subtype == "compromise_assessment": + pipeline = event.get("evidence_pipeline", {}) + if pipeline.get("clearance_allowed") is True and evidence.get("sensor_state") in {"degraded", "blind", "missing"}: + errors.append("evidence pipeline cannot clear compromise with degraded/blind/missing sensors") + + return errors + + +def validate_base_event(event: dict[str, Any]) -> list[str]: + return _schema_errors(base_validator, event) + semantic_errors_base(event) + + +def validate_umbrella_event(event: dict[str, Any]) -> list[str]: + return _schema_errors(umbrella_validator, event) + semantic_errors_umbrella(event) + + +def _validate_examples(label: str, path: Path, validator_fn) -> bool: + failed = False + examples = json.loads(path.read_text())["examples"] + for idx, event in enumerate(examples): + errors = validator_fn(event) + if errors: + failed = True + print(f"FAIL {label} example[{idx}] {event.get('schema')} {event.get('subtype', '')}") + for err in errors: + print(f" - {err}") + else: + print(f"PASS {label} example[{idx}] {event.get('schema')} {event.get('subtype', '')}") + return failed + + +def _validate_anti_patterns(label: str, path: Path, validator_fn) -> bool: + failed = False + anti_patterns = json.loads(path.read_text())["anti_patterns"] + for idx, item in enumerate(anti_patterns): + event = item["event"] + errors = validator_fn(event) + if errors: + print(f"PASS {label} anti-pattern rejected[{idx}] {item['name']}") + else: + failed = True + print(f"FAIL {label} anti-pattern unexpectedly valid[{idx}] {item['name']}") + return failed + + +def main() -> int: + failed = False + failed |= _validate_examples("base", BASE_EXAMPLES_PATH, validate_base_event) + failed |= _validate_anti_patterns("base", BASE_ANTI_PATH, validate_base_event) + failed |= _validate_examples("umbrella", UMBRELLA_EXAMPLES_PATH, validate_umbrella_event) + failed |= _validate_anti_patterns("umbrella", UMBRELLA_ANTI_PATH, validate_umbrella_event) + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main())