feat: v0.4 — desktop kind (spec + Node producer + FixtureBackend + MCP tools)

schema/agentmark-v0.4.json

@@ -0,0 +1,127 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://agentmark.dev/schema/v0.4.json",
+    "title": "agentmark v0.4 frontmatter",
+    "description": "v0.4 extends v0.3 with `kind: desktop`, `desktop_meta`, and the [WINDOW] / [ELEMENT] body tags for native applications captured via OS accessibility APIs (Windows UIA, macOS AXAPI, Linux AT-SPI). v0.3 docs continue to validate.",
+    "type": "object",
+    "required": ["agentmark", "url", "title"],
+    "properties": {
+        "agentmark": { "type": "string", "pattern": "^[0-9]+\\.[0-9]+(\\.[0-9]+)?$" },
+        "kind": { "enum": ["webpage", "document", "form", "audio", "video", "desktop"] },
+        "url": { "type": "string", "format": "uri" },
+        "title": { "type": "string", "maxLength": 512 },
+        "captured_at": { "type": "string", "format": "date-time" },
+        "expires_at": { "type": "string", "format": "date-time" },
+        "source": { "enum": ["rendered", "declared", "hybrid"] },
+        "language": { "type": "string" },
+        "direction": { "enum": ["ltr", "rtl"] },
+        "state": { "type": "object" },
+        "actions": { "type": "object" },
+        "media": { "type": "object" },
+        "document": { "$ref": "#/$defs/document" },
+        "media_meta": { "$ref": "#/$defs/media_meta" },
+        "desktop_meta": { "$ref": "#/$defs/desktop_meta" },
+        "speakers": {
+            "type": "object",
+            "patternProperties": {
+                "^[a-z][a-z0-9_]{0,63}$": { "type": "string", "maxLength": 256 }
+            },
+            "additionalProperties": false
+        },
+        "signatures": {
+            "type": "object",
+            "patternProperties": {
+                "^[a-z][a-z0-9_]{0,63}$": { "$ref": "#/$defs/signature" }
+            },
+            "additionalProperties": false
+        },
+        "memory": { "type": "object" },
+        "capabilities": { "type": "object" },
+        "cookies": { "type": "object" },
+        "permissions": { "type": "object" }
+    },
+    "patternProperties": {
+        "^x-": {}
+    },
+    "$defs": {
+        "document": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+                "pages": { "type": "integer", "minimum": 1 },
+                "author": { "type": "string", "maxLength": 512 },
+                "created_at": { "type": "string", "format": "date-time" },
+                "modified_at": { "type": "string", "format": "date-time" },
+                "format": { "enum": ["pdf", "docx", "rtf", "txt", "html"] },
+                "format_version": { "type": "string", "maxLength": 32 },
+                "ocr_used": { "type": "boolean" }
+            }
+        },
+        "media_meta": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+                "duration_sec": { "type": "number", "minimum": 0 },
+                "format": { "type": "string", "maxLength": 32 },
+                "language": { "type": "string", "maxLength": 32 },
+                "transcribed": { "type": "boolean" },
+                "transcription_backend": { "type": "string", "maxLength": 64 },
+                "vision_backend": { "type": "string", "maxLength": 64 },
+                "speaker_count": { "type": "integer", "minimum": 0 },
+                "frame_count": { "type": "integer", "minimum": 0 }
+            }
+        },
+        "desktop_meta": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+                "platform": { "enum": ["windows", "macos", "linux"] },
+                "process_name": { "type": "string", "maxLength": 256 },
+                "process_id": { "type": "integer", "minimum": 0 },
+                "window_class": { "type": "string", "maxLength": 256 },
+                "focused_element_id": { "type": "string", "maxLength": 256 },
+                "a11y_backend": { "type": "string", "maxLength": 64 },
+                "tree_depth": { "type": "integer", "minimum": 0 },
+                "element_count": { "type": "integer", "minimum": 0 }
+            }
+        },
+        "signature": {
+            "type": "object",
+            "required": ["kind", "page", "confidence"],
+            "additionalProperties": false,
+            "properties": {
+                "kind": {
+                    "enum": [
+                        "widget_visible_signed",
+                        "widget_unsigned",
+                        "cryptographic",
+                        "image_handwritten",
+                        "image_typed",
+                        "docusign",
+                        "adobe_sign",
+                        "unknown"
+                    ]
+                },
+                "page": { "type": "integer", "minimum": 1 },
+                "rect": {
+                    "type": "object",
+                    "additionalProperties": false,
+                    "properties": {
+                        "x": { "type": "number" },
+                        "y": { "type": "number" },
+                        "width": { "type": "number" },
+                        "height": { "type": "number" }
+                    }
+                },
+                "field_name": { "type": "string", "maxLength": 256 },
+                "inferred_role": { "type": "string", "maxLength": 64 },
+                "signer_name": { "type": "string", "maxLength": 256 },
+                "signer_email": { "type": "string", "maxLength": 256 },
+                "signed_at": { "type": "string", "format": "date-time" },
+                "confidence": { "type": "number", "minimum": 0, "maximum": 1 },
+                "valid": { "type": "boolean" },
+                "notes": { "type": "string", "maxLength": 1024 }
+            }
+        }
+    }
+}

src/index.ts

@@ -8,6 +8,7 @@ export type {
     SnapshotSource,
     SnapshotKind,
     DocumentMeta,
+    DesktopMeta,
     PageState,
     ActionType,
     ActionCost,

src/types.ts

@@ -5,20 +5,22 @@
  * Producers build an `Snapshot`; serializers turn it into the wire format.
  */
 
-export const AGENTMARK_VERSION = '0.3' as const
+export const AGENTMARK_VERSION = '0.4' as const
 
 /** Spec versions this implementation can validate against. */
-export const SUPPORTED_SPEC_VERSIONS = ['0.1', '0.2', '0.3'] as const
+export const SUPPORTED_SPEC_VERSIONS = ['0.1', '0.2', '0.3', '0.4'] as const
 
 // ──────────────────────────────────────────────────────────────────────────
 // Frontmatter envelope
 // ──────────────────────────────────────────────────────────────────────────
 
 /**
- * Discriminator. v0.2 added `webpage|document|form`; v0.3 added `audio|video`.
+ * Discriminator. v0.2 added `webpage|document|form`; v0.3 added `audio|video`;
+ * v0.4 adds `desktop` for native application surfaces captured via OS
+ * accessibility APIs (Windows UIA, macOS AXAPI, Linux AT-SPI).
  * Defaults to 'webpage' when omitted (v0.1 compatibility).
  */
-export type SnapshotKind = 'webpage' | 'document' | 'form' | 'audio' | 'video'
+export type SnapshotKind = 'webpage' | 'document' | 'form' | 'audio' | 'video' | 'desktop'
 
 export interface Snapshot {
     /** Spec version, e.g. "0.1" or "0.2" */
@@ -58,6 +60,9 @@ export interface Snapshot {
     /** Speaker labels keyed by ID (v0.3+, audio/video). Map ID → display name. */
     speakers?: Record<string, string>
 
+    /** Desktop-specific metadata (v0.4+, populated when kind === 'desktop'). */
+    desktop_meta?: DesktopMeta
+
     /**
      * Detected signatures on the document, keyed by signature ID
      * (e.g. `sig_1`). Body uses `[SIGNATURE:sig_1]` to reference them.
@@ -118,6 +123,39 @@ export interface DocumentMeta {
     ocr_used?: boolean
 }
 
+/**
+ * Desktop metadata captured from OS accessibility APIs (v0.4+). Producer
+ * walks the platform's accessibility tree (Windows UIA, macOS AXAPI, Linux
+ * AT-SPI) and emits a Snapshot describing one or more application windows.
+ * Interactive elements are exposed through the standard `actions` map; the
+ * fields here are descriptive metadata only.
+ *
+ * All fields are optional — backends populate what they can.
+ */
+export interface DesktopMeta {
+    /** Operating system the snapshot was captured on. */
+    platform?: 'windows' | 'macos' | 'linux'
+    /** Process name owning the focused window (e.g. 'EXCEL.EXE', 'Slack'). */
+    process_name?: string
+    /** OS process id of the captured window's owning process. */
+    process_id?: number
+    /** Toolkit / window class hint — Windows: UIA control type or Win32
+     *  class (e.g. 'XLMAIN'); macOS: AXSubrole; Linux: AT-SPI role. */
+    window_class?: string
+    /** Stable accessibility identifier of the currently focused element.
+     *  On Windows this is typically the UIA AutomationId; on macOS the
+     *  AXIdentifier; on Linux the AT-SPI accessible-id. */
+    focused_element_id?: string
+    /** Accessibility backend that produced the snapshot. Helps consumers
+     *  understand the fidelity of the captured data. */
+    a11y_backend?: 'windows_uia' | 'macos_axapi' | 'linux_atspi' | 'vision_fallback' | string
+    /** Maximum depth of the captured accessibility tree (debugging /
+     *  cardinality hint for renderers). */
+    tree_depth?: number
+    /** Total interactive elements extracted into the `actions` map. */
+    element_count?: number
+}
+
 // ──────────────────────────────────────────────────────────────────────────
 // Page state
 // ──────────────────────────────────────────────────────────────────────────
@@ -277,6 +315,15 @@ export type BodyTagKind =
     /** v0.3+: video frame reference. Payload is a frame ID (`f_42`) whose
      *  thumbnail + caption live in the `media` map. */
     | 'FRAME'
+    /** v0.4+: window boundary marker for `kind: 'desktop'`. Payload is a
+     *  window identifier (e.g. `w_1`) — used when a single snapshot spans
+     *  multiple application windows. */
+    | 'WINDOW'
+    /** v0.4+: non-interactive accessibility element reference for
+     *  `kind: 'desktop'`. Payload is an element ID (e.g. `e_42`) that
+     *  matches the element's AutomationId / AXIdentifier. Interactive
+     *  controls (buttons, inputs, etc.) continue to use ACTION / INPUT. */
+    | 'ELEMENT'
 
 /**
  * Descriptor for a detected signature. Lives in `Snapshot.signatures` keyed

src/validators/schema-validator.ts

@@ -14,15 +14,16 @@ const validatorCache = new Map<string, ValidateFunction>()
  * Unknown versions fall back to the highest known schema and emit a warning
  * elsewhere — see `validateSnapshot` cross-field check 2e.
  */
-function resolveSchemaVersion(declared: string): '0.1' | '0.2' | '0.3' {
+function resolveSchemaVersion(declared: string): '0.1' | '0.2' | '0.3' | '0.4' {
     const [major, minor] = declared.split('.')
     const minorMajor = `${major}.${minor}`
     if (minorMajor === '0.1') return '0.1'
     if (minorMajor === '0.2') return '0.2'
-    return '0.3'
+    if (minorMajor === '0.3') return '0.3'
+    return '0.4'
 }
 
-function loadValidator(version: '0.1' | '0.2' | '0.3'): ValidateFunction {
+function loadValidator(version: '0.1' | '0.2' | '0.3' | '0.4'): ValidateFunction {
     const cached = validatorCache.get(version)
     if (cached) return cached
 
@@ -95,7 +96,9 @@ export function validateSnapshot(snapshot: Snapshot): ValidationResult {
     //   PAGE    — v0.2, page boundary marker (p_n)
     //   TIME    — v0.3, timestamp marker for audio/video (t_seconds)
     //   SPEAKER — v0.3, speaker label (resolves to envelope.speakers map)
-    const STRUCTURAL_TAGS = new Set(['PAGE', 'TIME', 'SPEAKER'])
+    //   WINDOW  — v0.4, window boundary marker for desktop (w_n)
+    //   ELEMENT — v0.4, non-interactive accessibility element ref (e_n)
+    const STRUCTURAL_TAGS = new Set(['PAGE', 'TIME', 'SPEAKER', 'WINDOW', 'ELEMENT'])
 
     const signatureIds = new Set(Object.keys(snapshot.signatures ?? {}))
 
@@ -188,11 +191,11 @@ export function validateSnapshot(snapshot: Snapshot): ValidationResult {
     // 2e. version compatibility
     const major = parseInt(snapshot.agentmark.split('.')[0], 10)
     const minor = parseInt(snapshot.agentmark.split('.')[1] ?? '0', 10)
-    if (major > 0 || minor > 2) {
+    if (major > 0 || minor > 4) {
         warnings.push({
             severity: 'warning',
             path: '/agentmark',
-            message: `This validator implements v0.1 + v0.2; snapshot declares v${snapshot.agentmark}. Validated against v0.2 schema.`,
+            message: `This validator implements v0.1 + v0.2 + v0.3 + v0.4; snapshot declares v${snapshot.agentmark}. Validated against v0.4 schema.`,
         })
     }
 

test/audio/audio-converter.test.ts

@@ -42,7 +42,7 @@ describe('convertAudio', () => {
 
         const snap = parseSnapshot(agentmark)
         expect(snap.kind).toBe('audio')
-        expect(snap.agentmark).toBe('0.3')
+        expect(snap.agentmark).toBe('0.4')
         expect(snap.media_meta?.duration_sec).toBe(12.5)
         expect(snap.media_meta?.transcribed).toBe(true)
         expect(snap.media_meta?.transcription_backend).toBe('fake_transcribe')

test/build-snapshot.test.ts

@@ -45,7 +45,7 @@ describe('buildSnapshot', () => {
 
     it('sets agentmark version, source, and timestamps', () => {
         const snap = buildSnapshot(fakeExtraction())
-        expect(snap.agentmark).toBe('0.3')
+        expect(snap.agentmark).toBe('0.4')
         expect(snap.source).toBe('rendered')
         expect(snap.captured_at).toBeDefined()
         expect(snap.expires_at).toBeDefined()

test/pdf/pdf-converter.test.ts

@@ -183,7 +183,7 @@ describe('convertPdf', () => {
         // Snapshot is parseable + validates against v0.2 schema
         const snap = parseSnapshot(agentmark)
         expect(snap.kind).toBe('document')
-        expect(snap.agentmark).toBe('0.3')
+        expect(snap.agentmark).toBe('0.4')
         expect(snap.url).toBe('file:///tmp/annual-report.pdf')
         expect(snap.title).toBe('Annual Report')
 

test/spec-v0.2.test.ts

@@ -5,12 +5,12 @@ import type { Snapshot } from '../src/types'
 import { SUPPORTED_SPEC_VERSIONS, AGENTMARK_VERSION } from '../src/types'
 
 describe('Spec v0.2 — kind discriminator', () => {
-    it('default version is 0.3 in this implementation (v0.3 ships with audio support)', () => {
-        expect(AGENTMARK_VERSION).toBe('0.3')
+    it('default version is 0.4 in this implementation (v0.4 ships with desktop support)', () => {
+        expect(AGENTMARK_VERSION).toBe('0.4')
     })
 
-    it('reports v0.1, v0.2, and v0.3 as supported', () => {
-        expect(SUPPORTED_SPEC_VERSIONS).toEqual(['0.1', '0.2', '0.3'])
+    it('reports v0.1 through v0.4 as supported', () => {
+        expect(SUPPORTED_SPEC_VERSIONS).toEqual(['0.1', '0.2', '0.3', '0.4'])
     })
 
     it('v0.1 snapshots without kind still validate (backwards compat)', () => {