From e6708588794ca8109e419edc81148b8db1694dbd Mon Sep 17 00:00:00 2001 From: rrader26 Date: Mon, 11 May 2026 09:48:58 -0400 Subject: [PATCH] =?UTF-8?q?feat(spec):=20v0.4=20=E2=80=94=20add=20`desktop?= =?UTF-8?q?`=20kind=20for=20native=20applications?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the AgentMark spec to native desktop surfaces captured via OS accessibility APIs (Windows UIA, macOS AXAPI, Linux AT-SPI). Adds a new SnapshotKind `'desktop'` with a matching DesktopMeta envelope describing platform, process, window class, focused element, and backend metadata. Producers will arrive in subsequent PRs (Node-side converter + per-OS bridge processes). This PR is the spec foundation — purely additive, zero functional change. v0.3 snapshots continue to validate against v0.4. Changes: - AGENTMARK_VERSION bumped to '0.4' - SUPPORTED_SPEC_VERSIONS extended to include '0.4' - SnapshotKind union extended with 'desktop' - New DesktopMeta interface (mirrors MediaMeta pattern) - desktop_meta?: DesktopMeta on Snapshot - BodyTagKind extended with WINDOW (window boundary) and ELEMENT (non-interactive accessibility element reference) - New schema/agentmark-v0.4.json - Validator resolveSchemaVersion + STRUCTURAL_TAGS + version-compat warning all updated for v0.4 - DesktopMeta exported from src/index.ts - New test/spec-v0.4.test.ts — 7 tests covering valid desktop snapshots on Windows + macOS, invalid platform rejection, invalid extra fields, WINDOW/ELEMENT body tags, YAML round-trip, v0.3 backward compat - Three existing tests with hardcoded version strings bumped 0.3 → 0.4 Co-Authored-By: Claude Opus 4.7 (1M context) --- schema/agentmark-v0.4.json | 127 +++++++++++++++++++++++++++ src/index.ts | 1 + src/types.ts | 55 +++++++++++- src/validators/schema-validator.ts | 15 ++-- test/audio/audio-converter.test.ts | 2 +- test/build-snapshot.test.ts | 2 +- test/pdf/pdf-converter.test.ts | 2 +- test/spec-v0.2.test.ts | 8 +- test/spec-v0.4.test.ts | 135 +++++++++++++++++++++++++++++ 9 files changed, 330 insertions(+), 17 deletions(-) create mode 100644 schema/agentmark-v0.4.json create mode 100644 test/spec-v0.4.test.ts diff --git a/schema/agentmark-v0.4.json b/schema/agentmark-v0.4.json new file mode 100644 index 0000000..c6b5d3b --- /dev/null +++ b/schema/agentmark-v0.4.json @@ -0,0 +1,127 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentmark.dev/schema/v0.4.json", + "title": "agentmark v0.4 frontmatter", + "description": "v0.4 extends v0.3 with `kind: desktop`, `desktop_meta`, and the [WINDOW] / [ELEMENT] body tags for native applications captured via OS accessibility APIs (Windows UIA, macOS AXAPI, Linux AT-SPI). v0.3 docs continue to validate.", + "type": "object", + "required": ["agentmark", "url", "title"], + "properties": { + "agentmark": { "type": "string", "pattern": "^[0-9]+\\.[0-9]+(\\.[0-9]+)?$" }, + "kind": { "enum": ["webpage", "document", "form", "audio", "video", "desktop"] }, + "url": { "type": "string", "format": "uri" }, + "title": { "type": "string", "maxLength": 512 }, + "captured_at": { "type": "string", "format": "date-time" }, + "expires_at": { "type": "string", "format": "date-time" }, + "source": { "enum": ["rendered", "declared", "hybrid"] }, + "language": { "type": "string" }, + "direction": { "enum": ["ltr", "rtl"] }, + "state": { "type": "object" }, + "actions": { "type": "object" }, + "media": { "type": "object" }, + "document": { "$ref": "#/$defs/document" }, + "media_meta": { "$ref": "#/$defs/media_meta" }, + "desktop_meta": { "$ref": "#/$defs/desktop_meta" }, + "speakers": { + "type": "object", + "patternProperties": { + "^[a-z][a-z0-9_]{0,63}$": { "type": "string", "maxLength": 256 } + }, + "additionalProperties": false + }, + "signatures": { + "type": "object", + "patternProperties": { + "^[a-z][a-z0-9_]{0,63}$": { "$ref": "#/$defs/signature" } + }, + "additionalProperties": false + }, + "memory": { "type": "object" }, + "capabilities": { "type": "object" }, + "cookies": { "type": "object" }, + "permissions": { "type": "object" } + }, + "patternProperties": { + "^x-": {} + }, + "$defs": { + "document": { + "type": "object", + "additionalProperties": false, + "properties": { + "pages": { "type": "integer", "minimum": 1 }, + "author": { "type": "string", "maxLength": 512 }, + "created_at": { "type": "string", "format": "date-time" }, + "modified_at": { "type": "string", "format": "date-time" }, + "format": { "enum": ["pdf", "docx", "rtf", "txt", "html"] }, + "format_version": { "type": "string", "maxLength": 32 }, + "ocr_used": { "type": "boolean" } + } + }, + "media_meta": { + "type": "object", + "additionalProperties": false, + "properties": { + "duration_sec": { "type": "number", "minimum": 0 }, + "format": { "type": "string", "maxLength": 32 }, + "language": { "type": "string", "maxLength": 32 }, + "transcribed": { "type": "boolean" }, + "transcription_backend": { "type": "string", "maxLength": 64 }, + "vision_backend": { "type": "string", "maxLength": 64 }, + "speaker_count": { "type": "integer", "minimum": 0 }, + "frame_count": { "type": "integer", "minimum": 0 } + } + }, + "desktop_meta": { + "type": "object", + "additionalProperties": false, + "properties": { + "platform": { "enum": ["windows", "macos", "linux"] }, + "process_name": { "type": "string", "maxLength": 256 }, + "process_id": { "type": "integer", "minimum": 0 }, + "window_class": { "type": "string", "maxLength": 256 }, + "focused_element_id": { "type": "string", "maxLength": 256 }, + "a11y_backend": { "type": "string", "maxLength": 64 }, + "tree_depth": { "type": "integer", "minimum": 0 }, + "element_count": { "type": "integer", "minimum": 0 } + } + }, + "signature": { + "type": "object", + "required": ["kind", "page", "confidence"], + "additionalProperties": false, + "properties": { + "kind": { + "enum": [ + "widget_visible_signed", + "widget_unsigned", + "cryptographic", + "image_handwritten", + "image_typed", + "docusign", + "adobe_sign", + "unknown" + ] + }, + "page": { "type": "integer", "minimum": 1 }, + "rect": { + "type": "object", + "additionalProperties": false, + "properties": { + "x": { "type": "number" }, + "y": { "type": "number" }, + "width": { "type": "number" }, + "height": { "type": "number" } + } + }, + "field_name": { "type": "string", "maxLength": 256 }, + "inferred_role": { "type": "string", "maxLength": 64 }, + "signer_name": { "type": "string", "maxLength": 256 }, + "signer_email": { "type": "string", "maxLength": 256 }, + "signed_at": { "type": "string", "format": "date-time" }, + "confidence": { "type": "number", "minimum": 0, "maximum": 1 }, + "valid": { "type": "boolean" }, + "notes": { "type": "string", "maxLength": 1024 } + } + } + } +} diff --git a/src/index.ts b/src/index.ts index 8f33b30..91ca6ad 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,6 +8,7 @@ export type { SnapshotSource, SnapshotKind, DocumentMeta, + DesktopMeta, PageState, ActionType, ActionCost, diff --git a/src/types.ts b/src/types.ts index 89dcdcd..a09a2cb 100644 --- a/src/types.ts +++ b/src/types.ts @@ -5,20 +5,22 @@ * Producers build an `Snapshot`; serializers turn it into the wire format. */ -export const AGENTMARK_VERSION = '0.3' as const +export const AGENTMARK_VERSION = '0.4' as const /** Spec versions this implementation can validate against. */ -export const SUPPORTED_SPEC_VERSIONS = ['0.1', '0.2', '0.3'] as const +export const SUPPORTED_SPEC_VERSIONS = ['0.1', '0.2', '0.3', '0.4'] as const // ────────────────────────────────────────────────────────────────────────── // Frontmatter envelope // ────────────────────────────────────────────────────────────────────────── /** - * Discriminator. v0.2 added `webpage|document|form`; v0.3 added `audio|video`. + * Discriminator. v0.2 added `webpage|document|form`; v0.3 added `audio|video`; + * v0.4 adds `desktop` for native application surfaces captured via OS + * accessibility APIs (Windows UIA, macOS AXAPI, Linux AT-SPI). * Defaults to 'webpage' when omitted (v0.1 compatibility). */ -export type SnapshotKind = 'webpage' | 'document' | 'form' | 'audio' | 'video' +export type SnapshotKind = 'webpage' | 'document' | 'form' | 'audio' | 'video' | 'desktop' export interface Snapshot { /** Spec version, e.g. "0.1" or "0.2" */ @@ -58,6 +60,9 @@ export interface Snapshot { /** Speaker labels keyed by ID (v0.3+, audio/video). Map ID → display name. */ speakers?: Record + /** Desktop-specific metadata (v0.4+, populated when kind === 'desktop'). */ + desktop_meta?: DesktopMeta + /** * Detected signatures on the document, keyed by signature ID * (e.g. `sig_1`). Body uses `[SIGNATURE:sig_1]` to reference them. @@ -118,6 +123,39 @@ export interface DocumentMeta { ocr_used?: boolean } +/** + * Desktop metadata captured from OS accessibility APIs (v0.4+). Producer + * walks the platform's accessibility tree (Windows UIA, macOS AXAPI, Linux + * AT-SPI) and emits a Snapshot describing one or more application windows. + * Interactive elements are exposed through the standard `actions` map; the + * fields here are descriptive metadata only. + * + * All fields are optional — backends populate what they can. + */ +export interface DesktopMeta { + /** Operating system the snapshot was captured on. */ + platform?: 'windows' | 'macos' | 'linux' + /** Process name owning the focused window (e.g. 'EXCEL.EXE', 'Slack'). */ + process_name?: string + /** OS process id of the captured window's owning process. */ + process_id?: number + /** Toolkit / window class hint — Windows: UIA control type or Win32 + * class (e.g. 'XLMAIN'); macOS: AXSubrole; Linux: AT-SPI role. */ + window_class?: string + /** Stable accessibility identifier of the currently focused element. + * On Windows this is typically the UIA AutomationId; on macOS the + * AXIdentifier; on Linux the AT-SPI accessible-id. */ + focused_element_id?: string + /** Accessibility backend that produced the snapshot. Helps consumers + * understand the fidelity of the captured data. */ + a11y_backend?: 'windows_uia' | 'macos_axapi' | 'linux_atspi' | 'vision_fallback' | string + /** Maximum depth of the captured accessibility tree (debugging / + * cardinality hint for renderers). */ + tree_depth?: number + /** Total interactive elements extracted into the `actions` map. */ + element_count?: number +} + // ────────────────────────────────────────────────────────────────────────── // Page state // ────────────────────────────────────────────────────────────────────────── @@ -277,6 +315,15 @@ export type BodyTagKind = /** v0.3+: video frame reference. Payload is a frame ID (`f_42`) whose * thumbnail + caption live in the `media` map. */ | 'FRAME' + /** v0.4+: window boundary marker for `kind: 'desktop'`. Payload is a + * window identifier (e.g. `w_1`) — used when a single snapshot spans + * multiple application windows. */ + | 'WINDOW' + /** v0.4+: non-interactive accessibility element reference for + * `kind: 'desktop'`. Payload is an element ID (e.g. `e_42`) that + * matches the element's AutomationId / AXIdentifier. Interactive + * controls (buttons, inputs, etc.) continue to use ACTION / INPUT. */ + | 'ELEMENT' /** * Descriptor for a detected signature. Lives in `Snapshot.signatures` keyed diff --git a/src/validators/schema-validator.ts b/src/validators/schema-validator.ts index d078048..4646bb6 100644 --- a/src/validators/schema-validator.ts +++ b/src/validators/schema-validator.ts @@ -14,15 +14,16 @@ const validatorCache = new Map() * Unknown versions fall back to the highest known schema and emit a warning * elsewhere — see `validateSnapshot` cross-field check 2e. */ -function resolveSchemaVersion(declared: string): '0.1' | '0.2' | '0.3' { +function resolveSchemaVersion(declared: string): '0.1' | '0.2' | '0.3' | '0.4' { const [major, minor] = declared.split('.') const minorMajor = `${major}.${minor}` if (minorMajor === '0.1') return '0.1' if (minorMajor === '0.2') return '0.2' - return '0.3' + if (minorMajor === '0.3') return '0.3' + return '0.4' } -function loadValidator(version: '0.1' | '0.2' | '0.3'): ValidateFunction { +function loadValidator(version: '0.1' | '0.2' | '0.3' | '0.4'): ValidateFunction { const cached = validatorCache.get(version) if (cached) return cached @@ -95,7 +96,9 @@ export function validateSnapshot(snapshot: Snapshot): ValidationResult { // PAGE — v0.2, page boundary marker (p_n) // TIME — v0.3, timestamp marker for audio/video (t_seconds) // SPEAKER — v0.3, speaker label (resolves to envelope.speakers map) - const STRUCTURAL_TAGS = new Set(['PAGE', 'TIME', 'SPEAKER']) + // WINDOW — v0.4, window boundary marker for desktop (w_n) + // ELEMENT — v0.4, non-interactive accessibility element ref (e_n) + const STRUCTURAL_TAGS = new Set(['PAGE', 'TIME', 'SPEAKER', 'WINDOW', 'ELEMENT']) const signatureIds = new Set(Object.keys(snapshot.signatures ?? {})) @@ -188,11 +191,11 @@ export function validateSnapshot(snapshot: Snapshot): ValidationResult { // 2e. version compatibility const major = parseInt(snapshot.agentmark.split('.')[0], 10) const minor = parseInt(snapshot.agentmark.split('.')[1] ?? '0', 10) - if (major > 0 || minor > 2) { + if (major > 0 || minor > 4) { warnings.push({ severity: 'warning', path: '/agentmark', - message: `This validator implements v0.1 + v0.2; snapshot declares v${snapshot.agentmark}. Validated against v0.2 schema.`, + message: `This validator implements v0.1 + v0.2 + v0.3 + v0.4; snapshot declares v${snapshot.agentmark}. Validated against v0.4 schema.`, }) } diff --git a/test/audio/audio-converter.test.ts b/test/audio/audio-converter.test.ts index 7ed79c5..0f867c0 100644 --- a/test/audio/audio-converter.test.ts +++ b/test/audio/audio-converter.test.ts @@ -42,7 +42,7 @@ describe('convertAudio', () => { const snap = parseSnapshot(agentmark) expect(snap.kind).toBe('audio') - expect(snap.agentmark).toBe('0.3') + expect(snap.agentmark).toBe('0.4') expect(snap.media_meta?.duration_sec).toBe(12.5) expect(snap.media_meta?.transcribed).toBe(true) expect(snap.media_meta?.transcription_backend).toBe('fake_transcribe') diff --git a/test/build-snapshot.test.ts b/test/build-snapshot.test.ts index d1d14f3..c826090 100644 --- a/test/build-snapshot.test.ts +++ b/test/build-snapshot.test.ts @@ -45,7 +45,7 @@ describe('buildSnapshot', () => { it('sets agentmark version, source, and timestamps', () => { const snap = buildSnapshot(fakeExtraction()) - expect(snap.agentmark).toBe('0.3') + expect(snap.agentmark).toBe('0.4') expect(snap.source).toBe('rendered') expect(snap.captured_at).toBeDefined() expect(snap.expires_at).toBeDefined() diff --git a/test/pdf/pdf-converter.test.ts b/test/pdf/pdf-converter.test.ts index d8db0dc..f5ec242 100644 --- a/test/pdf/pdf-converter.test.ts +++ b/test/pdf/pdf-converter.test.ts @@ -183,7 +183,7 @@ describe('convertPdf', () => { // Snapshot is parseable + validates against v0.2 schema const snap = parseSnapshot(agentmark) expect(snap.kind).toBe('document') - expect(snap.agentmark).toBe('0.3') + expect(snap.agentmark).toBe('0.4') expect(snap.url).toBe('file:///tmp/annual-report.pdf') expect(snap.title).toBe('Annual Report') diff --git a/test/spec-v0.2.test.ts b/test/spec-v0.2.test.ts index 149a23c..b4a27eb 100644 --- a/test/spec-v0.2.test.ts +++ b/test/spec-v0.2.test.ts @@ -5,12 +5,12 @@ import type { Snapshot } from '../src/types' import { SUPPORTED_SPEC_VERSIONS, AGENTMARK_VERSION } from '../src/types' describe('Spec v0.2 — kind discriminator', () => { - it('default version is 0.3 in this implementation (v0.3 ships with audio support)', () => { - expect(AGENTMARK_VERSION).toBe('0.3') + it('default version is 0.4 in this implementation (v0.4 ships with desktop support)', () => { + expect(AGENTMARK_VERSION).toBe('0.4') }) - it('reports v0.1, v0.2, and v0.3 as supported', () => { - expect(SUPPORTED_SPEC_VERSIONS).toEqual(['0.1', '0.2', '0.3']) + it('reports v0.1 through v0.4 as supported', () => { + expect(SUPPORTED_SPEC_VERSIONS).toEqual(['0.1', '0.2', '0.3', '0.4']) }) it('v0.1 snapshots without kind still validate (backwards compat)', () => { diff --git a/test/spec-v0.4.test.ts b/test/spec-v0.4.test.ts new file mode 100644 index 0000000..78ee2ef --- /dev/null +++ b/test/spec-v0.4.test.ts @@ -0,0 +1,135 @@ +import { describe, it, expect } from 'vitest' +import { validateSnapshot } from '../src/validators/schema-validator' +import { serializeSnapshot, parseSnapshot } from '../src/serializers/yaml-frontmatter' +import type { Snapshot } from '../src/types' + +describe('Spec v0.4 — desktop kind', () => { + it('accepts a minimal desktop snapshot with desktop_meta', () => { + const snap: Snapshot = { + agentmark: '0.4', + kind: 'desktop', + url: 'desktop://localhost/window/12345', + title: 'Microsoft Excel - Book1', + desktop_meta: { + platform: 'windows', + process_name: 'EXCEL.EXE', + process_id: 12345, + window_class: 'XLMAIN', + a11y_backend: 'windows_uia', + tree_depth: 8, + element_count: 47, + }, + actions: { + act_save: { type: 'click', label: 'Save' }, + }, + body: '# Microsoft Excel — Book1\n\n[ACTION:act_save]', + } + const result = validateSnapshot(snap) + expect(result.errors).toEqual([]) + expect(result.valid).toBe(true) + }) + + it('accepts a desktop snapshot on macOS via AXAPI', () => { + const snap: Snapshot = { + agentmark: '0.4', + kind: 'desktop', + url: 'desktop://localhost/window/pages-1', + title: 'Pages - Untitled', + desktop_meta: { + platform: 'macos', + process_name: 'Pages', + a11y_backend: 'macos_axapi', + element_count: 12, + }, + body: '# Pages — Untitled\n\nEmpty document.', + } + const result = validateSnapshot(snap) + expect(result.valid).toBe(true) + }) + + it('rejects desktop_meta with unknown platform', () => { + const snap = { + agentmark: '0.4', + kind: 'desktop', + url: 'desktop://localhost/window/1', + title: 'Bad', + desktop_meta: { + // @ts-expect-error — intentionally invalid for the test + platform: 'beos', + }, + body: 'x', + } as unknown as Snapshot + const result = validateSnapshot(snap) + expect(result.valid).toBe(false) + expect(result.errors.some(e => e.path.includes('desktop_meta'))).toBe(true) + }) + + it('rejects desktop_meta with extra fields (closed object)', () => { + const snap = { + agentmark: '0.4', + kind: 'desktop', + url: 'desktop://localhost/window/1', + title: 'Bad', + desktop_meta: { + platform: 'windows', + // @ts-expect-error — additional properties not allowed + weird_field: 'nope', + }, + body: 'x', + } as unknown as Snapshot + const result = validateSnapshot(snap) + expect(result.valid).toBe(false) + }) + + it('accepts [WINDOW:w_1] and [ELEMENT:e_42] body tags without action lookup', () => { + // WINDOW and ELEMENT are structural — payload doesn't need to resolve + // to anything in actions/media/signatures. + const snap: Snapshot = { + agentmark: '0.4', + kind: 'desktop', + url: 'desktop://localhost/window/123', + title: 'Multi-window snapshot', + desktop_meta: { platform: 'windows', a11y_backend: 'windows_uia' }, + body: '[WINDOW:w_1]\n\n## Excel\n\n[ELEMENT:e_42] read-only cell value\n\n[WINDOW:w_2]\n\n## Outlook', + } + const result = validateSnapshot(snap) + expect(result.valid).toBe(true) + }) + + it('round-trips a desktop snapshot through YAML serialization', () => { + const original: Snapshot = { + agentmark: '0.4', + kind: 'desktop', + url: 'desktop://localhost/window/abc', + title: 'NowCerts Customer Detail', + desktop_meta: { + platform: 'windows', + process_name: 'NowCerts.exe', + a11y_backend: 'windows_uia', + element_count: 23, + }, + actions: { + act_search: { type: 'type', label: 'Search', placeholder: 'Search customers...' }, + }, + body: '# NowCerts\n\n[INPUT:act_search]', + } + const serialized = serializeSnapshot(original) + const reparsed = parseSnapshot(serialized) + expect(reparsed.kind).toBe('desktop') + expect(reparsed.desktop_meta?.platform).toBe('windows') + expect(reparsed.desktop_meta?.process_name).toBe('NowCerts.exe') + expect(reparsed.actions?.act_search?.type).toBe('type') + }) + + it('v0.3 snapshots continue to validate against v0.4 (backwards compat)', () => { + const snap: Snapshot = { + agentmark: '0.3', + kind: 'webpage', + url: 'https://example.com/', + title: 'Example', + body: '# Hello', + } + const result = validateSnapshot(snap) + expect(result.valid).toBe(true) + }) +})