From df0049018f3eea0dbbb055ec36802e9eece56c3f Mon Sep 17 00:00:00 2001 From: rrader26 Date: Sun, 10 May 2026 14:48:20 -0400 Subject: [PATCH 1/5] feat(signatures): v0.8 signature detection (AcroForm + label-pattern + heuristic image) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Detects signatures across the three real-world patterns we hit on insurance + government docs and surfaces them in the AgentMark snapshot envelope so agents can answer "who signed what". Architecture - SignatureDetector interface — same plug-and-play shape as OcrBackend. - DetectedSignature record with: kind, page, rect, field_name, inferred_role, signer_name, signer_email, signed_at, confidence, valid, notes. - SignatureDescriptor type added to Snapshot.signatures (additive to v0.2 spec). - New body tag [SIGNATURE:sig_n] with cross-field validation. Three reference detectors (default chain) 1. AcroFormSignatureDetector Walks AcroForm /Sig widgets via the existing extractor. Distinguishes signed vs unsigned via value presence; role from field name. 2. LabelPatternSignatureDetector Catches the gov-form pattern: text fields labeled "Signature Of Employee" etc. that aren't actual /Sig widgets. Critical for IRS/USCIS forms. 3. HeuristicImageSignatureDetector Walks the page operator list, computes image XObject rects from CTM (full PDF affine matrix multiply), filters by signature shape (aspect ratio 1.2-12, width 60-400pt, height 12-100pt), requires proximity to either a "Signature/Sign here/X" label OR a role keyword. Catches hand-signed scanned PDFs that the AcroForm/label detectors miss. Role inference (free, deterministic) - inferRoleFromFieldName: 22 role-keyword patterns matched against normalized snake/camel/kebab/dotted field names. Covers client, agent, broker, buyer, seller, tenant, landlord, witness, notary, guarantor, attorney, employer, employee, applicant, beneficiary, insured/insurer, policyholder, principal, authorized signator(y|ies), co-buyer/co-seller. - inferRoleFromNearbyText: scans text items in the label zone above a signature region (radius=60pt vertically, ±60pt horizontally). Returns matching role + diagnostic snippet for transparency. Pipeline - detectSignatures(input, detectors?) runs all detectors in parallel, dedupes by (page, IoU > 0.5) keeping higher confidence, renumbers IDs to clean sig_1..sig_N. Detector failure is per-detector best-effort (one failing doesn't abort the others). Wired into convertPdf - New options.signatureDetectors (custom chain or [] to disable). - snapshot.signatures populated when ≥1 detection. - Body tag [SIGNATURE:sig_n] insertion deferred to a follow-up — for now signatures live in the envelope only and are LLM-discoverable via the snapshot. Spec + schema updates - agentmark-v0.2.json gains the signatures map + signature $def with full field validation (kind enum, page minimum, confidence range). - BodyTagKind union adds 'SIGNATURE'. - Validator rejects body refs to undefined signature IDs. Tests (243 passing, 34 new) - Role-inference: 23 field-name + 5 nearby-text tests - Pipeline: 6 tests covering empty doc, ID renumbering, IoU dedup, non-overlapping retention, default-chain composition, fault tolerance Validation against real corpus pending — running diagnostic against user's insurance + government + new files in background. Co-Authored-By: Claude Opus 4.7 (1M context) --- schema/agentmark-v0.2.json | 45 +++ src/index.ts | 20 ++ src/pdf/index.ts | 18 ++ src/pdf/pdf-converter.ts | 53 ++++ src/pdf/signatures/acroform-detector.ts | 63 ++++ .../signatures/heuristic-image-detector.ts | 249 ++++++++++++++++ src/pdf/signatures/index.ts | 110 +++++++ src/pdf/signatures/label-pattern-detector.ts | 74 +++++ src/pdf/signatures/role-inference.ts | 129 +++++++++ src/pdf/signatures/types.ts | 81 ++++++ src/types.ts | 37 +++ src/validators/schema-validator.ts | 10 + test/pdf/signatures.test.ts | 268 ++++++++++++++++++ 13 files changed, 1157 insertions(+) create mode 100644 src/pdf/signatures/acroform-detector.ts create mode 100644 src/pdf/signatures/heuristic-image-detector.ts create mode 100644 src/pdf/signatures/index.ts create mode 100644 src/pdf/signatures/label-pattern-detector.ts create mode 100644 src/pdf/signatures/role-inference.ts create mode 100644 src/pdf/signatures/types.ts create mode 100644 test/pdf/signatures.test.ts diff --git a/schema/agentmark-v0.2.json b/schema/agentmark-v0.2.json index c552137..eed0a07 100644 --- a/schema/agentmark-v0.2.json +++ b/schema/agentmark-v0.2.json @@ -44,6 +44,13 @@ "additionalProperties": false }, "document": { "$ref": "#/$defs/document" }, + "signatures": { + "type": "object", + "patternProperties": { + "^[a-z][a-z0-9_]{0,63}$": { "$ref": "#/$defs/signature" } + }, + "additionalProperties": false + }, "memory": { "type": "object" }, "capabilities": { "type": "object" }, "cookies": { "type": "object" }, @@ -133,6 +140,44 @@ "format_version": { "type": "string", "maxLength": 32 }, "ocr_used": { "type": "boolean" } } + }, + "signature": { + "type": "object", + "required": ["kind", "page", "confidence"], + "additionalProperties": false, + "properties": { + "kind": { + "enum": [ + "widget_visible_signed", + "widget_unsigned", + "cryptographic", + "image_handwritten", + "image_typed", + "docusign", + "adobe_sign", + "unknown" + ] + }, + "page": { "type": "integer", "minimum": 1 }, + "rect": { + "type": "object", + "additionalProperties": false, + "properties": { + "x": { "type": "number" }, + "y": { "type": "number" }, + "width": { "type": "number" }, + "height": { "type": "number" } + } + }, + "field_name": { "type": "string", "maxLength": 256 }, + "inferred_role": { "type": "string", "maxLength": 64 }, + "signer_name": { "type": "string", "maxLength": 256 }, + "signer_email": { "type": "string", "maxLength": 256 }, + "signed_at": { "type": "string", "format": "date-time" }, + "confidence": { "type": "number", "minimum": 0, "maximum": 1 }, + "valid": { "type": "boolean" }, + "notes": { "type": "string", "maxLength": 1024 } + } } } } diff --git a/src/index.ts b/src/index.ts index ec8e332..c7ad13d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -149,3 +149,23 @@ export type { PdfDocumentSnapshot, SaveOptions, } from './pdf' + +// ── v0.8: Signature detection ──────────────────────────────────────────── + +export { + detectSignatures, + defaultDetectors, + AcroFormSignatureDetector, + HeuristicImageSignatureDetector, + inferRoleFromFieldName, + inferRoleFromNearbyText, +} from './pdf' +export type { + DetectedSignature, + SignatureDetector, + SignatureDetectorInput, + SignatureKind, + SignatureRole, + HeuristicImageDetectorOptions, +} from './pdf' +export type { SignatureDescriptor } from './types' diff --git a/src/pdf/index.ts b/src/pdf/index.ts index 08a7564..3986e9f 100644 --- a/src/pdf/index.ts +++ b/src/pdf/index.ts @@ -50,3 +50,21 @@ export type { PdfDocumentSnapshot, SaveOptions, } from './forms' + +// ── v0.8: Signature detection ──────────────────────────────────────────── +export { + detectSignatures, + defaultDetectors, + AcroFormSignatureDetector, + HeuristicImageSignatureDetector, + inferRoleFromFieldName, + inferRoleFromNearbyText, +} from './signatures' +export type { + DetectedSignature, + SignatureDetector, + SignatureDetectorInput, + SignatureKind, + SignatureRole, + HeuristicImageDetectorOptions, +} from './signatures' diff --git a/src/pdf/pdf-converter.ts b/src/pdf/pdf-converter.ts index 15a1509..1420c1e 100644 --- a/src/pdf/pdf-converter.ts +++ b/src/pdf/pdf-converter.ts @@ -35,6 +35,13 @@ import type { import type { ExtractedPdf, PdfTextItem } from './types' import { extractAcroForm } from './forms/acroform-extractor' import type { AcroFormField } from './forms/types' +import { + detectSignatures, + defaultDetectors, + type DetectedSignature, + type SignatureDetector, +} from './signatures' +import type { SignatureDescriptor } from '../types' export interface ConvertPdfOptions { /** Raw PDF bytes (from `readFile`, `fetch`, etc.). */ @@ -63,6 +70,12 @@ export interface ConvertPdfOptions { * PdfDocument before body-building. */ ocr?: OcrPipelineOptions + /** + * Custom signature-detector chain. When omitted, runs the default + * detectors (AcroForm Sig widgets + heuristic image signatures). + * Pass an empty array to disable signature detection entirely. + */ + signatureDetectors?: SignatureDetector[] } /** @@ -101,6 +114,26 @@ export async function convertPdf(options: ConvertPdfOptions): Promise 0 + ? await detectSignatures( + { extracted, rawBytes, password: options.password }, + detectorChain, + ).catch((err: Error) => { + logger.warn('signatures.detect.failed', { error: err.message }) + return [] + }) + : [] + const segments = buildBodyFromPdf(extracted, options.body ?? {}) const body = buildBody(segments) @@ -129,6 +162,25 @@ export async function convertPdf(options: ConvertPdfOptions): Promise = {} + for (const sig of signatures) { + signaturesMap[sig.id] = stripUndefined({ + kind: sig.kind, + page: sig.page, + rect: sig.rect, + field_name: sig.field_name, + inferred_role: sig.inferred_role, + signer_name: sig.signer_name, + signer_email: sig.signer_email, + signed_at: sig.signed_at, + confidence: sig.confidence, + valid: sig.valid, + notes: sig.notes, + }) + } + const snapshot: Snapshot = { agentmark: AGENTMARK_VERSION, kind, @@ -140,6 +192,7 @@ export async function convertPdf(options: ConvertPdfOptions): Promise 0 ? signaturesMap : undefined, capabilities: { preview_media: false, expand_disclosures: false, diff --git a/src/pdf/signatures/acroform-detector.ts b/src/pdf/signatures/acroform-detector.ts new file mode 100644 index 0000000..8a988c2 --- /dev/null +++ b/src/pdf/signatures/acroform-detector.ts @@ -0,0 +1,63 @@ +/** + * AcroForm signature widget detector. + * + * Walks the PDF's AcroForm fields looking for `/Sig` widgets. Each becomes + * a DetectedSignature with role inferred from the field name. + * + * Distinguishing signed vs unsigned without reading the cryptographic + * signature dictionary is approximate: pdfjs-dist's `getFieldObjects()` + * exposes a `value` field on signed widgets that's typically null/empty + * when unsigned. We use this as the heuristic; cryptographic verification + * comes in a follow-up release via Poppler's `pdfsig`. + */ + +import { extractAcroForm } from '../forms/acroform-extractor' +import { inferRoleFromFieldName } from './role-inference' +import type { + DetectedSignature, + SignatureDetector, + SignatureDetectorInput, +} from './types' + +export class AcroFormSignatureDetector implements SignatureDetector { + readonly name = 'acroform_widget' + + async detect(input: SignatureDetectorInput): Promise { + const acroform = await extractAcroForm({ + data: input.rawBytes, + password: input.password, + }).catch(() => ({ fields: [], hasFields: false })) + + const detections: DetectedSignature[] = [] + let counter = 0 + for (const field of acroform.fields) { + if (field.kind !== 'signature') continue + counter++ + + // Heuristic for signed vs unsigned: if pdfjs surfaced any value + // we treat it as signed; empty string / undefined → unsigned. + const valuePresent = + typeof field.value === 'string' && field.value.length > 0 + const kind = valuePresent ? 'widget_visible_signed' : 'widget_unsigned' + + const inferred_role = inferRoleFromFieldName(field.fieldName) + const confidence = valuePresent + ? inferred_role ? 0.9 : 0.7 + : inferred_role ? 0.85 : 0.6 + + detections.push({ + id: `sig_a_${counter}`, + kind, + page: field.page, + rect: field.rect, + field_name: field.fieldName, + inferred_role, + confidence, + notes: inferred_role + ? `Role inferred from field name: "${field.fieldName}"` + : `Sig widget — field name "${field.fieldName}" did not match any role pattern`, + }) + } + return detections + } +} diff --git a/src/pdf/signatures/heuristic-image-detector.ts b/src/pdf/signatures/heuristic-image-detector.ts new file mode 100644 index 0000000..4587d12 --- /dev/null +++ b/src/pdf/signatures/heuristic-image-detector.ts @@ -0,0 +1,249 @@ +/** + * Heuristic image-signature detector. + * + * Walks each PDF page's operator list looking for `paintImageXObject` ops, + * computes the image's user-space rectangle from the current transform + * matrix, then filters down to signature-shaped images (aspect ratio, + * size range, position on page) AND requires proximity to either: + * - explicit signature labels ("Signature", "Sign here", "X") + * - any role keyword ("Tenant", "Buyer", etc.) + * + * Heuristic-only — no vision model. False positives are bounded by the + * proximity-to-label requirement; a vision detector layer can be added + * later for higher recall on docs with no labels. + */ + +import { loadPdfjs } from '../pdfjs-loader' +import { inferRoleFromNearbyText } from './role-inference' +import type { + DetectedSignature, + SignatureDetector, + SignatureDetectorInput, +} from './types' + +export interface HeuristicImageDetectorOptions { + /** Max signature aspect ratio (width / height). Default: 12 (very wide is OK; signatures are wider than tall). */ + maxAspectRatio?: number + /** Min signature aspect ratio. Default: 1.2. */ + minAspectRatio?: number + /** Min width in PDF points. Default: 60 (~0.83 inch). */ + minWidth?: number + /** Max width. Default: 400 (~5.5 inch). */ + maxWidth?: number + /** Min height. Default: 12 (~0.17 inch). */ + minHeight?: number + /** Max height. Default: 100 (~1.4 inch). */ + maxHeight?: number +} + +const DEFAULTS: Required = { + maxAspectRatio: 12, + minAspectRatio: 1.2, + minWidth: 60, + maxWidth: 400, + minHeight: 12, + maxHeight: 100, +} + +export class HeuristicImageSignatureDetector implements SignatureDetector { + readonly name = 'heuristic_image' + private readonly opts: Required + + constructor(options: HeuristicImageDetectorOptions = {}) { + this.opts = { ...DEFAULTS, ...options } + } + + async detect(input: SignatureDetectorInput): Promise { + const detections: DetectedSignature[] = [] + const pdfjs = await loadPdfjs() + + // Defensive copy — pdfjs may detach the buffer. + const data = new Uint8Array(input.rawBytes) + const doc = await pdfjs.getDocument({ data, verbosity: 0 }).promise + + try { + const ops = pdfjs.OPS as Record + const PAINT = ops.paintImageXObject + const PAINT_INLINE = ops.paintInlineImageXObject + const TRANSFORM = ops.transform + const SAVE = ops.save + const RESTORE = ops.restore + + for (let pageNum = 1; pageNum <= doc.numPages; pageNum++) { + const page = await doc.getPage(pageNum) + const opList = await page.getOperatorList() + const fns = opList.fnArray + const args = opList.argsArray + + // Walk operators tracking the current transform matrix (CTM). + // This is a simplified model that handles the common forms; + // pdfjs's actual coordinate handling is more elaborate but + // for image-position heuristics this suffices. + const stack: number[][] = [identity()] + let ctm = stack[0] + + let imageCounter = 0 + for (let i = 0; i < fns.length; i++) { + const fn = fns[i] + if (fn === SAVE) { + ctm = clone(ctm) + stack.push(ctm) + } else if (fn === RESTORE) { + stack.pop() + ctm = stack[stack.length - 1] ?? identity() + } else if (fn === TRANSFORM) { + const m = args[i] as number[] + if (m && m.length >= 6) ctm = multiply(ctm, m) + } else if (fn === PAINT || fn === PAINT_INLINE) { + // Image XObject is painted with the current CTM scaled + // to fit a unit-square (0,0)-(1,1). + const rect = ctmToRect(ctm) + if (this.looksLikeSignature(rect)) { + imageCounter++ + const detection = this.tryDetect( + input, + pageNum, + rect, + imageCounter, + ) + if (detection) detections.push(detection) + } + } + } + page.cleanup() + } + } finally { + await doc.destroy() + } + return detections + } + + private looksLikeSignature(rect: { x: number; y: number; width: number; height: number }): boolean { + const { width, height } = rect + if (width <= 0 || height <= 0) return false + if (width < this.opts.minWidth || width > this.opts.maxWidth) return false + if (height < this.opts.minHeight || height > this.opts.maxHeight) return false + const aspect = width / height + if (aspect < this.opts.minAspectRatio || aspect > this.opts.maxAspectRatio) return false + return true + } + + private tryDetect( + input: SignatureDetectorInput, + page: number, + rect: { x: number; y: number; width: number; height: number }, + counter: number, + ): DetectedSignature | null { + // Check proximity to a signature-related label or role keyword. + const roleHit = inferRoleFromNearbyText(input.extracted, { page, rect }) + const sigLabelHit = hasSignatureLabel(input.extracted, page, rect) + + // Require AT LEAST one positive signal. An anonymous image + // somewhere on the page is too noisy to call a signature. + if (!roleHit && !sigLabelHit) return null + + const confidence = roleHit && sigLabelHit + ? 0.85 + : roleHit + ? 0.7 + : 0.55 + + const notes = roleHit + ? `Role from nearby text: "${roleHit.snippet}"` + : 'Image is signature-shaped near a "Signature/Sign/X" label, but no role inferred' + + return { + id: `sig_h_${page}_${counter}`, + kind: 'image_handwritten', + page, + rect, + inferred_role: roleHit?.role, + confidence, + notes, + } + } +} + +// ────────────────────────────────────────────────────────────────────────── +// CTM helpers +// ────────────────────────────────────────────────────────────────────────── + +function identity(): number[] { + return [1, 0, 0, 1, 0, 0] +} + +function clone(m: number[]): number[] { + return [m[0], m[1], m[2], m[3], m[4], m[5]] +} + +/** + * PDF matrix multiplication (3×3 affine, encoded as [a b c d e f]): + * + * | a b 0 | + * | c d 0 | + * | e f 1 | + * + * `m1` is the existing CTM, `m2` is being concat'd onto it. + */ +function multiply(m1: number[], m2: number[]): number[] { + return [ + m1[0] * m2[0] + m1[2] * m2[1], + m1[1] * m2[0] + m1[3] * m2[1], + m1[0] * m2[2] + m1[2] * m2[3], + m1[1] * m2[2] + m1[3] * m2[3], + m1[0] * m2[4] + m1[2] * m2[5] + m1[4], + m1[1] * m2[4] + m1[3] * m2[5] + m1[5], + ] +} + +/** + * Convert a CTM into the user-space rect of the unit square (0,0)-(1,1). + * Image XObjects are by convention drawn into the unit square; the + * transform matrix encodes their actual size + position. + */ +function ctmToRect(ctm: number[]): { x: number; y: number; width: number; height: number } { + // The unit square's corners are (0,0), (1,0), (0,1), (1,1). + // After transform: each corner is (a*x + c*y + e, b*x + d*y + f). + const [a, b, c, d, e, f] = ctm + const xs = [e, a + e, c + e, a + c + e] + const ys = [f, b + f, d + f, b + d + f] + const minX = Math.min(...xs) + const maxX = Math.max(...xs) + const minY = Math.min(...ys) + const maxY = Math.max(...ys) + return { + x: minX, + y: minY, + width: maxX - minX, + height: maxY - minY, + } +} + +// ────────────────────────────────────────────────────────────────────────── +// Label proximity helper +// ────────────────────────────────────────────────────────────────────────── + +const SIGNATURE_LABEL_RE = /\b(signature|signed|sign\s*here|x\s*[:_]|initials?)\b/i + +function hasSignatureLabel( + pdf: import('../types').ExtractedPdf, + pageNum: number, + rect: { x: number; y: number; width: number; height: number }, +): boolean { + const page = pdf.pages.find((p) => p.number === pageNum) + if (!page) return false + + const radius = 80 + const top = rect.y + rect.height + radius + const bottom = rect.y - radius * 0.25 + const left = rect.x - radius + const right = rect.x + rect.width + radius + + for (const item of page.items) { + if (item.y < bottom || item.y > top) continue + const itemRight = item.x + (item.width || 0) + if (itemRight < left || item.x > right) continue + if (SIGNATURE_LABEL_RE.test(item.text)) return true + } + return false +} diff --git a/src/pdf/signatures/index.ts b/src/pdf/signatures/index.ts new file mode 100644 index 0000000..08768c3 --- /dev/null +++ b/src/pdf/signatures/index.ts @@ -0,0 +1,110 @@ +/** + * Signature-detection module. + * + * Public API: + * - detectSignatures(input, detectors?) — runs all configured detectors + * and merges results + * - AcroFormSignatureDetector / HeuristicImageSignatureDetector — bundled + * reference implementations + * - SignatureDetector / DetectedSignature / SignatureKind / SignatureRole types + */ + +import type { + DetectedSignature, + SignatureDetector, + SignatureDetectorInput, +} from './types' +import { AcroFormSignatureDetector } from './acroform-detector' +import { HeuristicImageSignatureDetector } from './heuristic-image-detector' +import { LabelPatternSignatureDetector } from './label-pattern-detector' + +export type { + DetectedSignature, + SignatureDetector, + SignatureDetectorInput, + SignatureKind, + SignatureRole, +} from './types' +export { AcroFormSignatureDetector } from './acroform-detector' +export { + HeuristicImageSignatureDetector, +} from './heuristic-image-detector' +export type { + HeuristicImageDetectorOptions, +} from './heuristic-image-detector' +export { LabelPatternSignatureDetector } from './label-pattern-detector' +export { + inferRoleFromFieldName, + inferRoleFromNearbyText, +} from './role-inference' + +/** + * Default detector chain — runs AcroForm detection first (cheap + reliable), + * then heuristic image detection. Override by passing a custom array. + */ +export function defaultDetectors(): SignatureDetector[] { + return [ + new AcroFormSignatureDetector(), + new LabelPatternSignatureDetector(), + new HeuristicImageSignatureDetector(), + ] +} + +/** + * Run a chain of detectors and merge results, deduping overlapping + * detections by (page, IoU > 0.5). Renumbers IDs to a clean `sig_1` … + * `sig_N` ordering across all detectors. + */ +export async function detectSignatures( + input: SignatureDetectorInput, + detectors: SignatureDetector[] = defaultDetectors(), +): Promise { + const all: DetectedSignature[] = [] + for (const d of detectors) { + try { + const found = await d.detect(input) + for (const f of found) all.push(f) + } catch { + // Detectors are best-effort; one failing should not abort the others. + } + } + const merged = deduplicate(all) + // Renumber to clean sig_1 .. sig_N + return merged.map((sig, i) => ({ ...sig, id: `sig_${i + 1}` })) +} + +/** + * Drop duplicates: when two detections on the same page overlap by IoU > 0.5, + * keep the one with higher confidence. + */ +function deduplicate(detections: DetectedSignature[]): DetectedSignature[] { + const sorted = [...detections].sort((a, b) => b.confidence - a.confidence) + const kept: DetectedSignature[] = [] + for (const candidate of sorted) { + const overlap = kept.find( + (k) => k.page === candidate.page && k.rect && candidate.rect && iou(k.rect, candidate.rect) > 0.5, + ) + if (!overlap) kept.push(candidate) + } + return kept +} + +interface Rect { x: number; y: number; width: number; height: number } + +function iou(a: Rect, b: Rect): number { + const ax2 = a.x + a.width + const ay2 = a.y + a.height + const bx2 = b.x + b.width + const by2 = b.y + b.height + const ix1 = Math.max(a.x, b.x) + const iy1 = Math.max(a.y, b.y) + const ix2 = Math.min(ax2, bx2) + const iy2 = Math.min(ay2, by2) + const iw = Math.max(0, ix2 - ix1) + const ih = Math.max(0, iy2 - iy1) + const inter = iw * ih + const aArea = a.width * a.height + const bArea = b.width * b.height + const union = aArea + bArea - inter + return union <= 0 ? 0 : inter / union +} diff --git a/src/pdf/signatures/label-pattern-detector.ts b/src/pdf/signatures/label-pattern-detector.ts new file mode 100644 index 0000000..56001cc --- /dev/null +++ b/src/pdf/signatures/label-pattern-detector.ts @@ -0,0 +1,74 @@ +/** + * Label-pattern signature detector. + * + * Many government and legacy forms use plain *text* AcroForm fields labeled + * "Signature" / "Signed by" / "X" / role+signature instead of the proper + * `/Sig` widget type. The AcroForm detector only catches `/Sig` fields; + * this one finds the text-field-as-signature pattern. + * + * Heuristic: any AcroForm text field whose name OR label contains a + * signature keyword. Combined with role inference from the same name/label + * to figure out who's signing. + */ + +import { extractAcroForm } from '../forms/acroform-extractor' +import { inferRoleFromFieldName } from './role-inference' +import type { + DetectedSignature, + SignatureDetector, + SignatureDetectorInput, +} from './types' + +const SIGNATURE_LABEL_RE = + /\b(signature|signed\s*by|sign\s*here|initial(?:s|ed)?|autograph|sign-?off)\b/i + +export class LabelPatternSignatureDetector implements SignatureDetector { + readonly name = 'label_pattern' + + async detect(input: SignatureDetectorInput): Promise { + const acroform = await extractAcroForm({ + data: input.rawBytes, + password: input.password, + }).catch(() => ({ fields: [], hasFields: false })) + + const detections: DetectedSignature[] = [] + let counter = 0 + for (const field of acroform.fields) { + // Only text-type fields — Sig widgets handled by AcroForm detector. + if (field.kind !== 'text' && field.kind !== 'unknown') continue + + const haystack = `${field.fieldName} ${field.label}` + if (!SIGNATURE_LABEL_RE.test(haystack)) continue + + counter++ + const inferred_role = + inferRoleFromFieldName(field.fieldName) + ?? inferRoleFromFieldName(field.label) + + const valuePresent = + typeof field.value === 'string' && field.value.length > 0 + + // Treat as widget_unsigned (text-field-as-signature is unsigned by + // design — there's nothing cryptographic about it). When the user + // has typed a name into the field, kind stays unsigned but the + // value flows through normally. + const confidence = inferred_role + ? valuePresent ? 0.85 : 0.7 + : 0.55 + + detections.push({ + id: `sig_l_${counter}`, + kind: 'widget_unsigned', + page: field.page, + rect: field.rect, + field_name: field.fieldName, + inferred_role, + confidence, + notes: + `Text-field-as-signature: name="${field.fieldName}", ` + + `label="${field.label}"${inferred_role ? `, role inferred from name/label` : ''}`, + }) + } + return detections + } +} diff --git a/src/pdf/signatures/role-inference.ts b/src/pdf/signatures/role-inference.ts new file mode 100644 index 0000000..9abbaba --- /dev/null +++ b/src/pdf/signatures/role-inference.ts @@ -0,0 +1,129 @@ +/** + * Infer the role of a signer (client, agent, witness, etc.) from a field + * name OR from text near the signature region. + * + * Two-tier strategy: + * 1. Pattern match on field name — covers most AcroForm Sig widgets. + * 2. Scan nearby text for role keywords — covers image / scanned signatures. + * + * No LLM call here — purely string heuristics. An LLM-backed detector can + * be added as a higher-confidence layer later. + */ + +import type { ExtractedPdf, PdfTextItem } from '../types' +import type { SignatureRole } from './types' + +/** + * Canonical role tokens. Ordering matters when multiple match — earlier + * entries take precedence. Compound roles (e.g. "co-buyer") fall back to + * their primary role ("buyer") via the substring check. + */ +const ROLE_PATTERNS: Array<{ role: SignatureRole; regex: RegExp }> = [ + { role: 'notary', regex: /\b(notary|notar(?:y|ies))\b/i }, + { role: 'witness', regex: /\bwitness(es)?\b/i }, + { role: 'broker', regex: /\b(broker|brokerage)\b/i }, + { role: 'agent', regex: /\b(agent|representative|rep\.?)\b/i }, + { role: 'attorney', regex: /\b(attorney|counsel|lawyer)\b/i }, + { role: 'co-buyer', regex: /\bco[- ]?buyer\b/i }, + { role: 'co-seller', regex: /\bco[- ]?seller\b/i }, + { role: 'buyer', regex: /\bbuyer\b/i }, + { role: 'seller', regex: /\bseller\b/i }, + { role: 'tenant', regex: /\b(tenant|lessee)\b/i }, + { role: 'landlord', regex: /\b(landlord|lessor)\b/i }, + { role: 'guarantor', regex: /\b(guarantor|co[- ]?signer|cosigner)\b/i }, + { role: 'employer', regex: /\bemployer\b/i }, + { role: 'employee', regex: /\bemployee\b/i }, + { role: 'applicant', regex: /\bapplicant\b/i }, + { role: 'beneficiary', regex: /\bbeneficiary\b/i }, + { role: 'insured', regex: /\b(insured|policyholder|policy.?holder)\b/i }, + { role: 'insurer', regex: /\b(insurer|underwriter)\b/i }, + { role: 'client', regex: /\b(client|customer)\b/i }, + { role: 'principal', regex: /\bprincipal\b/i }, + { role: 'authorized', regex: /\bauthoriz(ed|ing) signator(?:y|ies)\b/i }, +] + +/** + * Look up a role from a field name. Field names are typically snake_case, + * camelCase, kebab-case, or use dot notation. Normalize to spaces and + * scan against ROLE_PATTERNS. + */ +export function inferRoleFromFieldName(fieldName: string): SignatureRole | undefined { + if (!fieldName) return undefined + const normalized = fieldName + .replace(/[._-]+/g, ' ') + .replace(/([a-z])([A-Z])/g, '$1 $2') + .toLowerCase() + for (const { role, regex } of ROLE_PATTERNS) { + if (regex.test(normalized)) return role + } + return undefined +} + +export interface NearbyTextLookupOptions { + /** Page number (1-indexed) the signature is on. */ + page: number + /** Bounding rect of the signature region in PDF user-space. */ + rect: { x: number; y: number; width: number; height: number } + /** + * Search radius (PDF points). Typical body text is ~11pt; we look up to + * 60pt above the signature (about 4 lines) and ~10pt left/right of the + * signature's left/right edges. + */ + radius?: number +} + +/** + * Find a role keyword in text near a signature region. + * + * Scans text items on the same page that fall within a "label zone": + * - vertically: from `rect.y + rect.height` (the top of the signature) + * up to `rect.y + rect.height + radius` (above the signature) + * - horizontally: from `rect.x - radius` to `rect.x + rect.width + radius` + * + * Returns the first matching role plus the matched text snippet for + * diagnostic notes. The label is typically immediately above the signature + * line ("Tenant Signature:" / "Buyer:"). + */ +export function inferRoleFromNearbyText( + pdf: ExtractedPdf, + opts: NearbyTextLookupOptions, +): { role: SignatureRole; snippet: string } | undefined { + const radius = opts.radius ?? 60 + const page = pdf.pages.find((p) => p.number === opts.page) + if (!page) return undefined + + const top = opts.rect.y + opts.rect.height + const bottom = opts.rect.y - radius * 0.25 // tolerate small overlap + const left = opts.rect.x - radius + const right = opts.rect.x + opts.rect.width + radius + const labelZoneTop = top + radius + + // Collect items in the zone (above the signature, with some horizontal + // overlap). PDF origin is bottom-left so larger Y = higher on page. + const candidates: PdfTextItem[] = [] + for (const item of page.items) { + if (item.y < bottom || item.y > labelZoneTop) continue + const itemRight = item.x + (item.width || 0) + if (itemRight < left || item.x > right) continue + candidates.push(item) + } + if (candidates.length === 0) return undefined + + // Sort candidates so items closest to the signature (smallest |item.y - top|) + // are scanned first. This biases toward the immediate label. + candidates.sort((a, b) => Math.abs(a.y - top) - Math.abs(b.y - top)) + + // Concatenate item text in reading order for snippet building, but match + // against full concatenation so multi-word labels ("co-buyer") are found. + const concatenated = candidates.map((c) => c.text).join(' ') + for (const { role, regex } of ROLE_PATTERNS) { + const match = concatenated.match(regex) + if (match) { + // Return a short snippet around the match for diagnostic notes. + const start = Math.max(0, concatenated.indexOf(match[0]) - 20) + const end = Math.min(concatenated.length, start + 80) + return { role, snippet: concatenated.slice(start, end).trim() } + } + } + return undefined +} diff --git a/src/pdf/signatures/types.ts b/src/pdf/signatures/types.ts new file mode 100644 index 0000000..044491a --- /dev/null +++ b/src/pdf/signatures/types.ts @@ -0,0 +1,81 @@ +/** + * Signature-detection types. + * + * AgentMark surfaces signatures (digital + hand-drawn + cryptographic) found + * in a document so an agent can answer "who signed this and in what role?" + * without reading every page. + */ + +import type { ExtractedPdf } from '../types' + +export type SignatureKind = + /** AcroForm `/Sig` widget that has been signed (has appearance + cert). */ + | 'widget_visible_signed' + /** AcroForm `/Sig` widget that is empty / awaiting a signature. */ + | 'widget_unsigned' + /** Cryptographic PKCS#7 signature on the document — verified or not (kind says nothing about validity; see `valid` field). */ + | 'cryptographic' + /** A signature-shaped image embedded on the page (hand-drawn scan or tablet capture). */ + | 'image_handwritten' + /** Typed text in a script-style font near a "Signature:" label. */ + | 'image_typed' + /** DocuSign envelope-style signature with audit trail. */ + | 'docusign' + /** Adobe Sign envelope-style signature with audit trail. */ + | 'adobe_sign' + /** Detected as something signature-shaped but the kind couldn't be narrowed. */ + | 'unknown' + +/** + * Inferred role of the signer. Free-form lowercase string. Common values: + * client, agent, broker, buyer, seller, tenant, landlord, witness, notary, + * guarantor, employer, employee, attorney, applicant. Use 'unknown' when + * the role can't be inferred. + */ +export type SignatureRole = string + +export interface DetectedSignature { + /** Stable ID, e.g. `sig_1`. Match against `[SIGNATURE:sig_1]` body tags. */ + id: string + kind: SignatureKind + /** 1-indexed page where the signature was detected. */ + page: number + /** Position on the page (PDF user space, page-local), when known. */ + rect?: { x: number; y: number; width: number; height: number } + /** Original PDF field name when sourced from an AcroForm Sig widget. */ + field_name?: string + /** Inferred role of the signer (client, agent, witness, etc.). */ + inferred_role?: SignatureRole + /** Signer's name — from a cert subject, surrounding text, or DocuSign audit. */ + signer_name?: string + /** Signer's email — from a cert or audit trail. */ + signer_email?: string + /** ISO 8601 timestamp when the signature was applied, when known. */ + signed_at?: string + /** Confidence the detection is real and the role/name are correct (0-1). */ + confidence: number + /** For cryptographic sigs: validation result, when checked. */ + valid?: boolean + /** Free-form notes — surrounding text snippet, label match, etc. */ + notes?: string +} + +/** + * A detector turns extracted PDF data into a list of DetectedSignature. + * Implementations run independently; the pipeline merges results across + * detectors and deduplicates overlapping detections by page + rect overlap. + */ +export interface SignatureDetector { + /** Implementation name — surfaced in detection notes for debugging. */ + readonly name: string + detect(input: SignatureDetectorInput): Promise +} + +export interface SignatureDetectorInput { + /** Extracted PDF (text items per page, metadata). */ + extracted: ExtractedPdf + /** Raw PDF bytes — needed by detectors that call back into pdfjs/poppler. */ + rawBytes: Uint8Array + /** Optional password for encrypted PDFs. */ + password?: string +} diff --git a/src/types.ts b/src/types.ts index 7da17aa..4a23dfc 100644 --- a/src/types.ts +++ b/src/types.ts @@ -52,6 +52,14 @@ export interface Snapshot { /** Document-specific metadata (v0.2+, populated when kind === 'document'). */ document?: DocumentMeta + /** + * Detected signatures on the document, keyed by signature ID + * (e.g. `sig_1`). Body uses `[SIGNATURE:sig_1]` to reference them. + * Populated by the signature-detection pipeline (v0.8+) when the + * source surface is a PDF. + */ + signatures?: Record + /** The Markdown body */ body: string @@ -229,6 +237,35 @@ export type BodyTagKind = /** v0.2+: page boundary marker for `kind: 'document'`. Payload is a * page identifier like `p_1` whose number maps to the source PDF page. */ | 'PAGE' + /** v0.8+: signature reference. Payload is a signature ID (e.g. `sig_1`) + * whose details live in the `signatures` map of the envelope. */ + | 'SIGNATURE' + +/** + * Descriptor for a detected signature. Lives in `Snapshot.signatures` keyed + * by ID. Body references via `[SIGNATURE:sig_1]`. + */ +export interface SignatureDescriptor { + kind: + | 'widget_visible_signed' + | 'widget_unsigned' + | 'cryptographic' + | 'image_handwritten' + | 'image_typed' + | 'docusign' + | 'adobe_sign' + | 'unknown' + page: number + rect?: { x: number; y: number; width: number; height: number } + field_name?: string + inferred_role?: string + signer_name?: string + signer_email?: string + signed_at?: string + confidence: number + valid?: boolean + notes?: string +} export interface BodyTagReference { kind: BodyTagKind diff --git a/src/validators/schema-validator.ts b/src/validators/schema-validator.ts index 3f9753f..1132e9f 100644 --- a/src/validators/schema-validator.ts +++ b/src/validators/schema-validator.ts @@ -88,11 +88,14 @@ export function validateSnapshot(snapshot: Snapshot): ValidationResult { // No-payload tags: AUTH_WALL const ACTION_RESOLVING = new Set(['ACTION', 'INPUT', 'NAV', 'TOAST']) const MEDIA_RESOLVING = new Set(['MEDIA']) + const SIGNATURE_RESOLVING = new Set(['SIGNATURE']) const PAYLOAD_TAGS = new Set(['ERROR', 'CHALLENGE']) // PAGE is a v0.2 structural marker for `kind: 'document'` — payload is a // page identifier (e.g. p_1) that does not resolve to any envelope entry. const STRUCTURAL_TAGS = new Set(['PAGE']) + const signatureIds = new Set(Object.keys(snapshot.signatures ?? {})) + const bodyRefs = extractTagReferences(body) for (const ref of bodyRefs) { if (!ref.payload) continue // AUTH_WALL etc. @@ -112,6 +115,13 @@ export function validateSnapshot(snapshot: Snapshot): ValidationResult { message: `Body references ${ref.kind}:${ref.payload} but no matching media is defined`, }) } + if (SIGNATURE_RESOLVING.has(ref.kind) && !signatureIds.has(ref.payload)) { + errors.push({ + severity: 'error', + path: `body[${ref.position}]`, + message: `Body references ${ref.kind}:${ref.payload} but no matching signature is defined in the envelope`, + }) + } // MODAL/TAB/DISCLOSURE refs are structural — payload is a label, // optionally matched to an action via region_id but not required to. } diff --git a/test/pdf/signatures.test.ts b/test/pdf/signatures.test.ts new file mode 100644 index 0000000..52e30f3 --- /dev/null +++ b/test/pdf/signatures.test.ts @@ -0,0 +1,268 @@ +/** + * Signature detection tests. + * + * Builds fillable PDFs with signature widgets via pdf-lib (which doesn't + * expose Sig fields in its high-level API) — so we use AcroForm text-style + * proxies named like "client_signature" to exercise the role-inference and + * AcroForm-detector codepaths with deterministic fixtures. + * + * For the heuristic image detector we'd need to embed actual image XObjects; + * that's covered by an end-to-end run against a real corpus rather than + * synthetic fixtures. + */ + +import { describe, it, expect } from 'vitest' +import { + inferRoleFromFieldName, + inferRoleFromNearbyText, +} from '../../src/pdf/signatures/role-inference' +import { detectSignatures, defaultDetectors } from '../../src/pdf/signatures' +import type { ExtractedPdf } from '../../src/pdf/types' + +// ────────────────────────────────────────────────────────────────────────── +// inferRoleFromFieldName +// ────────────────────────────────────────────────────────────────────────── + +describe('inferRoleFromFieldName', () => { + const cases: Array<[string, string | undefined]> = [ + ['client_signature', 'client'], + ['ClientSignature', 'client'], + ['client.sig', 'client'], + ['agent_sig', 'agent'], + ['broker_signature', 'broker'], + ['tenant_sig', 'tenant'], + ['landlord-signature', 'landlord'], + ['buyer_initials', 'buyer'], + ['seller_signature', 'seller'], + ['co_buyer_sig', 'co-buyer'], + ['witness_1_sig', 'witness'], + ['notary_block', 'notary'], + ['guarantor_signature', 'guarantor'], + ['cosigner_sig', 'guarantor'], + ['employee_signature', 'employee'], + ['employer_sig', 'employer'], + ['attorney_signature', 'attorney'], + ['policyholder_sig', 'insured'], + ['insured_signature', 'insured'], + ['Customer_Signature', 'client'], + ['just_a_field', undefined], + ['', undefined], + ['form_field_42', undefined], + ] + + for (const [input, expected] of cases) { + it(`maps "${input}" → ${expected ?? 'undefined'}`, () => { + expect(inferRoleFromFieldName(input)).toBe(expected) + }) + } +}) + +// ────────────────────────────────────────────────────────────────────────── +// inferRoleFromNearbyText +// ────────────────────────────────────────────────────────────────────────── + +function fakePdf(items: Array<{ text: string; x: number; y: number; width?: number }>): ExtractedPdf { + return { + pages: [ + { + number: 1, + width: 612, + height: 792, + items: items.map((it) => ({ + text: it.text, + fontSize: 11, + fontName: 'Helvetica', + x: it.x, + y: it.y, + width: it.width ?? it.text.length * 5.5, + hasEol: false, + })), + }, + ], + metadata: { pages: 1, format: 'pdf' }, + } +} + +describe('inferRoleFromNearbyText', () => { + it('finds a label directly above the signature region', () => { + const pdf = fakePdf([ + { text: 'Tenant Signature:', x: 50, y: 200 }, + ]) + const result = inferRoleFromNearbyText(pdf, { + page: 1, + rect: { x: 60, y: 170, width: 200, height: 25 }, + }) + expect(result?.role).toBe('tenant') + }) + + it('returns undefined when no role keyword is in the label zone', () => { + const pdf = fakePdf([ + { text: 'Some unrelated header', x: 50, y: 200 }, + ]) + const result = inferRoleFromNearbyText(pdf, { + page: 1, + rect: { x: 60, y: 170, width: 200, height: 25 }, + }) + expect(result).toBeUndefined() + }) + + it('finds a role even when the label is several lines above (within radius)', () => { + const pdf = fakePdf([ + { text: 'BUYER', x: 50, y: 230 }, + { text: 'Print name:', x: 50, y: 215 }, + ]) + const result = inferRoleFromNearbyText(pdf, { + page: 1, + rect: { x: 50, y: 170, width: 200, height: 25 }, + radius: 80, + }) + expect(result?.role).toBe('buyer') + }) + + it('uses ROLE_PATTERNS precedence (notary beats client when both present)', () => { + const pdf = fakePdf([ + { text: 'Client and notary', x: 50, y: 200 }, + ]) + const result = inferRoleFromNearbyText(pdf, { + page: 1, + rect: { x: 50, y: 170, width: 200, height: 25 }, + }) + // Earlier patterns win — notary precedes client in the table. + expect(result?.role).toBe('notary') + }) + + it('ignores items outside the horizontal zone', () => { + const pdf = fakePdf([ + // Far to the right — outside horizontal zone of the signature + { text: 'Tenant', x: 500, y: 200 }, + ]) + const result = inferRoleFromNearbyText(pdf, { + page: 1, + rect: { x: 50, y: 170, width: 200, height: 25 }, + radius: 60, + }) + expect(result).toBeUndefined() + }) +}) + +// ────────────────────────────────────────────────────────────────────────── +// Detection pipeline +// ────────────────────────────────────────────────────────────────────────── + +describe('detectSignatures pipeline', () => { + it('runs default detectors without throwing on a doc with no signatures', async () => { + const pdf = fakePdf([{ text: 'No signatures here', x: 50, y: 700 }]) + const result = await detectSignatures( + { extracted: pdf, rawBytes: new Uint8Array() }, + // Empty array — disables both detectors but still returns [] + [], + ) + expect(result).toEqual([]) + }) + + it('renumbers IDs across detectors (sig_1, sig_2, ...)', async () => { + // Use non-overlapping positions per detector so dedup doesn't merge. + const detectorAt = (offsetX: number, count: number) => ({ + name: `fake_${offsetX}`, + async detect() { + return Array.from({ length: count }, (_, i) => ({ + id: `original_${i}`, + kind: 'unknown' as const, + page: 1, + confidence: 0.8 - i * 0.05, + rect: { x: offsetX + i * 200, y: 100, width: 100, height: 30 }, + })) + }, + }) + + const result = await detectSignatures( + { extracted: fakePdf([]), rawBytes: new Uint8Array() }, + [detectorAt(0, 2), detectorAt(50, 3)], + ) + // Expected: 2 detections at x=0,200 from the first; 3 at x=50,250,450 + // from the second. The pairs (x=0, x=50) and (x=200, x=250) overlap + // (50pt out of 100pt width = 33% IoU which is below the 50% threshold). + // So all 5 survive. + expect(result).toHaveLength(5) + expect(result.map((s) => s.id)).toEqual(['sig_1', 'sig_2', 'sig_3', 'sig_4', 'sig_5']) + }) + + it('deduplicates overlapping detections by IoU > 0.5, keeping higher confidence', async () => { + const overlapping = (id: string, x: number, conf: number) => ({ + name: `det_${id}`, + async detect() { + return [{ + id, + kind: 'unknown' as const, + page: 1, + rect: { x, y: 100, width: 100, height: 30 }, + confidence: conf, + }] + }, + }) + + const result = await detectSignatures( + { extracted: fakePdf([]), rawBytes: new Uint8Array() }, + [ + overlapping('low', 100, 0.5), + overlapping('high', 105, 0.9), // overlaps the first by ~95% + ], + ) + // Only one survives — the higher-confidence detection. + expect(result.length).toBe(1) + expect(result[0].confidence).toBe(0.9) + }) + + it('keeps non-overlapping detections from the same page', async () => { + const at = (x: number, conf: number) => ({ + name: `det_${x}`, + async detect() { + return [{ + id: 'sig', + kind: 'unknown' as const, + page: 1, + rect: { x, y: 100, width: 100, height: 30 }, + confidence: conf, + }] + }, + }) + + const result = await detectSignatures( + { extracted: fakePdf([]), rawBytes: new Uint8Array() }, + [at(50, 0.8), at(400, 0.7)], + ) + expect(result.length).toBe(2) + }) + + it('default detector chain has both AcroForm and heuristic image detectors', () => { + const detectors = defaultDetectors() + const names = detectors.map((d) => d.name) + expect(names).toEqual(expect.arrayContaining(['acroform_widget', 'heuristic_image'])) + }) + + it('one detector throwing does not abort the others', async () => { + const flaky = { + name: 'flaky', + async detect() { + throw new Error('intermittent failure') + }, + } + const reliable = { + name: 'reliable', + async detect() { + return [{ + id: 'r_1', + kind: 'unknown' as const, + page: 1, + confidence: 0.9, + }] + }, + } + const result = await detectSignatures( + { extracted: fakePdf([]), rawBytes: new Uint8Array() }, + [flaky, reliable], + ) + expect(result.length).toBe(1) + expect(result[0].confidence).toBe(0.9) + }) +}) From 76b6883fc1c130afe5b1b5f973c5378a781d9b0b Mon Sep 17 00:00:00 2001 From: rrader26 Date: Sun, 10 May 2026 14:53:07 -0400 Subject: [PATCH 2/5] =?UTF-8?q?fix(signatures):=20three-tier=20role=20infe?= =?UTF-8?q?rence=20(name=20=E2=86=92=20label=20=E2=86=92=20nearby=20text)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real-world finding from running the v0.8 detectors against a corpus of ~50 mixed personal/business PDFs (insurance, rentals, gov forms, scans): - Form-builder-generated AcroForm Sig widgets often have random field names (e.g. "HelloSignature_79936460") that no role pattern matches. - Text-field-as-signature labels are sometimes generic ("Provide R Signature") rather than role-bearing. Both AcroFormSignatureDetector and LabelPatternSignatureDetector now fall back to inferRoleFromNearbyText when neither field name nor label yields a role. Found "LANDLORD:" near a HelloSign widget in the rental contract test → correctly inferred role: landlord. Notes field now records which tier of inference was used (field name / label / nearby text + snippet) for transparency. No new tests — existing role-inference tests cover the helpers; this change is composition only. Validated via re-running the corpus diagnostic. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/pdf/signatures/acroform-detector.ts | 33 +++++++++++++++++--- src/pdf/signatures/label-pattern-detector.ts | 25 +++++++++++++-- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/src/pdf/signatures/acroform-detector.ts b/src/pdf/signatures/acroform-detector.ts index 8a988c2..420aabd 100644 --- a/src/pdf/signatures/acroform-detector.ts +++ b/src/pdf/signatures/acroform-detector.ts @@ -12,7 +12,7 @@ */ import { extractAcroForm } from '../forms/acroform-extractor' -import { inferRoleFromFieldName } from './role-inference' +import { inferRoleFromFieldName, inferRoleFromNearbyText } from './role-inference' import type { DetectedSignature, SignatureDetector, @@ -40,7 +40,32 @@ export class AcroFormSignatureDetector implements SignatureDetector { typeof field.value === 'string' && field.value.length > 0 const kind = valuePresent ? 'widget_visible_signed' : 'widget_unsigned' - const inferred_role = inferRoleFromFieldName(field.fieldName) + // Three-tier role inference. Field name is fastest + most reliable + // when the name has semantic meaning ("client_signature"); fall + // back to label, then to nearby text. The third tier catches + // form-builder-generated random IDs (HelloSign, etc.). + let inferred_role = inferRoleFromFieldName(field.fieldName) + let role_source = inferred_role ? 'field name' : '' + if (!inferred_role) { + const labelRole = inferRoleFromFieldName(field.label) + if (labelRole) { + inferred_role = labelRole + role_source = 'label' + } + } + let nearbySnippet: string | undefined + if (!inferred_role && field.rect) { + const nearby = inferRoleFromNearbyText(input.extracted, { + page: field.page, + rect: field.rect, + }) + if (nearby) { + inferred_role = nearby.role + nearbySnippet = nearby.snippet + role_source = 'nearby text' + } + } + const confidence = valuePresent ? inferred_role ? 0.9 : 0.7 : inferred_role ? 0.85 : 0.6 @@ -54,8 +79,8 @@ export class AcroFormSignatureDetector implements SignatureDetector { inferred_role, confidence, notes: inferred_role - ? `Role inferred from field name: "${field.fieldName}"` - : `Sig widget — field name "${field.fieldName}" did not match any role pattern`, + ? `Role from ${role_source}${nearbySnippet ? `: "${nearbySnippet}"` : ` "${field.fieldName}"`}` + : `Sig widget "${field.fieldName}" — no role pattern matched`, }) } return detections diff --git a/src/pdf/signatures/label-pattern-detector.ts b/src/pdf/signatures/label-pattern-detector.ts index 56001cc..636f87e 100644 --- a/src/pdf/signatures/label-pattern-detector.ts +++ b/src/pdf/signatures/label-pattern-detector.ts @@ -12,7 +12,7 @@ */ import { extractAcroForm } from '../forms/acroform-extractor' -import { inferRoleFromFieldName } from './role-inference' +import { inferRoleFromFieldName, inferRoleFromNearbyText } from './role-inference' import type { DetectedSignature, SignatureDetector, @@ -41,9 +41,23 @@ export class LabelPatternSignatureDetector implements SignatureDetector { if (!SIGNATURE_LABEL_RE.test(haystack)) continue counter++ - const inferred_role = + let inferred_role = inferRoleFromFieldName(field.fieldName) ?? inferRoleFromFieldName(field.label) + // Fall back to surrounding-text inference when neither name nor + // label contains a role keyword — covers form-builder-generated + // random IDs and labels like "Provide R Signature". + let nearbySnippet: string | undefined + if (!inferred_role && field.rect) { + const nearby = inferRoleFromNearbyText(input.extracted, { + page: field.page, + rect: field.rect, + }) + if (nearby) { + inferred_role = nearby.role + nearbySnippet = nearby.snippet + } + } const valuePresent = typeof field.value === 'string' && field.value.length > 0 @@ -66,7 +80,12 @@ export class LabelPatternSignatureDetector implements SignatureDetector { confidence, notes: `Text-field-as-signature: name="${field.fieldName}", ` - + `label="${field.label}"${inferred_role ? `, role inferred from name/label` : ''}`, + + `label="${field.label}"` + + (inferred_role + ? nearbySnippet + ? `, role from nearby text: "${nearbySnippet}"` + : `, role from field` + : ''), }) } return detections From 98c1ebd9dbaf4241400ba3e2b83975dd2ca62b73 Mon Sep 17 00:00:00 2001 From: rrader26 Date: Sun, 10 May 2026 15:17:39 -0400 Subject: [PATCH 3/5] feat(vision): VisionBackend + VisionSignatureDetector (v0.9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vision-based detection unlocks the cases v0.8 heuristics miss: - Hand-signed scans where the whole page is a single rasterized image - Signatures without nearby text labels - Typed cursive-name signatures - Anywhere bbox + role inference can be done from rendered pixels Architecture - VisionBackend interface — same plug-and-play shape as OcrBackend. Generic enough to be reused by video frame captioning in v0.11. - ClaudeVisionBackend — calls Anthropic /v1/messages with image input + tool-use for structured output. No SDK dep — uses fetch. Default model: claude-haiku-4-5-20251001. - OpenAiVisionBackend — calls /v1/chat/completions with image_url + response_format: json_schema. No SDK dep. Default: gpt-4o-mini. - Both authenticate via env (ANTHROPIC_API_KEY / OPENAI_API_KEY) or constructor option. VisionSignatureDetector - Composes any RenderBackend (Poppler/pdfjs) with any VisionBackend. - Cost-conscious by default: pages='last' scans only the last 2 pages where signatures usually live. Override via 'all' / 'flagged' (pages whose extracted text contains a signature label) / number[]. - Renders → vision JSON-schema query → DetectedSignature[] with bbox converted from normalized 0-1 to PDF user-space. - Min confidence threshold filters low-confidence vision detections. - Per-page failures don't abort the run. Validation contract - Schema enforces kind enum + 0-1 ranges on bbox + confidence so the vision model's response can be trusted as structured. - "unknown" role is dropped (not surfaced to the user). - inferred_role is lowercased for downstream consistency. Tests (9 new, 262 total) - 'last' / 'all' / 'flagged' / explicit page-list modes - bbox → rect coordinate conversion - minConfidence filtering - role lowercase + 'unknown' drop - per-page failure tolerance Wired into the public API - Exported from src/pdf/vision/index.ts and re-exported at src/pdf and src/index. Both Claude + OpenAI backends ship as ready-to-use reference implementations; bring-your-own (Anthropic Claude Code, Apple Vision, Gemini, etc.) by implementing the interface. Not yet wired - VisionSignatureDetector is NOT in defaultDetectors() — it costs API $$ per page. Callers opt in by adding it to the chain. Future MCP pdf_open could expose enable_vision_signatures=true. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/index.ts | 19 +- src/pdf/index.ts | 16 ++ src/pdf/signatures/index.ts | 2 + src/pdf/signatures/vision-detector.ts | 225 +++++++++++++++++++++++ src/pdf/vision/claude-backend.ts | 154 ++++++++++++++++ src/pdf/vision/index.ts | 14 ++ src/pdf/vision/openai-backend.ts | 136 ++++++++++++++ src/pdf/vision/types.ts | 55 ++++++ test/pdf/vision-signature.test.ts | 251 ++++++++++++++++++++++++++ 9 files changed, 871 insertions(+), 1 deletion(-) create mode 100644 src/pdf/signatures/vision-detector.ts create mode 100644 src/pdf/vision/claude-backend.ts create mode 100644 src/pdf/vision/index.ts create mode 100644 src/pdf/vision/openai-backend.ts create mode 100644 src/pdf/vision/types.ts create mode 100644 test/pdf/vision-signature.test.ts diff --git a/src/index.ts b/src/index.ts index c7ad13d..46fb5c0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -150,13 +150,15 @@ export type { SaveOptions, } from './pdf' -// ── v0.8: Signature detection ──────────────────────────────────────────── +// ── v0.8 + v0.9: Signature detection (heuristic + vision) ──────────────── export { detectSignatures, defaultDetectors, AcroFormSignatureDetector, HeuristicImageSignatureDetector, + LabelPatternSignatureDetector, + VisionSignatureDetector, inferRoleFromFieldName, inferRoleFromNearbyText, } from './pdf' @@ -167,5 +169,20 @@ export type { SignatureKind, SignatureRole, HeuristicImageDetectorOptions, + VisionSignatureDetectorOptions, } from './pdf' export type { SignatureDescriptor } from './types' + +// ── v0.9: Vision backends ──────────────────────────────────────────────── + +export { + ClaudeVisionBackend, + OpenAiVisionBackend, +} from './pdf' +export type { + VisionBackend, + AnalyzeOptions, + AnalyzeResult, + ClaudeVisionOptions, + OpenAiVisionOptions, +} from './pdf' diff --git a/src/pdf/index.ts b/src/pdf/index.ts index 3986e9f..9e1ba31 100644 --- a/src/pdf/index.ts +++ b/src/pdf/index.ts @@ -57,6 +57,8 @@ export { defaultDetectors, AcroFormSignatureDetector, HeuristicImageSignatureDetector, + LabelPatternSignatureDetector, + VisionSignatureDetector, inferRoleFromFieldName, inferRoleFromNearbyText, } from './signatures' @@ -67,4 +69,18 @@ export type { SignatureKind, SignatureRole, HeuristicImageDetectorOptions, + VisionSignatureDetectorOptions, } from './signatures' + +// ── v0.9: Vision backends (used by signatures + video frame captioning) ── +export { + ClaudeVisionBackend, + OpenAiVisionBackend, +} from './vision' +export type { + VisionBackend, + AnalyzeOptions, + AnalyzeResult, + ClaudeVisionOptions, + OpenAiVisionOptions, +} from './vision' diff --git a/src/pdf/signatures/index.ts b/src/pdf/signatures/index.ts index 08768c3..ffb9cf6 100644 --- a/src/pdf/signatures/index.ts +++ b/src/pdf/signatures/index.ts @@ -33,6 +33,8 @@ export type { HeuristicImageDetectorOptions, } from './heuristic-image-detector' export { LabelPatternSignatureDetector } from './label-pattern-detector' +export { VisionSignatureDetector } from './vision-detector' +export type { VisionSignatureDetectorOptions } from './vision-detector' export { inferRoleFromFieldName, inferRoleFromNearbyText, diff --git a/src/pdf/signatures/vision-detector.ts b/src/pdf/signatures/vision-detector.ts new file mode 100644 index 0000000..4f02ad3 --- /dev/null +++ b/src/pdf/signatures/vision-detector.ts @@ -0,0 +1,225 @@ +/** + * Vision-based signature detector. + * + * Renders each candidate page to an image and asks a vision model + * (Claude / OpenAI / etc.) to identify signatures with bounding boxes, + * inferred role, and signer name when visible. + * + * This is the only signature detector that catches: + * - Hand-signed scans (whole page is one rasterized image) + * - Signatures without nearby labels + * - Typed cursive-font signatures + * - "X______" lines marked as signed in their visual context + * + * Cost-conscious by default: scans only the LAST `maxPages` pages where + * signatures usually live. Override with `pages: 'all'` to scan everywhere. + */ + +import type { RenderBackend } from '../ocr/types' +import type { VisionBackend } from '../vision/types' +import type { + DetectedSignature, + SignatureDetector, + SignatureDetectorInput, +} from './types' + +export interface VisionSignatureDetectorOptions { + /** Renders PDF pages to images. */ + render: RenderBackend + /** Vision model used for analysis. */ + vision: VisionBackend + /** + * Which pages to scan: + * - 'all' — every page + * - 'last' — only the last 2 pages (default; signatures usually here) + * - 'flagged' — only pages whose extracted text contains a signature label + * - number[] — specific 1-indexed page numbers + */ + pages?: 'all' | 'last' | 'flagged' | number[] + /** Number of pages to scan when pages='last'. Default: 2. */ + lastPagesCount?: number + /** DPI for rasterization. Default: 150. */ + dpi?: number + /** Min confidence threshold from the vision model. Default: 0.5. */ + minConfidence?: number +} + +interface VisionResponse { + signatures: Array<{ + kind?: + | 'image_handwritten' + | 'image_typed' + | 'widget_unsigned' + | 'unknown' + bbox?: { x: number; y: number; width: number; height: number } // normalized 0-1 + inferred_role?: string + signer_name?: string + confidence: number + notes?: string + }> +} + +const SIGNATURE_HINT_RE = /\b(signature|signed\s*by|sign\s*here|initial(?:s|ed)?|x\s*[:_])\b/i + +const SCHEMA = { + type: 'object', + additionalProperties: false, + required: ['signatures'], + properties: { + signatures: { + type: 'array', + items: { + type: 'object', + additionalProperties: false, + required: ['confidence'], + properties: { + kind: { + type: 'string', + enum: ['image_handwritten', 'image_typed', 'widget_unsigned', 'unknown'], + }, + bbox: { + type: 'object', + additionalProperties: false, + required: ['x', 'y', 'width', 'height'], + properties: { + x: { type: 'number', minimum: 0, maximum: 1 }, + y: { type: 'number', minimum: 0, maximum: 1 }, + width: { type: 'number', minimum: 0, maximum: 1 }, + height: { type: 'number', minimum: 0, maximum: 1 }, + }, + }, + inferred_role: { type: 'string' }, + signer_name: { type: 'string' }, + confidence: { type: 'number', minimum: 0, maximum: 1 }, + notes: { type: 'string' }, + }, + }, + }, + }, +} + +const PROMPT = + 'You are analyzing a single page of a document to find SIGNATURES on it. ' + + 'A signature is a handwritten name, a typed cursive name acting as a signature, ' + + 'or a clearly visible signature widget that has been signed. Do NOT report ' + + 'empty signature lines, signature boxes that have not been signed, or ' + + 'signature *labels* (the text "Signature:" alone is not a signature). ' + + 'For EACH signature you find, report:\n' + + ' - kind: image_handwritten | image_typed | widget_unsigned | unknown\n' + + ' - bbox: normalized [0,1] x/y/width/height (origin top-left)\n' + + ' - inferred_role: the role of the signer based on the page\'s context ' + + '(client, agent, broker, buyer, seller, tenant, landlord, witness, ' + + 'notary, attorney, employee, employer, applicant, etc). Use lowercase. ' + + 'Use "unknown" if you cannot tell.\n' + + ' - signer_name: the name of the person if visible (e.g. printed below ' + + 'the signature, or the typed cursive itself). Omit if not visible.\n' + + ' - confidence: 0-1 (your confidence this is actually a signature, ' + + 'not just an empty line or label).\n' + + ' - notes: 1 short sentence with anything useful (\"signature is on the ' + + 'right side of the page next to a tenant label\").\n\n' + + 'If there are NO signatures on this page, return an empty array. Do not ' + + 'invent signatures. Be precise with bbox coordinates.' + +export class VisionSignatureDetector implements SignatureDetector { + readonly name = 'vision' + private readonly opts: VisionSignatureDetectorOptions + + constructor(opts: VisionSignatureDetectorOptions) { + this.opts = opts + } + + async detect(input: SignatureDetectorInput): Promise { + const totalPages = input.extracted.pages.length + if (totalPages === 0) return [] + + const pages = this.resolvePagesToScan(input, totalPages) + if (pages.length === 0) return [] + + const minConfidence = this.opts.minConfidence ?? 0.5 + const dpi = this.opts.dpi ?? 150 + + const detections: DetectedSignature[] = [] + let counter = 0 + + for (const pageNum of pages) { + const rendered = await this.opts.render.renderPage(input.rawBytes, { + pageNumber: pageNum, + dpi, + format: 'png', + }).catch(() => null) + if (!rendered) continue + + const result = await this.opts.vision + .analyze({ + image: rendered.image, + mimeType: rendered.mimeType, + prompt: PROMPT, + schema: SCHEMA, + schemaName: 'report_signatures', + maxTokens: 1024, + }) + .catch(() => null) + + const signatures = result?.structured?.signatures ?? [] + const page = input.extracted.pages.find((p) => p.number === pageNum) + const pageWidth = page?.width ?? rendered.width + const pageHeight = page?.height ?? rendered.height + + for (const sig of signatures) { + if (sig.confidence < minConfidence) continue + counter++ + + // Convert normalized 0-1 bbox to PDF user-space rect. + // pdf-extractor uses origin bottom-left so we flip Y. + let rect: DetectedSignature['rect'] + if (sig.bbox) { + rect = { + x: sig.bbox.x * pageWidth, + y: pageHeight - (sig.bbox.y + sig.bbox.height) * pageHeight, + width: sig.bbox.width * pageWidth, + height: sig.bbox.height * pageHeight, + } + } + + detections.push({ + id: `sig_v_${counter}`, + kind: sig.kind ?? 'image_handwritten', + page: pageNum, + rect, + inferred_role: sig.inferred_role && sig.inferred_role !== 'unknown' + ? sig.inferred_role.toLowerCase() + : undefined, + signer_name: sig.signer_name, + confidence: sig.confidence, + notes: sig.notes + ? `Vision (${this.opts.vision.name}): ${sig.notes}` + : `Detected via ${this.opts.vision.name} vision`, + }) + } + } + return detections + } + + private resolvePagesToScan( + input: SignatureDetectorInput, + totalPages: number, + ): number[] { + const mode = this.opts.pages ?? 'last' + if (Array.isArray(mode)) { + return mode.filter((p) => p >= 1 && p <= totalPages) + } + if (mode === 'all') { + return Array.from({ length: totalPages }, (_, i) => i + 1) + } + if (mode === 'last') { + const count = Math.min(this.opts.lastPagesCount ?? 2, totalPages) + return Array.from({ length: count }, (_, i) => totalPages - count + 1 + i) + } + if (mode === 'flagged') { + return input.extracted.pages + .filter((p) => p.items.some((it) => SIGNATURE_HINT_RE.test(it.text))) + .map((p) => p.number) + } + return [] + } +} diff --git a/src/pdf/vision/claude-backend.ts b/src/pdf/vision/claude-backend.ts new file mode 100644 index 0000000..e0c02a6 --- /dev/null +++ b/src/pdf/vision/claude-backend.ts @@ -0,0 +1,154 @@ +/** + * Claude vision backend. + * + * Calls https://api.anthropic.com/v1/messages with an image + prompt. + * Uses Claude's tool-use feature for structured output when a schema is + * provided — most reliable way to get JSON back consistently. + * + * No SDK dependency — uses the global `fetch`. Authenticate via + * `ANTHROPIC_API_KEY` env var or the constructor option. + */ + +import { SnapshotError } from '../../errors' +import type { AnalyzeOptions, AnalyzeResult, VisionBackend } from './types' + +export interface ClaudeVisionOptions { + /** Anthropic API key. Defaults to env ANTHROPIC_API_KEY. */ + apiKey?: string + /** Override the API base URL (e.g. for a self-hosted proxy). */ + endpoint?: string + /** Model identifier. Default: 'claude-haiku-4-5-20251001'. */ + model?: string + /** Anthropic API version header. Default: '2023-06-01'. */ + anthropicVersion?: string +} + +interface AnthropicMessagesResponse { + content: Array< + | { type: 'text'; text: string } + | { type: 'tool_use'; id: string; name: string; input: unknown } + > + usage?: { input_tokens?: number; output_tokens?: number } + stop_reason?: string +} + +export class ClaudeVisionBackend implements VisionBackend { + readonly name = 'claude' + private readonly apiKey: string + private readonly endpoint: string + private readonly model: string + private readonly anthropicVersion: string + + constructor(options: ClaudeVisionOptions = {}) { + const apiKey = options.apiKey ?? process.env.ANTHROPIC_API_KEY + if (!apiKey) { + throw new SnapshotError( + 'ClaudeVisionBackend requires an API key. Set ANTHROPIC_API_KEY ' + + 'in the environment or pass { apiKey } to the constructor.', + ) + } + this.apiKey = apiKey + this.endpoint = options.endpoint ?? 'https://api.anthropic.com/v1/messages' + this.model = options.model ?? 'claude-haiku-4-5-20251001' + this.anthropicVersion = options.anthropicVersion ?? '2023-06-01' + } + + async analyze(opts: AnalyzeOptions): Promise> { + const mediaType = opts.mimeType ?? sniffMimeType(opts.image) + const base64 = Buffer.from(opts.image).toString('base64') + + const content: Array> = [ + { + type: 'image', + source: { type: 'base64', media_type: mediaType, data: base64 }, + }, + { type: 'text', text: opts.prompt }, + ] + + const body: Record = { + model: this.model, + max_tokens: opts.maxTokens ?? 1024, + messages: [{ role: 'user', content }], + } + if (opts.system) body.system = opts.system + + // Use tool use to coerce structured output when a schema is given. + const schemaName = opts.schemaName ?? 'extract' + if (opts.schema) { + body.tools = [ + { + name: schemaName, + description: 'Return the analysis result in this schema.', + input_schema: opts.schema, + }, + ] + body.tool_choice = { type: 'tool', name: schemaName } + } + + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), opts.timeoutMs ?? 60_000) + + let response: Response + try { + response = await fetch(this.endpoint, { + method: 'POST', + headers: { + 'x-api-key': this.apiKey, + 'anthropic-version': this.anthropicVersion, + 'content-type': 'application/json', + }, + body: JSON.stringify(body), + signal: controller.signal, + }) + } catch (err) { + const e = err as Error & { name?: string } + if (e.name === 'AbortError') { + throw new SnapshotError( + `Claude vision request timed out after ${opts.timeoutMs ?? 60_000}ms`, + e, + ) + } + throw new SnapshotError(`Claude vision request failed: ${e.message}`, e) + } finally { + clearTimeout(timer) + } + + if (!response.ok) { + const text = await response.text().catch(() => '') + throw new SnapshotError( + `Claude vision returned ${response.status} ${response.statusText}: ${text.slice(0, 500)}`, + ) + } + + const json = (await response.json()) as AnthropicMessagesResponse + + let structured: T | undefined + let textOut = '' + for (const block of json.content) { + if (block.type === 'tool_use' && block.name === schemaName) { + structured = block.input as T + } else if (block.type === 'text') { + textOut += block.text + } + } + + return { + structured, + text: textOut || (structured ? JSON.stringify(structured) : ''), + tokens: { + input: json.usage?.input_tokens ?? 0, + output: json.usage?.output_tokens ?? 0, + }, + } + } +} + +function sniffMimeType(bytes: Uint8Array): 'image/png' | 'image/jpeg' | 'image/webp' { + if (bytes.length >= 4 && bytes[0] === 0x89 && bytes[1] === 0x50 && bytes[2] === 0x4e && bytes[3] === 0x47) { + return 'image/png' + } + if (bytes.length >= 12 && bytes[8] === 0x57 && bytes[9] === 0x45 && bytes[10] === 0x42 && bytes[11] === 0x50) { + return 'image/webp' + } + return 'image/jpeg' +} diff --git a/src/pdf/vision/index.ts b/src/pdf/vision/index.ts new file mode 100644 index 0000000..b13afca --- /dev/null +++ b/src/pdf/vision/index.ts @@ -0,0 +1,14 @@ +/** + * Vision-backend module — used by signature detection (v0.9) and video + * frame captioning (v0.11). + */ + +export type { + VisionBackend, + AnalyzeOptions, + AnalyzeResult, +} from './types' +export { ClaudeVisionBackend } from './claude-backend' +export type { ClaudeVisionOptions } from './claude-backend' +export { OpenAiVisionBackend } from './openai-backend' +export type { OpenAiVisionOptions } from './openai-backend' diff --git a/src/pdf/vision/openai-backend.ts b/src/pdf/vision/openai-backend.ts new file mode 100644 index 0000000..f2adbb4 --- /dev/null +++ b/src/pdf/vision/openai-backend.ts @@ -0,0 +1,136 @@ +/** + * OpenAI vision backend. + * + * Calls https://api.openai.com/v1/chat/completions with an image attachment. + * Uses the `response_format: { type: 'json_schema' }` feature for + * structured output when a schema is provided. + * + * No SDK dependency — uses the global `fetch`. Authenticate via + * `OPENAI_API_KEY` env var or the constructor option. + */ + +import { SnapshotError } from '../../errors' +import type { AnalyzeOptions, AnalyzeResult, VisionBackend } from './types' + +export interface OpenAiVisionOptions { + /** OpenAI API key. Defaults to env OPENAI_API_KEY. */ + apiKey?: string + /** Override the API base URL (e.g. for a self-hosted proxy). */ + endpoint?: string + /** Model identifier. Default: 'gpt-4o-mini'. */ + model?: string +} + +interface OpenAiChatResponse { + choices: Array<{ message: { content: string | null } }> + usage?: { prompt_tokens?: number; completion_tokens?: number } +} + +export class OpenAiVisionBackend implements VisionBackend { + readonly name = 'openai' + private readonly apiKey: string + private readonly endpoint: string + private readonly model: string + + constructor(options: OpenAiVisionOptions = {}) { + const apiKey = options.apiKey ?? process.env.OPENAI_API_KEY + if (!apiKey) { + throw new SnapshotError( + 'OpenAiVisionBackend requires an API key. Set OPENAI_API_KEY ' + + 'in the environment or pass { apiKey } to the constructor.', + ) + } + this.apiKey = apiKey + this.endpoint = options.endpoint ?? 'https://api.openai.com/v1/chat/completions' + this.model = options.model ?? 'gpt-4o-mini' + } + + async analyze(opts: AnalyzeOptions): Promise> { + const mediaType = opts.mimeType ?? 'image/png' + const base64 = Buffer.from(opts.image).toString('base64') + const dataUrl = `data:${mediaType};base64,${base64}` + + const content: Array> = [ + { type: 'text', text: opts.prompt }, + { type: 'image_url', image_url: { url: dataUrl } }, + ] + + const messages: Array> = [ + { role: 'user', content }, + ] + if (opts.system) { + messages.unshift({ role: 'system', content: opts.system }) + } + + const body: Record = { + model: this.model, + max_tokens: opts.maxTokens ?? 1024, + messages, + } + if (opts.schema) { + body.response_format = { + type: 'json_schema', + json_schema: { + name: opts.schemaName ?? 'extract', + strict: true, + schema: opts.schema, + }, + } + } + + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), opts.timeoutMs ?? 60_000) + + let response: Response + try { + response = await fetch(this.endpoint, { + method: 'POST', + headers: { + Authorization: `Bearer ${this.apiKey}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(body), + signal: controller.signal, + }) + } catch (err) { + const e = err as Error & { name?: string } + if (e.name === 'AbortError') { + throw new SnapshotError( + `OpenAI vision request timed out after ${opts.timeoutMs ?? 60_000}ms`, + e, + ) + } + throw new SnapshotError(`OpenAI vision request failed: ${e.message}`, e) + } finally { + clearTimeout(timer) + } + + if (!response.ok) { + const text = await response.text().catch(() => '') + throw new SnapshotError( + `OpenAI vision returned ${response.status} ${response.statusText}: ${text.slice(0, 500)}`, + ) + } + + const json = (await response.json()) as OpenAiChatResponse + const messageText = json.choices[0]?.message?.content ?? '' + + let structured: T | undefined + if (opts.schema) { + try { + structured = JSON.parse(messageText) as T + } catch { + // Fall back to text-only mode when the model didn't return valid JSON. + } + } + + return { + structured, + text: messageText, + tokens: { + input: json.usage?.prompt_tokens ?? 0, + output: json.usage?.completion_tokens ?? 0, + }, + } + } +} diff --git a/src/pdf/vision/types.ts b/src/pdf/vision/types.ts new file mode 100644 index 0000000..79a92e2 --- /dev/null +++ b/src/pdf/vision/types.ts @@ -0,0 +1,55 @@ +/** + * Vision backend interface — used by both signature detection (v0.9) and + * video frame captioning (v0.11). One backend, two callers. + * + * Implementations should accept an image (PNG/JPEG bytes), a system prompt, + * a user prompt, and an optional JSON schema for structured output. The + * vision provider returns either freeform text or a parsed JSON object + * matching the schema. + */ + +export interface VisionBackend { + /** Implementation name — surfaces in detection notes for debugging. */ + readonly name: string + + /** + * Run a vision query against an image. When `schema` is provided the + * implementation must return an object matching it (Claude tool use, + * OpenAI json_schema response format, etc.). Without `schema` returns + * the model's freeform text. + */ + analyze(opts: AnalyzeOptions): Promise> + + /** Optional cleanup. */ + close?(): Promise +} + +export interface AnalyzeOptions { + /** Image bytes — typically PNG or JPEG. */ + image: Uint8Array + mimeType?: 'image/png' | 'image/jpeg' | 'image/webp' + /** Instruction prompt for the model. */ + prompt: string + /** Optional system message — useful for tone / role control. */ + system?: string + /** + * If provided, the model is asked to return a JSON object matching + * this schema (via tool use / response_format depending on provider). + */ + schema?: Record + /** Schema name when `schema` is given (default: 'extract'). */ + schemaName?: string + /** Max tokens. Default: 1024. */ + maxTokens?: number + /** Per-request timeout (ms). Default: 60000. */ + timeoutMs?: number +} + +export interface AnalyzeResult { + /** Structured output when `schema` was provided. */ + structured?: T + /** Freeform text. Always populated. */ + text: string + /** Approx tokens consumed (when the provider reports them). */ + tokens?: { input: number; output: number } +} diff --git a/test/pdf/vision-signature.test.ts b/test/pdf/vision-signature.test.ts new file mode 100644 index 0000000..a8f638c --- /dev/null +++ b/test/pdf/vision-signature.test.ts @@ -0,0 +1,251 @@ +/** + * Vision-based signature detection tests. + * + * Mocks both the RenderBackend and the VisionBackend so the test suite + * stays deterministic + offline (real Claude/OpenAI calls are cost + + * network-dependent). + */ + +import { describe, it, expect, vi } from 'vitest' +import { VisionSignatureDetector } from '../../src/pdf/signatures/vision-detector' +import type { RenderBackend, RenderedPage } from '../../src/pdf/ocr/types' +import type { VisionBackend, AnalyzeResult } from '../../src/pdf/vision/types' +import type { ExtractedPdf } from '../../src/pdf/types' + +function fakePdf(pageCount: number, withSignatureLabel?: number[]): ExtractedPdf { + const pages = Array.from({ length: pageCount }, (_, i) => { + const number = i + 1 + const items = withSignatureLabel?.includes(number) + ? [{ + text: 'Signature:', + fontSize: 11, + fontName: 'Helvetica', + x: 50, + y: 100, + width: 80, + hasEol: false, + }] + : [] + return { number, width: 612, height: 792, items } + }) + return { pages, metadata: { pages: pageCount, format: 'pdf' } } +} + +function tinyPng(): Uint8Array { + return new Uint8Array([ + 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, + 0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0, 0, 0, 31, 21, 196, 137, + 0, 0, 0, 13, 73, 68, 65, 84, 8, 153, 99, 248, 255, 255, 63, 0, + 5, 0, 1, 254, 215, 17, 196, 70, 0, 0, 0, 0, 73, 69, 78, 68, + 174, 66, 96, 130, + ]) +} + +function fakeRender(): RenderBackend & { calls: number[] } { + return { + name: 'fake_render', + calls: [] as number[], + async renderPage(_data, opts): Promise { + this.calls.push(opts.pageNumber) + return { + image: tinyPng(), + mimeType: 'image/png', + width: 612 * 2, + height: 792 * 2, + dpi: 144, + } + }, + } as RenderBackend & { calls: number[] } +} + +interface VisionResp { + signatures: Array<{ + kind?: string + bbox?: { x: number; y: number; width: number; height: number } + inferred_role?: string + signer_name?: string + confidence: number + notes?: string + }> +} + +function fakeVision(response: VisionResp): VisionBackend & { callCount: number } { + return { + name: 'fake_vision', + callCount: 0, + async analyze(): Promise> { + this.callCount++ + return { structured: response, text: JSON.stringify(response) } + }, + } as VisionBackend & { callCount: number } +} + +describe('VisionSignatureDetector', () => { + it('returns empty when no pages', async () => { + const detector = new VisionSignatureDetector({ + render: fakeRender(), + vision: fakeVision({ signatures: [] }), + }) + const result = await detector.detect({ + extracted: { pages: [], metadata: { pages: 0, format: 'pdf' } }, + rawBytes: new Uint8Array(), + }) + expect(result).toEqual([]) + }) + + it('default mode "last" scans the last 2 pages', async () => { + const render = fakeRender() + const vision = fakeVision({ signatures: [] }) + const detector = new VisionSignatureDetector({ render, vision }) + await detector.detect({ + extracted: fakePdf(5), + rawBytes: new Uint8Array(), + }) + expect(render.calls).toEqual([4, 5]) + expect(vision.callCount).toBe(2) + }) + + it('mode "all" scans every page', async () => { + const render = fakeRender() + const vision = fakeVision({ signatures: [] }) + const detector = new VisionSignatureDetector({ + render, + vision, + pages: 'all', + }) + await detector.detect({ + extracted: fakePdf(3), + rawBytes: new Uint8Array(), + }) + expect(render.calls).toEqual([1, 2, 3]) + }) + + it('mode "flagged" scans only pages with signature label text', async () => { + const render = fakeRender() + const vision = fakeVision({ signatures: [] }) + const detector = new VisionSignatureDetector({ + render, + vision, + pages: 'flagged', + }) + await detector.detect({ + extracted: fakePdf(5, [2, 4]), + rawBytes: new Uint8Array(), + }) + expect(render.calls).toEqual([2, 4]) + }) + + it('explicit page-list mode scans only those pages', async () => { + const render = fakeRender() + const vision = fakeVision({ signatures: [] }) + const detector = new VisionSignatureDetector({ + render, + vision, + pages: [1, 3], + }) + await detector.detect({ + extracted: fakePdf(5), + rawBytes: new Uint8Array(), + }) + expect(render.calls).toEqual([1, 3]) + }) + + it('emits a DetectedSignature for each model-found signature', async () => { + const detector = new VisionSignatureDetector({ + render: fakeRender(), + vision: fakeVision({ + signatures: [ + { + kind: 'image_handwritten', + bbox: { x: 0.1, y: 0.8, width: 0.3, height: 0.05 }, + inferred_role: 'tenant', + signer_name: 'Jane Doe', + confidence: 0.92, + notes: 'Bottom-left of the page near a tenant label.', + }, + ], + }), + }) + const result = await detector.detect({ + extracted: fakePdf(1), + rawBytes: new Uint8Array(), + }) + expect(result).toHaveLength(1) + const sig = result[0] + expect(sig.kind).toBe('image_handwritten') + expect(sig.inferred_role).toBe('tenant') + expect(sig.signer_name).toBe('Jane Doe') + expect(sig.confidence).toBe(0.92) + expect(sig.notes).toMatch(/Vision \(fake_vision\):/) + expect(sig.rect).toBeDefined() + // bbox(x=0.1, y=0.8, w=0.3, h=0.05) on 612×792 page + // PDF origin is bottom-left so y_pdf = 792 - (0.8 + 0.05) * 792 + expect(sig.rect!.x).toBeCloseTo(61.2, 0) + expect(sig.rect!.width).toBeCloseTo(183.6, 0) + expect(sig.rect!.height).toBeCloseTo(39.6, 0) + }) + + it('filters out detections below minConfidence', async () => { + const detector = new VisionSignatureDetector({ + render: fakeRender(), + vision: fakeVision({ + signatures: [ + { confidence: 0.3 }, + { confidence: 0.55 }, + { confidence: 0.8 }, + ], + }), + minConfidence: 0.6, + }) + const result = await detector.detect({ + extracted: fakePdf(1), + rawBytes: new Uint8Array(), + }) + expect(result.length).toBe(1) + expect(result[0].confidence).toBe(0.8) + }) + + it('lowercases inferred_role and drops "unknown"', async () => { + const detector = new VisionSignatureDetector({ + render: fakeRender(), + vision: fakeVision({ + signatures: [ + { confidence: 0.9, inferred_role: 'TENANT' }, + { confidence: 0.9, inferred_role: 'unknown' }, + ], + }), + }) + const result = await detector.detect({ + extracted: fakePdf(1), + rawBytes: new Uint8Array(), + }) + expect(result[0].inferred_role).toBe('tenant') + expect(result[1].inferred_role).toBeUndefined() + }) + + it('continues when one page render or analyze fails', async () => { + let calls = 0 + const flakyVision: VisionBackend = { + name: 'flaky', + async analyze() { + calls++ + if (calls === 1) throw new Error('model 500') + return { + structured: { signatures: [{ confidence: 0.9 }] }, + text: '', + } as unknown as AnalyzeResult + }, + } + const detector = new VisionSignatureDetector({ + render: fakeRender(), + vision: flakyVision, + pages: 'all', + }) + const result = await detector.detect({ + extracted: fakePdf(2), + rawBytes: new Uint8Array(), + }) + // Page 1 failed, page 2 returned 1 signature. + expect(result.length).toBe(1) + }) +}) From fe4a54c9ee3daf4c274ddb39f68ae4690d5c9efc Mon Sep 17 00:00:00 2001 From: rrader26 Date: Sun, 10 May 2026 15:23:00 -0400 Subject: [PATCH 4/5] feat(audio): convertAudio() + WhisperApiBackend + spec v0.3 (v0.10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the full audio surface: audio bytes → TranscriptionBackend → AgentMark snapshot (kind: 'audio') with timestamps, optional speaker diarization, and the same wire format any AI client already knows. Customer calls, voicemails, meeting recordings, podcasts — all become structured AgentMark. Spec v0.3 - kind enum extended: webpage|document|form|audio|video - New `media_meta` envelope field: duration_sec, format, language, transcribed, transcription_backend, vision_backend, speaker_count, frame_count - New `speakers` map: speaker IDs → display names - Three new body tags: [TIME:t_N] timestamp marker (N = seconds-from-start) [SPEAKER:s_X] speaker label, resolves to envelope.speakers [FRAME:f_N] video frame ref (reserved for v0.11) - agentmark-v0.3.json schema; validator picks v0.1/v0.2/v0.3 by declared version. v0.2 docs continue to validate unchanged. - AGENTMARK_VERSION bumped 0.2 → 0.3. TranscriptionBackend interface - transcribe({ data, mimeType, language, diarize }) → segments + full_text - WhisperApiBackend: OpenAI /v1/audio/transcriptions multipart/form-data + verbose_json for segment-level timestamps no SDK dep, fetch-based, OPENAI_API_KEY auth - Sniffs MIME from magic bytes: mp3/wav/ogg/m4a/flac/webm - Bring-your-own AssemblyAI / Deepgram / Apple Speech / whisper.cpp convertAudio() - Same contract as convertPdf — bytes in, ConversionResult out. - Body builder emits one [TIME:t_N] per segment + [SPEAKER:s_X] when diarized. Same-speaker continuing segments omit the SPEAKER tag for compactness. Body text escaped (matches web body builder rules) so transcripts containing literal "[PAGE:p_1]" don't become bogus refs. - Falls back to URL basename for title; counts distinct speakers when no speakers map provided; preserves vendor (x-) extensions. Tests (271 passing, 9 new for audio) - Round-trip: backend → snapshot → parsed → validated v0.3 - Diarized + non-diarized paths - Empty-segments fallback to full_text - Speaker dedup logic - Title fallback - Vendor extensions - Backend-failure SnapshotError wrapping - Body-text escaping (literal [TAG] doesn't become a ref) - Updated 3 prior tests to expect new agentmark version 0.3 Co-Authored-By: Claude Opus 4.7 (1M context) --- schema/agentmark-v0.3.json | 112 ++++++++++++++ src/audio/audio-converter.ts | 227 +++++++++++++++++++++++++++++ src/audio/index.ts | 14 ++ src/audio/types.ts | 47 ++++++ src/audio/whisper-api-backend.ts | 160 ++++++++++++++++++++ src/index.ts | 13 ++ src/types.ts | 47 +++++- src/validators/schema-validator.ts | 17 ++- test/audio/audio-converter.test.ts | 189 ++++++++++++++++++++++++ test/build-snapshot.test.ts | 2 +- test/pdf/pdf-converter.test.ts | 2 +- test/spec-v0.2.test.ts | 8 +- 12 files changed, 820 insertions(+), 18 deletions(-) create mode 100644 schema/agentmark-v0.3.json create mode 100644 src/audio/audio-converter.ts create mode 100644 src/audio/index.ts create mode 100644 src/audio/types.ts create mode 100644 src/audio/whisper-api-backend.ts create mode 100644 test/audio/audio-converter.test.ts diff --git a/schema/agentmark-v0.3.json b/schema/agentmark-v0.3.json new file mode 100644 index 0000000..a184028 --- /dev/null +++ b/schema/agentmark-v0.3.json @@ -0,0 +1,112 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentmark.dev/schema/v0.3.json", + "title": "agentmark v0.3 frontmatter", + "description": "v0.3 extends v0.2 with `kind: audio | video`, `media_meta`, `speakers`, and the [TIME] / [SPEAKER] / [FRAME] body tags. v0.2 docs continue to validate.", + "type": "object", + "required": ["agentmark", "url", "title"], + "properties": { + "agentmark": { "type": "string", "pattern": "^[0-9]+\\.[0-9]+(\\.[0-9]+)?$" }, + "kind": { "enum": ["webpage", "document", "form", "audio", "video"] }, + "url": { "type": "string", "format": "uri" }, + "title": { "type": "string", "maxLength": 512 }, + "captured_at": { "type": "string", "format": "date-time" }, + "expires_at": { "type": "string", "format": "date-time" }, + "source": { "enum": ["rendered", "declared", "hybrid"] }, + "language": { "type": "string" }, + "direction": { "enum": ["ltr", "rtl"] }, + "state": { "type": "object" }, + "actions": { "type": "object" }, + "media": { "type": "object" }, + "document": { "$ref": "#/$defs/document" }, + "media_meta": { "$ref": "#/$defs/media_meta" }, + "speakers": { + "type": "object", + "patternProperties": { + "^[a-z][a-z0-9_]{0,63}$": { "type": "string", "maxLength": 256 } + }, + "additionalProperties": false + }, + "signatures": { + "type": "object", + "patternProperties": { + "^[a-z][a-z0-9_]{0,63}$": { "$ref": "#/$defs/signature" } + }, + "additionalProperties": false + }, + "memory": { "type": "object" }, + "capabilities": { "type": "object" }, + "cookies": { "type": "object" }, + "permissions": { "type": "object" } + }, + "patternProperties": { + "^x-": {} + }, + "$defs": { + "document": { + "type": "object", + "additionalProperties": false, + "properties": { + "pages": { "type": "integer", "minimum": 1 }, + "author": { "type": "string", "maxLength": 512 }, + "created_at": { "type": "string", "format": "date-time" }, + "modified_at": { "type": "string", "format": "date-time" }, + "format": { "enum": ["pdf", "docx", "rtf", "txt", "html"] }, + "format_version": { "type": "string", "maxLength": 32 }, + "ocr_used": { "type": "boolean" } + } + }, + "media_meta": { + "type": "object", + "additionalProperties": false, + "properties": { + "duration_sec": { "type": "number", "minimum": 0 }, + "format": { "type": "string", "maxLength": 32 }, + "language": { "type": "string", "maxLength": 32 }, + "transcribed": { "type": "boolean" }, + "transcription_backend": { "type": "string", "maxLength": 64 }, + "vision_backend": { "type": "string", "maxLength": 64 }, + "speaker_count": { "type": "integer", "minimum": 0 }, + "frame_count": { "type": "integer", "minimum": 0 } + } + }, + "signature": { + "type": "object", + "required": ["kind", "page", "confidence"], + "additionalProperties": false, + "properties": { + "kind": { + "enum": [ + "widget_visible_signed", + "widget_unsigned", + "cryptographic", + "image_handwritten", + "image_typed", + "docusign", + "adobe_sign", + "unknown" + ] + }, + "page": { "type": "integer", "minimum": 1 }, + "rect": { + "type": "object", + "additionalProperties": false, + "properties": { + "x": { "type": "number" }, + "y": { "type": "number" }, + "width": { "type": "number" }, + "height": { "type": "number" } + } + }, + "field_name": { "type": "string", "maxLength": 256 }, + "inferred_role": { "type": "string", "maxLength": 64 }, + "signer_name": { "type": "string", "maxLength": 256 }, + "signer_email": { "type": "string", "maxLength": 256 }, + "signed_at": { "type": "string", "format": "date-time" }, + "confidence": { "type": "number", "minimum": 0, "maximum": 1 }, + "valid": { "type": "boolean" }, + "notes": { "type": "string", "maxLength": 1024 } + } + } + } +} diff --git a/src/audio/audio-converter.ts b/src/audio/audio-converter.ts new file mode 100644 index 0000000..668321e --- /dev/null +++ b/src/audio/audio-converter.ts @@ -0,0 +1,227 @@ +/** + * `convertAudio()` — convert audio bytes into an AgentMark snapshot with + * `kind: 'audio'`. Mirrors convertPdf's contract. + * + * Output body grammar: + * + * [TIME:t_0] + * [SPEAKER:s_1] First spoken segment text. + * + * [TIME:t_4] + * [SPEAKER:s_2] Second spoken segment text. + * + * ... + * + * - [TIME:t_N] markers carry seconds-from-start as the numeric suffix. + * Agents can correlate timestamps to body text directly. + * - [SPEAKER:s_X] markers identify speakers when the transcription + * backend supports diarization. Without diarization, no SPEAKER tags + * are emitted. + * + * The transcript is the body — no extra structure is inferred. Agents + * can summarize, extract action items, etc. directly. + */ + +import { + AGENTMARK_VERSION, + type ConversionResult, + type MediaMeta, + type Snapshot, +} from '../types' +import { serializeSnapshot } from '../serializers/yaml-frontmatter' +import { InMemoryActionBinding } from '../binding/action-binding' +import { noopLogger, type Logger } from '../observability/logger' +import { SnapshotError } from '../errors' +import type { + TranscriptionBackend, + TranscriptionResult, + TranscriptionSegment, +} from './types' + +export interface ConvertAudioOptions { + /** Raw audio bytes. */ + data: Uint8Array | ArrayBuffer + /** URL or `file://` URI identifying the audio source. */ + sourceUrl: string + /** Transcription backend (Whisper API, AssemblyAI, etc.). */ + transcribe: TranscriptionBackend + /** Override the document title. Default: source URL basename. */ + title?: string + /** BCP-47 language hint passed to the backend. */ + language?: string + /** Request speaker diarization. Default: false (most backends ignore). */ + diarize?: boolean + /** TTL for `expires_at` (ms). Default: 24 hours — audio doesn't change. */ + ttlMs?: number + /** Logger for structured events. */ + logger?: Logger + /** Vendor extensions (`x-` prefixed fields). */ + vendorExtensions?: Record + /** MIME type override (otherwise sniffed). */ + mimeType?: string +} + +export async function convertAudio(options: ConvertAudioOptions): Promise { + const logger = options.logger ?? noopLogger + const ttlMs = options.ttlMs ?? 24 * 60 * 60_000 // 24 h + + logger.debug('snapshot.capture.start', { source: options.sourceUrl, kind: 'audio' }) + + const data = options.data instanceof ArrayBuffer + ? new Uint8Array(options.data) + : new Uint8Array(options.data.buffer, options.data.byteOffset, options.data.byteLength) + + let transcription: TranscriptionResult + try { + transcription = await options.transcribe.transcribe({ + data, + mimeType: options.mimeType, + language: options.language, + diarize: options.diarize, + }) + } catch (err) { + logger.error('snapshot.failed', { error: (err as Error).message }) + if (err instanceof SnapshotError) throw err + throw new SnapshotError( + `Audio transcription failed: ${(err as Error).message}`, + err as Error, + ) + } + + const captured_at = new Date().toISOString() + const expires_at = new Date(Date.now() + ttlMs).toISOString() + + const speakers = transcription.speakers + const speakerCount = speakers ? Object.keys(speakers).length : countDistinctSpeakers(transcription.segments) + + const mediaMeta: MediaMeta = { + duration_sec: transcription.duration_sec, + format: deriveFormat(options.mimeType ?? sniffFormat(data)), + language: transcription.language ?? options.language, + transcribed: true, + transcription_backend: options.transcribe.name, + speaker_count: speakerCount > 0 ? speakerCount : undefined, + } + + const title = options.title ?? deriveTitleFromUrl(options.sourceUrl) + const body = buildAudioBody(transcription) + + const snapshot: Snapshot = { + agentmark: AGENTMARK_VERSION, + kind: 'audio', + url: options.sourceUrl, + title, + captured_at, + expires_at, + source: 'declared', + language: mediaMeta.language, + media_meta: stripUndefined(mediaMeta), + speakers: speakers && Object.keys(speakers).length > 0 ? speakers : undefined, + capabilities: { + preview_media: false, + expand_disclosures: false, + paginate: false, + scroll: true, + keyboard: false, + drag: false, + ocr: false, + vision: false, + }, + body, + } + + if (options.vendorExtensions) { + for (const [k, v] of Object.entries(options.vendorExtensions)) { + if (k.startsWith('x-')) (snapshot as unknown as Record)[k] = v + } + } + + const text = serializeSnapshot(snapshot) + logger.info('snapshot.captured', { + source: options.sourceUrl, + kind: 'audio', + duration_sec: mediaMeta.duration_sec, + segments: transcription.segments.length, + bytes: text.length, + }) + + return { agentmark: text, binding: new InMemoryActionBinding() } +} + +// ────────────────────────────────────────────────────────────────────────── +// Body builder +// ────────────────────────────────────────────────────────────────────────── + +function buildAudioBody(t: TranscriptionResult): string { + if (t.segments.length === 0) { + // No segments — just emit the full text under a single TIME marker. + const startStamp = `[TIME:t_0]` + return `${startStamp}\n\n${escapeBody(t.full_text || '')}\n` + } + + const lines: string[] = [] + let lastSpeaker: string | undefined + for (const seg of t.segments) { + const timeId = `t_${Math.round(seg.start)}` + lines.push(`[TIME:${timeId}]`) + if (seg.speaker && seg.speaker !== lastSpeaker) { + lines.push(`[SPEAKER:${seg.speaker}] ${escapeBody(seg.text)}`) + lastSpeaker = seg.speaker + } else if (seg.speaker) { + // Same speaker continuing — omit the SPEAKER tag for compactness. + lines.push(escapeBody(seg.text)) + } else { + lines.push(escapeBody(seg.text)) + } + lines.push('') // blank line between segments + } + return lines.join('\n') +} + +function escapeBody(text: string): string { + // Same rules as web body builder: escape `[` followed by uppercase + // (could otherwise create accidental tag references) and backslashes. + return text.replace(/\\/g, '\\\\').replace(/\[(?=[A-Z])/g, '\\[') +} + +function countDistinctSpeakers(segments: TranscriptionSegment[]): number { + const set = new Set() + for (const s of segments) if (s.speaker) set.add(s.speaker) + return set.size +} + +function deriveTitleFromUrl(url: string): string { + try { + const u = new URL(url) + const last = u.pathname.split('/').filter(Boolean).pop() ?? '(untitled)' + return decodeURIComponent(last).replace(/\.[a-z0-9]+$/i, '') || '(untitled)' + } catch { + return '(untitled)' + } +} + +function deriveFormat(mimeType: string | undefined): string | undefined { + if (!mimeType) return undefined + if (mimeType.includes('mpeg') || mimeType.includes('mp3')) return 'mp3' + if (mimeType.includes('wav')) return 'wav' + if (mimeType.includes('m4a')) return 'm4a' + if (mimeType.includes('ogg')) return 'ogg' + if (mimeType.includes('webm')) return 'webm' + if (mimeType.includes('flac')) return 'flac' + return undefined +} + +function sniffFormat(bytes: Uint8Array): string | undefined { + if (bytes.length >= 3 && bytes[0] === 0x49 && bytes[1] === 0x44 && bytes[2] === 0x33) return 'audio/mpeg' + if (bytes.length >= 12 && bytes[0] === 0x52 && bytes[1] === 0x49 && bytes[2] === 0x46 && bytes[3] === 0x46) return 'audio/wav' + if (bytes.length >= 4 && bytes[0] === 0x4f && bytes[1] === 0x67 && bytes[2] === 0x67) return 'audio/ogg' + return undefined +} + +function stripUndefined(obj: T): T { + const out: Record = {} + for (const [k, v] of Object.entries(obj)) { + if (v !== undefined) out[k] = v + } + return out as T +} diff --git a/src/audio/index.ts b/src/audio/index.ts new file mode 100644 index 0000000..038ba57 --- /dev/null +++ b/src/audio/index.ts @@ -0,0 +1,14 @@ +/** + * Audio support — convertAudio() + transcription backends. + */ + +export { convertAudio } from './audio-converter' +export type { ConvertAudioOptions } from './audio-converter' +export { WhisperApiBackend } from './whisper-api-backend' +export type { WhisperApiOptions } from './whisper-api-backend' +export type { + TranscriptionBackend, + TranscriptionResult, + TranscriptionSegment, + TranscribeOptions, +} from './types' diff --git a/src/audio/types.ts b/src/audio/types.ts new file mode 100644 index 0000000..c75a7f9 --- /dev/null +++ b/src/audio/types.ts @@ -0,0 +1,47 @@ +/** + * Audio support — types for transcription backends and the structured + * output convertAudio() emits. + */ + +export interface TranscriptionBackend { + readonly name: string + transcribe(opts: TranscribeOptions): Promise + close?(): Promise +} + +export interface TranscribeOptions { + /** Audio bytes — common formats (mp3/wav/m4a/ogg/flac/webm). */ + data: Uint8Array + /** MIME type. Default sniffed from bytes. */ + mimeType?: string + /** BCP-47 language hint. Default: auto-detect. */ + language?: string + /** When true, attempt speaker diarization. Default: false. */ + diarize?: boolean + /** Per-request timeout (ms). Default: 600000 (10 min). */ + timeoutMs?: number +} + +export interface TranscriptionResult { + /** Detected language (BCP-47) — when reported. */ + language?: string + /** Total duration in seconds — when reported. */ + duration_sec?: number + /** Transcript broken into time-aligned segments. */ + segments: TranscriptionSegment[] + /** Joined plain text — convenience. */ + full_text: string + /** Speaker labels keyed by ID, when diarized. */ + speakers?: Record +} + +export interface TranscriptionSegment { + /** Start time in seconds. */ + start: number + /** End time in seconds. */ + end: number + /** Text spoken in this segment. */ + text: string + /** Optional speaker ID (e.g. 's_alice') when diarized. */ + speaker?: string +} diff --git a/src/audio/whisper-api-backend.ts b/src/audio/whisper-api-backend.ts new file mode 100644 index 0000000..ef87a27 --- /dev/null +++ b/src/audio/whisper-api-backend.ts @@ -0,0 +1,160 @@ +/** + * OpenAI Whisper API transcription backend. + * + * POST https://api.openai.com/v1/audio/transcriptions + * + * Sends a multipart/form-data request with the audio file. Asks for + * `verbose_json` so we get word/segment-level timestamps. + * + * No SDK dependency — uses the global `fetch` + `FormData`. + * Authenticate via `OPENAI_API_KEY` env var or constructor option. + * + * Diarization: Whisper API does NOT do speaker diarization itself. + * For diarized transcripts, use a backend that supports it (AssemblyAI, + * Deepgram, etc.) — the interface is identical, just bring-your-own. + */ + +import { SnapshotError } from '../errors' +import type { + TranscriptionBackend, + TranscriptionResult, + TranscriptionSegment, + TranscribeOptions, +} from './types' + +export interface WhisperApiOptions { + /** OpenAI API key. Defaults to env OPENAI_API_KEY. */ + apiKey?: string + /** Override the API base URL. */ + endpoint?: string + /** Whisper model identifier. Default: 'whisper-1'. */ + model?: string +} + +interface VerboseJsonResponse { + text: string + language?: string + duration?: number + segments?: Array<{ + id: number + start: number + end: number + text: string + }> + words?: Array<{ word: string; start: number; end: number }> +} + +export class WhisperApiBackend implements TranscriptionBackend { + readonly name = 'whisper-api' + private readonly apiKey: string + private readonly endpoint: string + private readonly model: string + + constructor(options: WhisperApiOptions = {}) { + const apiKey = options.apiKey ?? process.env.OPENAI_API_KEY + if (!apiKey) { + throw new SnapshotError( + 'WhisperApiBackend requires an API key. Set OPENAI_API_KEY ' + + 'in the environment or pass { apiKey } to the constructor.', + ) + } + this.apiKey = apiKey + this.endpoint = options.endpoint + ?? 'https://api.openai.com/v1/audio/transcriptions' + this.model = options.model ?? 'whisper-1' + } + + async transcribe(opts: TranscribeOptions): Promise { + const mimeType = opts.mimeType ?? sniffAudioMimeType(opts.data) + const filename = filenameForMime(mimeType) + + const form = new FormData() + form.append('model', this.model) + if (opts.language) form.append('language', opts.language) + form.append('response_format', 'verbose_json') + form.append('timestamp_granularities[]', 'segment') + // Whisper accepts a Blob; convert from Uint8Array. + const blob = new Blob([opts.data as BlobPart], { type: mimeType }) + form.append('file', blob, filename) + + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), opts.timeoutMs ?? 600_000) + + let response: Response + try { + response = await fetch(this.endpoint, { + method: 'POST', + headers: { Authorization: `Bearer ${this.apiKey}` }, + body: form, + signal: controller.signal, + }) + } catch (err) { + const e = err as Error & { name?: string } + if (e.name === 'AbortError') { + throw new SnapshotError( + `Whisper API request timed out after ${opts.timeoutMs ?? 600_000}ms`, + e, + ) + } + throw new SnapshotError(`Whisper API request failed: ${e.message}`, e) + } finally { + clearTimeout(timer) + } + + if (!response.ok) { + const text = await response.text().catch(() => '') + throw new SnapshotError( + `Whisper API returned ${response.status} ${response.statusText}: ${text.slice(0, 500)}`, + ) + } + + const json = (await response.json()) as VerboseJsonResponse + const segments: TranscriptionSegment[] = (json.segments ?? []).map((s) => ({ + start: s.start, + end: s.end, + text: s.text.trim(), + })) + + return { + language: json.language, + duration_sec: json.duration, + segments, + full_text: json.text ?? segments.map((s) => s.text).join(' '), + } + } +} + +function sniffAudioMimeType(bytes: Uint8Array): string { + // ID3v2 / MP3 magic + if (bytes.length >= 3 && bytes[0] === 0x49 && bytes[1] === 0x44 && bytes[2] === 0x33) return 'audio/mpeg' + if (bytes.length >= 2 && bytes[0] === 0xff && (bytes[1] & 0xe0) === 0xe0) return 'audio/mpeg' + // RIFF / WAV + if (bytes.length >= 12 && bytes[0] === 0x52 && bytes[1] === 0x49 && bytes[2] === 0x46 && bytes[3] === 0x46 + && bytes[8] === 0x57 && bytes[9] === 0x41 && bytes[10] === 0x56 && bytes[11] === 0x45) { + return 'audio/wav' + } + // OggS + if (bytes.length >= 4 && bytes[0] === 0x4f && bytes[1] === 0x67 && bytes[2] === 0x67 && bytes[3] === 0x53) { + return 'audio/ogg' + } + // ftyp / M4A + if (bytes.length >= 8 && bytes[4] === 0x66 && bytes[5] === 0x74 && bytes[6] === 0x79 && bytes[7] === 0x70) { + return 'audio/m4a' + } + // FLaC + if (bytes.length >= 4 && bytes[0] === 0x66 && bytes[1] === 0x4c && bytes[2] === 0x61 && bytes[3] === 0x43) { + return 'audio/flac' + } + // Default: webm (most common browser-recorded format) + return 'audio/webm' +} + +function filenameForMime(mimeType: string): string { + if (mimeType === 'audio/mpeg') return 'audio.mp3' + if (mimeType === 'audio/wav') return 'audio.wav' + if (mimeType === 'audio/ogg') return 'audio.ogg' + if (mimeType === 'audio/m4a') return 'audio.m4a' + if (mimeType === 'audio/flac') return 'audio.flac' + if (mimeType === 'audio/webm') return 'audio.webm' + return 'audio.bin' +} diff --git a/src/index.ts b/src/index.ts index 46fb5c0..1181573 100644 --- a/src/index.ts +++ b/src/index.ts @@ -186,3 +186,16 @@ export type { ClaudeVisionOptions, OpenAiVisionOptions, } from './pdf' + +// ── v0.10: Audio support (kind: 'audio') ───────────────────────────────── + +export { convertAudio, WhisperApiBackend } from './audio' +export type { + ConvertAudioOptions, + WhisperApiOptions, + TranscriptionBackend, + TranscriptionResult, + TranscriptionSegment, + TranscribeOptions, +} from './audio' +export type { MediaMeta } from './types' diff --git a/src/types.ts b/src/types.ts index 4a23dfc..89dcdcd 100644 --- a/src/types.ts +++ b/src/types.ts @@ -5,20 +5,20 @@ * Producers build an `Snapshot`; serializers turn it into the wire format. */ -export const AGENTMARK_VERSION = '0.2' as const +export const AGENTMARK_VERSION = '0.3' as const /** Spec versions this implementation can validate against. */ -export const SUPPORTED_SPEC_VERSIONS = ['0.1', '0.2'] as const +export const SUPPORTED_SPEC_VERSIONS = ['0.1', '0.2', '0.3'] as const // ────────────────────────────────────────────────────────────────────────── // Frontmatter envelope // ────────────────────────────────────────────────────────────────────────── /** - * Discriminator added in v0.2. Identifies what kind of surface the snapshot - * was extracted from. Defaults to 'webpage' when omitted (v0.1 compatibility). + * Discriminator. v0.2 added `webpage|document|form`; v0.3 added `audio|video`. + * Defaults to 'webpage' when omitted (v0.1 compatibility). */ -export type SnapshotKind = 'webpage' | 'document' | 'form' +export type SnapshotKind = 'webpage' | 'document' | 'form' | 'audio' | 'video' export interface Snapshot { /** Spec version, e.g. "0.1" or "0.2" */ @@ -52,6 +52,12 @@ export interface Snapshot { /** Document-specific metadata (v0.2+, populated when kind === 'document'). */ document?: DocumentMeta + /** Audio/video-specific metadata (v0.3+, populated when kind === 'audio' | 'video'). */ + media_meta?: MediaMeta + + /** Speaker labels keyed by ID (v0.3+, audio/video). Map ID → display name. */ + speakers?: Record + /** * Detected signatures on the document, keyed by signature ID * (e.g. `sig_1`). Body uses `[SIGNATURE:sig_1]` to reference them. @@ -69,6 +75,28 @@ export interface Snapshot { export type SnapshotSource = 'rendered' | 'declared' | 'hybrid' +/** + * Media (audio/video) metadata. + */ +export interface MediaMeta { + /** Total duration in seconds. */ + duration_sec?: number + /** Format identifier (e.g. 'mp3', 'wav', 'mp4', 'webm'). */ + format?: string + /** BCP-47 language tag of the spoken content. */ + language?: string + /** Whether the source was transcribed (audio) or transcribed+frame-captioned (video). */ + transcribed?: boolean + /** When transcribed: name of the transcription backend used. */ + transcription_backend?: string + /** When video frames were captioned: name of the vision backend used. */ + vision_backend?: string + /** Number of speakers identified (when diarized). */ + speaker_count?: number + /** Number of frames captioned (video only). */ + frame_count?: number +} + /** * Document metadata extracted from PDF (or other document) backends. All * fields optional — backends populate what they can. @@ -240,6 +268,15 @@ export type BodyTagKind = /** v0.8+: signature reference. Payload is a signature ID (e.g. `sig_1`) * whose details live in the `signatures` map of the envelope. */ | 'SIGNATURE' + /** v0.3+: timestamp marker for `kind: 'audio' | 'video'`. Payload is + * a time identifier (`t_0`, `t_120`) whose number is seconds-from-start. */ + | 'TIME' + /** v0.3+: speaker label for `kind: 'audio' | 'video'`. Payload is a + * speaker ID (`s_alice`) keyed in the `speakers` map. */ + | 'SPEAKER' + /** v0.3+: video frame reference. Payload is a frame ID (`f_42`) whose + * thumbnail + caption live in the `media` map. */ + | 'FRAME' /** * Descriptor for a detected signature. Lives in `Snapshot.signatures` keyed diff --git a/src/validators/schema-validator.ts b/src/validators/schema-validator.ts index 1132e9f..d078048 100644 --- a/src/validators/schema-validator.ts +++ b/src/validators/schema-validator.ts @@ -14,14 +14,15 @@ const validatorCache = new Map() * Unknown versions fall back to the highest known schema and emit a warning * elsewhere — see `validateSnapshot` cross-field check 2e. */ -function resolveSchemaVersion(declared: string): '0.1' | '0.2' { +function resolveSchemaVersion(declared: string): '0.1' | '0.2' | '0.3' { const [major, minor] = declared.split('.') const minorMajor = `${major}.${minor}` if (minorMajor === '0.1') return '0.1' - return '0.2' + if (minorMajor === '0.2') return '0.2' + return '0.3' } -function loadValidator(version: '0.1' | '0.2'): ValidateFunction { +function loadValidator(version: '0.1' | '0.2' | '0.3'): ValidateFunction { const cached = validatorCache.get(version) if (cached) return cached @@ -87,12 +88,14 @@ export function validateSnapshot(snapshot: Snapshot): ValidationResult { // Payload-carrying tags (no lookup): ERROR, CHALLENGE // No-payload tags: AUTH_WALL const ACTION_RESOLVING = new Set(['ACTION', 'INPUT', 'NAV', 'TOAST']) - const MEDIA_RESOLVING = new Set(['MEDIA']) + const MEDIA_RESOLVING = new Set(['MEDIA', 'FRAME']) const SIGNATURE_RESOLVING = new Set(['SIGNATURE']) const PAYLOAD_TAGS = new Set(['ERROR', 'CHALLENGE']) - // PAGE is a v0.2 structural marker for `kind: 'document'` — payload is a - // page identifier (e.g. p_1) that does not resolve to any envelope entry. - const STRUCTURAL_TAGS = new Set(['PAGE']) + // Structural tags whose payload doesn't resolve to any envelope entry: + // PAGE — v0.2, page boundary marker (p_n) + // TIME — v0.3, timestamp marker for audio/video (t_seconds) + // SPEAKER — v0.3, speaker label (resolves to envelope.speakers map) + const STRUCTURAL_TAGS = new Set(['PAGE', 'TIME', 'SPEAKER']) const signatureIds = new Set(Object.keys(snapshot.signatures ?? {})) diff --git a/test/audio/audio-converter.test.ts b/test/audio/audio-converter.test.ts new file mode 100644 index 0000000..7ed79c5 --- /dev/null +++ b/test/audio/audio-converter.test.ts @@ -0,0 +1,189 @@ +/** + * Audio support tests with a mocked transcription backend. + */ + +import { describe, it, expect } from 'vitest' +import { convertAudio } from '../../src/audio/audio-converter' +import { parseSnapshot } from '../../src/serializers/yaml-frontmatter' +import { validateSnapshot } from '../../src/validators/schema-validator' +import type { + TranscriptionBackend, + TranscriptionResult, +} from '../../src/audio/types' + +function fakeTranscription(result: TranscriptionResult): TranscriptionBackend { + return { + name: 'fake_transcribe', + async transcribe() { + return result + }, + } +} + +describe('convertAudio', () => { + it('produces kind: "audio" snapshot with timestamps and speakers', async () => { + const transcribe = fakeTranscription({ + language: 'en', + duration_sec: 12.5, + segments: [ + { start: 0, end: 3, text: 'Hi, thanks for calling.', speaker: 's_alice' }, + { start: 3, end: 7, text: 'I have a question.', speaker: 's_bob' }, + { start: 7, end: 12, text: 'Sure, go ahead.', speaker: 's_alice' }, + ], + full_text: 'Hi, thanks for calling. I have a question. Sure, go ahead.', + speakers: { s_alice: 'Alice (Support)', s_bob: 'Bob (Customer)' }, + }) + + const { agentmark } = await convertAudio({ + data: new Uint8Array([0x49, 0x44, 0x33, 0x04]), // ID3v2 header (mp3-ish) + sourceUrl: 'file:///tmp/call.mp3', + transcribe, + }) + + const snap = parseSnapshot(agentmark) + expect(snap.kind).toBe('audio') + expect(snap.agentmark).toBe('0.3') + expect(snap.media_meta?.duration_sec).toBe(12.5) + expect(snap.media_meta?.transcribed).toBe(true) + expect(snap.media_meta?.transcription_backend).toBe('fake_transcribe') + expect(snap.media_meta?.speaker_count).toBe(2) + expect(snap.speakers).toEqual({ + s_alice: 'Alice (Support)', + s_bob: 'Bob (Customer)', + }) + + // Body has TIME + SPEAKER markers + escaped speech + expect(agentmark).toMatch(/\[TIME:t_0\]/) + expect(agentmark).toMatch(/\[TIME:t_3\]/) + expect(agentmark).toMatch(/\[TIME:t_7\]/) + expect(agentmark).toMatch(/\[SPEAKER:s_alice\] Hi, thanks for calling/) + expect(agentmark).toMatch(/\[SPEAKER:s_bob\] I have a question/) + // Same speaker continuing — second alice segment OMITS the speaker tag + expect(agentmark).toMatch(/\[SPEAKER:s_alice\] Sure, go ahead/) + }) + + it('validates against the v0.3 schema', async () => { + const transcribe = fakeTranscription({ + duration_sec: 5, + segments: [{ start: 0, end: 5, text: 'Hello world.' }], + full_text: 'Hello world.', + }) + const { agentmark } = await convertAudio({ + data: new Uint8Array(), + sourceUrl: 'file:///tmp/x.wav', + transcribe, + }) + const snap = parseSnapshot(agentmark) + const result = validateSnapshot(snap) + expect(result.errors).toEqual([]) + }) + + it('handles transcription with no segments (full_text only)', async () => { + const transcribe = fakeTranscription({ + segments: [], + full_text: 'A short utterance.', + }) + const { agentmark } = await convertAudio({ + data: new Uint8Array(), + sourceUrl: 'file:///tmp/x.mp3', + transcribe, + }) + expect(agentmark).toContain('[TIME:t_0]') + expect(agentmark).toContain('A short utterance.') + }) + + it('omits speakers map when transcription has none', async () => { + const transcribe = fakeTranscription({ + duration_sec: 3, + segments: [{ start: 0, end: 3, text: 'Solo speech.' }], + full_text: 'Solo speech.', + }) + const { agentmark } = await convertAudio({ + data: new Uint8Array(), + sourceUrl: 'file:///tmp/x.mp3', + transcribe, + }) + const snap = parseSnapshot(agentmark) + expect(snap.speakers).toBeUndefined() + expect(agentmark).not.toMatch(/\[SPEAKER:/) + }) + + it('falls back to URL basename when title is omitted', async () => { + const transcribe = fakeTranscription({ + segments: [{ start: 0, end: 1, text: 'x' }], + full_text: 'x', + }) + const { agentmark } = await convertAudio({ + data: new Uint8Array(), + sourceUrl: 'file:///tmp/customer-call-2026-05-10.mp3', + transcribe, + }) + const snap = parseSnapshot(agentmark) + expect(snap.title).toBe('customer-call-2026-05-10') + }) + + it('counts distinct speakers when no speakers map provided', async () => { + const transcribe = fakeTranscription({ + segments: [ + { start: 0, end: 3, text: 'A', speaker: 's_1' }, + { start: 3, end: 6, text: 'B', speaker: 's_2' }, + { start: 6, end: 9, text: 'C', speaker: 's_1' }, + ], + full_text: 'A B C', + }) + const { agentmark } = await convertAudio({ + data: new Uint8Array(), + sourceUrl: 'file:///tmp/x.mp3', + transcribe, + }) + const snap = parseSnapshot(agentmark) + expect(snap.media_meta?.speaker_count).toBe(2) + }) + + it('wraps backend failures in SnapshotError', async () => { + const broken: TranscriptionBackend = { + name: 'broken', + async transcribe() { + throw new Error('API exploded') + }, + } + await expect( + convertAudio({ + data: new Uint8Array(), + sourceUrl: 'file:///tmp/x.mp3', + transcribe: broken, + }), + ).rejects.toThrow(/transcription failed/) + }) + + it('escapes [TAG] sequences in transcript text so they do not become tag refs', async () => { + const transcribe = fakeTranscription({ + segments: [ + { start: 0, end: 3, text: 'I read [PAGE:p_1] in the document' }, + ], + full_text: 'I read [PAGE:p_1] in the document', + }) + const { agentmark } = await convertAudio({ + data: new Uint8Array(), + sourceUrl: 'file:///tmp/x.mp3', + transcribe, + }) + // Escaped form + expect(agentmark).toMatch(/I read \\\[PAGE:p_1\] in the document/) + }) + + it('preserves vendor extensions', async () => { + const transcribe = fakeTranscription({ + segments: [{ start: 0, end: 1, text: 'x' }], + full_text: 'x', + }) + const { agentmark } = await convertAudio({ + data: new Uint8Array(), + sourceUrl: 'file:///tmp/x.mp3', + transcribe, + vendorExtensions: { 'x-call-id': 'abc-123' }, + }) + expect(agentmark).toContain('x-call-id') + expect(agentmark).toContain('abc-123') + }) +}) diff --git a/test/build-snapshot.test.ts b/test/build-snapshot.test.ts index 45ae96a..d1d14f3 100644 --- a/test/build-snapshot.test.ts +++ b/test/build-snapshot.test.ts @@ -45,7 +45,7 @@ describe('buildSnapshot', () => { it('sets agentmark version, source, and timestamps', () => { const snap = buildSnapshot(fakeExtraction()) - expect(snap.agentmark).toBe('0.2') + expect(snap.agentmark).toBe('0.3') expect(snap.source).toBe('rendered') expect(snap.captured_at).toBeDefined() expect(snap.expires_at).toBeDefined() diff --git a/test/pdf/pdf-converter.test.ts b/test/pdf/pdf-converter.test.ts index 92537b0..d8db0dc 100644 --- a/test/pdf/pdf-converter.test.ts +++ b/test/pdf/pdf-converter.test.ts @@ -183,7 +183,7 @@ describe('convertPdf', () => { // Snapshot is parseable + validates against v0.2 schema const snap = parseSnapshot(agentmark) expect(snap.kind).toBe('document') - expect(snap.agentmark).toBe('0.2') + expect(snap.agentmark).toBe('0.3') expect(snap.url).toBe('file:///tmp/annual-report.pdf') expect(snap.title).toBe('Annual Report') diff --git a/test/spec-v0.2.test.ts b/test/spec-v0.2.test.ts index b230af4..149a23c 100644 --- a/test/spec-v0.2.test.ts +++ b/test/spec-v0.2.test.ts @@ -5,12 +5,12 @@ import type { Snapshot } from '../src/types' import { SUPPORTED_SPEC_VERSIONS, AGENTMARK_VERSION } from '../src/types' describe('Spec v0.2 — kind discriminator', () => { - it('default version is 0.2 in this implementation', () => { - expect(AGENTMARK_VERSION).toBe('0.2') + it('default version is 0.3 in this implementation (v0.3 ships with audio support)', () => { + expect(AGENTMARK_VERSION).toBe('0.3') }) - it('reports both v0.1 and v0.2 as supported', () => { - expect(SUPPORTED_SPEC_VERSIONS).toEqual(['0.1', '0.2']) + it('reports v0.1, v0.2, and v0.3 as supported', () => { + expect(SUPPORTED_SPEC_VERSIONS).toEqual(['0.1', '0.2', '0.3']) }) it('v0.1 snapshots without kind still validate (backwards compat)', () => { From 4ba2a4611e984c8617f3552cd667bd53e4453a1f Mon Sep 17 00:00:00 2001 From: rrader26 Date: Sun, 10 May 2026 15:26:32 -0400 Subject: [PATCH 5/5] feat(video): convertVideo() + FfmpegFrameBackend + spec v0.3 wire-up (v0.11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The final media surface. Video bytes become an AgentMark snapshot with kind: 'video' that interleaves transcript and visually-captioned frames in timeline order — so an LLM reading the snapshot sees both audio and visual context aligned in time. [TIME:t_0] [SPEAKER:s_alice] Welcome to the demo. [TIME:t_30] [FRAME:f_2] (frame caption: Architecture diagram with three boxes labeled A, B, C.) [TIME:t_60] [SPEAKER:s_alice] Today we'll cover the three pieces. Reuses the v0.10 audio + v0.9 vision pipelines — no new long-lived backend interfaces. Just a frame extractor + an orchestrator. FrameExtractionBackend interface - extractFrames({ data, sampling, width, format }) → ExtractedFrame[] - Sampling strategies: { every: N seconds } | { count: N frames } | { keyframes: true } FfmpegFrameBackend (reference impl) - Shells out to system `ffmpeg` (and `ffprobe` for duration). - Required: brew install ffmpeg / apt-get install ffmpeg. - Throws clean SnapshotError with install instructions if missing. - Atomic temp dir per call; cleaned up on completion. - Emits jpegs by default at 800px width — good vision-LLM input. convertVideo() orchestrator - Runs frame extraction + transcription in parallel (when transcribe is configured). - Captions each extracted frame in series via the caller-provided VisionBackend (the same interface introduced in v0.9 for signature detection — no new vision impl needed). - Builds a unified timeline of {transcript, frame} events sorted by timestamp, emits [TIME] / [SPEAKER] / [FRAME] tags as appropriate. - Frames go into the existing `media` map as `image` entries with captions; [FRAME:f_n] resolves through the existing MEDIA-resolving validator path (no new validator code). - Per-frame caption failures are logged + skipped (graceful). - transcribe=null → frames-only video (no audio extraction). - caption=null → [FRAME] markers without descriptions (cheaper). - Both null → fail fast with SnapshotError ("no frames and no transcript"). Validator - [FRAME] body tag added to MEDIA_RESOLVING set — must reference an envelope.media entry (cross-field check). Tests (278 total, 7 new for video — 252 → 268 → 271 → 278) - Interleaved transcript + frames in correct timeline order - v0.3 schema validation passes - caption=null path - transcribe=null path - both=null fails fast - Per-frame caption failure tolerated (graceful, frame still emitted) - Out-of-order insertion still produces ordered timeline Public API - convertVideo + FfmpegFrameBackend + types exported at /src and /index. - ffmpeg is NOT a peer dep (it's a system binary). Documented in TESTING.md / README. Version bump - @thinkfleet/agentmark: 0.7.0 → 0.11.0 (skipping 0.8/0.9/0.10 since signatures + vision sigs + audio + video all landed together in this iteration cycle and we haven't published yet). - Description rewritten to capture the full surface coverage. Co-Authored-By: Claude Opus 4.7 (1M context) --- package.json | 4 +- src/index.ts | 11 + src/video/ffmpeg-frame-backend.ts | 171 +++++++++++++++ src/video/index.ts | 13 ++ src/video/types.ts | 41 ++++ src/video/video-converter.ts | 334 +++++++++++++++++++++++++++++ test/video/video-converter.test.ts | 221 +++++++++++++++++++ 7 files changed, 793 insertions(+), 2 deletions(-) create mode 100644 src/video/ffmpeg-frame-backend.ts create mode 100644 src/video/index.ts create mode 100644 src/video/types.ts create mode 100644 src/video/video-converter.ts create mode 100644 test/video/video-converter.test.ts diff --git a/package.json b/package.json index f314d6c..5102faa 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@thinkfleet/agentmark", - "version": "0.7.0", - "description": "AI browser + document + form library + MCP server — convert any web page or PDF into a compact AgentMark snapshot, then drive it from any MCP client (Claude Desktop, Cursor, Claude Code) or directly via the SDK.", + "version": "0.11.0", + "description": "AI library for any AI-readable surface — web pages, PDFs (text/scanned/AcroForm), audio (transcribed), video (transcribed + frame-captioned), with signature detection. One compact wire format any AI client can read or drive via MCP.", "type": "commonjs", "main": "./dist/src/index.js", "types": "./dist/src/index.d.ts", diff --git a/src/index.ts b/src/index.ts index 1181573..8f33b30 100644 --- a/src/index.ts +++ b/src/index.ts @@ -199,3 +199,14 @@ export type { TranscribeOptions, } from './audio' export type { MediaMeta } from './types' + +// ── v0.11: Video support (kind: 'video') ───────────────────────────────── + +export { convertVideo, FfmpegFrameBackend } from './video' +export type { + ConvertVideoOptions, + FfmpegFrameBackendOptions, + FrameExtractionBackend, + ExtractFramesOptions, + ExtractedFrame, +} from './video' diff --git a/src/video/ffmpeg-frame-backend.ts b/src/video/ffmpeg-frame-backend.ts new file mode 100644 index 0000000..2e0c6bc --- /dev/null +++ b/src/video/ffmpeg-frame-backend.ts @@ -0,0 +1,171 @@ +/** + * Frame-extraction backend that shells out to `ffmpeg`. + * + * Requires ffmpeg installed system-wide: + * macOS: brew install ffmpeg + * Linux: apt-get install ffmpeg + * Windows: choco / scoop / official binaries + * + * If ffmpeg is missing, `extractFrames()` throws a SnapshotError with + * installation instructions on the first call. + */ + +import { spawn } from 'node:child_process' +import { writeFile, mkdtemp, readdir, readFile, rm } from 'node:fs/promises' +import * as os from 'node:os' +import * as path from 'node:path' +import { SnapshotError } from '../errors' +import type { + ExtractedFrame, + ExtractFramesOptions, + FrameExtractionBackend, +} from './types' + +export interface FfmpegFrameBackendOptions { + /** Override the ffmpeg binary path. Default: 'ffmpeg' on $PATH. */ + binary?: string + /** Override the ffprobe binary path (used to read video duration). */ + ffprobeBinary?: string +} + +export class FfmpegFrameBackend implements FrameExtractionBackend { + readonly name = 'ffmpeg' + private readonly binary: string + private readonly ffprobeBinary: string + private binaryChecked = false + + constructor(options: FfmpegFrameBackendOptions = {}) { + this.binary = options.binary ?? 'ffmpeg' + this.ffprobeBinary = options.ffprobeBinary ?? 'ffprobe' + } + + async extractFrames(opts: ExtractFramesOptions): Promise { + await this.ensureBinaryAvailable() + const tmpDir = await mkdtemp(path.join(os.tmpdir(), 'agentmark-ffmpeg-')) + const inputPath = path.join(tmpDir, 'input') + const ext = opts.format ?? 'jpeg' + + try { + await writeFile(inputPath, opts.data) + + const sampling = opts.sampling + const args: string[] = ['-y', '-loglevel', 'error', '-i', inputPath] + if ('every' in sampling) { + args.push('-vf', `fps=1/${sampling.every}`) + } else if ('count' in sampling) { + const duration = await this.probeDuration(inputPath).catch(() => 0) + if (duration > 0) { + const interval = duration / Math.max(sampling.count, 1) + args.push('-vf', `fps=1/${interval.toFixed(2)}`) + } else { + args.push('-vf', 'thumbnail') // best fallback for "give me N frames" + args.push('-frames:v', String(sampling.count)) + } + } else if (sampling.keyframes) { + args.push('-vf', "select='eq(pict_type,I)'", '-vsync', 'vfr') + } + if (opts.width) args.push('-vf', `${args[args.length - 1] === ',' ? '' : ''}scale=${opts.width}:-1`) + args.push('-q:v', '4') // jpeg quality 1-31, lower = better + const outputPattern = path.join(tmpDir, `frame-%04d.${ext === 'png' ? 'png' : 'jpg'}`) + args.push(outputPattern) + + await this.spawnFfmpeg(args) + + // Find emitted frames + const entries = await readdir(tmpDir) + const frameFiles = entries + .filter((f) => f.startsWith('frame-') && (f.endsWith('.jpg') || f.endsWith('.png'))) + .sort() + + if (frameFiles.length === 0) return [] + + // Compute timestamps for the emitted frames + const timestamps = await this.computeTimestamps(sampling, frameFiles.length, inputPath) + + const frames: ExtractedFrame[] = [] + for (let i = 0; i < frameFiles.length; i++) { + const buf = await readFile(path.join(tmpDir, frameFiles[i])) + frames.push({ + timestamp: timestamps[i] ?? 0, + image: new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength), + mimeType: ext === 'png' ? 'image/png' : 'image/jpeg', + }) + } + return frames + } finally { + await rm(tmpDir, { recursive: true, force: true }).catch(() => {}) + } + } + + private async ensureBinaryAvailable(): Promise { + if (this.binaryChecked) return + try { + await this.spawnFfmpeg(['-version']) + this.binaryChecked = true + } catch (err) { + throw new SnapshotError( + `Could not run "${this.binary}". Install ffmpeg:\n` + + ` macOS: brew install ffmpeg\n` + + ` Linux: apt-get install ffmpeg\n` + + ` Windows: choco install ffmpeg / scoop install ffmpeg`, + err as Error, + ) + } + } + + private spawnFfmpeg(args: string[]): Promise { + return new Promise((resolve, reject) => { + const proc = spawn(this.binary, args, { stdio: ['ignore', 'ignore', 'pipe'] }) + let stderr = '' + proc.stderr?.on('data', (chunk: Buffer) => { stderr += chunk.toString() }) + proc.on('error', reject) + proc.on('close', (code) => { + if (code === 0) resolve() + else reject(new Error(`ffmpeg exited ${code}: ${stderr.trim().slice(0, 1000)}`)) + }) + }) + } + + private async probeDuration(inputPath: string): Promise { + return await new Promise((resolve, reject) => { + const proc = spawn(this.ffprobeBinary, [ + '-v', 'error', + '-show_entries', 'format=duration', + '-of', 'default=noprint_wrappers=1:nokey=1', + inputPath, + ], { stdio: ['ignore', 'pipe', 'ignore'] }) + let stdout = '' + proc.stdout?.on('data', (chunk: Buffer) => { stdout += chunk.toString() }) + proc.on('error', reject) + proc.on('close', () => { + const dur = parseFloat(stdout.trim()) + resolve(Number.isFinite(dur) ? dur : 0) + }) + }) + } + + private async computeTimestamps( + sampling: ExtractFramesOptions['sampling'], + frameCount: number, + inputPath: string, + ): Promise { + if ('every' in sampling) { + return Array.from({ length: frameCount }, (_, i) => i * sampling.every) + } + if ('count' in sampling) { + const duration = await this.probeDuration(inputPath).catch(() => 0) + if (duration > 0 && frameCount > 0) { + const interval = duration / frameCount + return Array.from({ length: frameCount }, (_, i) => Math.round((i + 0.5) * interval)) + } + } + // keyframes — without parsing frame metadata we can't know exact timestamps; + // return evenly distributed estimates. + const duration = await this.probeDuration(inputPath).catch(() => 0) + if (duration > 0 && frameCount > 0) { + const interval = duration / frameCount + return Array.from({ length: frameCount }, (_, i) => Math.round(i * interval)) + } + return Array.from({ length: frameCount }, () => 0) + } +} diff --git a/src/video/index.ts b/src/video/index.ts new file mode 100644 index 0000000..33de8fd --- /dev/null +++ b/src/video/index.ts @@ -0,0 +1,13 @@ +/** + * Video support — convertVideo() + frame extraction backends. + */ + +export { convertVideo } from './video-converter' +export type { ConvertVideoOptions } from './video-converter' +export { FfmpegFrameBackend } from './ffmpeg-frame-backend' +export type { FfmpegFrameBackendOptions } from './ffmpeg-frame-backend' +export type { + FrameExtractionBackend, + ExtractFramesOptions, + ExtractedFrame, +} from './types' diff --git a/src/video/types.ts b/src/video/types.ts new file mode 100644 index 0000000..c0b0222 --- /dev/null +++ b/src/video/types.ts @@ -0,0 +1,41 @@ +/** + * Video support — frame-extraction interface + convertVideo() output types. + */ + +export interface FrameExtractionBackend { + readonly name: string + /** + * Extract a sample of frames from video bytes. Implementations should + * yield evenly-spaced frames OR detect scene changes; the interface + * doesn't dictate. + */ + extractFrames(opts: ExtractFramesOptions): Promise + close?(): Promise +} + +export interface ExtractFramesOptions { + /** Video bytes — typically mp4/webm/mov/mkv. */ + data: Uint8Array + /** MIME type. Default: sniffed from bytes. */ + mimeType?: string + /** + * Frame sampling strategy: + * - { every: N } — sample every N seconds + * - { count: N } — sample N evenly-spaced frames + * - { keyframes: true } — keyframes only + */ + sampling: { every: number } | { count: number } | { keyframes: true } + /** Per-frame output format. Default: 'jpeg'. */ + format?: 'jpeg' | 'png' + /** Per-frame output width in pixels. Default: 800. */ + width?: number +} + +export interface ExtractedFrame { + /** Timestamp in seconds where this frame was sampled. */ + timestamp: number + /** Image bytes. */ + image: Uint8Array + /** MIME type of `image`. */ + mimeType: 'image/jpeg' | 'image/png' +} diff --git a/src/video/video-converter.ts b/src/video/video-converter.ts new file mode 100644 index 0000000..8887b29 --- /dev/null +++ b/src/video/video-converter.ts @@ -0,0 +1,334 @@ +/** + * `convertVideo()` — convert video bytes into an AgentMark snapshot with + * `kind: 'video'`. Combines: + * + * 1. Audio transcription (TranscriptionBackend) → [TIME] + [SPEAKER] markers + * 2. Frame extraction (FrameExtractionBackend) → keyframes / sampled frames + * 3. Frame captioning (VisionBackend) → text descriptions per frame + * + * Output body interleaves transcript segments and frame captions in + * timestamp order so an LLM reading the snapshot sees both audio and + * visual context aligned in time: + * + * [TIME:t_0] + * [SPEAKER:s_alice] Welcome to the demo. + * + * [TIME:t_5] [FRAME:f_1] + * (frame caption: "Slide showing pricing tiers: Free, Pro, Enterprise") + * + * [TIME:t_8] + * [SPEAKER:s_alice] Today we'll cover three pricing options... + */ + +import { + AGENTMARK_VERSION, + type ConversionResult, + type MediaDefinition, + type MediaMeta, + type Snapshot, +} from '../types' +import { serializeSnapshot } from '../serializers/yaml-frontmatter' +import { InMemoryActionBinding } from '../binding/action-binding' +import { noopLogger, type Logger } from '../observability/logger' +import { SnapshotError } from '../errors' +import type { TranscriptionBackend, TranscriptionSegment } from '../audio/types' +import type { VisionBackend } from '../pdf/vision/types' +import type { + ExtractedFrame, + ExtractFramesOptions, + FrameExtractionBackend, +} from './types' + +export interface ConvertVideoOptions { + data: Uint8Array | ArrayBuffer + sourceUrl: string + /** Audio transcription backend. Required — the spoken track is the + * spine of the body. Pass `null` to skip transcription entirely. */ + transcribe: TranscriptionBackend | null + /** Frame extraction backend (FfmpegFrameBackend or custom). */ + frames: FrameExtractionBackend + /** Vision backend used to caption each extracted frame. Pass `null` + * to skip captions and just emit [FRAME] markers without text. */ + caption: VisionBackend | null + /** Frame sampling strategy. Default: { every: 30 } (one per 30s). */ + sampling?: ExtractFramesOptions['sampling'] + /** Width for extracted frames. Default: 800px. */ + frameWidth?: number + /** Override title. Default: source URL basename. */ + title?: string + language?: string + diarize?: boolean + ttlMs?: number + logger?: Logger + vendorExtensions?: Record + mimeType?: string +} + +interface TimelineEvent { + time: number + type: 'transcript' | 'frame' + transcript?: TranscriptionSegment + frame?: { id: string; index: number; caption?: string } +} + +export async function convertVideo(options: ConvertVideoOptions): Promise { + const logger = options.logger ?? noopLogger + const ttlMs = options.ttlMs ?? 24 * 60 * 60_000 + + logger.debug('snapshot.capture.start', { source: options.sourceUrl, kind: 'video' }) + + const data = options.data instanceof ArrayBuffer + ? new Uint8Array(options.data) + : new Uint8Array(options.data.buffer, options.data.byteOffset, options.data.byteLength) + const sampling = options.sampling ?? { every: 30 } + + // Run frame extraction and (optionally) transcription in parallel. + const [framesResult, transcriptionResult] = await Promise.all([ + options.frames + .extractFrames({ + data, + mimeType: options.mimeType, + sampling, + width: options.frameWidth ?? 800, + format: 'jpeg', + }) + .catch((err: Error) => { + logger.warn('video.frames.failed', { error: err.message }) + return [] as ExtractedFrame[] + }), + options.transcribe + ? options.transcribe.transcribe({ + data, + mimeType: options.mimeType, + language: options.language, + diarize: options.diarize, + }).catch((err: Error) => { + logger.warn('video.transcribe.failed', { error: err.message }) + return null + }) + : Promise.resolve(null), + ]) + + if (framesResult.length === 0 && !transcriptionResult) { + throw new SnapshotError('Video conversion produced no frames and no transcript') + } + + // Caption frames in series (vision API rate limits + token cost). Skip + // when caption=null. + const captionedFrames: Array<{ frame: ExtractedFrame; caption?: string; id: string }> = [] + for (let i = 0; i < framesResult.length; i++) { + const frame = framesResult[i] + const id = `f_${i + 1}` + let caption: string | undefined + if (options.caption) { + try { + const result = await options.caption.analyze({ + image: frame.image, + mimeType: frame.mimeType, + prompt: + 'Describe this video frame in 1-2 short sentences. Focus on ' + + 'what is most informative for someone who cannot see it: ' + + 'on-screen text, the subject, the setting, key visual cues. ' + + 'Be concrete and specific. No editorializing.', + maxTokens: 200, + }) + caption = result.text.trim() + } catch (err) { + logger.warn('video.caption.failed', { + frame: id, + error: err instanceof Error ? err.message : String(err), + }) + } + } + captionedFrames.push({ frame, caption, id }) + } + + // Build interleaved timeline + const events: TimelineEvent[] = [] + if (transcriptionResult) { + for (const seg of transcriptionResult.segments) { + events.push({ time: seg.start, type: 'transcript', transcript: seg }) + } + } + for (let i = 0; i < captionedFrames.length; i++) { + const cf = captionedFrames[i] + events.push({ + time: cf.frame.timestamp, + type: 'frame', + frame: { id: cf.id, index: i, caption: cf.caption }, + }) + } + events.sort((a, b) => a.time - b.time) + + const body = buildVideoBody(events) + + // Frames go into the `media` map so [FRAME:f_n] resolves through the + // existing MEDIA-resolving validator path. + const media: Record = {} + for (const cf of captionedFrames) { + media[cf.id] = { + type: 'image', + caption: cf.caption ?? null, + } + } + + const captured_at = new Date().toISOString() + const expires_at = new Date(Date.now() + ttlMs).toISOString() + const speakers = transcriptionResult?.speakers + const speakerCount = speakers + ? Object.keys(speakers).length + : countDistinctSpeakers(transcriptionResult?.segments ?? []) + + const mediaMeta: MediaMeta = { + duration_sec: transcriptionResult?.duration_sec, + format: deriveFormat(options.mimeType ?? sniffFormat(data)), + language: transcriptionResult?.language ?? options.language, + transcribed: transcriptionResult !== null, + transcription_backend: options.transcribe?.name, + vision_backend: options.caption?.name, + speaker_count: speakerCount > 0 ? speakerCount : undefined, + frame_count: captionedFrames.length, + } + + const title = options.title ?? deriveTitleFromUrl(options.sourceUrl) + + const snapshot: Snapshot = { + agentmark: AGENTMARK_VERSION, + kind: 'video', + url: options.sourceUrl, + title, + captured_at, + expires_at, + source: 'declared', + language: mediaMeta.language, + media_meta: stripUndefined(mediaMeta), + speakers: speakers && Object.keys(speakers).length > 0 ? speakers : undefined, + media: Object.keys(media).length > 0 ? media : undefined, + capabilities: { + preview_media: true, + expand_disclosures: false, + paginate: false, + scroll: true, + keyboard: false, + drag: false, + ocr: false, + vision: options.caption !== null, + }, + body, + } + + if (options.vendorExtensions) { + for (const [k, v] of Object.entries(options.vendorExtensions)) { + if (k.startsWith('x-')) (snapshot as unknown as Record)[k] = v + } + } + + const text = serializeSnapshot(snapshot) + logger.info('snapshot.captured', { + source: options.sourceUrl, + kind: 'video', + duration_sec: mediaMeta.duration_sec, + frames: captionedFrames.length, + segments: transcriptionResult?.segments.length ?? 0, + bytes: text.length, + }) + + // Best-effort cleanup of long-lived backends. + try { await Promise.resolve(options.transcribe?.close?.()) } catch { /* ignore */ } + try { await Promise.resolve(options.frames.close?.()) } catch { /* ignore */ } + try { await Promise.resolve(options.caption?.close?.()) } catch { /* ignore */ } + + return { agentmark: text, binding: new InMemoryActionBinding() } +} + +// ────────────────────────────────────────────────────────────────────────── +// Body builder +// ────────────────────────────────────────────────────────────────────────── + +function buildVideoBody(events: TimelineEvent[]): string { + if (events.length === 0) { + return '[TIME:t_0]\n\n(No transcript or frames extracted.)\n' + } + const lines: string[] = [] + let lastSpeaker: string | undefined + for (const event of events) { + const timeId = `t_${Math.round(event.time)}` + if (event.type === 'transcript' && event.transcript) { + lines.push(`[TIME:${timeId}]`) + const seg = event.transcript + if (seg.speaker && seg.speaker !== lastSpeaker) { + lines.push(`[SPEAKER:${seg.speaker}] ${escapeBody(seg.text)}`) + lastSpeaker = seg.speaker + } else { + lines.push(escapeBody(seg.text)) + } + lines.push('') + } else if (event.type === 'frame' && event.frame) { + lines.push(`[TIME:${timeId}] [FRAME:${event.frame.id}]`) + if (event.frame.caption) { + lines.push(`(frame caption: ${escapeBody(event.frame.caption)})`) + } + lines.push('') + } + } + return lines.join('\n') +} + +// ────────────────────────────────────────────────────────────────────────── +// Helpers (copied from audio-converter — small, not worth a shared module) +// ────────────────────────────────────────────────────────────────────────── + +function escapeBody(text: string): string { + return text.replace(/\\/g, '\\\\').replace(/\[(?=[A-Z])/g, '\\[') +} + +function countDistinctSpeakers(segments: TranscriptionSegment[]): number { + const set = new Set() + for (const s of segments) if (s.speaker) set.add(s.speaker) + return set.size +} + +function deriveTitleFromUrl(url: string): string { + try { + const u = new URL(url) + const last = u.pathname.split('/').filter(Boolean).pop() ?? '(untitled)' + return decodeURIComponent(last).replace(/\.[a-z0-9]+$/i, '') || '(untitled)' + } catch { + return '(untitled)' + } +} + +function deriveFormat(mimeType: string | undefined): string | undefined { + if (!mimeType) return undefined + if (mimeType.includes('mp4')) return 'mp4' + if (mimeType.includes('webm')) return 'webm' + if (mimeType.includes('quicktime') || mimeType.includes('mov')) return 'mov' + if (mimeType.includes('matroska') || mimeType.includes('mkv')) return 'mkv' + if (mimeType.includes('avi')) return 'avi' + return undefined +} + +function sniffFormat(bytes: Uint8Array): string | undefined { + // ftyp signature at offset 4 + if (bytes.length >= 12 && bytes[4] === 0x66 && bytes[5] === 0x74 && bytes[6] === 0x79 && bytes[7] === 0x70) { + return 'video/mp4' + } + // EBML header for webm/mkv + if (bytes.length >= 4 && bytes[0] === 0x1a && bytes[1] === 0x45 && bytes[2] === 0xdf && bytes[3] === 0xa3) { + return 'video/webm' + } + // RIFF + AVI + if (bytes.length >= 12 && bytes[0] === 0x52 && bytes[1] === 0x49 && bytes[2] === 0x46 && bytes[3] === 0x46 + && bytes[8] === 0x41 && bytes[9] === 0x56 && bytes[10] === 0x49) { + return 'video/avi' + } + return undefined +} + +function stripUndefined(obj: T): T { + const out: Record = {} + for (const [k, v] of Object.entries(obj)) { + if (v !== undefined) out[k] = v + } + return out as T +} diff --git a/test/video/video-converter.test.ts b/test/video/video-converter.test.ts new file mode 100644 index 0000000..4bcb335 --- /dev/null +++ b/test/video/video-converter.test.ts @@ -0,0 +1,221 @@ +/** + * Video converter tests with mocked transcription / frame / vision backends. + * + * Real ffmpeg + Whisper + Claude run via the kitchen-sink demo / manual + * smoke test against actual video files; this suite locks the + * orchestration logic. + */ + +import { describe, it, expect } from 'vitest' +import { convertVideo } from '../../src/video/video-converter' +import { parseSnapshot } from '../../src/serializers/yaml-frontmatter' +import { validateSnapshot } from '../../src/validators/schema-validator' +import type { + TranscriptionBackend, + TranscriptionResult, +} from '../../src/audio/types' +import type { + FrameExtractionBackend, + ExtractedFrame, +} from '../../src/video/types' +import type { + AnalyzeOptions, + AnalyzeResult, + VisionBackend, +} from '../../src/pdf/vision/types' + +function tinyJpeg(): Uint8Array { + return new Uint8Array([ + 0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00, 0x01, + 0x01, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0xff, 0xd9, + ]) +} + +function fakeTranscribe(result: TranscriptionResult): TranscriptionBackend { + return { + name: 'fake_whisper', + async transcribe() { return result }, + } +} + +function fakeFrames(frames: ExtractedFrame[]): FrameExtractionBackend { + return { + name: 'fake_ffmpeg', + async extractFrames() { return frames }, + } +} + +function fakeCaption(captions: string[]): VisionBackend & { calls: number } { + let i = 0 + return { + name: 'fake_vision', + calls: 0, + async analyze(_opts: AnalyzeOptions): Promise { + this.calls++ + return { text: captions[i++ % captions.length] ?? 'caption', structured: undefined } + }, + } as VisionBackend & { calls: number } +} + +const SAMPLE_FRAMES = (count: number, every = 30): ExtractedFrame[] => + Array.from({ length: count }, (_, i) => ({ + timestamp: i * every, + image: tinyJpeg(), + mimeType: 'image/jpeg' as const, + })) + +describe('convertVideo', () => { + it('produces kind: "video" snapshot interleaving transcript + frames', async () => { + const transcribe = fakeTranscribe({ + duration_sec: 90, + segments: [ + { start: 0, end: 5, text: 'Hello.', speaker: 's_alice' }, + { start: 60, end: 65, text: 'Now the demo.', speaker: 's_alice' }, + ], + full_text: 'Hello. Now the demo.', + speakers: { s_alice: 'Alice (Presenter)' }, + }) + const frames = fakeFrames(SAMPLE_FRAMES(3, 30)) // 0, 30, 60 + const caption = fakeCaption([ + 'Title slide reading "Q4 Demo".', + 'Architecture diagram with three boxes.', + 'Closing slide with contact info.', + ]) + + const { agentmark } = await convertVideo({ + data: new Uint8Array([0, 0, 0, 0, 0x66, 0x74, 0x79, 0x70]), // ftyp magic + sourceUrl: 'file:///tmp/demo.mp4', + transcribe, + frames, + caption, + }) + + const snap = parseSnapshot(agentmark) + expect(snap.kind).toBe('video') + expect(snap.media_meta?.duration_sec).toBe(90) + expect(snap.media_meta?.transcribed).toBe(true) + expect(snap.media_meta?.transcription_backend).toBe('fake_whisper') + expect(snap.media_meta?.vision_backend).toBe('fake_vision') + expect(snap.media_meta?.frame_count).toBe(3) + expect(snap.speakers?.s_alice).toBe('Alice (Presenter)') + + // Body has TIME + SPEAKER + FRAME tags interleaved by time + expect(agentmark).toMatch(/\[TIME:t_0\]/) + expect(agentmark).toMatch(/\[SPEAKER:s_alice\] Hello/) + expect(agentmark).toMatch(/\[TIME:t_30\] \[FRAME:f_2\]/) + expect(agentmark).toMatch(/Architecture diagram with three boxes/) + expect(agentmark).toMatch(/\[TIME:t_60\]/) + + // media map populated with frame entries + expect(snap.media?.f_1?.type).toBe('image') + expect(snap.media?.f_1?.caption).toMatch(/Title slide/) + expect(caption.calls).toBe(3) + }) + + it('validates against the v0.3 schema', async () => { + const { agentmark } = await convertVideo({ + data: new Uint8Array(), + sourceUrl: 'file:///tmp/x.mp4', + transcribe: fakeTranscribe({ + segments: [{ start: 0, end: 1, text: 'x' }], + full_text: 'x', + }), + frames: fakeFrames(SAMPLE_FRAMES(1)), + caption: fakeCaption(['caption']), + }) + const snap = parseSnapshot(agentmark) + const result = validateSnapshot(snap) + expect(result.errors).toEqual([]) + }) + + it('runs without captions when caption=null (FRAME tags but no descriptions)', async () => { + const { agentmark } = await convertVideo({ + data: new Uint8Array(), + sourceUrl: 'file:///tmp/x.mp4', + transcribe: fakeTranscribe({ + segments: [{ start: 0, end: 1, text: 'speech' }], + full_text: 'speech', + }), + frames: fakeFrames(SAMPLE_FRAMES(2)), + caption: null, + }) + expect(agentmark).toMatch(/\[FRAME:f_1\]/) + expect(agentmark).not.toMatch(/frame caption:/) + }) + + it('runs without transcription when transcribe=null', async () => { + const { agentmark } = await convertVideo({ + data: new Uint8Array(), + sourceUrl: 'file:///tmp/x.mp4', + transcribe: null, + frames: fakeFrames(SAMPLE_FRAMES(1)), + caption: fakeCaption(['Just a frame.']), + }) + const snap = parseSnapshot(agentmark) + expect(snap.media_meta?.transcribed).toBe(false) + expect(agentmark).not.toMatch(/\[SPEAKER:/) + expect(agentmark).toMatch(/Just a frame/) + }) + + it('throws when both frames and transcript are empty', async () => { + await expect( + convertVideo({ + data: new Uint8Array(), + sourceUrl: 'file:///tmp/x.mp4', + transcribe: null, + frames: fakeFrames([]), + caption: null, + }), + ).rejects.toThrow(/no frames and no transcript/) + }) + + it('continues when caption fails for one frame', async () => { + let i = 0 + const flaky: VisionBackend = { + name: 'flaky', + async analyze() { + i++ + if (i === 2) throw new Error('rate limit') + return { text: `caption ${i}`, structured: undefined } + }, + } + const { agentmark } = await convertVideo({ + data: new Uint8Array(), + sourceUrl: 'file:///tmp/x.mp4', + transcribe: null, + frames: fakeFrames(SAMPLE_FRAMES(3)), + caption: flaky, + }) + const snap = parseSnapshot(agentmark) + // Frame 1 + frame 3 captioned; frame 2 has no caption (graceful) + expect(snap.media?.f_1?.caption).toBe('caption 1') + expect(snap.media?.f_2?.caption).toBeNull() + expect(snap.media?.f_3?.caption).toMatch(/caption 3/) + }) + + it('orders timeline events by timestamp regardless of insertion order', async () => { + // Transcript at t=0, t=60. Frames at t=30, t=90. Should interleave. + const { agentmark } = await convertVideo({ + data: new Uint8Array(), + sourceUrl: 'file:///tmp/x.mp4', + transcribe: fakeTranscribe({ + segments: [ + { start: 0, end: 5, text: 'speak 0' }, + { start: 60, end: 65, text: 'speak 60' }, + ], + full_text: '', + }), + frames: fakeFrames([ + { timestamp: 30, image: tinyJpeg(), mimeType: 'image/jpeg' }, + { timestamp: 90, image: tinyJpeg(), mimeType: 'image/jpeg' }, + ]), + caption: fakeCaption(['frame_30', 'frame_90']), + }) + + const positions = ['t_0', 't_30', 't_60', 't_90'].map((id) => agentmark.indexOf(`[TIME:${id}]`)) + expect(positions[0]).toBeGreaterThan(0) + expect(positions[1]).toBeGreaterThan(positions[0]) + expect(positions[2]).toBeGreaterThan(positions[1]) + expect(positions[3]).toBeGreaterThan(positions[2]) + }) +})