From e6ff690a1a890c9e5d0cd5af552a30f869016870 Mon Sep 17 00:00:00 2001 From: Rupert Germann Date: Fri, 8 May 2026 10:04:58 +0200 Subject: [PATCH 01/29] feat: add motion capture posture analysis PRD --- docs/prd-mocap-posture.md | 154 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 docs/prd-mocap-posture.md diff --git a/docs/prd-mocap-posture.md b/docs/prd-mocap-posture.md new file mode 100644 index 0000000..3cfa08d --- /dev/null +++ b/docs/prd-mocap-posture.md @@ -0,0 +1,154 @@ +# PRD: Motion-capture posture analysis (freemocap integration) + +## Problem Statement + +I row indoors and the app already gives me deep performance analytics from SmartRow CSV (pace, power, stroke rate, PRs, training load). But the data is blind to **how I move**. Bad posture and stroke sequencing silently cap my power output, waste effort, and risk injury (lower-back rounding at the catch, early arm bend, opening the back too soon, asymmetric drive). I have no coach watching me. I want the app to see me row and tell me what's wrong with my technique — both while I'm rowing and after the session, alongside the metrics I already track. + +## Solution + +Integrate markerless motion capture so the app analyzes the rower's posture and stroke mechanics from video, derives per-stroke biomechanical metrics, detects posture faults, and delivers actionable coaching cues. Two capture paths: + +1. **Browser path (default)** — single webcam, in-browser MediaPipe Pose. Zero install. Live feedback during rowing, plus replay after. +2. **Sidecar path (precision)** — local Python sidecar running [freemocap](https://github.com/freemocap/freemocap) with one or more cameras for 3D-accurate skeletons. For users who want serious technique work. + +Pose data is stored raw (keypoints per frame) and aligned to existing `RowingSession` / `StrokeData` so users can scrub video, see skeleton overlay, see flagged faults at exact stroke phases, and watch posture trends evolve next to power/pace trends. + +## User Stories + +1. As a rower, I want to start a webcam-based mocap session in one click, so that I can get posture feedback without installing software. +2. As a rower, I want the app to detect my rowing machine in frame and confirm I'm positioned correctly, so that capture quality is good before I start. +3. As a rower, I want a calibration step (sit at catch, sit at finish), so that the analyzer knows my anatomy and machine geometry. +4. As a rower, I want live audio + visual cues during rowing ("back rounded", "arms bent early", "slow down recovery"), so that I can correct in real time. +5. As a rower, I want live cues to be quiet and non-nagging by default, so that they don't break my flow. +6. As a rower, I want every stroke segmented into catch / drive / finish / recovery automatically, so that faults are reported at the exact phase they occur. +7. As a rower, I want per-stroke posture metrics computed (back angle at catch, shin vertical, hip-knee timing offset, layback angle, sequencing delay, left-right asymmetry), so that I have objective measurements not just opinions. +8. As a rower, I want detected faults categorized by severity (info / warning / critical), so that I focus on the worst issues first. +9. As a rower, I want a post-session posture replay screen, so that I can review my session video with skeleton overlay and fault annotations. +10. As a rower, I want the replay timeline aligned with my SmartRow `StrokeData`, so that I can correlate posture issues with power / split / stroke-rate dips. +11. As a rower, I want to scrub to any stroke and see the skeleton frozen at catch / finish, so that I can study my position. +12. As a rower, I want to compare a fault-heavy stroke against a clean stroke from the same session side-by-side, so that I see the contrast. +13. As a rower, I want a coaching summary at session end (top 3 issues, frequency, suggested drills), so that I leave the session with a clear next action. +14. As a rower, I want fault frequency tracked over time on the dashboard, so that I see whether my technique is actually improving. +15. As a rower, I want posture metrics surfaced inside the existing AI insights system, so that the AI can correlate posture with performance regressions. +16. As a serious user, I want to opt into the freemocap sidecar with multi-camera 3D capture, so that I get precision metrics for technical work. +17. As a serious user, I want clear setup instructions for the sidecar (Docker / Python), so that I can install it without expert help. +18. As a serious user, I want browser captures and sidecar captures to share the same analysis pipeline and UI, so that the experience is consistent. +19. As a privacy-conscious user, I want all video and pose data stored locally / in my own database by default, so that my body footage is not uploaded to third parties. +20. As a privacy-conscious user, I want explicit opt-in before any pose data is sent to cloud AI, so that I control externalization. +21. As a rower, I want to delete a mocap session (video + pose + metrics) with one action, so that I can clean up storage. +22. As a rower, I want to know which past `RowingSession` rows have linked mocap data, so that I can find sessions worth reviewing. +23. As a rower, I want to attach a mocap session to an existing CSV-imported `RowingSession`, so that historical sessions can also gain video context. +24. As a rower, I want fault thresholds to be configurable (e.g., "warn me only if back angle < 15° at catch"), so that the system adapts to my body and goals. +25. As a rower, I want sensible default thresholds based on standard rowing technique references, so that I don't have to tune anything to start. +26. As a rower, I want the app to flag if camera framing degrades mid-session (occlusion, low light, person leaves frame), so that I trust the metrics. +27. As a rower, I want capture FPS, model confidence, and tracked-keypoint counts shown as quality indicators, so that I can judge whether a session's analysis is reliable. +28. As a rower, I want webcam access permission to be requested only when I start a capture, so that the app doesn't ask for camera on every page load. +29. As a rower, I want to record without analysis if I'm just collecting footage, so that I can defer analysis to later. +30. As a rower, I want re-analysis on demand (e.g., after fault rules improve), so that old sessions benefit from updated detectors. +31. As a rower, I want the existing chat/AI to answer questions about my mocap data ("why does my back round at stroke 80?"), so that posture insights are conversational. +32. As a rower, I want training plans to incorporate posture goals ("reduce early-arm-bend faults below 10%/session"), so that technique is a first-class plan dimension. +33. As a rower, I want posture-derived achievements ("100 strokes with clean catch"), so that good technique earns recognition same as PRs. +34. As a rower on a phone/tablet, I want a graceful degraded mode that records video for later replay even if live analysis is too heavy, so that mobile is still useful. +35. As a developer, I want the analysis pipeline (segment → metrics → faults) to be pure / deterministic on a frame stream, so that fixture-driven tests verify correctness. +36. As a developer, I want browser capture and freemocap sidecar to emit the same `PoseFrameStream` schema, so that downstream code is source-agnostic. + +## Implementation Decisions + +### New deep modules + +- **PoseCaptureSource** — interface emitting `PoseFrameStream` (timestamped keypoint frames + confidence + source-quality signals). Two implementations: + - `BrowserPoseSource` — webcam + MediaPipe Pose (33-keypoint or BlazePose-Heavy) in browser, runs in Web Worker. + - `FreemocapSidecarSource` — WebSocket client to local freemocap sidecar. Sidecar is a separate Python service (Docker image) wrapping freemocap, exposing a streaming pose API. +- **StrokePhaseSegmenter** — pure function: `PoseFrameStream → Stroke[]`. Each `Stroke` has phase boundaries (catch, drive-start, finish, recovery-start) and frame indices. Detection rule based on hip-knee distance / handle-position proxy / seat travel. +- **PostureMetricsCalculator** — pure function: `Stroke → PostureMetrics`. Computes: back angle at catch, back angle at finish, layback angle, shin vertical at catch, hip-knee opening offset (drive sequence), arm-bend onset frame, left-right asymmetry index, knee track deviation. +- **PostureFaultDetector** — pure function: `PostureMetrics + thresholds → PostureFault[]`. Severity levels: info / warning / critical. Rule-based v1; pluggable so ML model can replace later. +- **CoachingAdvisor** — `PostureFault[] + session history → CoachingCue[]`. Rule-based default cues; cloud AI augmentation behind existing `cloudAI` gate. +- **PostureSessionRepository** — Prisma-hidden read/write of `MocapSession`, `PoseFrame`, `StrokePostureMetric`, `PostureFault`. + +### Modified modules + +- `aiAnalysis.ts` — extend prompt context to include posture summary when mocap data present. +- `AIInsight` generation — new `category: "posture"` insights. +- Dashboard / `analytics` — add posture-fault-frequency-over-time card. +- `trainingPlans` — optional posture goals on a `TrainingPlan`. +- `awards` — posture-derived achievement criteria. +- Chat tool surface — expose mocap query functions to AI. +- Existing `RowingSession` lookup — surface a "has mocap" badge. + +### Schema additions + +- `MocapSession` — id, userId, rowingSessionId? (nullable, can attach to existing CSV session), videoStoragePath, source ("browser" / "sidecar"), captureModelVersion, captureFps, durationSec, qualityScore, status, createdAt. +- `PoseFrame` — id, mocapSessionId, frameIndex, timestampMs, keypointsJson (compact array), confidenceJson, sourceFlags. Stored raw for full replay (per user choice). Indexed by `mocapSessionId, frameIndex`. +- `StrokePostureMetric` — id, mocapSessionId, strokeIndex, phaseBoundariesJson, metricsJson (back angle catch/finish, layback, shin-vertical, sequencing offsets, asymmetry, etc.), strokeDataId? (link to existing `StrokeData` row). +- `PostureFault` — id, mocapSessionId, strokeIndex, faultType, severity, evidenceJson (frame index + metric value + threshold), createdAt. +- `UserSettings.postureThresholds: Json?` — user-tunable rule thresholds. +- `UserSettings.mocapPreferences: Json?` — capture source default, live-cue verbosity, audio on/off. + +### Architectural decisions + +- **Pose source abstraction is the deep boundary.** Browser MediaPipe and freemocap sidecar both produce identical `PoseFrameStream` shape. All downstream analysis is source-agnostic. This is the core deepening play. +- **Analysis is pure.** `StrokePhaseSegmenter`, `PostureMetricsCalculator`, `PostureFaultDetector` are pure functions over data structures. No I/O, no DB. Tested with fixture frame streams. +- **Live and replay share the pipeline.** Live mode runs the same segmenter/metrics/detector incrementally as frames arrive; replay runs them on the stored stream. No duplicate logic. +- **Storage contract.** Raw `PoseFrame` rows are large but enable full replay + re-analysis when rules improve. Stored in Postgres (JSONB columns); video file stored on existing storage backend (`storage/` dir or Vercel Blob in deployed env). User may purge raw frames per session if storage pressure rises (derived metrics retained). +- **Sidecar contract.** freemocap sidecar is an opt-in local Docker service. Communicates via WebSocket on localhost. Versioned schema. App degrades cleanly if sidecar is offline. +- **Privacy.** Video + pose data are user-scoped, never sent to cloud unless cloud-AI is explicitly enabled in `UserSettings.cloudAIEnabled`. Coaching cues by default run on local rules. +- **Frame budget.** Browser path targets ≥ 24 fps on a mid-tier laptop. Heavier work (full re-analysis, summaries) deferred to post-session. Mobile falls back to record-only when CPU is insufficient. +- **Calibration.** Two-pose calibration (catch + finish) at session start establishes per-user joint baselines. Stored on user profile. Re-runnable. +- **Coordinate alignment.** Browser path = 2D side view + heuristics. Sidecar path = 3D. Metric extraction respects whichever is available; faults that require 3D are skipped on 2D path with a clear UI marker. +- **Stroke alignment with SmartRow.** When a `RowingSession` (CSV) is linked, the segmenter's stroke timeline is aligned to `StrokeData` timestamps via cross-correlation; metrics get joined to `StrokeData` rows. + +### API contracts + +- `POST /api/mocap/sessions` — create new mocap session (browser uploads chunked video + pose stream, or sidecar streams directly). +- `GET /api/mocap/sessions/:id` — full mocap detail incl. metrics + faults + frame index. +- `POST /api/mocap/sessions/:id/reanalyze` — re-run pipeline with current rules. +- `POST /api/mocap/sessions/:id/link/:rowingSessionId` — attach mocap to existing CSV session. +- `DELETE /api/mocap/sessions/:id` — cascade delete frames, metrics, faults, video. +- WebSocket `/api/mocap/live` — live capture stream from browser; server emits incremental faults / cues. +- Sidecar local URL configurable in settings; health-check endpoint required. + +## Testing Decisions + +A good test verifies external behavior of a deep module given a fixed input. It does not assert on private structure, intermediate variables, or implementation choices. The pose pipeline is uniquely well-suited because the core modules are pure functions over data. + +### Modules to test + +- **StrokePhaseSegmenter** — fixture: synthetic and recorded `PoseFrameStream` files representing clean strokes, missed catch, paused recovery, asymmetric drive. Assert: correct stroke count, phase boundary frame indices within tolerance. +- **PostureMetricsCalculator** — fixture: hand-labeled `Stroke` with known geometry. Assert: computed angles within ±2° of ground truth. +- **PostureFaultDetector** — fixture: `PostureMetrics` instances crafted to cross / not-cross thresholds. Assert: correct fault types and severities emitted; no false positives on clean reference. + +### Modules NOT tested at unit level + +- UI views (`LiveCoachingView`, `PostureReplayView`, `PostureTrendsCard`) — covered by manual QA + Playwright smoke tests for golden path. +- `BrowserPoseSource` / `FreemocapSidecarSource` — thin adapters; correctness depends on external libs / services. Smoke-test only. +- `PostureSessionRepository` — covered indirectly by API integration tests. + +### Prior art + +- Existing analytics tests (if present in `src/lib`) for trend computation patterns. +- Existing parser tests around `csvParser.ts` / `strokeParser.ts` style: pure-function over fixture data. + +### Test fixtures + +- Bundle 5–10 short pose recordings (≤ 30 s each) covering: clean rowing, early arm bend, rounded back, slow recovery, asymmetry, lost tracking. Stored as JSON `PoseFrameStream` snapshots in the test fixture directory. + +## Out of Scope + +- Multi-user / coach views over mocap data. +- Public sharing of mocap video or skeleton clips. +- Smartphone-only mode as primary target (mobile is degraded record-only). +- Hardware sensors (IMU on seat, force sensors on handle / footplate). +- Strava / TrainingPeaks export of posture data. +- ML-trained fault detectors (v1 is rule-based; ML pluggable later). +- On-water rowing capture. +- Real-time streaming of mocap to a remote coach. +- Custom freemocap installation flows beyond the documented Docker sidecar. + +## Further Notes + +- freemocap upstream is GPL-licensed Python. Sidecar runs as separate process; no GPL code is linked into Next.js app. Confirm license interaction during implementation. +- Browser MediaPipe Pose is Apache-2.0 and ships as JS package — no licensing concern. +- Phase 1 ships browser path + analysis pipeline + replay UI + dashboard widget. Phase 2 ships freemocap sidecar + 3D-aware metrics. Phase 3 ships AI-augmented cues + posture-aware training plans + posture achievements. +- Deep-module boundary (`PoseFrameStream` as universal contract between capture and analysis) is the key bet — lets source change without rewriting analysis, and lets analysis evolve without touching capture. Contract reviewed and stabilized first. +- Performance baseline must be measured early on target dev machine; if browser pipeline cannot hit 24 fps, live-cue feature ships as post-stroke (≤ 1 s lag) instead of intra-stroke. +- Storage growth: raw `PoseFrame` rows ~2–4 KB per frame × 24 fps × 30 min = ~100 MB per session worst-case. Retention policy (auto-purge raw frames after N days, keep metrics + video) is a likely follow-up. From f4e3604c0cd37af2c4eb168994110d21a73ec992 Mon Sep 17 00:00:00 2001 From: Rupert Germann Date: Fri, 8 May 2026 11:03:35 +0200 Subject: [PATCH 02/29] docs(mocap): add CONTEXT, ADRs, and resolved-decisions section Establish the canonical mocap domain vocabulary and the architectural decisions that issue #9 and follow-ups build on. - CONTEXT.md: glossary entries for CapturePerspective, StrokeSegmentationSource, MocapSession, CueLatencyBand, PoseFrameStream, PostureFault (v1 catalog), FaultThresholds, Calibration. - ADR-0001: store raw PoseFrameStream as a binary blob, not Postgres JSONB. Drops the per-frame Prisma table from the PRD draft. - ADR-0002: defer sidecar contract to Phase 2. v1 = browser-2D only. - ADR-0003: run analysis pipeline in the browser. WebSocket /api/mocap/live is persistence-only; server runs no analysis during capture. - ADR-0004: cloud-AI payload tiering for mocap data. - PRD: append "Resolved Decisions" section linking ADRs and locking v1 ship scope (US 1-11, 13, 19-30, 35, 36 ship; US 12, 14-18, 31-34 deferred to Phase 2). These are referenced by issue #9 and must land before any mocap implementation diff can be reviewed coherently. Co-Authored-By: Claude Opus 4.7 --- CONTEXT.md | 84 +++++++++++++++++++ .../0001-pose-frame-stream-as-binary-blob.md | 56 +++++++++++++ .../0002-defer-sidecar-contract-to-phase-2.md | 47 +++++++++++ .../0003-browser-side-analysis-pipeline.md | 54 ++++++++++++ docs/adr/0004-cloud-ai-mocap-payload-tiers.md | 62 ++++++++++++++ docs/prd-mocap-posture.md | 31 +++++++ 6 files changed, 334 insertions(+) create mode 100644 CONTEXT.md create mode 100644 docs/adr/0001-pose-frame-stream-as-binary-blob.md create mode 100644 docs/adr/0002-defer-sidecar-contract-to-phase-2.md create mode 100644 docs/adr/0003-browser-side-analysis-pipeline.md create mode 100644 docs/adr/0004-cloud-ai-mocap-payload-tiers.md diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 0000000..f20d772 --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,84 @@ +# Context + +Domain glossary for rowing-tracker. Terms here are canonical — use them in code, issues, and UI copy. If a concept isn't here, either it's not yet domain-load-bearing, or there's a gap to resolve via `/grill-with-docs`. + +## Glossary + +### CapturePerspective + +Which physical viewpoint a mocap session was recorded from. Determines which posture metrics are computable. + +- `side-left` — single webcam, rower's left side facing camera. Browser path. +- `side-right` — single webcam, rower's right side facing camera. Browser path. +- `sidecar-3d` — multi-camera 3D capture via freemocap sidecar. All metrics available. + +Browser path emits `side-left` or `side-right` only. Metrics that require `sidecar-3d` (left-right asymmetry, knee track deviation) are marked **unavailable** on side captures — never silently zeroed or estimated. UI surfaces the unavailable state as "requires multi-camera capture." + +### StrokeSegmentationSource + +How a mocap session's stroke phase boundaries (catch / drive / finish / recovery) were derived. + +- `pose-segmented` — boundaries computed from the pose stream alone (hip-knee distance signal). Used during live capture, when no CSV is available yet. Lower confidence; downstream metrics tagged accordingly. +- `csv-aligned` — boundaries taken from `StrokeData` rows (SmartRow ground truth), with the pose stream time-aligned to them via cross-correlation. Used in post-session replay after the user imports the SmartRow CSV and links it to the mocap session. + +A mocap session always begins life as `pose-segmented`. When linked to a `RowingSession` (CSV import), re-segmentation to `csv-aligned` is **mandatory** — re-run metrics and faults atomically with the link. Never leave a linked session at `pose-segmented`. + +### MocapSession + +A captured rowing session containing video + pose stream + derived metrics + faults. Independent of `RowingSession` (which is CSV/erg-derived). May exist standalone (no CSV linked) or be linked to exactly one `RowingSession`. + +Lifecycle states (proposed): `capturing` → `analyzing` → `ready` → optionally `linked` (when joined to a `RowingSession`). + +**Linking to a `RowingSession`** is bidirectional and exclusive — one `MocapSession` is linked to at most one `RowingSession` and vice versa. Either side may be unlinked; relinking is allowed. Linking triggers mandatory re-analysis (`pose-segmented` → `csv-aligned`) as a background job; the mocap row goes back to `analyzing` until it completes. Unlinking reverts to `pose-segmented` and re-runs metrics. CSV import auto-prompts to link when a capture window overlaps a new `RowingSession` by ±2 minutes — user confirms, never silent. + +### CueLatencyBand + +When a coaching cue is delivered relative to the stroke that triggered it. + +- `intra-stroke` — fired mid-stroke from per-frame rules. **Out of v1 scope.** Pose-segmented stroke boundaries are too noisy for reliable real-time fault attribution within a 2.5s stroke window. +- `post-stroke` — fired ≤1s after stroke completes. The "live coaching" experience in v1. +- `post-session` — surfaced in replay / coaching summary after capture ends. + +v1 fault detector runs at the **stroke** granularity only — one pass per closed stroke. No per-frame fault rule path. + +### PoseFrameStream + +A timestamped sequence of keypoint frames with confidence values, produced by a `PoseCaptureSource` and consumed by the analysis pipeline. + +**v1 shape:** 2D side-view keypoints — `{x, y, confidence}` per keypoint, plus per-frame source-quality flags. Browser path only (see ADR-0002). The sidecar / 3D path is deferred to Phase 2; when it lands, the schema version bumps and an optional `z` channel is added without breaking existing blobs. + +### PostureFault (v1 catalog) + +Stroke-granular faults the v1 detector emits. All computable from a 2D side-view `PoseFrameStream`. Each fault is named, attributed to a stroke phase, and has severity bands defined in `FaultThresholds`. + +| Fault key | Phase | Severity bands | +| --- | --- | --- | +| `rounded_back_at_catch` | catch | warning < 30°, critical < 20° (back angle) | +| `early_arm_bend` | drive | info / warning by frame-offset of arm-bend onset vs leg-extension completion | +| `back_opens_before_legs_drive` | drive | warning if torso angle changes before legs start extending | +| `excessive_layback` | finish | info > 30°, warning > 45° (torso past vertical) | +| `slow_recovery_ratio` | recovery | warning > 2.5, critical > 3.5 (recovery / drive duration ratio) | + +**Excluded from v1**, surfaced as "metric available, detection deferred" or "requires multi-camera capture": + +- `left_right_asymmetry` — needs front view or `sidecar-3d` +- `knee_track_deviation` — needs front view or `sidecar-3d` +- `shin_not_vertical_at_catch` — disambiguating near-side shin from far-side shin in 2D is unreliable + +This catalog is the canonical vocabulary. Test fixtures, threshold tuning, coaching cue copy, and AI prompt context all reference these exact keys. Anything outside this list is out of v1 scope. + +### FaultThresholds + +The numeric bands a `PostureFault` rule fires against (e.g. "back angle at catch < threshold → rounded-back fault"). Stored on `UserSettings.postureThresholds: Json?`. + +**Defaults are hand-coded, conservative, and versioned in code** (`postureThresholdsV1`, `postureThresholdsV2`, …). Each default carries a source comment citing rowing-technique references. Conservative bands = wide tolerances, fewer false positives, fewer angry users in v1. + +Migration: when a new defaults version ships, users who haven't touched their thresholds upgrade automatically. Users with `userOverridden: true` keep their custom values; never stomp explicit customisation. + +### Calibration + +A pair of reference pose frames captured before recording starts: one at **catch** position, one at **finish** position. Used as pixel-space baselines for downstream metric calculations (e.g. "back angle delta from this rower's own catch baseline"). + +**Stored per `MocapSession`, not per `User`.** 2D side-view baselines are only meaningful for the camera setup that produced them — angle, distance, and framing change between sessions even when the rower doesn't. Storing on the user would imply a stable camera position the system can't verify. Recapture (~10 s) is required at the start of each session. + +**Storage:** persisted as one binary blob per `MocapSession`, alongside the video file (see ADR-0001). Not a Postgres table. The `MocapSession` row points at it via `poseStreamPath`. Header carries `fps`, `keypointSchemaVersion`, `frameCount`. Random access by frame index = byte-range read. diff --git a/docs/adr/0001-pose-frame-stream-as-binary-blob.md b/docs/adr/0001-pose-frame-stream-as-binary-blob.md new file mode 100644 index 0000000..e2c5463 --- /dev/null +++ b/docs/adr/0001-pose-frame-stream-as-binary-blob.md @@ -0,0 +1,56 @@ +# ADR-0001: Store raw PoseFrameStream as a binary blob, not Postgres JSONB + +**Status:** Accepted +**Date:** 2026-05-08 +**Context owner:** mocap posture analysis (see `docs/prd-mocap-posture.md`) + +## Context + +Mocap sessions produce a stream of pose keypoint frames at ~24 fps. A 30-minute session yields ~43,200 frames at ~2-4 KB each — roughly 100 MB of raw pose data per session, on top of the recorded video. + +The original PRD draft proposed a `PoseFrame` table in Postgres with a JSONB `keypointsJson` column, indexed on `(mocapSessionId, frameIndex)`. This would let downstream code load any frame via Prisma the same way other entities are loaded. + +We considered the actual access patterns: + +- **Re-analysis** — stream all frames of one session through the pure pipeline. Sequential. +- **Replay scrubbing** — random access to a frame by index. Localised reads. +- **Stroke window read** — load all frames between two `frameIndex` values. Contiguous range. +- **Cross-session ad-hoc SQL on individual frames** — none. All queries that span sessions operate on derived rows (`StrokePostureMetric`, `PostureFault`), not raw frames. + +There are no query patterns that benefit from per-frame SQL. The JSONB design optimises for a use case that doesn't exist. + +## Decision + +Store the raw `PoseFrameStream` for each `MocapSession` as a **single binary blob** on the same storage backend as the recorded video — local `storage/` directory in development, Vercel Blob in deployed environments. The `MocapSession` row holds a `poseStreamPath: string` pointing at it. + +Format: packed Float32 array of keypoints, prefixed by a small header carrying `{ fps, keypointSchemaVersion, frameCount, keypointsPerFrame }`. Random access by frame index = byte-range read at `header_size + frameIndex * frame_stride`. + +Postgres holds only the **derived** rows: `StrokePostureMetric`, `PostureFault`, plus the `MocapSession` row itself. These are small, indexable, queryable, and cheap to back up. + +Re-analysis streams the blob through the pipeline and rewrites the derived rows atomically. Scrubbing fetches byte-ranges. The Prisma schema stays compact. + +## Consequences + +**Positive** + +- Postgres footprint stays proportional to derived data, not raw capture volume. No 43k-row inserts per session, no JSONB TOAST bloat, no expensive vacuum. +- Re-analysis is a sequential file read instead of a 43k-row Prisma scan. +- Backups, replication, and DB restore stay cheap as mocap usage grows. +- Pose-stream blob lives next to the video on the same storage backend, so retention / purge / quota logic is one decision, not two. + +**Negative** + +- A custom serialiser is required (header + packed Float32 layout, version field). Schema evolution of the keypoint format requires a versioned reader. +- No ad-hoc SQL on raw frames. If a future need emerges (unlikely, but e.g. cross-session frame-level statistics), it would mean adding an index/materialisation layer on top of the blobs. +- Storage backend abstraction must support byte-range reads (Vercel Blob does; local filesystem does trivially). + +**Neutral** + +- A user-initiated "purge raw frames, keep metrics" action becomes "delete the blob"; derived rows are untouched. This matches the PRD's footnote about retention pressure. +- The `PoseFrame` Prisma model in the PRD's schema-additions section is dropped; replaced by `MocapSession.poseStreamPath`. + +## Alternatives considered + +- **Postgres JSONB row-per-frame (the PRD's original proposal).** Rejected for the bloat / vacuum / re-analysis cost reasons above, given that no query pattern needs per-frame SQL. +- **Parquet / Arrow file format instead of packed Float32.** More tooling-friendly, but adds a dependency for a one-shape data stream where a 32-byte header + flat array is sufficient. Revisit if cross-session analytics on raw frames ever becomes a real need. +- **Object store with one file per frame.** Random access via filename, but 43k tiny files per session is a worse storage pattern than one ~100 MB blob. diff --git a/docs/adr/0002-defer-sidecar-contract-to-phase-2.md b/docs/adr/0002-defer-sidecar-contract-to-phase-2.md new file mode 100644 index 0000000..065dbe9 --- /dev/null +++ b/docs/adr/0002-defer-sidecar-contract-to-phase-2.md @@ -0,0 +1,47 @@ +# ADR-0002: Defer freemocap sidecar contract design to Phase 2 + +**Status:** Accepted +**Date:** 2026-05-08 +**Context owner:** mocap posture analysis (see `docs/prd-mocap-posture.md`) + +## Context + +The PRD names `PoseFrameStream` as "the universal contract between capture and analysis" and calls source-abstraction "the deepening play" — the architectural bet that lets the capture source change (browser webcam → freemocap sidecar) without rewriting the analysis pipeline. + +The PRD's Phase 1 ships browser-only. Phase 2 ships the sidecar. But the PRD designs the sidecar contract up front: WebSocket protocol, Docker image, health-check API, versioned schema, 3D depth-bearing keypoints. + +A contract designed against one real implementation and one imagined implementation is, in practice, a contract designed against the imagined one. We don't yet know: + +- The exact shape of keypoints freemocap emits (count, ordering, coordinate frame, confidence semantics). +- Which fault rules actually benefit from 3D depth in practice — vs. which are answered just as well from 2D side view with calibrated heuristics. +- Whether the live coaching path even applies to multi-camera 3D capture, or whether sidecar usage is exclusively post-session technique work. + +Locking those decisions now means picking guesses; widening the contract later is cheap because of ADR-0001 (the blob has a `keypointSchemaVersion` header field). + +## Decision + +In v1, design `PoseFrameStream` against the **browser path only**: 2D keypoints (x, y), per-keypoint confidence, source-quality flags. No depth field. No sidecar process, no WebSocket protocol, no Docker image, no health-check API. + +Treat the sidecar as not-yet-existent. Phase 2 will design its contract against the realities of freemocap's actual output and the lessons of v1's fault rules — at that point, the schema version on stored blobs bumps, the reader handles both versions, and the analysis pipeline gains a 3D-aware path where it matters. + +## Consequences + +**Positive** + +- One real implementation, no premature abstraction. The "interface" is just whatever shape `BrowserPoseSource` emits. +- Faster v1: no sidecar contract review, no Docker image, no two-process integration testing. +- Phase 2 sidecar work starts from real demand (specific fault rules that need depth) rather than a speculative interface. + +**Negative** + +- When Phase 2 lands, `PoseFrameStream` widens from `{x, y, conf}` to `{x, y, z?, conf}`. Versioned blob reader handles old captures; pipeline gains a depth-aware branch. +- Until Phase 2, the abstraction the PRD originally called "the deepening play" doesn't exist as such — the analysis pipeline is coupled to one source. Acceptable, because there is only one source. + +**Neutral** + +- Anything in the PRD's `### API contracts` section that is sidecar-specific (sidecar local URL, sidecar health-check) is out of v1 scope. The browser API (`POST /api/mocap/sessions`, `GET /api/mocap/sessions/:id`, `POST /api/mocap/sessions/:id/reanalyze`, `POST /api/mocap/sessions/:id/link/:rowingSessionId`, `DELETE /api/mocap/sessions/:id`, `WebSocket /api/mocap/live`) stays. + +## Alternatives considered + +- **Lock both contracts now (the PRD's original plan).** Rejected. Single-implementation abstraction is an interface waiting for its second implementation — and the second implementation's real shape is unknown until freemocap is wired up. +- **Build a stub sidecar in v1 to validate the contract.** Rejected as scope creep — a stub doesn't surface the schema mismatches that a real freemocap integration would. diff --git a/docs/adr/0003-browser-side-analysis-pipeline.md b/docs/adr/0003-browser-side-analysis-pipeline.md new file mode 100644 index 0000000..b8a47c4 --- /dev/null +++ b/docs/adr/0003-browser-side-analysis-pipeline.md @@ -0,0 +1,54 @@ +# ADR-0003: Run the analysis pipeline in the browser; server is dumb storage in live mode + +**Status:** Accepted +**Date:** 2026-05-08 +**Context owner:** mocap posture analysis (see `docs/prd-mocap-posture.md`) + +## Context + +The mocap pipeline has two consumer paths: + +- **Live capture** — pose inference at ~24 fps, stroke segmentation, per-stroke metrics, fault detection, coaching cues delivered with `post-stroke` latency (≤1 s after stroke end; see `CueLatencyBand` in CONTEXT.md). +- **Post-session re-analysis** — same pipeline rerun on stored frames after a `RowingSession` is linked, or after fault rules are updated. + +The PRD draft proposed a `WebSocket /api/mocap/live` where the server "emits incremental faults / cues." That implies server-side analysis during capture. Two issues: + +1. Round-tripping pose frames to a server adds tens to hundreds of milliseconds of network latency per stroke window, blowing the post-stroke budget. +2. The server has no information the client doesn't — pose frames originate in the browser, and analysis is pure functions over those frames. There's nothing for the server to compute that the client can't. + +The PRD also commits to a privacy stance: "all video and pose data stored locally / in my own database by default … explicit opt-in before any pose data is sent to cloud AI." Server-side live analysis routes pose data through the server unconditionally, conflicting with that stance for users who would otherwise capture and view locally. + +## Decision + +Run the full analysis pipeline (segmenter, metrics calculator, fault detector, coaching advisor) **in the browser** during live capture. The pipeline is implemented as pure functions over `PoseFrameStream`, with no I/O, no DB calls, and no server dependency. + +Pose inference runs in a **Web Worker with OffscreenCanvas** using MediaPipe Tasks (WASM build). WebGPU acceleration is future-opt-in, not v1 default. + +`WebSocket /api/mocap/live` is for **persistence only**: the browser streams frames to the server, the server appends them to the session's blob storage. The server does not run the analysis pipeline during live capture and does not emit faults or cues over the WebSocket. + +The same pipeline code runs server-side on demand for **re-analysis** — `POST /api/mocap/sessions/:id/reanalyze` reads the blob, runs the pure pipeline, rewrites derived rows. That's the only server-side execution path. + +## Consequences + +**Positive** + +- Live `post-stroke` latency budget is achievable — no network round trip for analysis. +- Privacy posture matches the PRD: faults and cues are computed locally; pose data only leaves the browser when the user has opted into persistence (and never reaches a third-party AI service unless `cloudAIEnabled` is set). +- The pipeline is one codebase. Pure functions over data structures are portable: same code runs in a Web Worker live and on a Vercel Function for re-analysis. +- If a user denies upload permission or the network drops, live coaching keeps working — the browser has everything it needs. + +**Negative** + +- Bundle size grows by the size of the analysis pipeline (segmenter, metrics, fault detector, default thresholds, coaching cue text). Mitigated by: pipeline is pure logic, no large model weights; lazy-loaded on the mocap route, not the dashboard. +- Re-analysis on the server requires a runtime that can execute the same TypeScript modules. Vercel Functions on Node.js handle this; no isomorphic concerns for pure code. +- If the pipeline ever needs heavy compute (ML model inference for fault detection — explicitly out of v1 scope per PRD), the browser path becomes constrained. At that point, this decision can be revisited for the heavy-compute branch only; the rule-based v1 path stays browser-side. + +**Neutral** + +- The PRD's `CoachingAdvisor` cloud-AI augmentation (behind `cloudAIEnabled`) is unaffected: that's an opt-in, post-session enrichment that already routes through existing `cloudAI.ts` infrastructure. + +## Alternatives considered + +- **Server-side analysis with live WebSocket fault stream (the PRD's draft).** Rejected for the latency and privacy reasons above. +- **Hybrid: client computes, server validates.** Rejected as duplicate work — server has nothing to validate against. The client's frames are ground truth. +- **Pose inference on the server (upload video, server runs MediaPipe).** Rejected: kills live latency, defeats the local-first privacy stance, and adds GPU/CPU server cost for compute the client can do. diff --git a/docs/adr/0004-cloud-ai-mocap-payload-tiers.md b/docs/adr/0004-cloud-ai-mocap-payload-tiers.md new file mode 100644 index 0000000..1ca15f4 --- /dev/null +++ b/docs/adr/0004-cloud-ai-mocap-payload-tiers.md @@ -0,0 +1,62 @@ +# ADR-0004: Cloud-AI mocap payload — fault-summary by default, detailed metrics opt-in, raw frames never + +**Status:** Accepted +**Date:** 2026-05-08 +**Context owner:** mocap posture analysis (see `docs/prd-mocap-posture.md`) + +## Context + +When `UserSettings.cloudAIEnabled` is on, the existing `aiAnalysis.ts` flow sends a context payload to a third-party LLM (Anthropic/OpenAI). The mocap PRD asks for posture data to be included in that context so the AI can correlate posture with performance. + +Pose data is unusually sensitive among the things this app handles. A raw `PoseFrameStream` is essentially a low-resolution biometric capture of a user's body in motion — sending it to a third-party API conflicts with the project's standing privacy posture (`prd.md` §13.4.1) even when the user has enabled cloud AI for textual training data. + +But "cloud AI off entirely for mocap" loses the feature's biggest payoff: the LLM correlating posture faults with power/pace dips and giving narrative coaching. + +There are three plausible payload tiers: + +1. **Raw `PoseFrameStream`** — keypoints over time. Biometric. Also useless to a text LLM, which can't reason over keypoint arrays. +2. **`StrokePostureMetric` rows** — angles, offsets, asymmetry numbers per stroke. Numeric, geometric, not directly biometric, but reconstructs a coarse body model. +3. **`PostureFault` summary** — fault counts by type, severity, phase, plus session-level quality flags. Most compressed; most LLM-friendly; shares no body geometry. + +## Decision + +Three tiers, three policies: + +- **Tier 1 (raw frames):** never sent to cloud AI. No flag, no opt-in. Hard wall. +- **Tier 3 (fault summary):** sent when `cloudAIEnabled` is true. This is the default mocap → cloud-AI payload. +- **Tier 2 (per-stroke metrics):** sent only when both `cloudAIEnabled` AND a new `UserSettings.mocapDetailedAIShare` flag are true. Off by default. + +The fault-summary contract (tier 3) is fixed: + +``` +Mocap summary: +- Faults: (, severity=) ... +- Quality: % tracked, , fps +- Strokes analyzed: +``` + +No keypoints, no per-frame data, no per-stroke geometry, no video. Future fault types extend this format; the contract stays additive. + +## Consequences + +**Positive** + +- Default cloud-AI behaviour preserves user privacy: aggregate fault counts share *no* reconstructable body geometry. +- Power-user "share more for better insights" is one explicit toggle away — clear consent path, not buried in cloud-AI's general flag. +- The LLM gets enough signal from tier 3 to write useful coaching ("you had 12 rounded-back faults at catch — work on lat engagement before the drive") without ever holding biometric data. +- Hard wall on tier 1 means no future bug or refactor can accidentally leak raw pose data — the code path doesn't exist. + +**Negative** + +- Two flags (`cloudAIEnabled`, `mocapDetailedAIShare`) instead of one. Slightly more UX surface in settings. +- Tier 3 may not be enough for the most fine-grained AI queries ("why did my back round more on stroke 80 specifically?"). Those queries require tier 2 — user opts in or the answer stays generic. + +**Neutral** + +- Existing `cloudAI.ts` and `aiAnalysis.ts` get a new context-builder for mocap that materialises only tier 3 by default; tier 2 enrichment is gated and additive. + +## Alternatives considered + +- **Single flag covering both tier 2 and tier 3.** Rejected: collapses two distinct privacy decisions ("share that I had faults" vs "share my exact body geometry") into one toggle that users can't reason about. +- **No detailed share at all (tier 3 only, ever).** Rejected: forecloses the per-stroke AI query feature without user input. The opt-in pathway preserves it for users who want it. +- **Anonymise tier 2 by removing identifying joint configuration.** Rejected: the data being shared *is* joint geometry. Anonymising it would mean removing the signal that makes it useful. diff --git a/docs/prd-mocap-posture.md b/docs/prd-mocap-posture.md index 3cfa08d..cd46238 100644 --- a/docs/prd-mocap-posture.md +++ b/docs/prd-mocap-posture.md @@ -144,6 +144,37 @@ A good test verifies external behavior of a deep module given a fixed input. It - Real-time streaming of mocap to a remote coach. - Custom freemocap installation flows beyond the documented Docker sidecar. +## Resolved Decisions (grilling 2026-05-08) + +This section reflects the outcome of `/grill-with-docs` against this PRD. Where it conflicts with the body of the PRD above, **this section wins** and the older sections will be updated lazily. + +### Architecture (see `docs/adr/`) + +- **ADR-0001** — raw `PoseFrameStream` is stored as one binary blob per `MocapSession` alongside the video, not as Postgres JSONB rows. The `PoseFrame` Prisma model from `### Schema additions` is dropped; replaced by `MocapSession.poseStreamPath`. +- **ADR-0002** — sidecar contract is deferred to Phase 2. v1 `PoseFrameStream` shape is browser-2D only (`{x, y, confidence}` per keypoint, plus quality flags). No Docker image, no WebSocket sidecar protocol, no health-check API in v1. +- **ADR-0003** — analysis pipeline runs in the browser (Web Worker, MediaPipe Tasks WASM). `WebSocket /api/mocap/live` is for persistence only; server does not emit faults during live capture. Server-side execution is for `POST /api/mocap/sessions/:id/reanalyze` only. +- **ADR-0004** — cloud-AI mocap payload is `PostureFault` summary (tier 3) by default; per-stroke metrics (tier 2) opt-in via `UserSettings.mocapDetailedAIShare`; raw frames (tier 1) never cross to cloud. + +### Domain terms (see `CONTEXT.md`) + +`CapturePerspective`, `StrokeSegmentationSource`, `MocapSession`, `CueLatencyBand`, `PoseFrameStream`, `Calibration`, `PostureFault` (v1 catalog), `FaultThresholds`. Use these exact terms in code, issues, and UI copy. + +### Locked design choices + +- **Browser path is side-view only.** `side-left` or `side-right`. Front view, asymmetry, knee-track deviation = `requires-multi-cam`, deferred to sidecar. +- **Live capture = `pose-segmented`.** Live SmartRow CSV streaming is not available; CSV arrives post-session. Linking a `RowingSession` to a `MocapSession` triggers mandatory atomic re-analysis to `csv-aligned` as a background job. +- **Live cues are `post-stroke`** (≤ 1 s after stroke completes), not intra-stroke. Fault detector runs at stroke granularity only. +- **Calibration is per-session, not per-user.** Two reference frames (catch, finish) captured at session start, stored on `MocapSession`. +- **Auto-link prompt** on CSV import when capture window overlaps within ±2 minutes; user always confirms, never silent. Linking is bidirectional, exclusive, reversible (`unlink` endpoint). +- **v1 fault catalog is fixed at 5 types**: `rounded_back_at_catch`, `early_arm_bend`, `back_opens_before_legs_drive`, `excessive_layback`, `slow_recovery_ratio`. Anything else is out of v1 scope. +- **Default thresholds are hand-coded and versioned** (`postureThresholdsV1`). Conservative bands. Auto-migrate on version bump unless user has set `userOverridden: true`. + +### v1 ship scope (Phase 1) + +**Ships:** US 1-11, 13, 19-30, 35, 36 (US 36 reduced to "single-source `PoseFrameStream` shape, versioned for future widening"). + +**Deferred to Phase 2:** US 12 (side-by-side compare), 14 (fault-frequency dashboard card), 15 (posture in `aiAnalysis.ts` insights), 16-18 (sidecar), 31 (chat tool exposure), 32-33 (posture in training plans / achievements), 34 (mobile degraded mode). + ## Further Notes - freemocap upstream is GPL-licensed Python. Sidecar runs as separate process; no GPL code is linked into Next.js app. Confirm license interaction during implementation. From 701e1444fdb375cf3c48c3e4aa90e996828551e6 Mon Sep 17 00:00:00 2001 From: Rupert Germann Date: Fri, 8 May 2026 11:04:14 +0200 Subject: [PATCH 03/29] feat(mocap): MocapSession schema, storage abstraction, create/delete API PR1 of issue #9: lay the foundation for browser-path webcam capture. Capture worker, append endpoint, and capture UI follow in PR2/PR3. - Prisma: add MocapSession model with unique rowingSessionId for the bidirectional-exclusive link mandated in CONTEXT.md. - src/lib/mocap/storage.ts: MocapStorage interface (path generation, byte-range read, append, delete, exists) per ADR-0001 storage contract. LocalDiskStorage impl writes under storage/mocap// / and guards against path traversal. Vercel Blob impl is stubbed behind the same interface and gated on MOCAP_STORAGE_BACKEND. - src/lib/mocap/poseFrameStream.ts: 32-byte header + 404-byte frames (33 keypoints * x,y,confidence Float32 + timestamp Float32 + qualityFlags Uint32). Reader rejects unknown magic / format / schema versions. frameCount uses an OPEN_FRAME_COUNT sentinel during streaming append, derivable from blob size on tab-close truncation (acceptance: no partial-frame corruption). - POST /api/mocap/sessions: validates source/perspective with zod, creates the row, allocates storage paths, writes the header into the pose-stream blob, returns id + paths. - DELETE /api/mocap/sessions/[id]: cascades to both blobs via the storage abstraction. - GET /api/mocap/sessions/[id]: read access used by PR2/PR3. Refs #9. Co-Authored-By: Claude Opus 4.7 --- .../migration.sql | 34 ++++ prisma/schema.prisma | 167 ++++++++++-------- src/app/api/mocap/sessions/[id]/route.ts | 51 ++++++ src/app/api/mocap/sessions/route.ts | 74 ++++++++ src/lib/mocap/poseFrameStream.ts | 152 ++++++++++++++++ src/lib/mocap/storage.ts | 115 ++++++++++++ 6 files changed, 524 insertions(+), 69 deletions(-) create mode 100644 prisma/migrations/20260508120000_add_mocap_session/migration.sql create mode 100644 src/app/api/mocap/sessions/[id]/route.ts create mode 100644 src/app/api/mocap/sessions/route.ts create mode 100644 src/lib/mocap/poseFrameStream.ts create mode 100644 src/lib/mocap/storage.ts diff --git a/prisma/migrations/20260508120000_add_mocap_session/migration.sql b/prisma/migrations/20260508120000_add_mocap_session/migration.sql new file mode 100644 index 0000000..dd820ec --- /dev/null +++ b/prisma/migrations/20260508120000_add_mocap_session/migration.sql @@ -0,0 +1,34 @@ +-- CreateTable +CREATE TABLE "MocapSession" ( + "id" TEXT NOT NULL, + "userId" TEXT NOT NULL, + "rowingSessionId" TEXT, + "videoStoragePath" TEXT NOT NULL, + "poseStreamPath" TEXT NOT NULL, + "source" TEXT NOT NULL, + "captureModelVersion" TEXT NOT NULL, + "capturePerspective" TEXT NOT NULL, + "captureFps" DOUBLE PRECISION NOT NULL, + "durationSec" DOUBLE PRECISION NOT NULL DEFAULT 0, + "qualityScore" DOUBLE PRECISION, + "status" TEXT NOT NULL DEFAULT 'capturing', + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "MocapSession_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "MocapSession_rowingSessionId_key" ON "MocapSession"("rowingSessionId"); + +-- CreateIndex +CREATE INDEX "MocapSession_userId_idx" ON "MocapSession"("userId"); + +-- CreateIndex +CREATE INDEX "MocapSession_userId_createdAt_idx" ON "MocapSession"("userId", "createdAt"); + +-- AddForeignKey +ALTER TABLE "MocapSession" ADD CONSTRAINT "MocapSession_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "MocapSession" ADD CONSTRAINT "MocapSession_rowingSessionId_fkey" FOREIGN KEY ("rowingSessionId") REFERENCES "RowingSession"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 70c5ecd..ab6ad87 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -31,6 +31,7 @@ model User { accounts Account[] sessions AuthSession[] rowingSessions RowingSession[] + mocapSessions MocapSession[] personalRecords PersonalRecord[] earnedAwards EarnedAward[] aiAwards AIAwardSuggestion[] @@ -101,30 +102,31 @@ model PasswordResetToken { // ============================================================================ model RowingSession { - id String @id @default(cuid()) - userId String - timestamp DateTime - distance Int - duration Int - energy Int - strokeCount Int - avgPower Float - maxPower Float - wattPerKg Float - avgSplit Float - minSplit Float - avgWork Float - avgStrokeLength Float - avgStrokeRate Float - maxStrokeRate Float - consistencyScore Float? // Pre-computed consistency score (0-100) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - importedAt DateTime @default(now()) - sourceFile String? + id String @id @default(cuid()) + userId String + timestamp DateTime + distance Int + duration Int + energy Int + strokeCount Int + avgPower Float + maxPower Float + wattPerKg Float + avgSplit Float + minSplit Float + avgWork Float + avgStrokeLength Float + avgStrokeRate Float + maxStrokeRate Float + consistencyScore Float? // Pre-computed consistency score (0-100) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + importedAt DateTime @default(now()) + sourceFile String? user User @relation(fields: [userId], references: [id], onDelete: Cascade) strokeData StrokeData[] + mocapSession MocapSession? personalRecords PersonalRecord[] trainingSessionLinks TrainingSessionLink[] @@ -135,7 +137,7 @@ model RowingSession { } model StrokeData { - id String @id @default(cuid()) + id String @id @default(cuid()) sessionId String strokeIndex Int time Float @@ -175,6 +177,33 @@ model PersonalRecord { @@index([userId]) } +// ============================================================================ +// MOCAP (motion-capture posture analysis — see docs/prd-mocap-posture.md) +// ============================================================================ + +model MocapSession { + id String @id @default(cuid()) + userId String + rowingSessionId String? @unique + videoStoragePath String + poseStreamPath String + source String + captureModelVersion String + capturePerspective String + captureFps Float + durationSec Float @default(0) + qualityScore Float? + status String @default("capturing") + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + rowingSession RowingSession? @relation(fields: [rowingSessionId], references: [id], onDelete: SetNull) + + @@index([userId]) + @@index([userId, createdAt]) +} + // ============================================================================ // AWARDS & ACHIEVEMENTS // ============================================================================ @@ -215,15 +244,15 @@ model AIAwardSuggestion { } model GeneratedAchievement { - id String @id @default(cuid()) - userId String - awardId String - story String? @db.Text - imageUrl String? - hasImage Boolean @default(false) + id String @id @default(cuid()) + userId String + awardId String + story String? @db.Text + imageUrl String? + hasImage Boolean @default(false) colorPalette String? @default("classic") - earnedAt DateTime? - generatedAt DateTime @default(now()) + earnedAt DateTime? + generatedAt DateTime @default(now()) user User @relation(fields: [userId], references: [id], onDelete: Cascade) @@ -372,7 +401,7 @@ model AIInsight { archived Boolean @default(false) dateGenerated DateTime @default(now()) archivedAt DateTime? - feedback String? // 'helpful' | 'not_helpful' | 'action_taken' + feedback String? // 'helpful' | 'not_helpful' | 'action_taken' feedbackAt DateTime? user User @relation(fields: [userId], references: [id], onDelete: Cascade) @@ -402,7 +431,7 @@ model MemoryDocument { status String? uploadedAt DateTime @default(now()) - user User @relation(fields: [userId], references: [id], onDelete: Cascade) + user User @relation(fields: [userId], references: [id], onDelete: Cascade) @@index([userId]) @@index([userId, type]) @@ -413,44 +442,44 @@ model MemoryDocument { // ============================================================================ model UserSettings { - id String @id @default(cuid()) - userId String @unique - theme String @default("system") - units String @default("metric") - dateFormat String @default("MM/DD/YYYY") - timeFormat String @default("24h") - language String @default("en") - timeZone String? @default("UTC") - defaultChartType String @default("line") - animationsEnabled Boolean @default(true) - showPromptSuggestions Boolean @default(true) - customPrompts String[] - trainingZones Json? - preferredMetrics String[] - weeklyGoalType String @default("sessions") - weeklyGoalTarget Int @default(3) - restDayAlerts Boolean @default(true) - adaptationEnabled Boolean @default(true) - sessionReminders Boolean @default(false) - weeklyProgress Boolean @default(true) - achievementAlerts Boolean @default(true) - planReminders Boolean @default(true) - adherenceAlerts Boolean @default(true) - cloudAIEnabled Boolean @default(false) - maxTokens Int @default(1500) - aiConfig Json? - customPromptsAi Json? - userProfileContext String? @db.Text - userProfileRawInput String? @db.Text - dashboardSettings Json? - sessionsViewSettings Json? + id String @id @default(cuid()) + userId String @unique + theme String @default("system") + units String @default("metric") + dateFormat String @default("MM/DD/YYYY") + timeFormat String @default("24h") + language String @default("en") + timeZone String? @default("UTC") + defaultChartType String @default("line") + animationsEnabled Boolean @default(true) + showPromptSuggestions Boolean @default(true) + customPrompts String[] + trainingZones Json? + preferredMetrics String[] + weeklyGoalType String @default("sessions") + weeklyGoalTarget Int @default(3) + restDayAlerts Boolean @default(true) + adaptationEnabled Boolean @default(true) + sessionReminders Boolean @default(false) + weeklyProgress Boolean @default(true) + achievementAlerts Boolean @default(true) + planReminders Boolean @default(true) + adherenceAlerts Boolean @default(true) + cloudAIEnabled Boolean @default(false) + maxTokens Int @default(1500) + aiConfig Json? + customPromptsAi Json? + userProfileContext String? @db.Text + userProfileRawInput String? @db.Text + dashboardSettings Json? + sessionsViewSettings Json? sessionAnalysisSettings Json? - chartSettings Json? - analyticsSettings Json? - sessionsRevision Int @default(0) - insightsRevision Int @default(0) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt + chartSettings Json? + analyticsSettings Json? + sessionsRevision Int @default(0) + insightsRevision Int @default(0) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt user User @relation(fields: [userId], references: [id], onDelete: Cascade) } diff --git a/src/app/api/mocap/sessions/[id]/route.ts b/src/app/api/mocap/sessions/[id]/route.ts new file mode 100644 index 0000000..dbb8302 --- /dev/null +++ b/src/app/api/mocap/sessions/[id]/route.ts @@ -0,0 +1,51 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { getMocapStorage } from "@/lib/mocap/storage"; + +export async function GET( + _req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const row = await prisma.mocapSession.findFirst({ + where: { id, userId: session.user.id }, + }); + if (!row) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + return NextResponse.json({ session: row }); +} + +export async function DELETE( + _req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const row = await prisma.mocapSession.findFirst({ + where: { id, userId: session.user.id }, + }); + if (!row) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + + const storage = getMocapStorage(); + await Promise.allSettled([ + storage.delete(row.videoStoragePath), + storage.delete(row.poseStreamPath), + ]); + await prisma.mocapSession.delete({ where: { id } }); + + return NextResponse.json({ success: true, id }); +} diff --git a/src/app/api/mocap/sessions/route.ts b/src/app/api/mocap/sessions/route.ts new file mode 100644 index 0000000..d340a4a --- /dev/null +++ b/src/app/api/mocap/sessions/route.ts @@ -0,0 +1,74 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { z } from "zod"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { getMocapStorage } from "@/lib/mocap/storage"; +import { encodeHeader } from "@/lib/mocap/poseFrameStream"; + +const CreateBody = z.object({ + source: z.enum(["browser"]), + captureModelVersion: z.string().min(1).max(120), + capturePerspective: z.enum(["side-left", "side-right"]), + captureFps: z.number().positive().max(240), +}); + +export async function POST(req: Request) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + let body: z.infer; + try { + body = CreateBody.parse(await req.json()); + } catch (err) { + return NextResponse.json( + { error: "Invalid request body", details: err instanceof Error ? err.message : String(err) }, + { status: 400 }, + ); + } + + const userId = session.user.id; + const storage = getMocapStorage(); + + const created = await prisma.$transaction(async (tx) => { + const row = await tx.mocapSession.create({ + data: { + userId, + source: body.source, + captureModelVersion: body.captureModelVersion, + capturePerspective: body.capturePerspective, + captureFps: body.captureFps, + videoStoragePath: "pending", + poseStreamPath: "pending", + status: "capturing", + }, + }); + const videoStoragePath = storage.videoPath(userId, row.id); + const poseStreamPath = storage.poseStreamPath(userId, row.id); + return tx.mocapSession.update({ + where: { id: row.id }, + data: { videoStoragePath, poseStreamPath }, + }); + }); + + try { + const header = encodeHeader({ fps: body.captureFps }); + await storage.appendBytes(created.poseStreamPath, header); + } catch (err) { + await prisma.mocapSession.delete({ where: { id: created.id } }).catch(() => {}); + return NextResponse.json( + { error: "Failed to initialize pose stream blob", details: err instanceof Error ? err.message : String(err) }, + { status: 500 }, + ); + } + + return NextResponse.json({ + id: created.id, + videoStoragePath: created.videoStoragePath, + poseStreamPath: created.poseStreamPath, + status: created.status, + createdAt: created.createdAt, + }); +} diff --git a/src/lib/mocap/poseFrameStream.ts b/src/lib/mocap/poseFrameStream.ts new file mode 100644 index 0000000..3ee2091 --- /dev/null +++ b/src/lib/mocap/poseFrameStream.ts @@ -0,0 +1,152 @@ +/** + * PoseFrameStream binary blob format (ADR-0001). + * + * Layout: [header: 32 bytes][frame 0][frame 1]...[frame N-1] + * + * Random access by frame index = byte-range read at + * HEADER_SIZE + frameIndex * BYTES_PER_FRAME + * + * Header `frameCount` is OPEN_FRAME_COUNT during streaming append and updated + * to the final count on session finalize. Readers must accept either form: when + * they see OPEN_FRAME_COUNT, they derive the count from file size. + */ + +export const MAGIC = new Uint8Array([0x4d, 0x4f, 0x50, 0x53]); // "MOPS" +export const HEADER_SIZE = 32; +export const FORMAT_VERSION = 1; +export const KEYPOINT_SCHEMA_V1 = 1; +export const KEYPOINTS_PER_FRAME_V1 = 33; +export const BYTES_PER_FRAME_V1 = + 4 /* timestampMs Float32 */ + + KEYPOINTS_PER_FRAME_V1 * 3 * 4 /* x, y, confidence Float32 */ + + 4; /* qualityFlags Uint32 */ +export const OPEN_FRAME_COUNT = 0xffffffff; + +export const QUALITY_FLAG = { + LOW_LIGHT: 1 << 0, + OCCLUDED: 1 << 1, + OUT_OF_FRAME: 1 << 2, + LOW_CONFIDENCE: 1 << 3, + CAMERA_MOTION: 1 << 4, +} as const; + +export interface PoseStreamHeader { + formatVersion: number; + keypointSchemaVersion: number; + fps: number; + keypointsPerFrame: number; + bytesPerFrame: number; + frameCount: number; // OPEN_FRAME_COUNT during streaming +} + +export interface PoseFrame { + timestampMs: number; + /** Length = keypointsPerFrame * 3 (x, y, confidence interleaved). */ + keypoints: Float32Array; + qualityFlags: number; +} + +export class PoseStreamFormatError extends Error { + constructor(message: string) { + super(message); + this.name = "PoseStreamFormatError"; + } +} + +export function encodeHeader(opts: { + fps: number; + keypointSchemaVersion?: number; + frameCount?: number; +}): Uint8Array { + const schema = opts.keypointSchemaVersion ?? KEYPOINT_SCHEMA_V1; + if (schema !== KEYPOINT_SCHEMA_V1) { + throw new PoseStreamFormatError( + `Unsupported keypointSchemaVersion ${schema}`, + ); + } + const buf = new Uint8Array(HEADER_SIZE); + const view = new DataView(buf.buffer); + buf.set(MAGIC, 0); + view.setUint16(4, FORMAT_VERSION, true); + view.setUint16(6, schema, true); + view.setFloat32(8, opts.fps, true); + view.setUint16(12, KEYPOINTS_PER_FRAME_V1, true); + view.setUint16(14, BYTES_PER_FRAME_V1, true); + view.setUint32(16, opts.frameCount ?? OPEN_FRAME_COUNT, true); + // bytes 20-31 reserved (zero) + return buf; +} + +export function decodeHeader(bytes: Uint8Array): PoseStreamHeader { + if (bytes.byteLength < HEADER_SIZE) { + throw new PoseStreamFormatError( + `Header too short: ${bytes.byteLength} < ${HEADER_SIZE}`, + ); + } + for (let i = 0; i < MAGIC.length; i++) { + if (bytes[i] !== MAGIC[i]) { + throw new PoseStreamFormatError("Bad magic; not a PoseFrameStream blob"); + } + } + const view = new DataView(bytes.buffer, bytes.byteOffset, HEADER_SIZE); + const formatVersion = view.getUint16(4, true); + if (formatVersion !== FORMAT_VERSION) { + throw new PoseStreamFormatError( + `Unsupported formatVersion ${formatVersion}; reader knows ${FORMAT_VERSION}`, + ); + } + const keypointSchemaVersion = view.getUint16(6, true); + if (keypointSchemaVersion !== KEYPOINT_SCHEMA_V1) { + throw new PoseStreamFormatError( + `Unsupported keypointSchemaVersion ${keypointSchemaVersion}; reader knows ${KEYPOINT_SCHEMA_V1}`, + ); + } + return { + formatVersion, + keypointSchemaVersion, + fps: view.getFloat32(8, true), + keypointsPerFrame: view.getUint16(12, true), + bytesPerFrame: view.getUint16(14, true), + frameCount: view.getUint32(16, true), + }; +} + +export function encodeFrame(frame: PoseFrame): Uint8Array { + if (frame.keypoints.length !== KEYPOINTS_PER_FRAME_V1 * 3) { + throw new PoseStreamFormatError( + `Expected ${KEYPOINTS_PER_FRAME_V1 * 3} keypoint floats, got ${frame.keypoints.length}`, + ); + } + const buf = new Uint8Array(BYTES_PER_FRAME_V1); + const view = new DataView(buf.buffer); + view.setFloat32(0, frame.timestampMs, true); + const floats = new Float32Array(buf.buffer, 4, KEYPOINTS_PER_FRAME_V1 * 3); + floats.set(frame.keypoints); + view.setUint32(BYTES_PER_FRAME_V1 - 4, frame.qualityFlags >>> 0, true); + return buf; +} + +export function decodeFrame(bytes: Uint8Array, offset = 0): PoseFrame { + if (bytes.byteLength - offset < BYTES_PER_FRAME_V1) { + throw new PoseStreamFormatError( + `Frame slice too short: ${bytes.byteLength - offset} < ${BYTES_PER_FRAME_V1}`, + ); + } + const view = new DataView(bytes.buffer, bytes.byteOffset + offset, BYTES_PER_FRAME_V1); + const timestampMs = view.getFloat32(0, true); + const keypoints = new Float32Array(KEYPOINTS_PER_FRAME_V1 * 3); + for (let i = 0; i < keypoints.length; i++) { + keypoints[i] = view.getFloat32(4 + i * 4, true); + } + const qualityFlags = view.getUint32(BYTES_PER_FRAME_V1 - 4, true); + return { timestampMs, keypoints, qualityFlags }; +} + +export function frameByteOffset(frameIndex: number): number { + return HEADER_SIZE + frameIndex * BYTES_PER_FRAME_V1; +} + +export function framesFromBlobSize(blobSize: number): number { + if (blobSize < HEADER_SIZE) return 0; + return Math.floor((blobSize - HEADER_SIZE) / BYTES_PER_FRAME_V1); +} diff --git a/src/lib/mocap/storage.ts b/src/lib/mocap/storage.ts new file mode 100644 index 0000000..e4601b1 --- /dev/null +++ b/src/lib/mocap/storage.ts @@ -0,0 +1,115 @@ +import { promises as fs } from "node:fs"; +import path from "node:path"; + +export type ByteRange = { start: number; end?: number }; + +export interface MocapStorage { + videoPath(userId: string, sessionId: string): string; + poseStreamPath(userId: string, sessionId: string): string; + appendBytes(storagePath: string, bytes: Uint8Array): Promise; + read(storagePath: string, range?: ByteRange): Promise; + size(storagePath: string): Promise; + exists(storagePath: string): Promise; + delete(storagePath: string): Promise; +} + +const MOCAP_ROOT = "mocap"; + +class LocalDiskStorage implements MocapStorage { + constructor(private readonly root: string) {} + + videoPath(userId: string, sessionId: string): string { + return path.posix.join(MOCAP_ROOT, userId, sessionId, "video.webm"); + } + + poseStreamPath(userId: string, sessionId: string): string { + return path.posix.join(MOCAP_ROOT, userId, sessionId, "pose-stream.bin"); + } + + private resolve(storagePath: string): string { + const abs = path.resolve(this.root, storagePath); + const rootAbs = path.resolve(this.root); + if (!abs.startsWith(rootAbs + path.sep) && abs !== rootAbs) { + throw new Error(`Path escapes storage root: ${storagePath}`); + } + return abs; + } + + async appendBytes(storagePath: string, bytes: Uint8Array): Promise { + const abs = this.resolve(storagePath); + await fs.mkdir(path.dirname(abs), { recursive: true }); + await fs.appendFile(abs, bytes); + } + + async read(storagePath: string, range?: ByteRange): Promise { + const abs = this.resolve(storagePath); + if (!range) { + const buf = await fs.readFile(abs); + return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength); + } + const fh = await fs.open(abs, "r"); + try { + const stat = await fh.stat(); + const end = range.end ?? stat.size; + const length = Math.max(0, end - range.start); + const buf = Buffer.alloc(length); + if (length > 0) { + await fh.read(buf, 0, length, range.start); + } + return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength); + } finally { + await fh.close(); + } + } + + async size(storagePath: string): Promise { + const abs = this.resolve(storagePath); + const stat = await fs.stat(abs); + return stat.size; + } + + async exists(storagePath: string): Promise { + try { + await fs.access(this.resolve(storagePath)); + return true; + } catch { + return false; + } + } + + async delete(storagePath: string): Promise { + const abs = this.resolve(storagePath); + await fs.rm(abs, { force: true }); + const dir = path.dirname(abs); + try { + const entries = await fs.readdir(dir); + if (entries.length === 0) { + await fs.rmdir(dir); + } + } catch { + // ignore + } + } +} + +let instance: MocapStorage | null = null; + +export function getMocapStorage(): MocapStorage { + if (instance) return instance; + + const backend = process.env.MOCAP_STORAGE_BACKEND ?? "local"; + if (backend === "local") { + const root = process.env.MOCAP_STORAGE_ROOT + ? path.resolve(process.env.MOCAP_STORAGE_ROOT) + : path.resolve(process.cwd(), "storage"); + instance = new LocalDiskStorage(root); + return instance; + } + + // Vercel Blob backend: deferred. Add @vercel/blob dependency and a + // VercelBlobStorage class implementing MocapStorage. Append uses overwrite + // via put({allowOverwrite: true}); byte-range read via fetch with Range header. + throw new Error( + `MOCAP_STORAGE_BACKEND="${backend}" not implemented. Set MOCAP_STORAGE_BACKEND=local for now.`, + ); +} From cf7f6c618ada419c9bae15b7886a6b6bb8413e73 Mon Sep 17 00:00:00 2001 From: Rupert Germann Date: Fri, 8 May 2026 11:53:23 +0200 Subject: [PATCH 04/29] Add mocap analysis pipeline --- .claude/settings.local.json | 17 +- package-lock.json | 527 ++++++ package.json | 4 + playwright.config.ts | 42 + .../api/mocap/sessions/[id]/finalize/route.ts | 113 ++ .../mocap/sessions/[id]/pose-stream/route.ts | 94 + src/app/api/mocap/sessions/[id]/pose/route.ts | 53 + .../api/mocap/sessions/[id]/video/route.ts | 43 + src/app/mocap/page.tsx | 383 ++++ src/components/navigation.tsx | 3 +- src/lib/mocap/analysis/index.ts | 5 + .../mocap/analysis/postureFaultDetector.ts | 160 ++ src/lib/mocap/analysis/postureMetrics.ts | 233 +++ src/lib/mocap/analysis/postureThresholds.ts | 154 ++ .../mocap/analysis/strokePhaseSegmenter.ts | 160 ++ src/lib/mocap/analysis/types.ts | 144 ++ src/lib/mocap/browserPoseSource.ts | 222 +++ src/lib/mocap/poseWorker.ts | 173 ++ src/lib/mocap/storage.ts | 16 + src/lib/mocap/videoUploader.ts | 45 + tests/e2e/mocap-capture.spec.ts | 38 + .../mocap/asymmetric-side-unavailable.json | 1549 +++++++++++++++++ .../mocap/back-opens-before-legs.json | 864 +++++++++ tests/fixtures/mocap/clean-reference.json | 859 +++++++++ tests/fixtures/mocap/early-arm-bend.json | 864 +++++++++ tests/fixtures/mocap/excessive-layback.json | 864 +++++++++ tests/fixtures/mocap/lost-tracking.json | 859 +++++++++ .../fixtures/mocap/rounded-back-critical.json | 864 +++++++++ .../mocap/slow-recovery-critical.json | 1548 ++++++++++++++++ tests/mocapAnalysis.test.ts | 195 +++ tests/poseFrameStream.test.ts | 116 ++ 31 files changed, 11209 insertions(+), 2 deletions(-) create mode 100644 playwright.config.ts create mode 100644 src/app/api/mocap/sessions/[id]/finalize/route.ts create mode 100644 src/app/api/mocap/sessions/[id]/pose-stream/route.ts create mode 100644 src/app/api/mocap/sessions/[id]/pose/route.ts create mode 100644 src/app/api/mocap/sessions/[id]/video/route.ts create mode 100644 src/app/mocap/page.tsx create mode 100644 src/lib/mocap/analysis/index.ts create mode 100644 src/lib/mocap/analysis/postureFaultDetector.ts create mode 100644 src/lib/mocap/analysis/postureMetrics.ts create mode 100644 src/lib/mocap/analysis/postureThresholds.ts create mode 100644 src/lib/mocap/analysis/strokePhaseSegmenter.ts create mode 100644 src/lib/mocap/analysis/types.ts create mode 100644 src/lib/mocap/browserPoseSource.ts create mode 100644 src/lib/mocap/poseWorker.ts create mode 100644 src/lib/mocap/videoUploader.ts create mode 100644 tests/e2e/mocap-capture.spec.ts create mode 100644 tests/fixtures/mocap/asymmetric-side-unavailable.json create mode 100644 tests/fixtures/mocap/back-opens-before-legs.json create mode 100644 tests/fixtures/mocap/clean-reference.json create mode 100644 tests/fixtures/mocap/early-arm-bend.json create mode 100644 tests/fixtures/mocap/excessive-layback.json create mode 100644 tests/fixtures/mocap/lost-tracking.json create mode 100644 tests/fixtures/mocap/rounded-back-critical.json create mode 100644 tests/fixtures/mocap/slow-recovery-critical.json create mode 100644 tests/mocapAnalysis.test.ts create mode 100644 tests/poseFrameStream.test.ts diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 6f12184..ec238ad 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -17,7 +17,22 @@ "Bash(npx tsx:*)", "Bash(npx eslint:*)", "Bash(find:*)", - "Bash(wc:*)" + "Bash(wc:*)", + "Bash(gh repo *)", + "Bash(gh label *)", + "Bash(gh issue *)", + "Bash(sed -i '' 's|BLOCKER_1|#9|' /tmp/issue-2-calibration.md)", + "Bash(sed -i '' 's|BLOCKER_3|#10|' /tmp/issue-6-thresholds.md)", + "Bash(sed -i '' 's|BLOCKER_3|#10|; s|BLOCKER_4|#13|' /tmp/issue-5-cues.md)", + "Bash(sed -i '' 's|BLOCKER_4|#13|' /tmp/issue-7-link.md)", + "Bash(sed -i '' 's|BLOCKER_4|#13|' /tmp/issue-8-lifecycle.md)", + "Bash(sed -i '' 's|BLOCKER_4|#13|' /tmp/issue-9-cloudai.md)", + "Bash(npx prisma *)", + "Bash(PGPASSWORD=rowing_dev_password psql -h localhost -U rowing -d rowing_tracker -f prisma/migrations/20260508120000_add_mocap_session/migration.sql)", + "Bash(PGPASSWORD=rowing_dev_password psql -h localhost -U rowing -d rowing_tracker -c \"INSERT INTO _prisma_migrations \\(id, checksum, finished_at, migration_name, started_at, applied_steps_count\\) VALUES \\(gen_random_uuid\\(\\)::text, md5\\('20260508120000_add_mocap_session'\\), now\\(\\), '20260508120000_add_mocap_session', now\\(\\), 1\\) ON CONFLICT DO NOTHING;\")", + "Bash(git commit *)", + "Bash(xargs -I{} echo {})", + "Bash(npm test *)" ] } } diff --git a/package-lock.json b/package-lock.json index 287aaf2..ee957d1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "license": "MIT", "dependencies": { "@auth/prisma-adapter": "^2.11.2", + "@mediapipe/tasks-vision": "^0.10.35", "@playwright/test": "^1.56.1", "@prisma/adapter-pg": "^7.8.0", "@prisma/client": "^7.8.0", @@ -70,6 +71,7 @@ "eslint": "^9", "eslint-config-next": "16.0.3", "tailwindcss": "^4", + "tsx": "^4.21.0", "tw-animate-css": "^1.4.0", "typescript": "^5" } @@ -540,6 +542,448 @@ "tslib": "^2.4.0" } }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", + "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", + "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", + "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", + "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", + "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", + "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", + "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", + "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", + "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", + "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", + "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", + "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", + "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", + "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", + "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", + "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", + "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", + "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", + "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", + "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", + "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", + "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", + "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", + "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", + "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", + "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.9.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.0.tgz", @@ -1308,6 +1752,12 @@ "integrity": "sha512-M5UknZPHRu3DEDWoipU6sE8PdkZ6Z/S+v4dD+Ke8IaNlpdSQah50lz1KtcFBa2vsdOnwbbnxJwVM4wty6udA5w==", "license": "MIT" }, + "node_modules/@mediapipe/tasks-vision": { + "version": "0.10.35", + "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.35.tgz", + "integrity": "sha512-HOvadwVRE6JC+45nyYhmnywnr5h/J8KZvOeUNVOG9q/0875pZgItznFB9bRTvLc264YSJqiZ1NsIpCStJw/egg==", + "license": "Apache-2.0" + }, "node_modules/@napi-rs/wasm-runtime": { "version": "0.2.12", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", @@ -5439,6 +5889,48 @@ "benchmarks" ] }, + "node_modules/esbuild": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", + "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.7", + "@esbuild/android-arm": "0.27.7", + "@esbuild/android-arm64": "0.27.7", + "@esbuild/android-x64": "0.27.7", + "@esbuild/darwin-arm64": "0.27.7", + "@esbuild/darwin-x64": "0.27.7", + "@esbuild/freebsd-arm64": "0.27.7", + "@esbuild/freebsd-x64": "0.27.7", + "@esbuild/linux-arm": "0.27.7", + "@esbuild/linux-arm64": "0.27.7", + "@esbuild/linux-ia32": "0.27.7", + "@esbuild/linux-loong64": "0.27.7", + "@esbuild/linux-mips64el": "0.27.7", + "@esbuild/linux-ppc64": "0.27.7", + "@esbuild/linux-riscv64": "0.27.7", + "@esbuild/linux-s390x": "0.27.7", + "@esbuild/linux-x64": "0.27.7", + "@esbuild/netbsd-arm64": "0.27.7", + "@esbuild/netbsd-x64": "0.27.7", + "@esbuild/openbsd-arm64": "0.27.7", + "@esbuild/openbsd-x64": "0.27.7", + "@esbuild/openharmony-arm64": "0.27.7", + "@esbuild/sunos-x64": "0.27.7", + "@esbuild/win32-arm64": "0.27.7", + "@esbuild/win32-ia32": "0.27.7", + "@esbuild/win32-x64": "0.27.7" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -10926,6 +11418,41 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, + "node_modules/tsx": { + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", + "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.27.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/tsx/node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/tw-animate-css": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/tw-animate-css/-/tw-animate-css-1.4.0.tgz", diff --git a/package.json b/package.json index 8e62b94..db64f5d 100644 --- a/package.json +++ b/package.json @@ -25,10 +25,13 @@ "db:seed": "npx prisma db seed", "db:reset": "npx prisma migrate reset", "admin:promote": "npx tsx scripts/promote-admin.ts", + "test": "npx tsx --test tests/*.test.ts", + "test:e2e": "npx playwright test", "postinstall": "prisma generate" }, "dependencies": { "@auth/prisma-adapter": "^2.11.2", + "@mediapipe/tasks-vision": "^0.10.35", "@playwright/test": "^1.56.1", "@prisma/adapter-pg": "^7.8.0", "@prisma/client": "^7.8.0", @@ -88,6 +91,7 @@ "eslint": "^9", "eslint-config-next": "16.0.3", "tailwindcss": "^4", + "tsx": "^4.21.0", "tw-animate-css": "^1.4.0", "typescript": "^5" }, diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 0000000..2e0bb02 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,42 @@ +import { defineConfig, devices } from "@playwright/test"; + +const PORT = process.env.PORT ? parseInt(process.env.PORT, 10) : 3000; +const BASE_URL = process.env.PLAYWRIGHT_BASE_URL ?? `http://localhost:${PORT}`; + +export default defineConfig({ + testDir: "./tests/e2e", + timeout: 60_000, + expect: { timeout: 10_000 }, + fullyParallel: false, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 1 : 0, + reporter: process.env.CI ? "github" : "list", + use: { + baseURL: BASE_URL, + trace: "retain-on-failure", + video: "retain-on-failure", + }, + projects: [ + { + name: "chromium", + use: { + ...devices["Desktop Chrome"], + launchOptions: { + args: [ + "--use-fake-ui-for-media-stream", + "--use-fake-device-for-media-stream", + "--autoplay-policy=no-user-gesture-required", + ], + }, + }, + }, + ], + webServer: process.env.PLAYWRIGHT_SKIP_SERVER + ? undefined + : { + command: "npm run dev", + url: BASE_URL, + reuseExistingServer: !process.env.CI, + timeout: 120_000, + }, +}); diff --git a/src/app/api/mocap/sessions/[id]/finalize/route.ts b/src/app/api/mocap/sessions/[id]/finalize/route.ts new file mode 100644 index 0000000..cfe2973 --- /dev/null +++ b/src/app/api/mocap/sessions/[id]/finalize/route.ts @@ -0,0 +1,113 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { z } from "zod"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { getMocapStorage } from "@/lib/mocap/storage"; +import { + BYTES_PER_FRAME_V1, + HEADER_SIZE, + framesFromBlobSize, +} from "@/lib/mocap/poseFrameStream"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +const FRAME_COUNT_OFFSET = 16; + +const Body = z.object({ + durationSec: z.number().nonnegative().max(60 * 60 * 8), + qualityScore: z.number().min(0).max(1).optional(), +}); + +export async function POST( + req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const row = await prisma.mocapSession.findFirst({ + where: { id, userId: session.user.id }, + select: { + id: true, + status: true, + poseStreamPath: true, + videoStoragePath: true, + }, + }); + if (!row) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + if (row.status !== "capturing") { + return NextResponse.json( + { error: `Session not capturing (status=${row.status})` }, + { status: 409 }, + ); + } + + let body: z.infer; + try { + body = Body.parse(await req.json()); + } catch (err) { + return NextResponse.json( + { + error: "Invalid request body", + details: err instanceof Error ? err.message : String(err), + }, + { status: 400 }, + ); + } + + const storage = getMocapStorage(); + + let poseSize = 0; + try { + poseSize = await storage.size(row.poseStreamPath); + } catch { + return NextResponse.json( + { error: "Pose stream missing" }, + { status: 500 }, + ); + } + if (poseSize < HEADER_SIZE) { + return NextResponse.json( + { error: "Pose stream truncated below header" }, + { status: 500 }, + ); + } + const trailing = (poseSize - HEADER_SIZE) % BYTES_PER_FRAME_V1; + if (trailing !== 0) { + return NextResponse.json( + { + error: `Pose stream has ${trailing} trailing bytes (corrupt)`, + }, + { status: 500 }, + ); + } + const frameCount = framesFromBlobSize(poseSize); + + const headerPatch = new Uint8Array(4); + new DataView(headerPatch.buffer).setUint32(0, frameCount, true); + await storage.writeAt(row.poseStreamPath, headerPatch, FRAME_COUNT_OFFSET); + + const updated = await prisma.mocapSession.update({ + where: { id: row.id }, + data: { + status: "ready", + durationSec: body.durationSec, + qualityScore: body.qualityScore ?? null, + }, + }); + + return NextResponse.json({ + id: updated.id, + status: updated.status, + durationSec: updated.durationSec, + frameCount, + poseStreamBytes: poseSize, + }); +} diff --git a/src/app/api/mocap/sessions/[id]/pose-stream/route.ts b/src/app/api/mocap/sessions/[id]/pose-stream/route.ts new file mode 100644 index 0000000..ab0618e --- /dev/null +++ b/src/app/api/mocap/sessions/[id]/pose-stream/route.ts @@ -0,0 +1,94 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { getMocapStorage } from "@/lib/mocap/storage"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +function parseRange( + header: string | null, + totalSize: number, +): { start: number; end: number } | null { + if (!header) return null; + const m = /^bytes=(\d*)-(\d*)$/.exec(header.trim()); + if (!m) return null; + const startStr = m[1]; + const endStr = m[2]; + if (startStr === "" && endStr === "") return null; + let start: number; + let end: number; + if (startStr === "") { + const suffix = parseInt(endStr, 10); + if (!Number.isFinite(suffix) || suffix <= 0) return null; + start = Math.max(0, totalSize - suffix); + end = totalSize - 1; + } else { + start = parseInt(startStr, 10); + end = endStr === "" ? totalSize - 1 : parseInt(endStr, 10); + } + if ( + !Number.isFinite(start) || + !Number.isFinite(end) || + start < 0 || + end < start || + start >= totalSize + ) { + return null; + } + end = Math.min(end, totalSize - 1); + return { start, end }; +} + +export async function GET( + req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const row = await prisma.mocapSession.findFirst({ + where: { id, userId: session.user.id }, + select: { poseStreamPath: true }, + }); + if (!row) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + + const storage = getMocapStorage(); + const totalSize = await storage.size(row.poseStreamPath); + const range = parseRange(req.headers.get("range"), totalSize); + + if (!range) { + const bytes = await storage.read(row.poseStreamPath); + const u8 = new Uint8Array(bytes); + return new Response(u8 as BodyInit, { + status: 200, + headers: { + "Content-Type": "application/octet-stream", + "Content-Length": String(totalSize), + "Accept-Ranges": "bytes", + "Cache-Control": "private, no-store", + }, + }); + } + + const slice = await storage.read(row.poseStreamPath, { + start: range.start, + end: range.end + 1, + }); + return new Response(new Uint8Array(slice) as BodyInit, { + status: 206, + headers: { + "Content-Type": "application/octet-stream", + "Content-Length": String(slice.byteLength), + "Content-Range": `bytes ${range.start}-${range.end}/${totalSize}`, + "Accept-Ranges": "bytes", + "Cache-Control": "private, no-store", + }, + }); +} diff --git a/src/app/api/mocap/sessions/[id]/pose/route.ts b/src/app/api/mocap/sessions/[id]/pose/route.ts new file mode 100644 index 0000000..c263381 --- /dev/null +++ b/src/app/api/mocap/sessions/[id]/pose/route.ts @@ -0,0 +1,53 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { getMocapStorage } from "@/lib/mocap/storage"; +import { BYTES_PER_FRAME_V1 } from "@/lib/mocap/poseFrameStream"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +export async function POST( + req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const row = await prisma.mocapSession.findFirst({ + where: { id, userId: session.user.id }, + select: { id: true, status: true, poseStreamPath: true }, + }); + if (!row) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + if (row.status !== "capturing") { + return NextResponse.json( + { error: `Session not capturing (status=${row.status})` }, + { status: 409 }, + ); + } + + const buf = new Uint8Array(await req.arrayBuffer()); + if (buf.byteLength === 0) { + return NextResponse.json({ appended: 0 }); + } + if (buf.byteLength % BYTES_PER_FRAME_V1 !== 0) { + return NextResponse.json( + { + error: `Body length ${buf.byteLength} not multiple of frame size ${BYTES_PER_FRAME_V1}`, + }, + { status: 400 }, + ); + } + + const storage = getMocapStorage(); + await storage.appendBytes(row.poseStreamPath, buf); + + const framesAppended = buf.byteLength / BYTES_PER_FRAME_V1; + return NextResponse.json({ appended: framesAppended }); +} diff --git a/src/app/api/mocap/sessions/[id]/video/route.ts b/src/app/api/mocap/sessions/[id]/video/route.ts new file mode 100644 index 0000000..a8e3c0b --- /dev/null +++ b/src/app/api/mocap/sessions/[id]/video/route.ts @@ -0,0 +1,43 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { getMocapStorage } from "@/lib/mocap/storage"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +export async function POST( + req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const row = await prisma.mocapSession.findFirst({ + where: { id, userId: session.user.id }, + select: { id: true, status: true, videoStoragePath: true }, + }); + if (!row) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + if (row.status !== "capturing") { + return NextResponse.json( + { error: `Session not capturing (status=${row.status})` }, + { status: 409 }, + ); + } + + const buf = new Uint8Array(await req.arrayBuffer()); + if (buf.byteLength === 0) { + return NextResponse.json({ appended: 0 }); + } + + const storage = getMocapStorage(); + await storage.appendBytes(row.videoStoragePath, buf); + + return NextResponse.json({ appended: buf.byteLength }); +} diff --git a/src/app/mocap/page.tsx b/src/app/mocap/page.tsx new file mode 100644 index 0000000..b42f787 --- /dev/null +++ b/src/app/mocap/page.tsx @@ -0,0 +1,383 @@ +"use client"; + +import { useCallback, useEffect, useRef, useState } from "react"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { + BrowserPoseSource, + type PoseSourceStatus, +} from "@/lib/mocap/browserPoseSource"; +import { VideoUploader } from "@/lib/mocap/videoUploader"; + +const CAPTURE_FPS = 30; +const CAPTURE_MODEL_VERSION = "mediapipe-pose-landmarker-lite@0.10.35"; +const VIDEO_TIMESLICE_MS = 1000; + +type CaptureState = + | { kind: "idle" } + | { kind: "starting" } + | { + kind: "capturing"; + sessionId: string; + startedAt: number; + } + | { + kind: "stopping"; + sessionId: string; + } + | { + kind: "done"; + sessionId: string; + durationSec: number; + frameCount: number; + } + | { kind: "error"; message: string }; + +export default function MocapCapturePage() { + const videoRef = useRef(null); + const streamRef = useRef(null); + const recorderRef = useRef(null); + const uploaderRef = useRef(null); + const sourceRef = useRef(null); + const startedAtRef = useRef(0); + + const [state, setState] = useState({ kind: "idle" }); + const [framesEncoded, setFramesEncoded] = useState(0); + const [poseStatus, setPoseStatus] = useState("idle"); + const [perspective, setPerspective] = useState<"side-left" | "side-right">( + "side-right", + ); + const [elapsedSec, setElapsedSec] = useState(0); + + useEffect(() => { + if (state.kind !== "capturing") return; + const t = setInterval(() => { + setElapsedSec((Date.now() - startedAtRef.current) / 1000); + }, 250); + return () => clearInterval(t); + }, [state.kind]); + + const teardown = useCallback(async () => { + sourceRef.current = null; + recorderRef.current = null; + uploaderRef.current = null; + if (streamRef.current) { + for (const track of streamRef.current.getTracks()) track.stop(); + streamRef.current = null; + } + if (videoRef.current) { + videoRef.current.srcObject = null; + } + }, []); + + const handleError = useCallback( + async (err: unknown, sessionId?: string) => { + const message = err instanceof Error ? err.message : String(err); + setState({ kind: "error", message }); + try { + await sourceRef.current?.stop(); + } catch { + // ignore + } + try { + recorderRef.current?.stop(); + } catch { + // ignore + } + await uploaderRef.current?.drain(); + await teardown(); + if (sessionId) { + // Best-effort delete of the abandoned row + fetch(`/api/mocap/sessions/${sessionId}`, { method: "DELETE" }).catch( + () => {}, + ); + } + }, + [teardown], + ); + + const start = useCallback(async () => { + setState({ kind: "starting" }); + let sessionId: string | undefined; + try { + const stream = await navigator.mediaDevices.getUserMedia({ + video: { width: 1280, height: 720, frameRate: CAPTURE_FPS }, + audio: false, + }); + streamRef.current = stream; + const video = videoRef.current!; + video.srcObject = stream; + await video.play(); + + const createRes = await fetch("/api/mocap/sessions", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + source: "browser", + captureModelVersion: CAPTURE_MODEL_VERSION, + capturePerspective: perspective, + captureFps: CAPTURE_FPS, + }), + }); + if (!createRes.ok) { + throw new Error(`Create session failed: ${createRes.status}`); + } + const created: { id: string } = await createRes.json(); + sessionId = created.id; + + uploaderRef.current = new VideoUploader(sessionId, (err) => + handleError(err, sessionId), + ); + + const mimeType = pickRecorderMime(); + const recorder = new MediaRecorder(stream, { + mimeType, + videoBitsPerSecond: 2_500_000, + }); + recorder.ondataavailable = (event) => { + if (event.data && event.data.size > 0) { + uploaderRef.current?.enqueue(event.data); + } + }; + recorder.onerror = (event) => { + handleError( + (event as ErrorEvent).error ?? new Error("MediaRecorder error"), + sessionId, + ); + }; + recorderRef.current = recorder; + + const source = new BrowserPoseSource({ + sessionId, + videoEl: video, + onStatus: (s) => setPoseStatus(s), + onFrame: (info) => setFramesEncoded(info.framesEncoded), + onError: (err) => handleError(err, sessionId), + }); + sourceRef.current = source; + await source.init(); + + recorder.start(VIDEO_TIMESLICE_MS); + source.start(); + startedAtRef.current = Date.now(); + setElapsedSec(0); + setFramesEncoded(0); + setState({ + kind: "capturing", + sessionId, + startedAt: startedAtRef.current, + }); + } catch (err) { + await handleError(err, sessionId); + } + }, [handleError, perspective]); + + const stop = useCallback(async () => { + if (state.kind !== "capturing") return; + const sessionId = state.sessionId; + setState({ kind: "stopping", sessionId }); + try { + const recorder = recorderRef.current; + if (recorder && recorder.state !== "inactive") { + await new Promise((resolve) => { + recorder.onstop = () => resolve(); + recorder.stop(); + }); + } + await sourceRef.current?.stop(); + await uploaderRef.current?.drain(); + + const durationSec = (Date.now() - startedAtRef.current) / 1000; + const finalizeRes = await fetch( + `/api/mocap/sessions/${sessionId}/finalize`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ durationSec }), + }, + ); + if (!finalizeRes.ok) { + throw new Error(`Finalize failed: ${finalizeRes.status}`); + } + const finalized: { + id: string; + durationSec: number; + frameCount: number; + } = await finalizeRes.json(); + await teardown(); + setState({ + kind: "done", + sessionId: finalized.id, + durationSec: finalized.durationSec, + frameCount: finalized.frameCount, + }); + } catch (err) { + await handleError(err, sessionId); + } + }, [state, handleError, teardown]); + + useEffect(() => { + return () => { + sourceRef.current?.stop().catch(() => {}); + recorderRef.current?.stop(); + teardown(); + }; + }, [teardown]); + + useEffect(() => { + if (state.kind !== "capturing") return; + const onUnload = () => { + try { + recorderRef.current?.requestData?.(); + recorderRef.current?.stop(); + } catch { + // ignore + } + }; + window.addEventListener("beforeunload", onUnload); + return () => window.removeEventListener("beforeunload", onUnload); + }, [state.kind]); + + return ( +
+ + + Motion capture session + + Single-webcam pose capture. Camera permission is requested only when + you click Start. + + + +
+ + {state.kind === "idle" || state.kind === "done" ? ( + + ) : null} + {state.kind === "starting" ? ( + + ) : null} + {state.kind === "capturing" ? ( + + ) : null} + {state.kind === "stopping" ? ( + + ) : null} +
+ +
+
+ +
+ + + + 0 + ? (framesEncoded / elapsedSec).toFixed(1) + : "0.0" + } + /> +
+ + {state.kind === "done" ? ( +
+
+ Session {state.sessionId} stored. +
+
+ {state.frameCount} pose frames · {state.durationSec.toFixed(1)}s + duration +
+
+ ) : null} + + {state.kind === "error" ? ( +
+ Error: {state.message} +
+ ) : null} + + +
+ ); +} + +function Stat({ label, value }: { label: string; value: string }) { + return ( +
+
{label}
+
{value}
+
+ ); +} + +function pickRecorderMime(): string { + const candidates = [ + "video/webm;codecs=vp9", + "video/webm;codecs=vp8", + "video/webm", + "video/mp4", + ]; + for (const c of candidates) { + if ( + typeof MediaRecorder !== "undefined" && + MediaRecorder.isTypeSupported(c) + ) { + return c; + } + } + return "video/webm"; +} diff --git a/src/components/navigation.tsx b/src/components/navigation.tsx index 05b45e6..2076c94 100644 --- a/src/components/navigation.tsx +++ b/src/components/navigation.tsx @@ -16,7 +16,7 @@ import { } from '@/components/ui/dropdown-menu'; import { cn } from '@/lib/utils'; import { settings } from '@/lib/settings'; -import { Upload, BarChart3, List, Trophy, MessageCircle, Target, Settings as SettingsIcon, Gauge, Brain, User, LogOut, UserCircle, Sun, Moon, Monitor, RefreshCw } from 'lucide-react'; +import { Upload, BarChart3, List, Trophy, MessageCircle, Target, Settings as SettingsIcon, Gauge, Brain, User, LogOut, UserCircle, Sun, Moon, Monitor, RefreshCw, Video } from 'lucide-react'; const navigation = [ { name: 'Dashboard', href: '/dashboard', icon: BarChart3 }, @@ -26,6 +26,7 @@ const navigation = [ { name: 'Personal Records', href: '/prs', icon: Trophy }, { name: 'Training Plans', href: '/plans', icon: Target }, { name: 'AI Coach', href: '/chat', icon: MessageCircle }, + { name: 'Mocap', href: '/mocap', icon: Video }, { name: 'Sync', href: '/sync', icon: RefreshCw }, ]; diff --git a/src/lib/mocap/analysis/index.ts b/src/lib/mocap/analysis/index.ts new file mode 100644 index 0000000..7939373 --- /dev/null +++ b/src/lib/mocap/analysis/index.ts @@ -0,0 +1,5 @@ +export * from "./types"; +export * from "./strokePhaseSegmenter"; +export * from "./postureMetrics"; +export * from "./postureFaultDetector"; +export * from "./postureThresholds"; diff --git a/src/lib/mocap/analysis/postureFaultDetector.ts b/src/lib/mocap/analysis/postureFaultDetector.ts new file mode 100644 index 0000000..405cea9 --- /dev/null +++ b/src/lib/mocap/analysis/postureFaultDetector.ts @@ -0,0 +1,160 @@ +import { postureThresholdsV1, type PostureThresholdBands } from "./postureThresholds"; +import type { PostureFault, PostureMetrics } from "./types"; + +export function PostureFaultDetector( + metrics: PostureMetrics, + thresholds: PostureThresholdBands = postureThresholdsV1.thresholds, +): PostureFault[] { + const faults: PostureFault[] = []; + + if ( + metrics.backAngleAtCatchDeg < + thresholds.rounded_back_at_catch.criticalBelowDeg + ) { + faults.push({ + strokeIndex: metrics.strokeIndex, + faultType: "rounded_back_at_catch", + severity: "critical", + phase: "catch", + evidence: { + metric: "backAngleAtCatchDeg", + value: metrics.backAngleAtCatchDeg, + threshold: thresholds.rounded_back_at_catch.criticalBelowDeg, + }, + }); + } else if ( + metrics.backAngleAtCatchDeg < + thresholds.rounded_back_at_catch.warningBelowDeg + ) { + faults.push({ + strokeIndex: metrics.strokeIndex, + faultType: "rounded_back_at_catch", + severity: "warning", + phase: "catch", + evidence: { + metric: "backAngleAtCatchDeg", + value: metrics.backAngleAtCatchDeg, + threshold: thresholds.rounded_back_at_catch.warningBelowDeg, + }, + }); + } + + const armLead = metrics.armBendBeforeLegsCompleteFrames; + if ( + armLead !== null && + armLead >= thresholds.early_arm_bend.warningBeforeLegsCompleteFrames + ) { + faults.push({ + strokeIndex: metrics.strokeIndex, + faultType: "early_arm_bend", + severity: "warning", + phase: "drive", + evidence: { + metric: "armBendBeforeLegsCompleteFrames", + value: armLead, + threshold: thresholds.early_arm_bend.warningBeforeLegsCompleteFrames, + frameIndex: metrics.armBendOnsetFrameIndex ?? undefined, + }, + }); + } else if ( + armLead !== null && + armLead >= thresholds.early_arm_bend.infoBeforeLegsCompleteFrames + ) { + faults.push({ + strokeIndex: metrics.strokeIndex, + faultType: "early_arm_bend", + severity: "info", + phase: "drive", + evidence: { + metric: "armBendBeforeLegsCompleteFrames", + value: armLead, + threshold: thresholds.early_arm_bend.infoBeforeLegsCompleteFrames, + frameIndex: metrics.armBendOnsetFrameIndex ?? undefined, + }, + }); + } + + const openingOffset = metrics.hipKneeOpeningOffsetFrames; + if ( + openingOffset !== null && + openingOffset <= + -thresholds.back_opens_before_legs_drive + .warningTorsoOpensBeforeLegsFrames + ) { + faults.push({ + strokeIndex: metrics.strokeIndex, + faultType: "back_opens_before_legs_drive", + severity: "warning", + phase: "drive", + evidence: { + metric: "hipKneeOpeningOffsetFrames", + value: openingOffset, + threshold: + -thresholds.back_opens_before_legs_drive + .warningTorsoOpensBeforeLegsFrames, + }, + }); + } + + if (metrics.laybackAngleDeg > thresholds.excessive_layback.warningAboveDeg) { + faults.push({ + strokeIndex: metrics.strokeIndex, + faultType: "excessive_layback", + severity: "warning", + phase: "finish", + evidence: { + metric: "laybackAngleDeg", + value: metrics.laybackAngleDeg, + threshold: thresholds.excessive_layback.warningAboveDeg, + }, + }); + } else if ( + metrics.laybackAngleDeg > thresholds.excessive_layback.infoAboveDeg + ) { + faults.push({ + strokeIndex: metrics.strokeIndex, + faultType: "excessive_layback", + severity: "info", + phase: "finish", + evidence: { + metric: "laybackAngleDeg", + value: metrics.laybackAngleDeg, + threshold: thresholds.excessive_layback.infoAboveDeg, + }, + }); + } + + if ( + metrics.recoveryDriveRatio > + thresholds.slow_recovery_ratio.criticalAboveRatio + ) { + faults.push({ + strokeIndex: metrics.strokeIndex, + faultType: "slow_recovery_ratio", + severity: "critical", + phase: "recovery", + evidence: { + metric: "recoveryDriveRatio", + value: metrics.recoveryDriveRatio, + threshold: thresholds.slow_recovery_ratio.criticalAboveRatio, + }, + }); + } else if ( + metrics.recoveryDriveRatio > + thresholds.slow_recovery_ratio.warningAboveRatio + ) { + faults.push({ + strokeIndex: metrics.strokeIndex, + faultType: "slow_recovery_ratio", + severity: "warning", + phase: "recovery", + evidence: { + metric: "recoveryDriveRatio", + value: metrics.recoveryDriveRatio, + threshold: thresholds.slow_recovery_ratio.warningAboveRatio, + }, + }); + } + + return faults; +} diff --git a/src/lib/mocap/analysis/postureMetrics.ts b/src/lib/mocap/analysis/postureMetrics.ts new file mode 100644 index 0000000..36e88c7 --- /dev/null +++ b/src/lib/mocap/analysis/postureMetrics.ts @@ -0,0 +1,233 @@ +import { + getPosePoint, + landmarkSide, + type Calibration, + type PoseAnalysisFrame, + type PoseFrameStream, + type PoseLandmarkName, + type PosePoint, + type PostureMetrics, + type Stroke, +} from "./types"; + +const MIN_CONFIDENCE = 0.25; + +export function PostureMetricsCalculator( + stream: PoseFrameStream, + stroke: Stroke, + calibration?: Calibration, +): PostureMetrics { + void calibration; + const catchFrame = frameAt(stream, stroke.catchFrameIndex); + const finishFrame = frameAt(stream, stroke.finishFrameIndex); + const backAngleAtCatchDeg = torsoBackAngleDeg(stream, catchFrame); + const backAngleAtFinishDeg = torsoBackAngleDeg(stream, finishFrame); + const laybackAngleDeg = Math.max(0, 90 - backAngleAtFinishDeg); + + const legSignal = legExtensionSignal(stream, stroke); + const catchLeg = legSignal[0]?.value ?? 0; + const finishLeg = legSignal[stroke.finishFrameIndex - stroke.catchFrameIndex] + ?.value ?? catchLeg; + const legRange = Math.max(0.0001, finishLeg - catchLeg); + + const legExtensionStartFrameIndex = firstSignalFrameAtLeast( + legSignal, + catchLeg + legRange * 0.2, + ); + const legExtensionCompleteFrameIndex = firstSignalFrameAtLeast( + legSignal, + catchLeg + legRange * 0.8, + ); + const torsoOpenFrameIndex = firstTorsoChangeFrame( + stream, + stroke, + backAngleAtCatchDeg, + ); + const armBendOnsetFrameIndex = firstArmBendFrame(stream, stroke); + + return { + strokeIndex: stroke.strokeIndex, + segmentationSource: stroke.segmentationSource, + backAngleAtCatchDeg, + backAngleAtFinishDeg, + laybackAngleDeg, + hipKneeOpeningOffsetFrames: + legExtensionStartFrameIndex === null || torsoOpenFrameIndex === null + ? null + : torsoOpenFrameIndex - legExtensionStartFrameIndex, + armBendOnsetFrameIndex, + legExtensionCompleteFrameIndex, + armBendBeforeLegsCompleteFrames: + armBendOnsetFrameIndex === null || legExtensionCompleteFrameIndex === null + ? null + : legExtensionCompleteFrameIndex - armBendOnsetFrameIndex, + recoveryDriveRatio: recoveryDriveRatio(stroke), + leftRightAsymmetry: + stream.capturePerspective === "sidecar-3d" + ? { available: false, reason: "insufficient-tracking" } + : { available: false, reason: "requires-sidecar-3d" }, + shinVerticalAtCatchDeg: + stream.capturePerspective === "sidecar-3d" + ? { available: false, reason: "insufficient-tracking" } + : { available: false, reason: "requires-sidecar-3d" }, + kneeTrackDeviation: + stream.capturePerspective === "sidecar-3d" + ? { available: false, reason: "insufficient-tracking" } + : { available: false, reason: "requires-sidecar-3d" }, + }; +} + +function frameAt(stream: PoseFrameStream, frameIndex: number): PoseAnalysisFrame { + const frame = stream.frames[frameIndex]; + if (!frame) { + throw new Error(`Frame ${frameIndex} is outside the PoseFrameStream`); + } + return frame; +} + +function sideNames(stream: PoseFrameStream): { + shoulder: PoseLandmarkName; + elbow: PoseLandmarkName; + wrist: PoseLandmarkName; + hip: PoseLandmarkName; + knee: PoseLandmarkName; +} { + const side = landmarkSide(stream.capturePerspective); + return { + shoulder: `${side}Shoulder` as PoseLandmarkName, + elbow: `${side}Elbow` as PoseLandmarkName, + wrist: `${side}Wrist` as PoseLandmarkName, + hip: `${side}Hip` as PoseLandmarkName, + knee: `${side}Knee` as PoseLandmarkName, + }; +} + +function requiredPoint( + frame: PoseAnalysisFrame, + name: PoseLandmarkName, +): PosePoint { + const point = getPosePoint(frame, name); + if (!point || point.confidence < MIN_CONFIDENCE) { + throw new Error(`Missing tracked landmark ${name}`); + } + return point; +} + +function torsoBackAngleDeg( + stream: PoseFrameStream, + frame: PoseAnalysisFrame, +): number { + const names = sideNames(stream); + const hip = requiredPoint(frame, names.hip); + const shoulder = requiredPoint(frame, names.shoulder); + const dx = shoulder.x - hip.x; + const dyUp = hip.y - shoulder.y; + const raw = radiansToDegrees(Math.atan2(dyUp, dx)); + const normalized = raw < 0 ? raw + 180 : raw; + return normalized > 90 ? 180 - normalized : normalized; +} + +function legExtensionSignal( + stream: PoseFrameStream, + stroke: Stroke, +): Array<{ frameIndex: number; value: number }> { + const names = sideNames(stream); + const signal: Array<{ frameIndex: number; value: number }> = []; + for ( + let frameIndex = stroke.catchFrameIndex; + frameIndex <= stroke.finishFrameIndex; + frameIndex++ + ) { + const frame = frameAt(stream, frameIndex); + const hip = requiredPoint(frame, names.hip); + const knee = requiredPoint(frame, names.knee); + signal.push({ + frameIndex, + value: Math.hypot(hip.x - knee.x, hip.y - knee.y), + }); + } + return signal; +} + +function firstSignalFrameAtLeast( + signal: Array<{ frameIndex: number; value: number }>, + threshold: number, +): number | null { + for (const point of signal) { + if (point.value >= threshold) return point.frameIndex; + } + return null; +} + +function firstTorsoChangeFrame( + stream: PoseFrameStream, + stroke: Stroke, + catchAngleDeg: number, +): number | null { + for ( + let frameIndex = stroke.catchFrameIndex + 1; + frameIndex <= stroke.finishFrameIndex; + frameIndex++ + ) { + const frame = frameAt(stream, frameIndex); + if (Math.abs(torsoBackAngleDeg(stream, frame) - catchAngleDeg) >= 5) { + return frameIndex; + } + } + return null; +} + +function firstArmBendFrame( + stream: PoseFrameStream, + stroke: Stroke, +): number | null { + const initialAngle = elbowAngleDeg(stream, frameAt(stream, stroke.catchFrameIndex)); + const threshold = Math.min(160, initialAngle - 15); + for ( + let frameIndex = stroke.catchFrameIndex + 1; + frameIndex <= stroke.finishFrameIndex; + frameIndex++ + ) { + const angle = elbowAngleDeg(stream, frameAt(stream, frameIndex)); + if (angle <= threshold) return frameIndex; + } + return null; +} + +function elbowAngleDeg( + stream: PoseFrameStream, + frame: PoseAnalysisFrame, +): number { + const names = sideNames(stream); + const shoulder = requiredPoint(frame, names.shoulder); + const elbow = requiredPoint(frame, names.elbow); + const wrist = requiredPoint(frame, names.wrist); + return angleAtPointDeg(shoulder, elbow, wrist); +} + +function angleAtPointDeg(a: PosePoint, vertex: PosePoint, b: PosePoint): number { + const ax = a.x - vertex.x; + const ay = a.y - vertex.y; + const bx = b.x - vertex.x; + const by = b.y - vertex.y; + const denom = Math.hypot(ax, ay) * Math.hypot(bx, by); + if (denom === 0) return 0; + const cos = Math.max(-1, Math.min(1, (ax * bx + ay * by) / denom)); + return radiansToDegrees(Math.acos(cos)); +} + +function recoveryDriveRatio(stroke: Stroke): number { + const driveFrames = Math.max( + 1, + stroke.finishFrameIndex - stroke.driveStartFrameIndex, + ); + const recoveryFrames = Math.max( + 1, + stroke.nextCatchFrameIndex - stroke.recoveryStartFrameIndex, + ); + return recoveryFrames / driveFrames; +} + +function radiansToDegrees(radians: number): number { + return (radians * 180) / Math.PI; +} diff --git a/src/lib/mocap/analysis/postureThresholds.ts b/src/lib/mocap/analysis/postureThresholds.ts new file mode 100644 index 0000000..ceef3f5 --- /dev/null +++ b/src/lib/mocap/analysis/postureThresholds.ts @@ -0,0 +1,154 @@ +import type { PostureFaultType } from "./types"; + +export type PostureThresholdVersion = "V1" | `V${number}`; + +export interface PostureThresholdBands { + rounded_back_at_catch: { + warningBelowDeg: number; + criticalBelowDeg: number; + }; + early_arm_bend: { + infoBeforeLegsCompleteFrames: number; + warningBeforeLegsCompleteFrames: number; + }; + back_opens_before_legs_drive: { + warningTorsoOpensBeforeLegsFrames: number; + }; + excessive_layback: { + infoAboveDeg: number; + warningAboveDeg: number; + }; + slow_recovery_ratio: { + warningAboveRatio: number; + criticalAboveRatio: number; + }; +} + +export interface VersionedPostureThresholds { + version: PostureThresholdVersion; + thresholds: PostureThresholdBands; +} + +export interface UserPostureThresholdSettings extends VersionedPostureThresholds { + userOverridden: boolean; +} + +export const POSTURE_FAULT_CATALOG_V1: readonly PostureFaultType[] = [ + "rounded_back_at_catch", + "early_arm_bend", + "back_opens_before_legs_drive", + "excessive_layback", + "slow_recovery_ratio", +]; + +export const postureThresholdsV1: VersionedPostureThresholds = { + version: "V1", + thresholds: { + // British Rowing Technique: catch/drive keeps the back straight and leaning + // forward; CONTEXT.md fixes the v1 warning/critical bands at 30/20 deg. + rounded_back_at_catch: { + warningBelowDeg: 30, + criticalBelowDeg: 20, + }, + // Concept2 Indoor Rowing Technique: drive sequence is legs, body, then arms; + // early elbow flexion before leg extension is therefore treated conservatively. + early_arm_bend: { + infoBeforeLegsCompleteFrames: 1, + warningBeforeLegsCompleteFrames: 4, + }, + // British Rowing and Concept2 both teach the drive as legs before body swing; + // any torso opening before the leg signal starts is a v1 warning. + back_opens_before_legs_drive: { + warningTorsoOpensBeforeLegsFrames: 1, + }, + // Concept2 finish guidance says the upper body leans back slightly; British + // Rowing calls leaning too far back a recovery-delaying fault. + excessive_layback: { + infoAboveDeg: 30, + warningAboveDeg: 45, + }, + // Concept2 frames recovery as the rest/preparation phase; ratios beyond 2.5x + // are deliberately conservative v1 flags for very slow recoveries. + slow_recovery_ratio: { + warningAboveRatio: 2.5, + criticalAboveRatio: 3.5, + }, + }, +}; + +export function defaultPostureThresholdSettings( + defaults: VersionedPostureThresholds = postureThresholdsV1, +): UserPostureThresholdSettings { + return { + version: defaults.version, + thresholds: cloneThresholds(defaults.thresholds), + userOverridden: false, + }; +} + +export function migratePostureThresholdSettings( + stored: unknown, + defaults: VersionedPostureThresholds = postureThresholdsV1, +): UserPostureThresholdSettings { + if (!isUserPostureThresholdSettings(stored)) { + return defaultPostureThresholdSettings(defaults); + } + + if (stored.version !== defaults.version && !stored.userOverridden) { + return defaultPostureThresholdSettings(defaults); + } + + return { + version: stored.version, + thresholds: cloneThresholds(stored.thresholds), + userOverridden: stored.userOverridden, + }; +} + +function isUserPostureThresholdSettings( + value: unknown, +): value is UserPostureThresholdSettings { + if (!value || typeof value !== "object") return false; + const candidate = value as Partial; + return ( + typeof candidate.version === "string" && + typeof candidate.userOverridden === "boolean" && + isPostureThresholdBands(candidate.thresholds) + ); +} + +function isPostureThresholdBands( + value: unknown, +): value is PostureThresholdBands { + if (!value || typeof value !== "object") return false; + const t = value as PostureThresholdBands; + return ( + isFiniteNumber(t.rounded_back_at_catch?.warningBelowDeg) && + isFiniteNumber(t.rounded_back_at_catch?.criticalBelowDeg) && + isFiniteNumber(t.early_arm_bend?.infoBeforeLegsCompleteFrames) && + isFiniteNumber(t.early_arm_bend?.warningBeforeLegsCompleteFrames) && + isFiniteNumber( + t.back_opens_before_legs_drive?.warningTorsoOpensBeforeLegsFrames, + ) && + isFiniteNumber(t.excessive_layback?.infoAboveDeg) && + isFiniteNumber(t.excessive_layback?.warningAboveDeg) && + isFiniteNumber(t.slow_recovery_ratio?.warningAboveRatio) && + isFiniteNumber(t.slow_recovery_ratio?.criticalAboveRatio) + ); +} + +function isFiniteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value); +} + +function cloneThresholds(thresholds: PostureThresholdBands): PostureThresholdBands { + return { + rounded_back_at_catch: { ...thresholds.rounded_back_at_catch }, + early_arm_bend: { ...thresholds.early_arm_bend }, + back_opens_before_legs_drive: { + ...thresholds.back_opens_before_legs_drive, + }, + excessive_layback: { ...thresholds.excessive_layback }, + slow_recovery_ratio: { ...thresholds.slow_recovery_ratio }, + }; +} diff --git a/src/lib/mocap/analysis/strokePhaseSegmenter.ts b/src/lib/mocap/analysis/strokePhaseSegmenter.ts new file mode 100644 index 0000000..0385e63 --- /dev/null +++ b/src/lib/mocap/analysis/strokePhaseSegmenter.ts @@ -0,0 +1,160 @@ +import { + getPosePoint, + landmarkSide, + type PoseFrameStream, + type PoseLandmarkName, + type Stroke, +} from "./types"; + +interface SignalPoint { + frameIndex: number; + value: number; + tracked: boolean; +} + +export function StrokePhaseSegmenter(stream: PoseFrameStream): Stroke[] { + if (stream.frames.length < 3) return []; + + const signal = smoothSignal(buildHipKneeDistanceSignal(stream), stream.fps); + const catches = findCatchCandidates(signal, stream.fps); + const strokes: Stroke[] = []; + + for (let i = 0; i < catches.length - 1; i++) { + const catchFrameIndex = catches[i]; + const nextCatchFrameIndex = catches[i + 1]; + if (nextCatchFrameIndex - catchFrameIndex < 3) continue; + + const finishFrameIndex = maxSignalFrame( + signal, + catchFrameIndex, + nextCatchFrameIndex, + ); + if (finishFrameIndex <= catchFrameIndex) continue; + + const recoveryStartFrameIndex = Math.min( + finishFrameIndex + 1, + nextCatchFrameIndex, + ); + const trackedFrames = signal + .slice(catchFrameIndex, nextCatchFrameIndex + 1) + .filter((p) => p.tracked).length; + + strokes.push({ + strokeIndex: strokes.length, + segmentationSource: "pose-segmented", + catchFrameIndex, + driveStartFrameIndex: catchFrameIndex, + finishFrameIndex, + recoveryStartFrameIndex, + nextCatchFrameIndex, + confidence: trackedFrames / (nextCatchFrameIndex - catchFrameIndex + 1), + }); + } + + return strokes; +} + +function buildHipKneeDistanceSignal(stream: PoseFrameStream): SignalPoint[] { + const side = landmarkSide(stream.capturePerspective); + const hipName = `${side}Hip` as PoseLandmarkName; + const kneeName = `${side}Knee` as PoseLandmarkName; + + return stream.frames.map((frame, frameIndex) => { + const hip = getPosePoint(frame, hipName); + const knee = getPosePoint(frame, kneeName); + if (!hip || !knee || hip.confidence < 0.25 || knee.confidence < 0.25) { + return { frameIndex, value: Number.NaN, tracked: false }; + } + const dx = hip.x - knee.x; + const dy = hip.y - knee.y; + return { + frameIndex, + value: Math.hypot(dx, dy), + tracked: true, + }; + }); +} + +function smoothSignal(signal: SignalPoint[], fps: number): SignalPoint[] { + const radius = Math.max(1, Math.round(fps * 0.06)); + return signal.map((point, i) => { + let total = 0; + let count = 0; + for ( + let j = Math.max(0, i - radius); + j <= Math.min(signal.length - 1, i + radius); + j++ + ) { + const value = signal[j].value; + if (Number.isFinite(value)) { + total += value; + count++; + } + } + return { + frameIndex: point.frameIndex, + value: count > 0 ? total / count : point.value, + tracked: point.tracked, + }; + }); +} + +function findCatchCandidates(signal: SignalPoint[], fps: number): number[] { + const values = signal + .map((p) => p.value) + .filter((value) => Number.isFinite(value)); + if (values.length < 3) return []; + + const min = Math.min(...values); + const max = Math.max(...values); + const range = Math.max(0.0001, max - min); + const lowThreshold = min + range * 0.35; + const minGap = Math.max(4, Math.round(fps * 0.4)); + const catches: number[] = []; + + for (let i = 0; i < signal.length; i++) { + const prev = signal[Math.max(0, i - 1)]?.value; + const cur = signal[i].value; + const next = signal[Math.min(signal.length - 1, i + 1)]?.value; + if (!Number.isFinite(cur) || cur > lowThreshold) continue; + + const isEndpointMinimum = + (i === 0 && Number.isFinite(next) && cur <= next) || + (i === signal.length - 1 && Number.isFinite(prev) && cur <= prev); + const isInteriorMinimum = + i > 0 && + i < signal.length - 1 && + Number.isFinite(prev) && + Number.isFinite(next) && + cur <= prev && + cur <= next && + (cur < prev || cur < next); + + if (!isEndpointMinimum && !isInteriorMinimum) continue; + const last = catches[catches.length - 1]; + if (last === undefined || i - last >= minGap) { + catches.push(i); + } else if (cur < signal[last].value) { + catches[catches.length - 1] = i; + } + } + + return catches; +} + +function maxSignalFrame( + signal: SignalPoint[], + startFrame: number, + endFrame: number, +): number { + let maxFrame = startFrame; + let maxValue = -Infinity; + for (let i = startFrame; i <= endFrame; i++) { + const value = signal[i]?.value; + if (Number.isFinite(value) && value > maxValue) { + maxValue = value; + maxFrame = i; + } + } + return maxFrame; +} diff --git a/src/lib/mocap/analysis/types.ts b/src/lib/mocap/analysis/types.ts new file mode 100644 index 0000000..d8ff2ae --- /dev/null +++ b/src/lib/mocap/analysis/types.ts @@ -0,0 +1,144 @@ +export type CapturePerspective = "side-left" | "side-right" | "sidecar-3d"; + +export type StrokeSegmentationSource = "pose-segmented" | "csv-aligned"; + +export type PostureFaultType = + | "rounded_back_at_catch" + | "early_arm_bend" + | "back_opens_before_legs_drive" + | "excessive_layback" + | "slow_recovery_ratio"; + +export type FaultSeverity = "info" | "warning" | "critical"; + +export type PoseLandmarkName = + | "leftShoulder" + | "rightShoulder" + | "leftElbow" + | "rightElbow" + | "leftWrist" + | "rightWrist" + | "leftHip" + | "rightHip" + | "leftKnee" + | "rightKnee" + | "leftAnkle" + | "rightAnkle"; + +export const POSE_LANDMARK_INDEX: Record = { + leftShoulder: 11, + rightShoulder: 12, + leftElbow: 13, + rightElbow: 14, + leftWrist: 15, + rightWrist: 16, + leftHip: 23, + rightHip: 24, + leftKnee: 25, + rightKnee: 26, + leftAnkle: 27, + rightAnkle: 28, +}; + +export interface PosePoint { + x: number; + y: number; + confidence: number; +} + +export type PoseKeypoints = + | readonly PosePoint[] + | Partial>; + +export interface PoseAnalysisFrame { + timestampMs: number; + keypoints: PoseKeypoints; + qualityFlags?: number; +} + +export interface PoseFrameStream { + fps: number; + capturePerspective: CapturePerspective; + frames: readonly PoseAnalysisFrame[]; +} + +export interface Stroke { + strokeIndex: number; + segmentationSource: StrokeSegmentationSource; + catchFrameIndex: number; + driveStartFrameIndex: number; + finishFrameIndex: number; + recoveryStartFrameIndex: number; + nextCatchFrameIndex: number; + confidence: number; +} + +export interface Calibration { + capturePerspective: CapturePerspective; + catchFrame?: PoseAnalysisFrame; + finishFrame?: PoseAnalysisFrame; +} + +export interface UnavailableMetric { + available: false; + reason: "requires-sidecar-3d" | "insufficient-tracking"; +} + +export interface AvailableMetric { + available: true; + value: T; +} + +export type MaybeMetric = AvailableMetric | UnavailableMetric; + +export interface PostureMetrics { + strokeIndex: number; + segmentationSource: StrokeSegmentationSource; + backAngleAtCatchDeg: number; + backAngleAtFinishDeg: number; + laybackAngleDeg: number; + hipKneeOpeningOffsetFrames: number | null; + armBendOnsetFrameIndex: number | null; + legExtensionCompleteFrameIndex: number | null; + armBendBeforeLegsCompleteFrames: number | null; + recoveryDriveRatio: number; + leftRightAsymmetry: MaybeMetric; + shinVerticalAtCatchDeg: MaybeMetric; + kneeTrackDeviation: MaybeMetric; +} + +export interface PostureFault { + strokeIndex: number; + faultType: PostureFaultType; + severity: FaultSeverity; + phase: "catch" | "drive" | "finish" | "recovery"; + evidence: { + metric: keyof PostureMetrics | "armBendBeforeLegsCompleteFrames"; + value: number; + threshold: number; + frameIndex?: number; + }; +} + +export function getPosePoint( + frame: PoseAnalysisFrame, + name: PoseLandmarkName, +): PosePoint | null { + const keypoints = frame.keypoints; + if (isPosePointArray(keypoints)) { + const point = keypoints[POSE_LANDMARK_INDEX[name]]; + return point?.confidence > 0 ? point : null; + } + const point = keypoints[name]; + return point?.confidence && point.confidence > 0 ? point : null; +} + +function isPosePointArray(keypoints: PoseKeypoints): keypoints is readonly PosePoint[] { + return Array.isArray(keypoints); +} + +export function landmarkSide( + perspective: CapturePerspective, +): "left" | "right" { + return perspective === "side-left" ? "left" : "right"; +} diff --git a/src/lib/mocap/browserPoseSource.ts b/src/lib/mocap/browserPoseSource.ts new file mode 100644 index 0000000..ee7bdd6 --- /dev/null +++ b/src/lib/mocap/browserPoseSource.ts @@ -0,0 +1,222 @@ +/** + * BrowserPoseSource: main-thread orchestration for the pose worker. + * + * Owns the worker, the upload queue, and the per-frame ImageBitmap pipeline. + * Frames flow: