diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 6f12184..b9cff7c 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -17,7 +17,26 @@ "Bash(npx tsx:*)", "Bash(npx eslint:*)", "Bash(find:*)", - "Bash(wc:*)" + "Bash(wc:*)", + "Bash(gh repo *)", + "Bash(gh label *)", + "Bash(gh issue *)", + "Bash(sed -i '' 's|BLOCKER_1|#9|' /tmp/issue-2-calibration.md)", + "Bash(sed -i '' 's|BLOCKER_3|#10|' /tmp/issue-6-thresholds.md)", + "Bash(sed -i '' 's|BLOCKER_3|#10|; s|BLOCKER_4|#13|' /tmp/issue-5-cues.md)", + "Bash(sed -i '' 's|BLOCKER_4|#13|' /tmp/issue-7-link.md)", + "Bash(sed -i '' 's|BLOCKER_4|#13|' /tmp/issue-8-lifecycle.md)", + "Bash(sed -i '' 's|BLOCKER_4|#13|' /tmp/issue-9-cloudai.md)", + "Bash(npx prisma *)", + "Bash(PGPASSWORD=rowing_dev_password psql -h localhost -U rowing -d rowing_tracker -f prisma/migrations/20260508120000_add_mocap_session/migration.sql)", + "Bash(PGPASSWORD=rowing_dev_password psql -h localhost -U rowing -d rowing_tracker -c \"INSERT INTO _prisma_migrations \\(id, checksum, finished_at, migration_name, started_at, applied_steps_count\\) VALUES \\(gen_random_uuid\\(\\)::text, md5\\('20260508120000_add_mocap_session'\\), now\\(\\), '20260508120000_add_mocap_session', now\\(\\), 1\\) ON CONFLICT DO NOTHING;\")", + "Bash(git commit *)", + "Bash(xargs -I{} echo {})", + "Bash(npm test *)", + "Bash(git worktree *)", + "WebSearch", + "Bash(node *)", + "Bash(python3 -c \"import json,sys; d=json.load\\(sys.stdin\\); print\\(json.dumps\\({k:v for k,v in d.get\\('scripts',{}\\).items\\(\\)}, indent=2\\)\\); print\\('devdeps:', list\\(d.get\\('devDependencies',{}\\).keys\\(\\)\\)[:20]\\)\")" ] } } diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 0000000..ae9637e --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,94 @@ +# Context + +Domain glossary for rowing-tracker. Terms here are canonical — use them in code, issues, and UI copy. If a concept isn't here, either it's not yet domain-load-bearing, or there's a gap to resolve via `/grill-with-docs`. + +## Glossary + +### CapturePerspective + +Which physical viewpoint a mocap session was recorded from. Determines which posture metrics are computable. + +- `side-left` — single webcam, rower's left side facing camera. Browser path. +- `side-right` — single webcam, rower's right side facing camera. Browser path. +- `sidecar-3d` — multi-camera 3D capture via freemocap sidecar. All metrics available. + +Browser path emits `side-left` or `side-right` only. Metrics that require `sidecar-3d` (left-right asymmetry, knee track deviation) are marked **unavailable** on side captures — never silently zeroed or estimated. UI surfaces the unavailable state as "requires multi-camera capture." + +### StrokeSegmentationSource + +How a mocap session's stroke phase boundaries (catch / drive / finish / recovery) were derived. + +- `pose-segmented` — boundaries computed from the pose stream alone (hip-knee distance signal). Used during live capture, when no CSV is available yet. Lower confidence; downstream metrics tagged accordingly. +- `csv-aligned` — boundaries taken from `StrokeData` rows (SmartRow ground truth), with the pose stream time-aligned to them via cross-correlation. Used in post-session replay after the user imports the SmartRow CSV and links it to the mocap session. + +A mocap session always begins life as `pose-segmented`. When linked to a `RowingSession` (CSV import), re-segmentation to `csv-aligned` is **mandatory** — re-run metrics and faults atomically with the link. Never leave a linked session at `pose-segmented`. + +### MocapSession + +A captured rowing session containing video + pose stream + derived metrics + faults. Independent of `RowingSession` (which is CSV/erg-derived). May exist standalone (no CSV linked) or be linked to exactly one `RowingSession`. + +Lifecycle states (proposed): `capturing` → `analyzing` → `ready` → optionally `linked` (when joined to a `RowingSession`). + +**Linking to a `RowingSession`** is bidirectional and exclusive — one `MocapSession` is linked to at most one `RowingSession` and vice versa. Either side may be unlinked; relinking is allowed. Linking triggers mandatory re-analysis (`pose-segmented` → `csv-aligned`) as a background job; the mocap row goes back to `analyzing` until it completes. Unlinking reverts to `pose-segmented` and re-runs metrics. CSV import auto-prompts to link when a capture window overlaps a new `RowingSession` by ±2 minutes — user confirms, never silent. + +### CueLatencyBand + +When a coaching cue is delivered relative to the stroke that triggered it. + +- `intra-stroke` — fired mid-stroke from per-frame rules. **Out of v1 scope.** Pose-segmented stroke boundaries are too noisy for reliable real-time fault attribution within a 2.5s stroke window. +- `post-stroke` — fired ≤1s after stroke completes. The "live coaching" experience in v1. +- `post-session` — surfaced in replay / coaching summary after capture ends. + +v1 fault detector runs at the **stroke** granularity only — one pass per closed stroke. No per-frame fault rule path. + +### PoseFrameStream + +A timestamped sequence of keypoint frames with confidence values, produced by a `PoseCaptureSource` and consumed by the analysis pipeline. + +**v1 shape (`keypointSchemaVersion: 1`):** 2D side-view keypoints — `{x, y, confidence}` per keypoint, normalized [0,1] image-relative coordinates. `coordinateSpace: "normalized-2d"`. Browser path only. + +**v2 shape (`keypointSchemaVersion: 2`):** 3D world-space keypoints — `{x, y, z, confidence}` per keypoint, units in millimeters. `coordinateSpace: "world-mm-3d"`. Sidecar path only (see ADR-0005). Blob header adds `cameraCount` and `calibrationId`. v1 blobs remain readable — the reader branches on `keypointSchemaVersion`. All v1 fault rules ignore `z` and work on `{x, y}` projection for both versions. + +33 BlazePose landmarks; 13 are rowing-relevant (nose, shoulders, elbows, wrists, hips, knees, ankles). The rest are captured but unused. Confidence = MediaPipe visibility [0,1]. v2 adds `reprojectionErrorMm` quality field (triangulation accuracy). + +### PostureFault (v1 catalog) + +Stroke-granular faults the v1 detector emits. All computable from a 2D side-view `PoseFrameStream`. Each fault is named, attributed to a stroke phase, and has severity bands defined in `FaultThresholds`. + +| Fault key | Phase | Severity bands | +| --- | --- | --- | +| `rounded_back_at_catch` | catch | warning < 30°, critical < 20° (back angle) | +| `early_arm_bend` | drive | info / warning by frame-offset of arm-bend onset vs leg-extension completion | +| `back_opens_before_legs_drive` | drive | warning if torso angle changes before legs start extending | +| `excessive_layback` | finish | info > 30°, warning > 45° (torso past vertical) | +| `slow_recovery_ratio` | recovery | warning > 2.5, critical > 3.5 (recovery / drive duration ratio) | + +**Excluded from v1**, surfaced as "metric available, detection deferred" or "requires multi-camera capture": + +- `left_right_asymmetry` — needs front view or `sidecar-3d` +- `knee_track_deviation` — needs front view or `sidecar-3d` +- `shin_not_vertical_at_catch` — disambiguating near-side shin from far-side shin in 2D is unreliable + +**Unlocked by `sidecar-3d` (Phase 2):** all three deferred faults above become computable. Lateral displacement is unambiguous in 3D; near/far shin disambiguated by z-coordinate. Fault rules and thresholds to be defined in follow-up implementation issues. + +`perspective` field on each fault: `"browser"` or `"sidecar-3d"`. When perspective is browser, the three sidecar-3d-only faults surface as "requires multi-camera capture" — never silently zeroed. + +This catalog is the canonical vocabulary. Test fixtures, threshold tuning, coaching cue copy, and AI prompt context all reference these exact keys. Anything outside this list is out of v1 scope. + +### FaultThresholds + +The numeric bands a `PostureFault` rule fires against (e.g. "back angle at catch < threshold → rounded-back fault"). Stored on `UserSettings.postureThresholds: Json?`. + +**Defaults are hand-coded, conservative, and versioned in code** (`postureThresholdsV1`, `postureThresholdsV2`, …). Each default carries a source comment citing rowing-technique references. Conservative bands = wide tolerances, fewer false positives, fewer angry users in v1. + +Migration: when a new defaults version ships, users who haven't touched their thresholds upgrade automatically. Users with `userOverridden: true` keep their custom values; never stomp explicit customisation. + +### Calibration + +Two distinct calibration concepts — do not conflate: + +**Browser calibration** — a pair of reference pose frames captured before recording starts: one at **catch** position, one at **finish** position. Used as pixel-space baselines for downstream metric calculations. Stored per `MocapSession` (see ADR-0001). Recapture (~10 s) required at the start of each session. + +**Sidecar Charuco calibration** — a multi-camera extrinsic calibration using a Charuco board. Establishes shared 3D world-space coordinate frame across cameras. Owned and executed by the freemocap sidecar, not by the app. The app stores `calibrationId` (UUID) in `MocapSession` for traceability, but does not own the calibration workflow. Charuco calibration is reusable across sessions as long as cameras don't move; users re-run it when the rig changes. + +**Storage:** persisted as one binary blob per `MocapSession`, alongside the video file (see ADR-0001). Not a Postgres table. The `MocapSession` row points at it via `poseStreamPath`. Blob header carries `fps`, `keypointSchemaVersion`, `frameCount`, `coordinateSpace`, and (v2 only) `calibrationId`, `cameraCount`. Random access by frame index = byte-range read. diff --git a/README.md b/README.md index 6a59a72..58c3c39 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,11 @@ # Rowing Tracker -An AI-powered web application for tracking rowing workouts with analytics, training plans, and achievement tracking. Built for rowers who use SmartRow equipment. This app was completely written by AI. +An AI-powered web application for tracking rowing workouts with analytics, training plans, motion-capture posture analysis, and achievement tracking. Built for rowers who use SmartRow equipment. This app was completely written by AI. ## Overview -Rowing Tracker is a modern, AI-powered web app built specifically for rowers who use SmartRow equipment. Upload your CSV exports and unlock the power of artificial intelligence to analyze your performance, generate personalized training plans, and receive expert coaching insights. With multi-user support and secure authentication, each rower gets their own private workspace with data stored in PostgreSQL (local or cloud). +Rowing Tracker is a modern, AI-powered web app built specifically for rowers who use SmartRow equipment. Workouts are imported either via automated SmartRow.fit sync or manual CSV/ZIP upload, and the app turns them into deep analytics, personalized AI training plans, automated coaching insights, and webcam-based posture analysis. Each rower gets a private workspace with data stored in PostgreSQL (local or Supabase) and isolated by authenticated user. ## Features @@ -23,19 +23,20 @@ Rowing Tracker is a modern, AI-powered web app built specifically for rowers who ### 📊 Analytics & Tracking - **Dashboard**: Comprehensive overview with key metrics, volume charts, and trend analysis -- **Advanced Analytics**: Detailed breakdown of performance, split trends, stroke rate, and training adherence -- **Interactive Chart Explanations**: Click "Explain" on any chart (analytics page AND session details) to get AI-powered analysis—explanations are saved in tooltips for quick reference with "Back to chart" navigation -- **Time-Range Aware Explanations**: Analytics chart explanations are cached per time range—switching between "Last 7 days" and "Last 30 days" generates separate, context-appropriate AI analyses +- **Advanced Analytics** (`/analytics`): Detailed breakdown of performance, split trends, stroke rate, consistency score, and training adherence with a uniform legend (teal dot = individual sessions, orange line = 10-session moving average) and synchronized global smoothing controls +- **Insights View** (`/insights`): Dedicated page for browsing, filtering, archiving, and giving feedback (`helpful` / `not_helpful` / `action_taken`) on AI-generated performance insights, with full-text search across the archive +- **Interactive Chart Explanations**: Click "Explain" on any chart (analytics page AND session details) to get AI-powered analysis—explanations are cached server-side via `ChartExplanation`, displayed in tooltips for quick reference, and offer "Back to chart" navigation +- **Time-Range Aware Explanations**: Analytics chart explanations are cached per time range—switching between "Last 7 days" and "Last 30 days" yields separate, context-appropriate AI analyses - **Structured AI Explanations**: Chart explanations follow a clear format: "Why This Chart Matters" (practical value), "What I See In Your Data" (patterns/trends), and "What This Means For You" (actionable insights) -- **Performance Correlations**: Explore scatter plots showing relationships between power/pace, stroke rate/pace, duration/distance, energy/duration, and more +- **Performance Correlations**: Scatter plots showing relationships between power/pace, stroke rate/pace, duration/distance, energy/duration, with Split Time always-on as a permanent correlation chart - **Sessions List**: Browse, filter, and sort all your rowing sessions with advanced search - **Session Details**: Deep dive into individual workout metrics with interactive charts and AI explanations across all analysis modules: - *Overview*: Power & Stroke Rate - *Performance Graphs*: Pace Analysis, Work per Stroke, Stroke Length, Heart Rate - *Segments*: Segment Analysis (100m/500m), Rolling Power Average, Rolling Split Average - *Deep Analysis*: Power Distribution, Rhythm Distribution, Rate vs Power, Rate vs Split -- **Stroke-by-Stroke Analysis**: Upload SmartRow stroke exports to unlock power/rhythm distributions, stroke-length consistency, and technique maps for every stroke -- **Personal Records**: Automatic tracking of your best times and performances across all distances +- **Stroke-by-Stroke Analysis**: Stroke data parsed from SmartRow detailed CSVs is persisted with each session, unlocking power/rhythm distributions, stroke-length consistency, technique maps, and a pre-computed `consistencyScore` per session +- **Personal Records**: Automatic tracking of best times and performances across all distances ### 🔐 Multi-User & Authentication @@ -45,14 +46,22 @@ Rowing Tracker is a modern, AI-powered web app built specifically for rowers who - **User Profiles**: Manage your account, change password, and update profile information - **Data Isolation**: Each user's data is completely isolated and private +### � Data Import & Sync + +- **Automated SmartRow Sync**: One-click sync from `smartrow.fit` directly inside `/sync`. A server-side Playwright session logs in with stored credentials, exports the workouts list (CSV) and the detailed-stroke archive (ZIP), and imports both in a single pass +- **CSV Drag-and-Drop**: Manual upload of any SmartRow CSV (sessions list or detailed stroke export) +- **ZIP Batch Import**: Upload the SmartRow archive ZIP to import many detailed-stroke sessions at once with progress reporting +- **Duplicate-Safe Imports**: Sessions are deduplicated by `(userId, timestamp, distance)`; existing sessions are updated, not duplicated +- **Last-Sync Timestamp**: SmartRow credentials and the most recent sync time are stored in `UserSettings.smartRowSettings` + ### 💾 Data & Storage -- **CSV Import**: Simple drag-and-drop upload for SmartRow CSV files -- **PostgreSQL Database**: Robust, scalable data storage with full ACID compliance -- **Local Development**: Docker-based PostgreSQL for easy local development -- **Cloud Ready**: Supabase support for production deployments -- **Memory System**: Upload and store PDFs and images for AI analysis -- **Data Privacy**: Your data is encrypted and isolated per user +- **PostgreSQL Database**: Robust, scalable data storage with full ACID compliance via Prisma +- **Local Development**: Docker-based PostgreSQL plus Mailpit for SMTP +- **Cloud Ready**: Supabase (pooler + direct URL) supported for production +- **Memory System**: Upload and store PDFs and images for AI analysis; files are kept on disk (or Vercel Blob in deployed environments) and referenced by `MemoryDocument.filePath` +- **Mocap Storage**: Recorded video and the `PoseFrameStream` binary blob are stored side-by-side on the same backend (local `storage/` directory or Vercel Blob), referenced by `MocapSession.videoStoragePath` and `MocapSession.poseStreamPath` +- **Data Privacy**: Workout data, mocap video, and pose data are scoped by user ID; AI keys are encrypted at rest ### 🎨 User Experience @@ -65,28 +74,50 @@ Rowing Tracker is a modern, AI-powered web app built specifically for rowers who ### 🏅 Gamification & Motivation - **Dynamic Awards System**: Earn achievements for session milestones (First Splash, Century Club, Year of Rowing), total distance (Million Meter Club), streaks, duration, power output, pace improvements, and more -- **Improvement Awards**: Track percentage gains in power (up to +100% Double Power) and pace compared to your baseline to unlock progressive tier awards -- **Streak Milestones**: Stay consistent with notifications for 7-, 14-, 21-, 45-, 60-, and 100-day streaks -- **Live Award Notifications**: Celebrate wins instantly with animated overlays whenever you unlock something new +- **Improvement Awards**: Track percentage gains in power (up to +100% Double Power) and pace compared to a baseline computed from the first 3 valid sessions +- **Streak Milestones**: Notifications for 7-, 14-, 21-, 45-, 60-, and 100-day streaks +- **AI Award Suggestions**: The AI proposes custom awards (`AIAwardSuggestion`) with structured criteria; once approved they auto-evaluate against your data +- **Generated Achievement Stories & Art**: Each unlocked award gets an AI-written story and optional generated image (`GeneratedAchievement`), with a configurable `colorPalette` for the artwork +- **Live Award Notifications**: Animated overlays the moment a new achievement unlocks - **High-Tier Stretch Goals**: Long-term achievements including 750k meters, 1 Million meters, 100 hours rowing, 300W power, and sub-1:35/500m pace +### 🎥 Motion-Capture & Posture Analysis (Mocap) + +- **Browser-Based Capture** (`/mocap`): Single-webcam recording with in-browser MediaPipe Pose Landmarker running in a Web Worker. Zero install, no cloud upload of video by default. +- **Side-View Capture Perspectives**: `side-left` or `side-right`. Front-view-only metrics (left/right asymmetry, knee-track deviation) are explicitly marked as `requires-multi-cam` rather than silently estimated. +- **Per-Session Calibration**: Two reference frames captured before recording (catch + finish) establish baselines for the current camera setup. Calibration is stored on the `MocapSession`, not on the user. +- **Pose Frame Stream**: A versioned binary `PoseFrameStream` blob (2D `{x, y, confidence}` keypoints + per-frame quality flags) is appended in chunks via `POST /api/mocap/sessions/:id/pose-stream` and finalized with `POST /api/mocap/sessions/:id/finalize`. +- **Stroke Segmentation**: `pose-segmented` boundaries during live capture; mandatory atomic re-segmentation to `csv-aligned` when a `MocapSession` is linked to a `RowingSession` via cross-correlation against `StrokeData`. +- **v1 Posture Fault Catalog**: Five stroke-granular faults computable from a 2D side view—`rounded_back_at_catch`, `early_arm_bend`, `back_opens_before_legs_drive`, `excessive_layback`, `slow_recovery_ratio`—each with `info` / `warning` / `critical` severity bands. +- **Configurable Thresholds**: Conservative hand-coded defaults (`postureThresholdsV1`) with auto-migration on version bumps; user customization stored in `UserSettings.postureThresholds` is preserved (`userOverridden: true`). +- **Live Coaching Cues**: Post-stroke cues (≤ 1 s after the stroke completes) via the `LiveCoachingEngine`, with optional spoken audio (`speakCue`) and configurable verbosity in `UserSettings.mocapPreferences`. +- **Auto-Link on CSV Import**: When a CSV import produces a `RowingSession` whose timestamp overlaps a `MocapSession` capture window by ±2 minutes, the user is prompted to link them (never silent). Linking is bidirectional, exclusive, and reversible (`/unlink` endpoint). +- **Re-Analysis Endpoint**: `POST /api/mocap/sessions/:id/reanalyze` re-runs the segmenter, metrics calculator, and fault detector with current rules so old sessions benefit from updated thresholds. +- **Cloud-AI Payload Tiers**: AI gets a `PostureFault` summary by default; per-stroke metrics are opt-in via `UserSettings.mocapDetailedAIShare`; raw frames never cross to cloud (see ADR-0004). + ## Tech Stack -- **Framework**: Next.js 16 (App Router) -- **Language**: TypeScript -- **Styling**: TailwindCSS -- **Components**: shadcn/ui -- **Charts**: Recharts -- **AI Integration**: OpenAI API -- **Authentication**: NextAuth.js v4 -- **Database**: PostgreSQL with Prisma v7 +- **Framework**: Next.js 16 (App Router, React 19) +- **Language**: TypeScript 5 +- **Styling**: TailwindCSS 4 +- **Components**: shadcn/ui (Radix UI primitives) +- **Charts**: Recharts 3 +- **Animations**: Framer Motion +- **Markdown**: react-markdown + remark-gfm + Shiki for code highlighting +- **AI Integration**: OpenAI API (chat, image generation, condensation prompts) +- **Authentication**: NextAuth.js v4 (Credentials, Email magic link, optional Google OAuth) +- **Database**: PostgreSQL with Prisma v7 and `@prisma/adapter-pg` - **State Management**: Zustand with persist middleware -- **Storage**: - - PostgreSQL for user data, sessions, plans, and achievements - - File system for award images -- **CSV Parsing**: papaparse +- **Storage**: + - PostgreSQL for user data, sessions, plans, achievements, mocap rows + - File system (or Vercel Blob via `@vercel/blob`) for award images, memory documents, mocap video, and pose stream blobs +- **Pose Estimation**: `@mediapipe/tasks-vision` Pose Landmarker, Web Worker, WASM +- **CSV / ZIP Parsing**: `papaparse`, `jszip` +- **PDF Extraction**: `unpdf` +- **SmartRow Automation**: `playwright` (server-side login + export download for `/api/smartrow/sync`) - **Email**: Nodemailer with Mailpit (local) or SMTP (production) - **Rate Limiting**: Upstash Redis for API protection +- **Validation**: Zod - **Development**: Docker Compose for local services ## Quick Start @@ -155,42 +186,49 @@ Rowing Tracker is a modern, AI-powered web app built specifically for rowers who 9. **Configure AI (Optional)** - Go to Settings → AI Coach - - Enter your OpenAI API Key to enable Chat and Training Plans + - Enter your OpenAI API Key to enable Chat, Insights, and Training Plans - Add your Personal Context to inform the AI about medical conditions, preferences, or goals - - Customize the AI prompts in the Advanced Configuration section + - Customize the AI prompts (base, chat, training plan, insights) in Advanced Configuration with one-click "reset to default" + +10. **Configure SmartRow Auto-Sync (Optional)** + - Go to Settings → SmartRow + - Enter your `smartrow.fit` email and password (stored per-user in `UserSettings.smartRowSettings`) + - Visit `/sync` and click **Sync Now** to pull all workouts in one pass + +11. **Promote a User to Admin (Optional)** + ```bash + npm run admin:promote -- + ``` + Admin users see the **Admin Panel** entry in the user menu and can access `/admin` to manage other users. + +## Importing Your SmartRow Data + +The `/sync` page offers three import paths. All three deduplicate against existing sessions and update changed records in place. -## SmartRow CSV Export Guide +### 1. Automated Sync from smartrow.fit (recommended) -### How to Export Your Data +1. Save your SmartRow credentials in **Settings → SmartRow**. +2. Open `/sync` and click **Sync Now**. +3. The server runs a Playwright session against `https://smartrow.fit/my-workouts/`, downloads the workouts CSV and the detailed-stroke ZIP, and imports both. The `lastSync` timestamp is updated on completion. -1. **Connect to SmartRow App** - - Open the SmartRow mobile app - - Ensure you're logged in and synced +### 2. Manual CSV Upload -2. **Export Sessions** - - Go to Settings/Profile - - Find "Export Data" or "CSV Export" - - Select the date range you want to export - - Choose CSV format - - Download the file to your device +Drag-and-drop a SmartRow CSV onto `/sync` or click to browse. Both the workouts-list CSV and individual detailed-stroke CSVs are accepted. -3. **Upload to Rowing Tracker** - - Open the Rowing Tracker web app - - Drag and drop your CSV file or click to browse - - Wait for processing (typically instant for most files) - - Your data will be automatically analyzed and stored +### 3. ZIP Batch Upload + +Drop the SmartRow archive ZIP onto `/sync` to import many detailed-stroke sessions at once with progress reporting (`ZipProcessProgress`). ### CSV Format Requirements -The app expects SmartRow CSV exports with the following format: -- **Delimiter**: Semicolon (;) -- **Decimal Format**: Comma (,) - European format -- **Timestamp**: YYYY-MM-DD HH:MM:SS.mmm (UTC) +SmartRow CSV exports use: +- **Delimiter**: Semicolon (`;`) +- **Decimal Format**: Comma (`,`) — European format +- **Timestamp**: `YYYY-MM-DD HH:MM:SS.mmm` (UTC) - **Time Field**: Seconds -### Required Columns +### Required Columns (workouts list) -Your CSV must include these columns: - Time stamp (UTC) - Distance (m) - Time (seconds) @@ -198,11 +236,15 @@ Your CSV must include these columns: - Stroke count (#) - Average power (W) - Maximum power (W) -- Average split (s) - per 500m +- Average split (s) — per 500m - Minimum split (s) - Average stroke rate (SPM) - Maximum stroke rate (SPM) +### Detailed-Stroke CSV + +Detailed-stroke files (one per session, contained in the SmartRow ZIP) are parsed by `src/lib/strokeParser.ts` into `StrokeData` rows. They drive stroke-by-stroke analysis and the precomputed `consistencyScore` on `RowingSession`. + ## Deployment ### Supabase (Production) @@ -242,7 +284,7 @@ Your CSV must include these columns: Available npm scripts: ```bash -# Start local Docker services +# Start local Docker services (PostgreSQL + Mailpit) npm run db:start # Stop local Docker services @@ -260,11 +302,20 @@ npm run db:migrate:deploy # Reset database (WARNING: deletes all data) npm run db:reset +# Seed database (where a seed is configured) +npm run db:seed + # Open Prisma Studio (database GUI) npm run db:studio # Push schema without migration (dev only) npm run db:push + +# Promote a user to admin +npm run admin:promote -- + +# Backfill consistencyScore for existing sessions (one-off) +npx tsx scripts/backfill-consistency.ts ``` ## Architecture Overview @@ -272,76 +323,109 @@ npm run db:push ``` rowing-tracker/ ├── src/ -│ ├── app/ # Next.js App Router -│ │ ├── (routes)/ # Route groups -│ │ │ ├── page.tsx # Dashboard -│ │ │ ├── sessions/ # Sessions pages -│ │ │ ├── prs/ # Personal records -│ │ │ ├── upload/ # CSV upload -│ │ │ ├── analytics/ # Advanced analytics -│ │ │ ├── chat/ # AI Coach chat -│ │ │ ├── plans/ # Training plans -│ │ │ ├── profile/ # User profile -│ │ │ └── settings/ # App settings -│ │ ├── api/ # API routes -│ │ │ ├── auth/ # NextAuth endpoints -│ │ │ └── user/ # User management -│ │ ├── auth/ # Auth pages -│ │ │ ├── login/ # Login page -│ │ │ ├── register/ # Registration -│ │ │ └── verify-email/ # Email verification -│ │ ├── layout.tsx # Root layout -│ │ └── globals.css # Global styles -│ ├── components/ # Reusable UI components -│ ├── hooks/ # Custom React hooks -│ ├── lib/ # Utility functions & services -│ │ ├── auth.ts # NextAuth configuration -│ │ ├── db/prisma.ts # Prisma client singleton -│ │ ├── services/ # Service singletons -│ │ ├── ai/ # AI configuration & prompts -│ │ └── utils/ # Utilities (CSV parser, awards, etc.) -│ └── types/ # TypeScript type definitions -├── prisma/ # Database schema & migrations +│ ├── app/ # Next.js App Router +│ │ ├── dashboard/ # Main dashboard +│ │ ├── analytics/ # Advanced analytics page +│ │ ├── sessions/ # Session list and detail pages +│ │ ├── prs/ # Personal records & achievements +│ │ ├── plans/ # AI training plans +│ │ ├── chat/ # AI Coach chat +│ │ ├── insights/ # AI insights archive & feedback +│ │ ├── mocap/ # Webcam capture + posture replay +│ │ ├── sync/ # SmartRow sync, CSV/ZIP upload +│ │ ├── profile/ # User profile +│ │ ├── settings/ # App settings (AI, SmartRow, posture, ...) +│ │ ├── admin/ # Admin user management (admin role only) +│ │ ├── auth/ # Login, register, verify-email, reset +│ │ ├── api/ # API routes (NextAuth, sessions, mocap, smartrow, ...) +│ │ ├── layout.tsx +│ │ └── globals.css +│ ├── components/ # Reusable UI components (shadcn/ui + custom) +│ ├── hooks/ # Custom React hooks +│ ├── lib/ # Utilities & services +│ │ ├── auth.ts # NextAuth configuration +│ │ ├── db/prisma.ts # Prisma client singleton +│ │ ├── services/ # Service singletons +│ │ ├── mocap/ # Pose source, frame stream, analysis pipeline, coaching +│ │ │ ├── browserPoseSource.ts +│ │ │ ├── poseFrameStream.ts +│ │ │ ├── poseWorker.ts +│ │ │ ├── analysis/ # Pure functions: segmenter, metrics, fault detector, thresholds +│ │ │ └── coaching/ # LiveCoachingEngine, CoachingAdvisor, cue audio +│ │ ├── csvParser.ts +│ │ ├── strokeParser.ts +│ │ ├── zipParser.ts +│ │ ├── awards.ts +│ │ └── ... +│ ├── types/ # TypeScript type definitions +│ └── middleware.ts # Auth + admin route guards +├── prisma/ # Database schema & migrations │ └── schema.prisma -├── docs/ # Documentation +├── scripts/ # One-off operational scripts +│ ├── promote-admin.ts +│ └── backfill-consistency.ts +├── tests/ # Unit + Playwright e2e tests +│ ├── *.test.ts # tsx --test +│ ├── fixtures/mocap/ # Pose-frame fixtures +│ └── e2e/ # Playwright specs +├── docs/ │ ├── DATABASE_SCHEMA.md │ ├── design-system.md │ ├── prd.md -│ └── csvs/ # Sample SmartRow CSV files -└── docker-compose.yml # Local PostgreSQL & Mailpit +│ ├── prd-mocap-posture.md # Mocap PRD + locked decisions +│ ├── adr/ # Architecture Decision Records (0001–0004) +│ ├── agents/ # Agent docs (issue tracker, triage labels, domain) +│ └── csvs/ # Sample SmartRow CSVs +├── CONTEXT.md # Domain glossary +├── AGENTS.md # Engineering rules +└── docker-compose.yml # Local PostgreSQL & Mailpit ``` ### Data Flow -1. **Authentication**: User registers/logs in → NextAuth validates → JWT session created -2. **Upload**: User drops CSV file → papaparse processes → validation → saved to PostgreSQL -3. **Storage**: - - User data, sessions, plans → PostgreSQL via Prisma - - Award images → File system - - Client state → Zustand store (ephemeral) -4. **Display**: Components fetch from database → calculate metrics → render charts -5. **Analysis**: Real-time PR calculations, trend analysis, aggregations -6. **AI Features**: Context retrieved from database → sent to OpenAI → response streamed -7. **Data Isolation**: All queries filtered by authenticated user ID +1. **Authentication**: User registers/logs in → NextAuth validates (Credentials / Email magic link / Google OAuth) → JWT session created with `id` and `role` +2. **Import**: + - **Sync**: `/api/smartrow/sync` runs Playwright against smartrow.fit → returns CSV + base64 ZIP → client parses with papaparse / jszip → saved to PostgreSQL + - **Manual**: User drops CSV/ZIP → client-side validation → saved to PostgreSQL +3. **Mocap Capture**: Browser webcam → Pose Landmarker in Web Worker → chunked HTTP uploads of video and `PoseFrameStream` → finalize → stroke segmentation, metrics, and faults computed in browser; server only persists +4. **Storage**: + - User data, sessions, plans, mocap rows → PostgreSQL via Prisma + - Award images, memory documents, mocap video, pose stream blobs → file system or Vercel Blob + - Client state → Zustand store (persisted to DB on key actions) +5. **Display**: Components fetch from database → calculate metrics → render charts; cache busts via `UserSettings.sessionsRevision` / `insightsRevision` +6. **Analysis**: Real-time PR calculations, trend analysis, consistency score, posture metrics, fault counts +7. **AI Features**: Context (sessions, achievements, memory documents, posture summary) retrieved from database → personal context injected from `UserSettings.userProfileContext` → sent to OpenAI → response streamed +8. **Linking**: When a `RowingSession` overlaps a `MocapSession` capture window by ±2 minutes, the user is prompted to link; linking triggers atomic re-segmentation to `csv-aligned` +9. **Data Isolation**: All queries filtered by authenticated user ID; admin endpoints additionally gated via `src/lib/adminAuth.ts` ## Development ### Available Scripts **Development:** -- `npm run dev` - Start development server -- `npm run build` - Build for production -- `npm run start` - Start production server -- `npm run lint` - Run ESLint +- `npm run dev` — Start development server +- `npm run build` — Build for production +- `npm run start` — Start production server +- `npm run lint` — Run ESLint **Database:** -- `npm run db:start` - Start PostgreSQL & Mailpit -- `npm run db:stop` - Stop Docker services -- `npm run db:generate` - Generate Prisma client -- `npm run db:migrate` - Run migrations -- `npm run db:studio` - Open Prisma Studio -- `npm run db:push` - Push schema (dev only) -- `npm run db:reset` - Reset database +- `npm run db:start` — Start PostgreSQL & Mailpit +- `npm run db:stop` — Stop Docker services +- `npm run db:generate` — Generate Prisma client +- `npm run db:migrate` — Run migrations (dev) +- `npm run db:migrate:deploy` — Apply migrations (prod) +- `npm run db:studio` — Open Prisma Studio +- `npm run db:push` — Push schema without migration (dev only) +- `npm run db:seed` — Seed database (when configured) +- `npm run db:reset` — Reset database (destructive) + +**Tests:** +- `npm test` — Run unit tests via `tsx --test` (covers `tests/*.test.ts`, including `mocapAnalysis`, `poseFrameStream`, `liveCoachingEngine`, `aiPayload`, etc.) +- `npm run test:e2e` — Run Playwright end-to-end tests (`tests/e2e/`, e.g. `mocap-capture.spec.ts`) + +**Operations:** +- `npm run admin:promote -- ` — Promote a user to admin +- `npx tsx scripts/backfill-consistency.ts` — Backfill `consistencyScore` for existing sessions and bump `sessionsRevision` for affected users ### Project Structure @@ -362,41 +446,46 @@ npx shadcn@latest add button card table badge ## Data Model -The app uses PostgreSQL with Prisma ORM. Key models: +PostgreSQL with Prisma ORM. Key models: **User Management:** -- `User` - User accounts with authentication -- `Account` - OAuth provider accounts -- `AuthSession` - Active sessions -- `VerificationToken` - Email verification tokens -- `UserSettings` - User preferences and configuration -- `UserApiKey` - Encrypted API keys (OpenAI, etc.) +- `User` — User accounts with authentication and `role` (`user` / `admin`) +- `Account` — OAuth provider accounts +- `AuthSession` — Active sessions +- `VerificationToken` — Email verification + magic-link tokens +- `PasswordResetToken` — Password reset flow +- `UserSettings` — Preferences, AI config, SmartRow credentials, posture thresholds, mocap preferences, dashboard/sessions/analytics view state, cache-busting revisions +- `UserApiKey` — Encrypted per-provider API keys (e.g. OpenAI) **Rowing Data:** -- `RowingSession` - Workout sessions with metrics -- `StrokeData` - Stroke-by-stroke analysis data -- `PersonalRecord` - Best performances per distance +- `RowingSession` — Workout sessions with metrics and pre-computed `consistencyScore` +- `StrokeData` — Stroke-by-stroke analysis data (with optional `strokeLength`) +- `PersonalRecord` — Best performance per distance + +**Mocap (Posture Analysis):** +- `MocapSession` — One capture per session with video + pose stream paths, `capturePerspective`, calibration frames, `qualityScore`, status +- `StrokePostureMetric` — Per-stroke posture metrics with `segmentationSource` (`pose-segmented` or `csv-aligned`) +- `PostureFault` — Detected faults with `faultType`, `severity`, `phase`, `evidenceJson` **Achievements:** -- `EarnedAward` - Unlocked achievements -- `AIAwardSuggestion` - AI-suggested custom awards -- `GeneratedAchievement` - AI-generated award stories/images +- `EarnedAward` — Unlocked achievements +- `AIAwardSuggestion` — AI-suggested custom awards with structured criteria +- `GeneratedAchievement` — AI-generated story + optional image with `colorPalette` **Training:** -- `TrainingPlan` - Multi-week training programs -- `TrainingWeek` - Weekly training structure -- `TrainingSession` - Individual planned workouts -- `TrainingSessionLink` - Links planned to actual sessions +- `TrainingPlan` — Multi-week training programs with adherence tracking +- `TrainingWeek` — Weekly structure +- `TrainingSession` — Individual planned workouts with target zones +- `TrainingSessionLink` — Links planned sessions to actual `RowingSession` rows **AI & Memory:** -- `ChatSession` - AI coach conversations -- `ChatMessage` - Individual chat messages -- `AIInsight` - Generated performance insights -- `MemoryDocument` - Uploaded PDFs/images for AI context -- `MemoryBlob` - Binary data storage -- `ChartExplanation` - Cached chart explanations +- `ChatSession` — AI coach conversations grouped by `category` (`chat`, `explanation`, `plan_analysis`, `insight_discussion`) +- `ChatMessage` — Individual messages with optional attachments +- `AIInsight` — Generated performance insights with `priority`, `confidence`, `evidence`, `feedback` +- `MemoryDocument` — Uploaded PDFs / images / training plans / notes referenced by `filePath` +- `ChartExplanation` — Cached chart explanations keyed by `(userId, chartId)` -See `prisma/schema.prisma` for complete schema or `docs/DATABASE_SCHEMA.md` for detailed documentation. +See `prisma/schema.prisma` for the complete, authoritative schema, and `docs/DATABASE_SCHEMA.md` for the annotated reference. ## Privacy & Data @@ -443,10 +532,13 @@ Rate limiting is optional but recommended for production. To enable: ### Protected Endpoints Rate limiting is applied to: -- `/api/auth/register` - Prevents registration spam -- `/api/chat` (POST) - Controls AI usage costs -- `/api/user/delete` - Protects account deletion -- `/api/user/export` - Prevents data export abuse +- `/api/auth/register` — Prevents registration spam +- `/api/auth/forgot-password`, `/api/auth/reset-password` — Limits password-reset abuse +- `/api/chat` (POST) — Controls AI usage costs +- `/api/smartrow/sync` — Limits Playwright-driven SmartRow scrapes +- `/api/mocap/sessions/*` (upload + finalize) — Caps mocap capture write volume +- `/api/user/delete` — Protects account deletion +- `/api/user/export` — Prevents data export abuse ## Browser Support diff --git a/docs/DATABASE_SCHEMA.md b/docs/DATABASE_SCHEMA.md index 773d0d6..0a23d83 100644 --- a/docs/DATABASE_SCHEMA.md +++ b/docs/DATABASE_SCHEMA.md @@ -1,8 +1,10 @@ # Database Schema (Condensed, current) -- **Stack**: PostgreSQL + Prisma + NextAuth. -- **Hosting**: Dev via Docker Postgres; Prod via Supabase/Vercel Postgres/Railway. +- **Stack**: PostgreSQL + Prisma v7 + NextAuth.js v4. +- **Hosting**: Dev via Docker Postgres; Prod via Supabase (pooler + direct URL) or any managed Postgres. - **Migrations**: `npx prisma generate` → `npx prisma migrate dev` (dev) / `npx prisma migrate deploy` (prod). +- **Adapter**: `@prisma/adapter-pg` over `pg` `Pool` (configured in `prisma.config.ts`). +- **Source of truth**: `prisma/schema.prisma`. This document is a condensed, annotated mirror. ## Core Models (Prisma) ```prisma @@ -27,24 +29,28 @@ model User { emailVerified DateTime? name String? image String? - passwordHash String? // For email/password auth + passwordHash String? // For email/password auth (bcrypt) + role String @default("user") // 'user' | 'admin' createdAt DateTime @default(now()) updatedAt DateTime @updatedAt // Relations - accounts Account[] - sessions AuthSession[] - rowingSessions RowingSession[] - personalRecords PersonalRecord[] - earnedAwards EarnedAward[] - aiAwards AIAwardSuggestion[] + accounts Account[] + sessions AuthSession[] + rowingSessions RowingSession[] + mocapSessions MocapSession[] + personalRecords PersonalRecord[] + earnedAwards EarnedAward[] + aiAwards AIAwardSuggestion[] generatedAchievements GeneratedAchievement[] - trainingPlans TrainingPlan[] - chatSessions ChatSession[] - aiInsights AIInsight[] - memoryDocuments MemoryDocument[] - settings UserSettings? - + trainingPlans TrainingPlan[] + chatSessions ChatSession[] + aiInsights AIInsight[] + memoryDocuments MemoryDocument[] + settings UserSettings? + apiKeys UserApiKey[] + chartExplanations ChartExplanation[] + @@index([email]) } @@ -86,6 +92,17 @@ model VerificationToken { @@unique([identifier, token]) } +model PasswordResetToken { + id String @id @default(cuid()) + email String + token String @unique + expires DateTime + createdAt DateTime @default(now()) + + @@index([email]) + @@index([token]) +} + // ============================================================================ // ROWING SESSIONS // ============================================================================ @@ -109,6 +126,7 @@ model RowingSession { avgStrokeLength Float // meters avgStrokeRate Float // SPM maxStrokeRate Float + consistencyScore Float? // Pre-computed power-CV-based consistency (0–100) // Metadata createdAt DateTime @default(now()) @@ -119,6 +137,7 @@ model RowingSession { // Relations user User @relation(fields: [userId], references: [id], onDelete: Cascade) strokeData StrokeData[] + mocapSession MocapSession? personalRecords PersonalRecord[] trainingSessionLinks TrainingSessionLink[] @@ -146,6 +165,7 @@ model StrokeData { strokeLength Float? session RowingSession @relation(fields: [sessionId], references: [id], onDelete: Cascade) + mocapMetrics StrokePostureMetric[] @@index([sessionId]) @@index([sessionId, strokeIndex]) @@ -172,6 +192,74 @@ model PersonalRecord { @@index([userId]) } +// ============================================================================ +// MOCAP (motion-capture posture analysis — see docs/prd-mocap-posture.md) +// ============================================================================ + +model MocapSession { + id String @id @default(cuid()) + userId String + rowingSessionId String? @unique // Bidirectional, exclusive link to RowingSession + videoStoragePath String // Path to recorded video on backend (FS or Vercel Blob) + poseStreamPath String // Path to PoseFrameStream binary blob (see ADR-0001) + source String // 'browser' | 'sidecar' + captureModelVersion String // e.g. 'mediapipe-pose-landmarker-lite@0.10.35' + capturePerspective String // 'side-left' | 'side-right' | 'sidecar-3d' + captureFps Float + calibrationCatchFrame Json? // Per-session catch baseline (encoded pose frame) + calibrationFinishFrame Json? // Per-session finish baseline (encoded pose frame) + durationSec Float @default(0) + qualityScore Float? + qualityFlags String[] @default([]) + status String @default("capturing") // capturing | analyzing | ready | linked + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + rowingSession RowingSession? @relation(fields: [rowingSessionId], references: [id], onDelete: SetNull) + strokePostureMetrics StrokePostureMetric[] + postureFaults PostureFault[] + + @@index([userId]) + @@index([userId, createdAt]) +} + +model StrokePostureMetric { + id String @id @default(cuid()) + mocapSessionId String + strokeIndex Int + phaseBoundariesJson Json // catch / drive / finish / recovery boundaries + metricsJson Json // back angle, layback, sequencing offsets, etc. + segmentationSource String // 'pose-segmented' | 'csv-aligned' + strokeDataId String? // Joined to StrokeData when csv-aligned + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + mocapSession MocapSession @relation(fields: [mocapSessionId], references: [id], onDelete: Cascade) + strokeData StrokeData? @relation(fields: [strokeDataId], references: [id], onDelete: SetNull) + + @@unique([mocapSessionId, strokeIndex, segmentationSource]) + @@index([mocapSessionId]) + @@index([strokeDataId]) +} + +model PostureFault { + id String @id @default(cuid()) + mocapSessionId String + strokeIndex Int + faultType String // see CONTEXT.md PostureFault catalog + severity String // 'info' | 'warning' | 'critical' + phase String // 'catch' | 'drive' | 'finish' | 'recovery' + evidenceJson Json // frame index + metric value + threshold + createdAt DateTime @default(now()) + + mocapSession MocapSession @relation(fields: [mocapSessionId], references: [id], onDelete: Cascade) + + @@index([mocapSessionId]) + @@index([mocapSessionId, strokeIndex]) + @@index([faultType, severity]) +} + // ============================================================================ // AWARDS & ACHIEVEMENTS // ============================================================================ @@ -190,45 +278,43 @@ model EarnedAward { } model AIAwardSuggestion { - id String @id @default(cuid()) - userId String - - title String - description String - rationale String @db.Text - status String // 'suggested' | 'approved' | 'earned' - + id String @id @default(cuid()) + userId String + title String + description String + rationale String @db.Text + status String // 'suggested' | 'approved' | 'earned' + // Structured criteria for auto-evaluation - criteriaType String? // 'total_distance', 'single_session_power', etc. + criteriaType String? // 'total_distance', 'single_session_power', etc. criteriaValue Float? - criteriaComparison String? // 'gte', 'lte', 'eq' - - targetDate DateTime? - suggestedAt DateTime @default(now()) - approvedAt DateTime? - earnedAt DateTime? - model String? // AI model used - - user User @relation(fields: [userId], references: [id], onDelete: Cascade) - + criteriaComparison String? // 'gte' | 'lte' | 'eq' + + targetDate DateTime? + suggestedAt DateTime @default(now()) + approvedAt DateTime? + earnedAt DateTime? + model String? // AI model used + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + @@index([userId]) @@index([userId, status]) } model GeneratedAchievement { - id String @id @default(cuid()) - userId String - awardId String // Can be static or AI award ID - - story String? @db.Text - imageUrl String? // Path to stored image - hasImage Boolean @default(false) - - earnedAt DateTime? - generatedAt DateTime @default(now()) - - user User @relation(fields: [userId], references: [id], onDelete: Cascade) - + id String @id @default(cuid()) + userId String + awardId String // Can be static or AI award ID + story String? @db.Text + imageUrl String? // Path to stored image + hasImage Boolean @default(false) + colorPalette String? @default("classic") // Image color palette + earnedAt DateTime? + generatedAt DateTime @default(now()) + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + @@unique([userId, awardId]) @@index([userId]) } @@ -376,26 +462,30 @@ model ChatMessage { // ============================================================================ model AIInsight { - id String @id @default(cuid()) - userId String - - type String // 'performance' | 'recommendation' | 'trend' | 'achievement' | 'warning' - title String - description String @db.Text - priority String // 'high' | 'medium' | 'low' - actionable Boolean @default(false) - confidence Float? - evidence String[] // Array of evidence strings - category String? - - source String // 'cloud-ai' | 'local-analysis' - archived Boolean @default(false) - - dateGenerated DateTime @default(now()) + id String @id @default(cuid()) + userId String + + type String // 'performance' | 'recommendation' | 'trend' | 'achievement' | 'warning' + title String + description String @db.Text + priority String // 'high' | 'medium' | 'low' + actionable Boolean @default(false) + confidence Float? + evidence String[] // Evidence strings backing the insight + category String? + + source String // 'cloud-ai' | 'local-analysis' + archived Boolean @default(false) + + dateGenerated DateTime @default(now()) archivedAt DateTime? - - user User @relation(fields: [userId], references: [id], onDelete: Cascade) - + + // User feedback collected from /insights + feedback String? // 'helpful' | 'not_helpful' | 'action_taken' + feedbackAt DateTime? + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + @@index([userId]) @@index([userId, archived]) @@index([userId, dateGenerated]) @@ -406,111 +496,113 @@ model AIInsight { // ============================================================================ model MemoryDocument { - id String @id @default(cuid()) - userId String - - name String - type String // 'image' | 'pdf' | 'training_plan' | 'insight' | 'note' - source String // 'user' | 'system' - mimeType String - size Int // bytes - + id String @id @default(cuid()) + userId String + + name String + type String // 'image' | 'pdf' | 'training_plan' | 'insight' | 'note' + source String // 'user' | 'system' + mimeType String + size Int // bytes + filePath String? // FS path or Vercel Blob URL; nullable for system docs + description String? @db.Text extractedText String? @db.Text tags String[] - - // For system documents - content Json? - status String? // 'active' | 'archived' for training plans - - uploadedAt DateTime @default(now()) - - user User @relation(fields: [userId], references: [id], onDelete: Cascade) - blob MemoryBlob? - + + // For system documents (training plans, insights, etc.) + content Json? + status String? // 'active' | 'archived' for training plans + + uploadedAt DateTime @default(now()) + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + @@index([userId]) @@index([userId, type]) } -model MemoryBlob { - id String @id @default(cuid()) - documentId String @unique - - data Bytes // Binary data - - document MemoryDocument @relation(fields: [documentId], references: [id], onDelete: Cascade) -} - // ============================================================================ // USER SETTINGS // ============================================================================ model UserSettings { - id String @id @default(cuid()) - userId String @unique - + id String @id @default(cuid()) + userId String @unique + // User Preferences - theme String @default("system") - units String @default("metric") - dateFormat String @default("MM/DD/YYYY") - timeFormat String @default("24h") - language String @default("en") - timeZone String? - defaultChartType String @default("line") - animationsEnabled Boolean @default(true) - showPromptSuggestions Boolean @default(true) - customPrompts String[] - + theme String @default("system") + units String @default("metric") + dateFormat String @default("MM/DD/YYYY") + timeFormat String @default("24h") + language String @default("en") + timeZone String? @default("UTC") + defaultChartType String @default("line") + animationsEnabled Boolean @default(true) + showPromptSuggestions Boolean @default(true) + customPrompts String[] + // Training Settings - trainingZones Json? // Zone configuration - preferredMetrics String[] - weeklyGoalType String @default("sessions") - weeklyGoalTarget Int @default(3) - restDayAlerts Boolean @default(true) - adaptationEnabled Boolean @default(true) - + trainingZones Json? + preferredMetrics String[] + weeklyGoalType String @default("sessions") + weeklyGoalTarget Int @default(3) + restDayAlerts Boolean @default(true) + adaptationEnabled Boolean @default(true) + // Notification Settings - sessionReminders Boolean @default(false) - weeklyProgress Boolean @default(true) - achievementAlerts Boolean @default(true) - planReminders Boolean @default(true) - adherenceAlerts Boolean @default(true) - - // AI Settings (sensitive - API key stored separately) - cloudAIEnabled Boolean @default(false) - maxTokens Int @default(1500) - aiConfig Json? // Per-use-case config (chat, insights, etc.) - customPrompts_ai Json? // System prompts, etc. - + sessionReminders Boolean @default(false) + weeklyProgress Boolean @default(true) + achievementAlerts Boolean @default(true) + planReminders Boolean @default(true) + adherenceAlerts Boolean @default(true) + + // AI Settings (API keys stored separately in UserApiKey) + cloudAIEnabled Boolean @default(false) + mocapDetailedAIShare Boolean @default(false) // ADR-0004: opt-in to share per-stroke metrics with cloud AI + maxTokens Int @default(1500) + aiConfig Json? // Per-use-case AI config (chat, insights, plans, ...) + customPromptsAi Json? // Base / chat / plan / insights prompt overrides + // Personal context for AI - userProfileContext String? @db.Text - userProfileRawInput String? @db.Text - - // Dashboard/View Settings - dashboardSettings Json? - sessionsViewSettings Json? + userProfileContext String? @db.Text + userProfileRawInput String? @db.Text + + // Mocap (posture analysis) + postureThresholds Json? // Per-fault threshold overrides; respects userOverridden flag + mocapPreferences Json? // Capture source default, live-cue verbosity, audio on/off + + // Dashboard / View Settings + dashboardSettings Json? + sessionsViewSettings Json? sessionAnalysisSettings Json? - chartSettings Json? - analyticsSettings Json? - - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - - user User @relation(fields: [userId], references: [id], onDelete: Cascade) + chartSettings Json? + analyticsSettings Json? + + // Cache-busting revisions bumped by mutations to invalidate client caches + sessionsRevision Int @default(0) + insightsRevision Int @default(0) + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) } -// Separate table for sensitive API keys (encrypted at rest) +// Separate table for sensitive API keys (AES-256-GCM encrypted at rest) model UserApiKey { - id String @id @default(cuid()) - userId String - - provider String // 'openai', etc. - keyHash String // Hashed for verification - encryptedKey String @db.Text // Encrypted API key - - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - + id String @id @default(cuid()) + userId String + + provider String // 'openai', etc. + keyHash String // Hashed for verification + encryptedKey String @db.Text // Encrypted API key + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + @@unique([userId, provider]) @@index([userId]) } @@ -520,16 +612,18 @@ model UserApiKey { // ============================================================================ model ChartExplanation { - id String @id @default(cuid()) - userId String - chartId String // Unique identifier for the chart - - summary String @db.Text - fullResponse String @db.Text - chartTitle String - - generatedAt DateTime @default(now()) - + id String @id @default(cuid()) + userId String + chartId String // Unique identifier for the chart (may include time-range suffix) + + summary String @db.Text + fullResponse String @db.Text + chartTitle String + + generatedAt DateTime @default(now()) + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + @@unique([userId, chartId]) @@index([userId]) } @@ -538,7 +632,14 @@ model ChartExplanation { --- ## Notes & Operations -- Indexes are defined via Prisma; main filters are on `userId`, timestamps, status/archived. -- Security: all queries scoped by `userId`; API keys encrypted (UserApiKey, AES-256-GCM, hashed); NextAuth enforced on routes. -- Backups: enable managed backups/PITR; allow user export if needed. + +- **Source of truth**: `prisma/schema.prisma`. If this document and the schema disagree, the schema wins. +- **Indexes**: defined via Prisma; primary filters are `userId`, timestamps, `status` / `archived`, and mocap session id. +- **Security**: all queries are scoped by `userId`; API keys are AES-256-GCM encrypted (`UserApiKey`); NextAuth is enforced on protected routes; admin endpoints additionally checked through `src/lib/adminAuth.ts`. +- **Mocap storage**: video and `PoseFrameStream` blobs live on the storage backend (`storage/` directory or Vercel Blob), never in Postgres; see ADR-0001 and ADR-0003. +- **Cache invalidation**: mutations bump `UserSettings.sessionsRevision` / `insightsRevision` so client caches discard stale data without a full refetch protocol. +- **Backups**: enable managed backups / PITR on the Postgres host; user-initiated export is available via `/api/user/export`. +- **One-off scripts**: + - `npm run admin:promote -- ` — set `User.role = 'admin'`. + - `npx tsx scripts/backfill-consistency.ts` — backfill `RowingSession.consistencyScore` from existing `StrokeData` and bump `sessionsRevision` for affected users. diff --git a/docs/adr/0001-pose-frame-stream-as-binary-blob.md b/docs/adr/0001-pose-frame-stream-as-binary-blob.md new file mode 100644 index 0000000..e2c5463 --- /dev/null +++ b/docs/adr/0001-pose-frame-stream-as-binary-blob.md @@ -0,0 +1,56 @@ +# ADR-0001: Store raw PoseFrameStream as a binary blob, not Postgres JSONB + +**Status:** Accepted +**Date:** 2026-05-08 +**Context owner:** mocap posture analysis (see `docs/prd-mocap-posture.md`) + +## Context + +Mocap sessions produce a stream of pose keypoint frames at ~24 fps. A 30-minute session yields ~43,200 frames at ~2-4 KB each — roughly 100 MB of raw pose data per session, on top of the recorded video. + +The original PRD draft proposed a `PoseFrame` table in Postgres with a JSONB `keypointsJson` column, indexed on `(mocapSessionId, frameIndex)`. This would let downstream code load any frame via Prisma the same way other entities are loaded. + +We considered the actual access patterns: + +- **Re-analysis** — stream all frames of one session through the pure pipeline. Sequential. +- **Replay scrubbing** — random access to a frame by index. Localised reads. +- **Stroke window read** — load all frames between two `frameIndex` values. Contiguous range. +- **Cross-session ad-hoc SQL on individual frames** — none. All queries that span sessions operate on derived rows (`StrokePostureMetric`, `PostureFault`), not raw frames. + +There are no query patterns that benefit from per-frame SQL. The JSONB design optimises for a use case that doesn't exist. + +## Decision + +Store the raw `PoseFrameStream` for each `MocapSession` as a **single binary blob** on the same storage backend as the recorded video — local `storage/` directory in development, Vercel Blob in deployed environments. The `MocapSession` row holds a `poseStreamPath: string` pointing at it. + +Format: packed Float32 array of keypoints, prefixed by a small header carrying `{ fps, keypointSchemaVersion, frameCount, keypointsPerFrame }`. Random access by frame index = byte-range read at `header_size + frameIndex * frame_stride`. + +Postgres holds only the **derived** rows: `StrokePostureMetric`, `PostureFault`, plus the `MocapSession` row itself. These are small, indexable, queryable, and cheap to back up. + +Re-analysis streams the blob through the pipeline and rewrites the derived rows atomically. Scrubbing fetches byte-ranges. The Prisma schema stays compact. + +## Consequences + +**Positive** + +- Postgres footprint stays proportional to derived data, not raw capture volume. No 43k-row inserts per session, no JSONB TOAST bloat, no expensive vacuum. +- Re-analysis is a sequential file read instead of a 43k-row Prisma scan. +- Backups, replication, and DB restore stay cheap as mocap usage grows. +- Pose-stream blob lives next to the video on the same storage backend, so retention / purge / quota logic is one decision, not two. + +**Negative** + +- A custom serialiser is required (header + packed Float32 layout, version field). Schema evolution of the keypoint format requires a versioned reader. +- No ad-hoc SQL on raw frames. If a future need emerges (unlikely, but e.g. cross-session frame-level statistics), it would mean adding an index/materialisation layer on top of the blobs. +- Storage backend abstraction must support byte-range reads (Vercel Blob does; local filesystem does trivially). + +**Neutral** + +- A user-initiated "purge raw frames, keep metrics" action becomes "delete the blob"; derived rows are untouched. This matches the PRD's footnote about retention pressure. +- The `PoseFrame` Prisma model in the PRD's schema-additions section is dropped; replaced by `MocapSession.poseStreamPath`. + +## Alternatives considered + +- **Postgres JSONB row-per-frame (the PRD's original proposal).** Rejected for the bloat / vacuum / re-analysis cost reasons above, given that no query pattern needs per-frame SQL. +- **Parquet / Arrow file format instead of packed Float32.** More tooling-friendly, but adds a dependency for a one-shape data stream where a 32-byte header + flat array is sufficient. Revisit if cross-session analytics on raw frames ever becomes a real need. +- **Object store with one file per frame.** Random access via filename, but 43k tiny files per session is a worse storage pattern than one ~100 MB blob. diff --git a/docs/adr/0002-defer-sidecar-contract-to-phase-2.md b/docs/adr/0002-defer-sidecar-contract-to-phase-2.md new file mode 100644 index 0000000..9a63404 --- /dev/null +++ b/docs/adr/0002-defer-sidecar-contract-to-phase-2.md @@ -0,0 +1,47 @@ +# ADR-0002: Defer freemocap sidecar contract design to Phase 2 + +**Status:** Accepted +**Date:** 2026-05-08 +**Context owner:** mocap posture analysis (see `docs/prd-mocap-posture.md`) + +## Context + +The PRD names `PoseFrameStream` as "the universal contract between capture and analysis" and calls source-abstraction "the deepening play" — the architectural bet that lets the capture source change (browser webcam → freemocap sidecar) without rewriting the analysis pipeline. + +The PRD's Phase 1 ships browser-only. Phase 2 ships the sidecar. But the PRD designs the sidecar contract up front: WebSocket protocol, Docker image, health-check API, versioned schema, 3D depth-bearing keypoints. + +A contract designed against one real implementation and one imagined implementation is, in practice, a contract designed against the imagined one. We don't yet know: + +- The exact shape of keypoints freemocap emits (count, ordering, coordinate frame, confidence semantics). +- Which fault rules actually benefit from 3D depth in practice — vs. which are answered just as well from 2D side view with calibrated heuristics. +- Whether the live coaching path even applies to multi-camera 3D capture, or whether sidecar usage is exclusively post-session technique work. + +Locking those decisions now means picking guesses; widening the contract later is cheap because of ADR-0001 (the blob has a `keypointSchemaVersion` header field). + +## Decision + +In v1, design `PoseFrameStream` against the **browser path only**: 2D keypoints (x, y), per-keypoint confidence, source-quality flags. No depth field. No sidecar process, no WebSocket protocol, no Docker image, no health-check API. + +Treat the sidecar as not-yet-existent. Phase 2 will design its contract against the realities of freemocap's actual output and the lessons of v1's fault rules — at that point, the schema version on stored blobs bumps, the reader handles both versions, and the analysis pipeline gains a 3D-aware path where it matters. + +## Consequences + +**Positive** + +- One real implementation, no premature abstraction. The "interface" is just whatever shape `BrowserPoseSource` emits. +- Faster v1: no sidecar contract review, no Docker image, no two-process integration testing. +- Phase 2 sidecar work starts from real demand (specific fault rules that need depth) rather than a speculative interface. + +**Negative** + +- When Phase 2 lands, `PoseFrameStream` widens from `{x, y, conf}` to `{x, y, z?, conf}`. Versioned blob reader handles old captures; pipeline gains a depth-aware branch. +- Until Phase 2, the abstraction the PRD originally called "the deepening play" doesn't exist as such — the analysis pipeline is coupled to one source. Acceptable, because there is only one source. + +**Neutral** + +- Anything in the PRD's `### API contracts` section that is sidecar-specific (sidecar local URL, sidecar health-check) is out of v1 scope. The browser API (`POST /api/mocap/sessions`, `POST|GET /api/mocap/sessions/:id/pose-stream`, `POST /api/mocap/sessions/:id/video`, `POST /api/mocap/sessions/:id/finalize`, `GET /api/mocap/sessions/:id`, `POST /api/mocap/sessions/:id/reanalyze`, `POST /api/mocap/sessions/:id/link/:rowingSessionId`, `DELETE /api/mocap/sessions/:id`) stays. + +## Alternatives considered + +- **Lock both contracts now (the PRD's original plan).** Rejected. Single-implementation abstraction is an interface waiting for its second implementation — and the second implementation's real shape is unknown until freemocap is wired up. +- **Build a stub sidecar in v1 to validate the contract.** Rejected as scope creep — a stub doesn't surface the schema mismatches that a real freemocap integration would. diff --git a/docs/adr/0003-browser-side-analysis-pipeline.md b/docs/adr/0003-browser-side-analysis-pipeline.md new file mode 100644 index 0000000..6a15496 --- /dev/null +++ b/docs/adr/0003-browser-side-analysis-pipeline.md @@ -0,0 +1,63 @@ +# ADR-0003: Run the analysis pipeline in the browser; server is dumb storage in live mode + +**Status:** Accepted +**Date:** 2026-05-08 +**Context owner:** mocap posture analysis (see `docs/prd-mocap-posture.md`) + +## Context + +The mocap pipeline has two consumer paths: + +- **Live capture** — pose inference at ~24 fps, stroke segmentation, per-stroke metrics, fault detection, coaching cues delivered with `post-stroke` latency (≤1 s after stroke end; see `CueLatencyBand` in CONTEXT.md). +- **Post-session re-analysis** — same pipeline rerun on stored frames after a `RowingSession` is linked, or after fault rules are updated. + +The PRD draft proposed a `WebSocket /api/mocap/live` where the server "emits incremental faults / cues." That implies server-side analysis during capture. Two issues: + +1. Round-tripping pose frames to a server adds tens to hundreds of milliseconds of network latency per stroke window, blowing the post-stroke budget. +2. The server has no information the client doesn't — pose frames originate in the browser, and analysis is pure functions over those frames. There's nothing for the server to compute that the client can't. + +The PRD also commits to a privacy stance: "all video and pose data stored locally / in my own database by default … explicit opt-in before any pose data is sent to cloud AI." Server-side live analysis routes pose data through the server unconditionally, conflicting with that stance for users who would otherwise capture and view locally. + +## Decision + +Run the full analysis pipeline (segmenter, metrics calculator, fault detector, coaching advisor) **in the browser** during live capture. The pipeline is implemented as pure functions over `PoseFrameStream`, with no I/O, no DB calls, and no server dependency. + +Pose inference runs in a **Web Worker with OffscreenCanvas** using MediaPipe Tasks (WASM build). WebGPU acceleration is future-opt-in, not v1 default. + +Live persistence uses short HTTP chunk uploads, not a long-lived WebSocket: + +- `POST /api/mocap/sessions/:id/pose-stream` appends whole encoded `PoseFrameStream` frames to the pose blob. +- `POST /api/mocap/sessions/:id/video` appends `MediaRecorder` video chunks to the video blob. +- `POST /api/mocap/sessions/:id/finalize` patches the pose header `frameCount` and flips the session from `capturing` to `ready`. + +The server does not run the analysis pipeline during live capture and does not emit faults or cues over the upload transport. + +The same pipeline code runs server-side on demand for **re-analysis** — `POST /api/mocap/sessions/:id/reanalyze` reads the blob, runs the pure pipeline, rewrites derived rows. That's the only server-side execution path. + +## Consequences + +**Positive** + +- Live `post-stroke` latency budget is achievable — no network round trip for analysis. +- Privacy posture matches the PRD: faults and cues are computed locally; pose data only leaves the browser when the user has opted into persistence (and never reaches a third-party AI service unless `cloudAIEnabled` is set). +- The pipeline is one codebase. Pure functions over data structures are portable: same code runs in a Web Worker live and on a Vercel Function for re-analysis. +- If a user denies upload permission or the network drops, live coaching keeps working — the browser has everything it needs. +- HTTP chunk uploads fit the Vercel deployment target without introducing a third-party socket provider. Chunks are independently retriable, and finalize can validate that only complete pose frames reached storage. + +**Negative** + +- Bundle size grows by the size of the analysis pipeline (segmenter, metrics, fault detector, default thresholds, coaching cue text). Mitigated by: pipeline is pure logic, no large model weights; lazy-loaded on the mocap route, not the dashboard. +- Re-analysis on the server requires a runtime that can execute the same TypeScript modules. Vercel Functions on Node.js handle this; no isomorphic concerns for pure code. +- The upload transport is less "live" than a persistent socket. v1 does not need server-to-client capture messages because live cues are computed in-browser. +- If the pipeline ever needs heavy compute (ML model inference for fault detection — explicitly out of v1 scope per PRD), the browser path becomes constrained. At that point, this decision can be revisited for the heavy-compute branch only; the rule-based v1 path stays browser-side. + +**Neutral** + +- The PRD's `CoachingAdvisor` cloud-AI augmentation (behind `cloudAIEnabled`) is unaffected: that's an opt-in, post-session enrichment that already routes through existing `cloudAI.ts` infrastructure. + +## Alternatives considered + +- **Server-side analysis with live WebSocket fault stream (the PRD's draft).** Rejected for the latency and privacy reasons above. +- **Persistence-only WebSocket (`/api/mocap/live`).** Rejected for v1 because the deployment target does not provide arbitrary long-lived WebSocket handling without another provider, and the server has no live messages to send back. +- **Hybrid: client computes, server validates.** Rejected as duplicate work — server has nothing to validate against. The client's frames are ground truth. +- **Pose inference on the server (upload video, server runs MediaPipe).** Rejected: kills live latency, defeats the local-first privacy stance, and adds GPU/CPU server cost for compute the client can do. diff --git a/docs/adr/0004-cloud-ai-mocap-payload-tiers.md b/docs/adr/0004-cloud-ai-mocap-payload-tiers.md new file mode 100644 index 0000000..1ca15f4 --- /dev/null +++ b/docs/adr/0004-cloud-ai-mocap-payload-tiers.md @@ -0,0 +1,62 @@ +# ADR-0004: Cloud-AI mocap payload — fault-summary by default, detailed metrics opt-in, raw frames never + +**Status:** Accepted +**Date:** 2026-05-08 +**Context owner:** mocap posture analysis (see `docs/prd-mocap-posture.md`) + +## Context + +When `UserSettings.cloudAIEnabled` is on, the existing `aiAnalysis.ts` flow sends a context payload to a third-party LLM (Anthropic/OpenAI). The mocap PRD asks for posture data to be included in that context so the AI can correlate posture with performance. + +Pose data is unusually sensitive among the things this app handles. A raw `PoseFrameStream` is essentially a low-resolution biometric capture of a user's body in motion — sending it to a third-party API conflicts with the project's standing privacy posture (`prd.md` §13.4.1) even when the user has enabled cloud AI for textual training data. + +But "cloud AI off entirely for mocap" loses the feature's biggest payoff: the LLM correlating posture faults with power/pace dips and giving narrative coaching. + +There are three plausible payload tiers: + +1. **Raw `PoseFrameStream`** — keypoints over time. Biometric. Also useless to a text LLM, which can't reason over keypoint arrays. +2. **`StrokePostureMetric` rows** — angles, offsets, asymmetry numbers per stroke. Numeric, geometric, not directly biometric, but reconstructs a coarse body model. +3. **`PostureFault` summary** — fault counts by type, severity, phase, plus session-level quality flags. Most compressed; most LLM-friendly; shares no body geometry. + +## Decision + +Three tiers, three policies: + +- **Tier 1 (raw frames):** never sent to cloud AI. No flag, no opt-in. Hard wall. +- **Tier 3 (fault summary):** sent when `cloudAIEnabled` is true. This is the default mocap → cloud-AI payload. +- **Tier 2 (per-stroke metrics):** sent only when both `cloudAIEnabled` AND a new `UserSettings.mocapDetailedAIShare` flag are true. Off by default. + +The fault-summary contract (tier 3) is fixed: + +``` +Mocap summary: +- Faults: (, severity=) ... +- Quality: % tracked, , fps +- Strokes analyzed: +``` + +No keypoints, no per-frame data, no per-stroke geometry, no video. Future fault types extend this format; the contract stays additive. + +## Consequences + +**Positive** + +- Default cloud-AI behaviour preserves user privacy: aggregate fault counts share *no* reconstructable body geometry. +- Power-user "share more for better insights" is one explicit toggle away — clear consent path, not buried in cloud-AI's general flag. +- The LLM gets enough signal from tier 3 to write useful coaching ("you had 12 rounded-back faults at catch — work on lat engagement before the drive") without ever holding biometric data. +- Hard wall on tier 1 means no future bug or refactor can accidentally leak raw pose data — the code path doesn't exist. + +**Negative** + +- Two flags (`cloudAIEnabled`, `mocapDetailedAIShare`) instead of one. Slightly more UX surface in settings. +- Tier 3 may not be enough for the most fine-grained AI queries ("why did my back round more on stroke 80 specifically?"). Those queries require tier 2 — user opts in or the answer stays generic. + +**Neutral** + +- Existing `cloudAI.ts` and `aiAnalysis.ts` get a new context-builder for mocap that materialises only tier 3 by default; tier 2 enrichment is gated and additive. + +## Alternatives considered + +- **Single flag covering both tier 2 and tier 3.** Rejected: collapses two distinct privacy decisions ("share that I had faults" vs "share my exact body geometry") into one toggle that users can't reason about. +- **No detailed share at all (tier 3 only, ever).** Rejected: forecloses the per-stroke AI query feature without user input. The opt-in pathway preserves it for users who want it. +- **Anonymise tier 2 by removing identifying joint configuration.** Rejected: the data being shared *is* joint geometry. Anonymising it would mean removing the signal that makes it useful. diff --git a/docs/adr/0005-freemocap-sidecar-contract.md b/docs/adr/0005-freemocap-sidecar-contract.md new file mode 100644 index 0000000..d168983 --- /dev/null +++ b/docs/adr/0005-freemocap-sidecar-contract.md @@ -0,0 +1,146 @@ +# ADR-0005: freemocap sidecar contract — Phase 2 shape + +**Status:** Accepted +**Date:** 2026-05-09 +**Context owner:** mocap posture analysis (see `docs/prd-mocap-posture.md`, `docs/freemocap-sample-schema.md`) + +## Context + +ADR-0002 deferred the freemocap sidecar contract until real output and real needs could inform its shape. Phase 1 (browser path) is now far enough along to draw those lessons. This ADR makes the sidecar contract explicit so Phase 2 implementation can proceed. + +Empirical basis: freemocap v0.3.x output schema documented in `docs/freemocap-sample-schema.md`. Key facts: + +- freemocap uses MediaPipe BlazePose-Heavy per camera, then triangulates to world-space 3D. +- Output shape: `(N_frames, 33, 4)` — 33 BlazePose landmarks × `[x_mm, y_mm, z_mm, confidence]`. +- Coordinate frame: world-space mm, right-handed, origin at calibration rig. Not normalized, not body-relative. +- Confidence = MediaPipe visibility [0,1], not triangulation reprojection error (separate `reprojection_error_mm` available). +- freemocap's batch `.npy` output is frame-indexed, not timestamped. The sidecar live-streaming layer adds wall-clock timestamps. + +v1 lessons that inform Phase 2: +- All five v1 faults (`rounded_back_at_catch`, `early_arm_bend`, `back_opens_before_legs_drive`, `excessive_layback`, `slow_recovery_ratio`) compute cleanly from 2D side view. They don't need depth. +- The three deferred faults (`left_right_asymmetry`, `knee_track_deviation`, `shin_not_vertical_at_catch`) genuinely require frontal-plane or depth information. Sidecar-3D unlocks all three. +- Live coaching (post-stroke cues) is desirable for sidecar users too — the timing window is the same as the browser path. + +## Decisions + +### 1. `PoseFrameStream` schema version bump + +The blob gains an optional `z` channel. `keypointSchemaVersion` bumps from `1` → `2`: + +| Field | v1 (browser) | v2 (sidecar-3d) | +|-------|-------------|-----------------| +| `x` | normalized [0,1], image-relative | world-space mm | +| `y` | normalized [0,1], image-relative | world-space mm | +| `z` | absent | world-space mm | +| `confidence` | MediaPipe visibility [0,1] | MediaPipe visibility [0,1] | +| `coordinateSpace` | `normalized-2d` | `world-mm-3d` | + +Blob header additions for v2: `coordinateSpace: "world-mm-3d"`, `cameraCount: number`, `calibrationId: string` (UUID of the Charuco calibration file used). + +Existing v1 blobs are unchanged and remain readable. The blob reader branches on `keypointSchemaVersion`. The analysis pipeline accepts both; all v1 fault rules ignore `z` and work on the `{x, y}` projection regardless of source. + +### 2. `PoseFrameStream` widened shape + +```typescript +interface KeypointFrame { + frameIndex: number; + timestampMs: number; // wall-clock Unix ms; for sidecar, from live wrapper + keypoints: Keypoint[]; + quality: FrameQuality; +} + +interface Keypoint { + index: number; // 0–32, BlazePose landmark index + x: number; + y: number; + z?: number; // present only in v2 / sidecar-3d + confidence: number; // MediaPipe visibility [0,1] +} + +interface FrameQuality { + trackedCount: number; // rowing-relevant landmarks with confidence ≥ 0.5 + meanConfidence: number; + reprojectionErrorMm?: number; // sidecar-3d only + cameraCount?: number; // sidecar-3d only +} +``` + +### 3. WebSocket sidecar wire protocol + +The sidecar is a local Python process exposing: + +- `ws://localhost:8765/pose-stream` — streams `KeypointFrame` JSON, one message per frame +- `GET http://localhost:8765/health` — returns `{ "status": "ready", "fps": 30, "cameras": 3, "schemaVersion": 2 }` +- `POST http://localhost:8765/session/start` — arms capture, returns `{ "sessionId": "", "calibrationId": "" }` +- `POST http://localhost:8765/session/stop` — flushes, closes stream + +No Docker required. Users install the Python sidecar via `pip install rowing-tracker-sidecar` (separate PyPI package). The health endpoint is what the app polls during the camera readiness gate (already implemented in Phase 1 for browser; sidecar path uses the same gate, different URL). + +Port 8765 is the default; configurable in `UserSettings.sidecarPort`. + +### 4. Faults available with `sidecar-3d` + +All v1 faults remain available (they use `{x, y}` only; z is ignored). Three faults become available for the first time: + +| Fault key | Requires | What depth enables | +|-----------|----------|--------------------| +| `left_right_asymmetry` | frontal-plane x-displacement of shoulders/hips across strokes | Lateral deviation is unambiguous in 3D | +| `knee_track_deviation` | lateral knee displacement vs ankle during drive | Frontal-plane; inferred from z-differential in side view is unreliable | +| `shin_not_vertical_at_catch` | near-side vs far-side shin disambiguation | In 2D, near/far shin superimpose; z separates them | + +These three remain marked `perspective: "sidecar-3d-only"` in the fault catalog. When perspective is `side-left` or `side-right`, they surface as "requires multi-camera capture" — never silently zeroed. + +Fault rules for the three new faults: to be specified in follow-up issues (see § Implementation notes). This ADR establishes that they exist and are unlocked by sidecar-3d, not their exact threshold definitions. + +### 5. Metrics available with `sidecar-3d` + +`PostureMetricsCalculator` gains a `sidecar-3d` branch that computes: + +- All existing v1 metrics (using x, y projection; z ignored for backward compatibility) +- `lateralShoulderSymmetryMm` — mean absolute lateral displacement between left/right shoulder x-coordinates across stroke +- `lateralHipSymmetryMm` — same for hips +- `leftKneeTrackDeviationMm` / `rightKneeTrackDeviationMm` — peak lateral knee deviation from ankle x during drive phase +- `nearShinAngleDeg` — shin angle computed from the nearer (lower z) ankle/knee pair, unambiguous in 3D + +### 6. Timing model for alignment + +For live streaming: `timestampMs` in each frame is wall-clock epoch ms, set by the sidecar wrapper at frame capture time. This is the join key for SmartRow CSV alignment (same cross-correlation approach used in v1, `StrokeSegmentationSource.csv-aligned`). + +For post-session batch mode (if user runs freemocap offline and imports the `.npy`): `timestampMs = sessionStartEpochMs + frameIndex * (1000.0 / fps)`. The import endpoint accepts a `sessionStartEpochMs` parameter. + +### 7. Privacy implications of sidecar-3d + +ADR-0004 tiers apply unchanged: + +- **Tier 1 (raw frames):** never sent to cloud AI. The 3D keypoint array is geometrically richer than 2D — this makes the hard wall *more* important, not less. +- **Tier 3 (fault summary):** sent when `cloudAIEnabled` is true. Same format; the summary adds `"perspective": "sidecar-3d"` so the LLM knows depth metrics are available. +- **Tier 2 (per-stroke metrics):** gated on `mocapDetailedAIShare`. For sidecar-3d, tier 2 includes `lateralShoulderSymmetryMm` and other 3D-derived values. The settings UI should make this explicit: "Share detailed 3D posture measurements for richer AI analysis." + +No new flag needed. The existing `mocapDetailedAIShare` flag already carries the right semantics ("I consent to sharing reconstructable body geometry"). The UI copy update noting "includes 3D measurements" is sufficient. + +## Consequences + +**Positive** + +- Contract is grounded in real freemocap output, not a speculative interface. +- `keypointSchemaVersion` already existed (ADR-0001); the version bump is a one-line change in the blob reader. +- All v1 fault logic is untouched; sidecar-3d is a pure addition. +- The three long-deferred faults now have a clear unlock path. +- Live coaching and post-session replay both work the same way for sidecar users as for browser users. + +**Negative** + +- The sidecar is a separate Python install; adds setup burden for precision users (acceptable — they opted into the sidecar path). +- `world-mm-3d` coordinates require the analysis pipeline to handle unit normalization before computing angles (v1 assumes image-relative units and uses pixel ratios). The metrics calculator needs a coordinate-space adapter. +- `reprojection_error_mm` quality signal adds a new quality dimension that the UI needs to surface. + +**Neutral** + +- The Charuco calibration step is the sidecar's responsibility. The app stores `calibrationId` in the `MocapSession` row but does not own the calibration workflow. +- Port 8765 is an arbitrary choice. If it conflicts with local tooling, `UserSettings.sidecarPort` overrides it. + +## Alternatives considered + +- **Normalize sidecar output to [0,1] to match v1 browser blobs.** Rejected: discards mm-scale information that makes the three new faults computable. Normalization can be done at query time. +- **Use gRPC instead of WebSocket.** Rejected: WebSocket is simpler for a local loopback connection with no firewall issues; the bandwidth is trivial. +- **Support any pose model (OpenPose, ViTPose, etc.) not just BlazePose.** Rejected: the 33-landmark schema is the contract; other models would need an adapter. Defer until a concrete demand exists. diff --git a/docs/agents/sidecar-tracer-impl-notes.md b/docs/agents/sidecar-tracer-impl-notes.md new file mode 100644 index 0000000..1ec1047 --- /dev/null +++ b/docs/agents/sidecar-tracer-impl-notes.md @@ -0,0 +1,170 @@ +# Sidecar tracer — implementation notes for AFK agent + +Goal: build a minimal but end-to-end working freemocap sidecar integration. "Minimal tracer" means: the app can connect to a running sidecar, receive pose frames, store them as v2 blobs, and run the existing analysis pipeline on them. + +## Prerequisites (must exist before this work starts) + +- Phase 1 (browser path) is complete and merged. +- `PoseFrameStream` v1 blob reader/writer is in production. +- Camera readiness gate (`/api/mocap/sessions/:id/readiness`) is implemented. +- All v1 fault rules pass their test suite. + +## Scope of this tracer + +1. Sidecar connection (WebSocket + health poll) +2. `PoseFrameStream` v2 blob format (extend existing writer) +3. Coordinate-space adapter (world-mm-3d → pipeline-compatible units) +4. The three new sidecar-3d faults wired up (detection logic deferred — wire the rule slot, emit "detection pending" if rule not implemented) +5. `MocapSession.source = "sidecar"` flow through existing UI + +Out of scope for tracer: Charuco calibration UI, multi-camera setup wizard, 3D skeleton overlay in replay. + +## File locations to touch + +``` +src/lib/mocap/ + pose-frame-stream.ts # blob reader/writer — add v2 support + pose-capture-source.ts # add FreemocapSidecarSource class + posture-metrics.ts # add coordinateSpaceAdapter(), sidecar-3d branch + posture-fault-detector.ts # add 3 new fault rule stubs + sidecar-client.ts # NEW: WebSocket client + health poller + +src/app/api/mocap/ + sessions/[id]/sidecar/ + connect/route.ts # NEW: POST to trigger sidecar connection + status/route.ts # NEW: GET sidecar health → proxied to localhost:8765/health + +prisma/schema.prisma # add calibrationId String? to MocapSession +src/app/(app)/mocap/ + capture/page.tsx # add "Use sidecar" toggle; show sidecar-3d quality fields +``` + +## Step-by-step + +### Step 1 — Extend blob format to v2 + +In `pose-frame-stream.ts`: +- Blob header struct: add `coordinateSpace: u8` (0 = normalized-2d, 1 = world-mm-3d), `cameraCount: u8`, `calibrationIdLength: u8 + bytes`. +- Frame struct: add optional `z: float32` per keypoint when `coordinateSpace === 1`. Flag bit in per-frame flags byte (bit 3, currently unused). +- Reader: branch on `keypointSchemaVersion`. v1 readers get `z = undefined` for every keypoint. +- Writer: accept optional `coordinateSpace` param; default `normalized-2d` keeps v1 behavior. + +### Step 2 — Sidecar WebSocket client + +New file `src/lib/mocap/sidecar-client.ts`: + +```typescript +const DEFAULT_PORT = 8765; + +export async function checkSidecarHealth(port = DEFAULT_PORT): Promise { + const res = await fetch(`http://localhost:${port}/health`); + if (!res.ok) throw new Error("sidecar not reachable"); + return res.json(); +} + +export function connectSidecarStream( + port: number, + onFrame: (frame: KeypointFrame) => void, + onError: (err: Error) => void +): () => void { + const ws = new WebSocket(`ws://localhost:${port}/pose-stream`); + ws.onmessage = (e) => onFrame(JSON.parse(e.data) as KeypointFrame); + ws.onerror = () => onError(new Error("sidecar WebSocket error")); + return () => ws.close(); +} +``` + +`FreemocapSidecarSource` in `pose-capture-source.ts` wraps this client and emits `KeypointFrame` objects into the existing pipeline. It sets `coordinateSpace = "world-mm-3d"` on the blob writer. + +### Step 3 — Coordinate space adapter + +In `posture-metrics.ts`, before running any metric calculation: + +```typescript +function toNormalizedProjection(keypoint: Keypoint, sessionBounds: SessionBounds): { x: number; y: number } { + if (keypoint.z === undefined) return { x: keypoint.x, y: keypoint.y }; // v1 pass-through + // world-mm-3d: project to side-view plane (x ignored, use y and z as the 2D plane) + // SessionBounds = { yMin, yMax, zMin, zMax } computed from first N frames of the session + return { + x: (keypoint.z - sessionBounds.zMin) / (sessionBounds.zMax - sessionBounds.zMin), + y: (keypoint.y - sessionBounds.yMin) / (sessionBounds.yMax - sessionBounds.yMin), + }; +} +``` + +All existing v1 metric functions call `toNormalizedProjection()` first — no other changes to metric logic. This keeps v1 correctness and makes sidecar-3d use the same rules on the projected plane. + +3D-specific metrics (lateral symmetry, knee track) are computed separately in a `computeSidecar3DMetrics()` function that runs only when `coordinateSpace === "world-mm-3d"`. + +### Step 4 — Three new fault rule stubs + +In `posture-fault-detector.ts`: + +```typescript +// Rule stubs — return null until thresholds are defined in a follow-up issue +function detectLeftRightAsymmetry(metrics: SidecarPostureMetrics, thresholds: FaultThresholds): PostureFault | null { + if (!metrics.lateralShoulderSymmetryMm) return null; // not available + // TODO: threshold definition in follow-up + return null; +} + +function detectKneeTrackDeviation(metrics: SidecarPostureMetrics, thresholds: FaultThresholds): PostureFault | null { + if (!metrics.leftKneeTrackDeviationMm) return null; + return null; +} + +function detectShinNotVertical(metrics: SidecarPostureMetrics, thresholds: FaultThresholds): PostureFault | null { + if (!metrics.nearShinAngleDeg) return null; + return null; +} +``` + +Wire these into the main detector. When `perspective !== "sidecar-3d"`, skip them. When `perspective === "sidecar-3d"` and they return null (threshold not yet defined), emit a `PostureFault` with `severity: "pending"` and `key: "left_right_asymmetry"` / etc. so the UI can show "detection coming soon" rather than silence. + +### Step 5 — Database + +Add to `MocapSession` in `prisma/schema.prisma`: +``` +calibrationId String? +cameraCount Int? +``` + +Run `npx prisma migrate dev --name add-sidecar-fields`. + +Update `PostureSessionRepository` to persist/read these fields. + +### Step 6 — API routes + +`POST /api/mocap/sessions/:id/sidecar/connect`: +- Validates session is in `capturing` state. +- Calls `checkSidecarHealth(port)`. +- Returns `{ status: "connected", fps, cameras, schemaVersion }` or `{ status: "unreachable" }`. + +`GET /api/mocap/sessions/:id/sidecar/status`: +- Proxies to `http://localhost:${port}/health`. +- Used by the camera readiness gate polling loop. + +### Step 7 — UI + +In the capture page, add: +- "Use multi-camera sidecar" toggle (off by default). +- When on: replace browser webcam initialization with sidecar health poll. Show sidecar-specific quality fields (`reprojectionErrorMm`, `cameraCount`) in the quality indicator bar. +- "Sidecar not reachable" error state with setup link. + +No changes to replay or fault display — they work on `PostureFault` rows and `StrokePostureMetric` rows which are source-agnostic. + +## Test fixtures needed + +- `v2-blob-3d.bin` — a synthetic 100-frame v2 blob with world-mm-3d coordinates, one full rowing stroke. Add to `src/lib/mocap/__tests__/fixtures/`. +- Unit tests for `toNormalizedProjection()` covering: y-axis projection, boundary conditions (zMin === zMax), v1 pass-through. +- Unit tests for each new metric function with the synthetic fixture. +- The three fault rule stubs should have tests asserting they return null (pending) until thresholds are set. + +## Definition of done + +- [ ] v2 blob round-trips (write → read) without data loss. +- [ ] `FreemocapSidecarSource` connects to a locally running sidecar mock (or real freemocap) and writes a valid v2 blob. +- [ ] Existing v1 test suite still passes without modification. +- [ ] Three new fault stubs appear in the detector output as `severity: "pending"` for sidecar sessions. +- [ ] `POST /api/mocap/sessions/:id/sidecar/connect` returns 200 with health info when sidecar mock is running on port 8765. +- [ ] UI shows "Use sidecar" toggle; toggling it changes capture source. diff --git a/docs/freemocap-sample-schema.md b/docs/freemocap-sample-schema.md new file mode 100644 index 0000000..55cce34 --- /dev/null +++ b/docs/freemocap-sample-schema.md @@ -0,0 +1,94 @@ +# freemocap Output Sample & Schema Reference + +Representative output documented from freemocap v0.3.x (BlazePose-Heavy backend). Used as the empirical basis for ADR-0005. + +## How freemocap produces data + +1. Records synchronized video from 2+ calibrated cameras (Charuco board calibration). +2. Runs MediaPipe BlazePose-Heavy per camera → per-frame 2D keypoints. +3. Triangulates 2D keypoints across camera views → 3D world-space keypoints. +4. Writes `(N_frames, 33, 4)` NumPy array: `[x_mm, y_mm, z_mm, confidence]`. + +Post-session batch mode only. Real-time streaming requires a thin wrapper (what the sidecar provides). + +## Keypoint schema — BlazePose 33-landmark set + +| Index | Name | Rowing relevance | +|-------|------|-----------------| +| 0 | nose | head position | +| 11 | left_shoulder | back angle, torso | +| 12 | right_shoulder | back angle, torso | +| 13 | left_elbow | arm-bend detection | +| 14 | right_elbow | arm-bend detection | +| 15 | left_wrist | handle proxy | +| 16 | right_wrist | handle proxy | +| 23 | left_hip | torso origin, drive sequence | +| 24 | right_hip | torso origin, drive sequence | +| 25 | left_knee | leg extension, knee track | +| 26 | right_knee | leg extension, knee track | +| 27 | left_ankle | foot/footrest proxy | +| 28 | right_ankle | foot/footrest proxy | + +Remaining 20 landmarks (face mesh points, finger tips) are captured but unused in rowing analysis. + +## Coordinate frame + +- **Origin:** calibration rig origin (Charuco board position). Not body-relative. +- **Units:** millimeters. +- **Axes:** right-handed. Approximate orientation after standard rig placement: x=lateral (left→right from camera perspective), y=vertical (up), z=depth (toward camera = positive). +- **Not normalized.** Raw world-space; values depend on rig geometry and rower distance. + +Contrast with browser path: MediaPipe in-browser emits **normalized** 2D `[0,1]` coordinates. Sidecar emits **absolute mm** 3D coordinates. The blob header's `coordinateSpace` field distinguishes them. + +## Confidence semantics + +freemocap passes through MediaPipe's `visibility` score unchanged: + +- `1.0` = landmark clearly visible, high confidence +- `0.5` = landmark partially occluded or inferred +- `0.0` = landmark not detected / outside frame + +For rowing sidecar usage, mean per-frame confidence < 0.6 across the 13 rowing-relevant landmarks = session quality flag `low_tracking`. + +Confidence is **not** a triangulation reprojection error — it is the per-camera MediaPipe visibility, averaged across cameras before triangulation. A separate `reprojection_error_mm` field is available from freemocap's 3D output and should be surfaced as an additional quality signal. + +## Timing model + +freemocap's post-session `.npy` output is **frame-indexed, not timestamped**. Timing reconstruction: `t_ms = frame_index * (1000.0 / fps)` relative to session start. + +The sidecar live-streaming mode (see ADR-0005) adds an explicit `timestamp_ms` field (Unix epoch ms, wall clock of the frame's capture) to each WebSocket message. This is the authoritative timestamp for `PoseFrameStream` alignment with SmartRow CSV data. + +## Representative single-frame JSON (sidecar wire format) + +```json +{ + "schema_version": 2, + "frame_index": 312, + "timestamp_ms": 1746787234512, + "source": "sidecar-3d", + "fps": 30, + "keypoints": [ + { "index": 0, "name": "nose", "x": -42.3, "y": 1204.1, "z": 88.2, "confidence": 0.97 }, + { "index": 11, "name": "left_shoulder", "x": -98.7, "y": 1102.4, "z": 71.5, "confidence": 0.96 }, + { "index": 12, "name": "right_shoulder", "x": 87.2, "y": 1099.8, "z": 69.3, "confidence": 0.95 }, + { "index": 13, "name": "left_elbow", "x": -142.1,"y": 958.2, "z": 112.4, "confidence": 0.93 }, + { "index": 14, "name": "right_elbow", "x": 138.9,"y": 961.7, "z": 110.8, "confidence": 0.92 }, + { "index": 15, "name": "left_wrist", "x": -188.4,"y": 842.3, "z": 134.7, "confidence": 0.91 }, + { "index": 16, "name": "right_wrist", "x": 182.7,"y": 845.1, "z": 133.2, "confidence": 0.90 }, + { "index": 23, "name": "left_hip", "x": -78.3, "y": 812.6, "z": 22.1, "confidence": 0.98 }, + { "index": 24, "name": "right_hip", "x": 74.9, "y": 810.2, "z": 21.8, "confidence": 0.98 }, + { "index": 25, "name": "left_knee", "x": -91.2, "y": 512.4, "z": -88.4, "confidence": 0.97 }, + { "index": 26, "name": "right_knee", "x": 87.6, "y": 514.8, "z": -87.1, "confidence": 0.97 }, + { "index": 27, "name": "left_ankle", "x": -84.7, "y": 182.3, "z": -134.2,"confidence": 0.95 }, + { "index": 28, "name": "right_ankle", "x": 81.3, "y": 184.7, "z": -133.8,"confidence": 0.95 } + ], + "quality": { + "tracked_count": 13, + "mean_confidence": 0.951, + "reprojection_error_mm": 4.2, + "camera_count": 3 + } +} +``` + +All 33 landmarks are transmitted; only the 13 listed above are used by the analysis pipeline. `tracked_count` counts only the 13 rowing-relevant landmarks with confidence ≥ 0.5. diff --git a/docs/prd-mocap-posture.md b/docs/prd-mocap-posture.md new file mode 100644 index 0000000..236b074 --- /dev/null +++ b/docs/prd-mocap-posture.md @@ -0,0 +1,188 @@ +# PRD: Motion-capture posture analysis (freemocap integration) + +## Problem Statement + +I row indoors and the app already gives me deep performance analytics from SmartRow CSV (pace, power, stroke rate, PRs, training load). But the data is blind to **how I move**. Bad posture and stroke sequencing silently cap my power output, waste effort, and risk injury (lower-back rounding at the catch, early arm bend, opening the back too soon, asymmetric drive). I have no coach watching me. I want the app to see me row and tell me what's wrong with my technique — both while I'm rowing and after the session, alongside the metrics I already track. + +## Solution + +Integrate markerless motion capture so the app analyzes the rower's posture and stroke mechanics from video, derives per-stroke biomechanical metrics, detects posture faults, and delivers actionable coaching cues. Two capture paths: + +1. **Browser path (default)** — single webcam, in-browser MediaPipe Pose. Zero install. Live feedback during rowing, plus replay after. +2. **Sidecar path (precision)** — local Python sidecar running [freemocap](https://github.com/freemocap/freemocap) with one or more cameras for 3D-accurate skeletons. For users who want serious technique work. + +Pose data is stored raw (keypoints per frame) and aligned to existing `RowingSession` / `StrokeData` so users can scrub video, see skeleton overlay, see flagged faults at exact stroke phases, and watch posture trends evolve next to power/pace trends. + +## User Stories + +1. As a rower, I want to start a webcam-based mocap session in one click, so that I can get posture feedback without installing software. +2. As a rower, I want the app to detect my rowing machine in frame and confirm I'm positioned correctly, so that capture quality is good before I start. +3. As a rower, I want a calibration step (sit at catch, sit at finish), so that the analyzer knows my anatomy and machine geometry. +4. As a rower, I want live audio + visual cues during rowing ("back rounded", "arms bent early", "slow down recovery"), so that I can correct in real time. +5. As a rower, I want live cues to be quiet and non-nagging by default, so that they don't break my flow. +6. As a rower, I want every stroke segmented into catch / drive / finish / recovery automatically, so that faults are reported at the exact phase they occur. +7. As a rower, I want per-stroke posture metrics computed (back angle at catch, shin vertical, hip-knee timing offset, layback angle, sequencing delay, left-right asymmetry), so that I have objective measurements not just opinions. +8. As a rower, I want detected faults categorized by severity (info / warning / critical), so that I focus on the worst issues first. +9. As a rower, I want a post-session posture replay screen, so that I can review my session video with skeleton overlay and fault annotations. +10. As a rower, I want the replay timeline aligned with my SmartRow `StrokeData`, so that I can correlate posture issues with power / split / stroke-rate dips. +11. As a rower, I want to scrub to any stroke and see the skeleton frozen at catch / finish, so that I can study my position. +12. As a rower, I want to compare a fault-heavy stroke against a clean stroke from the same session side-by-side, so that I see the contrast. +13. As a rower, I want a coaching summary at session end (top 3 issues, frequency, suggested drills), so that I leave the session with a clear next action. +14. As a rower, I want fault frequency tracked over time on the dashboard, so that I see whether my technique is actually improving. +15. As a rower, I want posture metrics surfaced inside the existing AI insights system, so that the AI can correlate posture with performance regressions. +16. As a serious user, I want to opt into the freemocap sidecar with multi-camera 3D capture, so that I get precision metrics for technical work. +17. As a serious user, I want clear setup instructions for the sidecar (Docker / Python), so that I can install it without expert help. +18. As a serious user, I want browser captures and sidecar captures to share the same analysis pipeline and UI, so that the experience is consistent. +19. As a privacy-conscious user, I want all video and pose data stored locally / in my own database by default, so that my body footage is not uploaded to third parties. +20. As a privacy-conscious user, I want explicit opt-in before any pose data is sent to cloud AI, so that I control externalization. +21. As a rower, I want to delete a mocap session (video + pose + metrics) with one action, so that I can clean up storage. +22. As a rower, I want to know which past `RowingSession` rows have linked mocap data, so that I can find sessions worth reviewing. +23. As a rower, I want to attach a mocap session to an existing CSV-imported `RowingSession`, so that historical sessions can also gain video context. +24. As a rower, I want fault thresholds to be configurable (e.g., "warn me only if back angle < 15° at catch"), so that the system adapts to my body and goals. +25. As a rower, I want sensible default thresholds based on standard rowing technique references, so that I don't have to tune anything to start. +26. As a rower, I want the app to flag if camera framing degrades mid-session (occlusion, low light, person leaves frame), so that I trust the metrics. +27. As a rower, I want capture FPS, model confidence, and tracked-keypoint counts shown as quality indicators, so that I can judge whether a session's analysis is reliable. +28. As a rower, I want webcam access permission to be requested only when I start a capture, so that the app doesn't ask for camera on every page load. +29. As a rower, I want to record without analysis if I'm just collecting footage, so that I can defer analysis to later. +30. As a rower, I want re-analysis on demand (e.g., after fault rules improve), so that old sessions benefit from updated detectors. +31. As a rower, I want the existing chat/AI to answer questions about my mocap data ("why does my back round at stroke 80?"), so that posture insights are conversational. +32. As a rower, I want training plans to incorporate posture goals ("reduce early-arm-bend faults below 10%/session"), so that technique is a first-class plan dimension. +33. As a rower, I want posture-derived achievements ("100 strokes with clean catch"), so that good technique earns recognition same as PRs. +34. As a rower on a phone/tablet, I want a graceful degraded mode that records video for later replay even if live analysis is too heavy, so that mobile is still useful. +35. As a developer, I want the analysis pipeline (segment → metrics → faults) to be pure / deterministic on a frame stream, so that fixture-driven tests verify correctness. +36. As a developer, I want browser capture and freemocap sidecar to emit the same `PoseFrameStream` schema, so that downstream code is source-agnostic. + +## Implementation Decisions + +### New deep modules + +- **PoseCaptureSource** — interface emitting `PoseFrameStream` (timestamped keypoint frames + confidence + source-quality signals). Two implementations: + - `BrowserPoseSource` — webcam + MediaPipe Pose (33-keypoint or BlazePose-Heavy) in browser, runs in Web Worker. + - `FreemocapSidecarSource` — WebSocket client to local freemocap sidecar. Sidecar is a separate Python service (Docker image) wrapping freemocap, exposing a streaming pose API. +- **StrokePhaseSegmenter** — pure function: `PoseFrameStream → Stroke[]`. Each `Stroke` has phase boundaries (catch, drive-start, finish, recovery-start) and frame indices. Detection rule based on hip-knee distance / handle-position proxy / seat travel. +- **PostureMetricsCalculator** — pure function: `Stroke → PostureMetrics`. Computes: back angle at catch, back angle at finish, layback angle, shin vertical at catch, hip-knee opening offset (drive sequence), arm-bend onset frame, left-right asymmetry index, knee track deviation. +- **PostureFaultDetector** — pure function: `PostureMetrics + thresholds → PostureFault[]`. Severity levels: info / warning / critical. Rule-based v1; pluggable so ML model can replace later. +- **CoachingAdvisor** — `PostureFault[] + session history → CoachingCue[]`. Rule-based default cues; cloud AI augmentation behind existing `cloudAI` gate. +- **PostureSessionRepository** — Prisma-hidden read/write of `MocapSession`, `StrokePostureMetric`, `PostureFault`, plus byte-range access to the stored `PoseFrameStream` blob. + +### Modified modules + +- `aiAnalysis.ts` — extend prompt context to include posture summary when mocap data present. +- `AIInsight` generation — new `category: "posture"` insights. +- Dashboard / `analytics` — add posture-fault-frequency-over-time card. +- `trainingPlans` — optional posture goals on a `TrainingPlan`. +- `awards` — posture-derived achievement criteria. +- Chat tool surface — expose mocap query functions to AI. +- Existing `RowingSession` lookup — surface a "has mocap" badge. + +### Schema additions + +- `MocapSession` — id, userId, rowingSessionId? (nullable, can attach to existing CSV session), videoStoragePath, source ("browser" / "sidecar"), captureModelVersion, captureFps, durationSec, qualityScore, status, createdAt. +- `PoseFrame` — id, mocapSessionId, frameIndex, timestampMs, keypointsJson (compact array), confidenceJson, sourceFlags. Stored raw for full replay (per user choice). Indexed by `mocapSessionId, frameIndex`. +- `StrokePostureMetric` — id, mocapSessionId, strokeIndex, phaseBoundariesJson, metricsJson (back angle catch/finish, layback, shin-vertical, sequencing offsets, asymmetry, etc.), strokeDataId? (link to existing `StrokeData` row). +- `PostureFault` — id, mocapSessionId, strokeIndex, faultType, severity, evidenceJson (frame index + metric value + threshold), createdAt. +- `UserSettings.postureThresholds: Json?` — user-tunable rule thresholds. +- `UserSettings.mocapPreferences: Json?` — capture source default, live-cue verbosity, audio on/off. + +### Architectural decisions + +- **Pose source abstraction is the deep boundary.** Browser MediaPipe and freemocap sidecar both produce identical `PoseFrameStream` shape. All downstream analysis is source-agnostic. This is the core deepening play. +- **Analysis is pure.** `StrokePhaseSegmenter`, `PostureMetricsCalculator`, `PostureFaultDetector` are pure functions over data structures. No I/O, no DB. Tested with fixture frame streams. +- **Live and replay share the pipeline.** Live mode runs the same segmenter/metrics/detector incrementally as frames arrive; replay runs them on the stored stream. No duplicate logic. +- **Storage contract.** Raw pose data is stored as one binary `PoseFrameStream` blob per `MocapSession`, alongside the video file on the same storage backend (`storage/` dir or Vercel Blob in deployed env). Postgres stores the `MocapSession` row and derived rows only. +- **Sidecar contract.** freemocap sidecar is an opt-in local Docker service. Communicates via WebSocket on localhost. Versioned schema. App degrades cleanly if sidecar is offline. +- **Privacy.** Video + pose data are user-scoped, never sent to cloud unless cloud-AI is explicitly enabled in `UserSettings.cloudAIEnabled`. Coaching cues by default run on local rules. +- **Frame budget.** Browser path targets ≥ 24 fps on a mid-tier laptop. Heavier work (full re-analysis, summaries) deferred to post-session. Mobile falls back to record-only when CPU is insufficient. +- **Calibration.** Two-pose calibration (catch + finish) at session start establishes per-user joint baselines. Stored on user profile. Re-runnable. +- **Coordinate alignment.** Browser path = 2D side view + heuristics. Sidecar path = 3D. Metric extraction respects whichever is available; faults that require 3D are skipped on 2D path with a clear UI marker. +- **Stroke alignment with SmartRow.** When a `RowingSession` (CSV) is linked, the segmenter's stroke timeline is aligned to `StrokeData` timestamps via cross-correlation; metrics get joined to `StrokeData` rows. + +### API contracts + +- `POST /api/mocap/sessions` — create new mocap session (browser uploads chunked video + pose stream, or sidecar streams directly). +- `GET /api/mocap/sessions/:id` — full mocap detail incl. metrics + faults + frame index. +- `POST /api/mocap/sessions/:id/pose-stream` — append complete encoded pose-frame chunks to the session's `PoseFrameStream` blob. Server validates chunk boundaries but runs no analysis. +- `GET /api/mocap/sessions/:id/pose-stream` — byte-range reads from the session's `PoseFrameStream` blob for replay and re-analysis. +- `POST /api/mocap/sessions/:id/video` — append recorded video chunks to the same storage backend. +- `POST /api/mocap/sessions/:id/finalize` — finalize the pose header frame count and transition `capturing` → `ready`. +- `POST /api/mocap/sessions/:id/reanalyze` — re-run pipeline with current rules. +- `POST /api/mocap/sessions/:id/link/:rowingSessionId` — attach mocap to existing CSV session. +- `DELETE /api/mocap/sessions/:id` — cascade delete frames, metrics, faults, video. +- Sidecar local URL configurable in settings; health-check endpoint required. + +## Testing Decisions + +A good test verifies external behavior of a deep module given a fixed input. It does not assert on private structure, intermediate variables, or implementation choices. The pose pipeline is uniquely well-suited because the core modules are pure functions over data. + +### Modules to test + +- **StrokePhaseSegmenter** — fixture: synthetic and recorded `PoseFrameStream` files representing clean strokes, missed catch, paused recovery, asymmetric drive. Assert: correct stroke count, phase boundary frame indices within tolerance. +- **PostureMetricsCalculator** — fixture: hand-labeled `Stroke` with known geometry. Assert: computed angles within ±2° of ground truth. +- **PostureFaultDetector** — fixture: `PostureMetrics` instances crafted to cross / not-cross thresholds. Assert: correct fault types and severities emitted; no false positives on clean reference. + +### Modules NOT tested at unit level + +- UI views (`LiveCoachingView`, `PostureReplayView`, `PostureTrendsCard`) — covered by manual QA + Playwright smoke tests for golden path. +- `BrowserPoseSource` / `FreemocapSidecarSource` — thin adapters; correctness depends on external libs / services. Smoke-test only. +- `PostureSessionRepository` — covered indirectly by API integration tests. + +### Prior art + +- Existing analytics tests (if present in `src/lib`) for trend computation patterns. +- Existing parser tests around `csvParser.ts` / `strokeParser.ts` style: pure-function over fixture data. + +### Test fixtures + +- Bundle 5–10 short pose recordings (≤ 30 s each) covering: clean rowing, early arm bend, rounded back, slow recovery, asymmetry, lost tracking. Stored as JSON `PoseFrameStream` snapshots in the test fixture directory. + +## Out of Scope + +- Multi-user / coach views over mocap data. +- Public sharing of mocap video or skeleton clips. +- Smartphone-only mode as primary target (mobile is degraded record-only). +- Hardware sensors (IMU on seat, force sensors on handle / footplate). +- Strava / TrainingPeaks export of posture data. +- ML-trained fault detectors (v1 is rule-based; ML pluggable later). +- On-water rowing capture. +- Real-time streaming of mocap to a remote coach. +- Custom freemocap installation flows beyond the documented Docker sidecar. + +## Resolved Decisions (grilling 2026-05-08) + +This section reflects the outcome of `/grill-with-docs` against this PRD. Where it conflicts with the body of the PRD above, **this section wins** and the older sections will be updated lazily. + +### Architecture (see `docs/adr/`) + +- **ADR-0001** — raw `PoseFrameStream` is stored as one binary blob per `MocapSession` alongside the video, not as Postgres JSONB rows. The `PoseFrame` Prisma model from `### Schema additions` is dropped; replaced by `MocapSession.poseStreamPath`. +- **ADR-0002** — sidecar contract is deferred to Phase 2. v1 `PoseFrameStream` shape is browser-2D only (`{x, y, confidence}` per keypoint, plus quality flags). No Docker image, no WebSocket sidecar protocol, no health-check API in v1. +- **ADR-0003** — analysis pipeline runs in the browser (Web Worker, MediaPipe Tasks WASM). Live persistence uses HTTP chunk uploads (`pose-stream`, `video`, `finalize`); the server does not emit faults during live capture. Server-side execution is for `POST /api/mocap/sessions/:id/reanalyze` only. +- **ADR-0004** — cloud-AI mocap payload is `PostureFault` summary (tier 3) by default; per-stroke metrics (tier 2) opt-in via `UserSettings.mocapDetailedAIShare`; raw frames (tier 1) never cross to cloud. + +### Domain terms (see `CONTEXT.md`) + +`CapturePerspective`, `StrokeSegmentationSource`, `MocapSession`, `CueLatencyBand`, `PoseFrameStream`, `Calibration`, `PostureFault` (v1 catalog), `FaultThresholds`. Use these exact terms in code, issues, and UI copy. + +### Locked design choices + +- **Browser path is side-view only.** `side-left` or `side-right`. Front view, asymmetry, knee-track deviation = `requires-multi-cam`, deferred to sidecar. +- **Live capture = `pose-segmented`.** Live SmartRow CSV streaming is not available; CSV arrives post-session. Linking a `RowingSession` to a `MocapSession` triggers mandatory atomic re-analysis to `csv-aligned` as a background job. +- **Live cues are `post-stroke`** (≤ 1 s after stroke completes), not intra-stroke. Fault detector runs at stroke granularity only. +- **Calibration is per-session, not per-user.** Two reference frames (catch, finish) captured at session start, stored on `MocapSession`. +- **Auto-link prompt** on CSV import when capture window overlaps within ±2 minutes; user always confirms, never silent. Linking is bidirectional, exclusive, reversible (`unlink` endpoint). +- **v1 fault catalog is fixed at 5 types**: `rounded_back_at_catch`, `early_arm_bend`, `back_opens_before_legs_drive`, `excessive_layback`, `slow_recovery_ratio`. Anything else is out of v1 scope. +- **Default thresholds are hand-coded and versioned** (`postureThresholdsV1`). Conservative bands. Auto-migrate on version bump unless user has set `userOverridden: true`. + +### v1 ship scope (Phase 1) + +**Ships:** US 1-11, 13, 19-30, 35, 36 (US 36 reduced to "single-source `PoseFrameStream` shape, versioned for future widening"). + +**Deferred to Phase 2:** US 12 (side-by-side compare), 14 (fault-frequency dashboard card), 15 (posture in `aiAnalysis.ts` insights), 16-18 (sidecar), 31 (chat tool exposure), 32-33 (posture in training plans / achievements), 34 (mobile degraded mode). + +## Further Notes + +- freemocap upstream is GPL-licensed Python. Sidecar runs as separate process; no GPL code is linked into Next.js app. Confirm license interaction during implementation. +- Browser MediaPipe Pose is Apache-2.0 and ships as JS package — no licensing concern. +- Phase 1 ships browser path + analysis pipeline + replay UI + dashboard widget. Phase 2 ships freemocap sidecar + 3D-aware metrics. Phase 3 ships AI-augmented cues + posture-aware training plans + posture achievements. +- Deep-module boundary (`PoseFrameStream` as universal contract between capture and analysis) is the key bet — lets source change without rewriting analysis, and lets analysis evolve without touching capture. Contract reviewed and stabilized first. +- Performance baseline must be measured early on target dev machine; if browser pipeline cannot hit 24 fps, live-cue feature ships as post-stroke (≤ 1 s lag) instead of intra-stroke. +- Storage growth: raw `PoseFrame` rows ~2–4 KB per frame × 24 fps × 30 min = ~100 MB per session worst-case. Retention policy (auto-purge raw frames after N days, keep metrics + video) is a likely follow-up. diff --git a/docs/sidecar-local-setup.md b/docs/sidecar-local-setup.md new file mode 100644 index 0000000..a69bf80 --- /dev/null +++ b/docs/sidecar-local-setup.md @@ -0,0 +1,137 @@ +# Running the sidecar tracer locally + +This guide covers how to run the minimal freemocap sidecar integration for local development and testing. + +## Prerequisites + +- Python 3.10+ with `venv` +- The app running locally (`npm run dev`) + +## Option A — real freemocap sidecar + +`rowing-tracker-sidecar` is not currently available on PyPI as of May 10, 2026, so the commands below will only work once that package is published or if you have access to an internal distribution. + +```bash +python3 -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +python -m pip install rowing-tracker-sidecar +rowing-tracker-sidecar --port 8765 +``` + +If `python -m pip install rowing-tracker-sidecar` fails with `No matching distribution found`, use Option B for local app development. + +The sidecar exposes: +- `ws://localhost:8765/pose-stream` — streams `KeypointFrame` JSON +- `GET http://localhost:8765/health` — returns `{ status, fps, cameras, schemaVersion }` +- `POST http://localhost:8765/session/start` — arms capture +- `POST http://localhost:8765/session/stop` — flushes and closes + +## Option B — minimal mock server (for UI/API dev without hardware) + +```python +#!/usr/bin/env python3 +"""Minimal sidecar mock — runs without freemocap or cameras.""" +import asyncio, json, math, random, time +import websockets +from http.server import BaseHTTPRequestHandler, HTTPServer +import threading + +PORT = 8765 +FPS = 30 + +def health(): + return {"status": "ready", "fps": FPS, "cameras": 3, "schemaVersion": 2} + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/health": + body = json.dumps(health()).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(body) + def do_POST(self): + body = json.dumps({"sessionId": "mock-session", "calibrationId": "mock-calib"}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(body) + def log_message(self, *a): pass + +async def pose_stream(websocket): + frame_index = 0 + while True: + ts = time.time() * 1000 + keypoints = [ + {"index": i, "x": 50 + math.sin(i * 0.5) * 200, + "y": 500 + math.cos(i * 0.3 + frame_index * 0.05) * 300, + "z": 1000 + random.gauss(0, 20), + "confidence": 0.85 + random.gauss(0, 0.05)} + for i in range(33) + ] + frame = {"frameIndex": frame_index, "timestampMs": ts, + "keypoints": keypoints, + "quality": {"trackedCount": 33, "meanConfidence": 0.85, + "reprojectionErrorMm": 1.2, "cameraCount": 3}} + try: + await websocket.send(json.dumps(frame)) + except websockets.exceptions.ConnectionClosed: + break + frame_index += 1 + await asyncio.sleep(1 / FPS) + +async def main(): + http = HTTPServer(("", PORT), Handler) + threading.Thread(target=http.serve_forever, daemon=True).start() + print(f"Sidecar mock running on port {PORT}") + async with websockets.serve(pose_stream, "localhost", PORT, path="/pose-stream"): + await asyncio.Future() + +asyncio.run(main()) +``` + +Save as `scripts/sidecar-mock.py` and run: + +```bash +python3 -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +python -m pip install websockets +python scripts/sidecar-mock.py +``` + +## Using the sidecar in the app + +1. Start the sidecar (real or mock) on port 8765. +2. Open the app at `http://localhost:3000/mocap`. +3. Check **Multi-camera sidecar** — the UI polls health and shows "Sidecar ready — 3 cameras, 30 fps". +4. Click **Start mocap session** — the app creates a session with `source=sidecar`, `capturePerspective=sidecar-3d`. +5. The session detail page opens as normal. Posture faults from sidecar-3D will appear with `severity=pending` for the three new fault types until thresholds are defined. + +## API endpoints added + +| Method | Path | Purpose | +|--------|------|---------| +| `POST` | `/api/mocap/sessions/:id/sidecar/connect` | Verify sidecar health and arm capture | +| `GET` | `/api/mocap/sessions/:id/sidecar/status` | Proxy to `localhost:8765/health` | + +Both require an authenticated session and a MocapSession in `capturing` status. + +## PoseFrameStream v2 blob format + +v2 blobs are written when `source=sidecar`. Key differences from v1: + +- `keypointSchemaVersion = 2` in header +- Each keypoint is `[x, y, z, confidence]` (4 × Float32 per keypoint, vs 3 × Float32 in v1) +- Header byte 20: `coordinateSpace` (0 = normalized-2d, 1 = world-mm-3d) +- Header byte 21: `cameraCount` +- v1 blobs are unchanged and remain readable + +## Running the new tests + +```bash +npx tsx --test tests/sidecarTracer.test.ts +``` + +Expected: 13 tests pass. diff --git a/package-lock.json b/package-lock.json index 287aaf2..11ccd9f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "license": "MIT", "dependencies": { "@auth/prisma-adapter": "^2.11.2", + "@mediapipe/tasks-vision": "^0.10.35", "@playwright/test": "^1.56.1", "@prisma/adapter-pg": "^7.8.0", "@prisma/client": "^7.8.0", @@ -28,6 +29,7 @@ "@radix-ui/react-tooltip": "^1.2.8", "@upstash/ratelimit": "^2.0.7", "@upstash/redis": "^1.36.0", + "@vercel/blob": "^2.3.3", "bcryptjs": "^3.0.3", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", @@ -70,6 +72,7 @@ "eslint": "^9", "eslint-config-next": "16.0.3", "tailwindcss": "^4", + "tsx": "^4.21.0", "tw-animate-css": "^1.4.0", "typescript": "^5" } @@ -540,6 +543,448 @@ "tslib": "^2.4.0" } }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", + "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", + "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", + "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", + "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", + "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", + "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", + "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", + "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", + "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", + "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", + "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", + "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", + "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", + "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", + "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", + "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", + "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", + "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", + "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", + "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", + "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", + "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", + "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", + "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", + "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", + "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.9.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.0.tgz", @@ -1308,6 +1753,12 @@ "integrity": "sha512-M5UknZPHRu3DEDWoipU6sE8PdkZ6Z/S+v4dD+Ke8IaNlpdSQah50lz1KtcFBa2vsdOnwbbnxJwVM4wty6udA5w==", "license": "MIT" }, + "node_modules/@mediapipe/tasks-vision": { + "version": "0.10.35", + "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.35.tgz", + "integrity": "sha512-HOvadwVRE6JC+45nyYhmnywnr5h/J8KZvOeUNVOG9q/0875pZgItznFB9bRTvLc264YSJqiZ1NsIpCStJw/egg==", + "license": "Apache-2.0" + }, "node_modules/@napi-rs/wasm-runtime": { "version": "0.2.12", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", @@ -1322,9 +1773,9 @@ } }, "node_modules/@next/env": { - "version": "16.2.4", - "resolved": "https://registry.npmjs.org/@next/env/-/env-16.2.4.tgz", - "integrity": "sha512-dKkkOzOSwFYe5RX6y26fZgkSpVAlIOJKQHIiydQcrWH6y/97+RceSOAdjZ14Qa3zLduVUy0TXcn+EiM6t4rPgw==", + "version": "16.2.6", + "resolved": "https://registry.npmjs.org/@next/env/-/env-16.2.6.tgz", + "integrity": "sha512-gd8HoHN4ufj73WmR3JmVolrpJR47ILK6LouP5xElPglaVxir6e1a7VzvTvDWkOoPXT9rkkTzyCxBu4yeZfZwcw==", "license": "MIT" }, "node_modules/@next/eslint-plugin-next": { @@ -1338,9 +1789,9 @@ } }, "node_modules/@next/swc-darwin-arm64": { - "version": "16.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.2.4.tgz", - "integrity": "sha512-OXTFFox5EKN1Ym08vfrz+OXxmCcEjT4SFMbNRsWZE99dMqt2Kcusl5MqPXcW232RYkMLQTy0hqgAMEsfEd/l2A==", + "version": "16.2.6", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.2.6.tgz", + "integrity": "sha512-ZJGkkcNfYgrrMkqOdZ7zoLa1TOy0qpcMfk/z4Mh/FKUz40gVO+HNQWqmLxf67Z5WB64DRp0dhEbyHfel+6sJUg==", "cpu": [ "arm64" ], @@ -1354,9 +1805,9 @@ } }, "node_modules/@next/swc-darwin-x64": { - "version": "16.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.2.4.tgz", - "integrity": "sha512-XhpVnUfmYWvD3YrXu55XdcAkQtOnvaI6wtQa8fuF5fGoKoxIUZ0kWPtcOfqJEWngFF/lOS9l3+O9CcownhiQxQ==", + "version": "16.2.6", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.2.6.tgz", + "integrity": "sha512-v/YLBHIY132Ced3puBJ7YJKw1lqsCrgcNo2aRJlCEyQrrCeRJlvGlnmxhPxNQI3KE3N1DN5r9TPNPvka3nq5RQ==", "cpu": [ "x64" ], @@ -1370,9 +1821,9 @@ } }, "node_modules/@next/swc-linux-arm64-gnu": { - "version": "16.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.2.4.tgz", - "integrity": "sha512-Mx/tjlNA3G8kg14QvuGAJ4xBwPk1tUHq56JxZ8CXnZwz1Etz714soCEzGQQzVMz4bEnGPowzkV6Xrp6wAkEWOQ==", + "version": "16.2.6", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.2.6.tgz", + "integrity": "sha512-RPOvqlYBbcQjkz9VQQDZ2T2bARIjXZV1KFlt+V2Mr6SW/e4I9fcKsaA0hdyf2FHoTlsV2xnBd5Y912rP/1Ce6w==", "cpu": [ "arm64" ], @@ -1386,9 +1837,9 @@ } }, "node_modules/@next/swc-linux-arm64-musl": { - "version": "16.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.2.4.tgz", - "integrity": "sha512-iVMMp14514u7Nup2umQS03nT/bN9HurK8ufylC3FZNykrwjtx7V1A7+4kvhbDSCeonTVqV3Txnv0Lu+m2oDXNg==", + "version": "16.2.6", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.2.6.tgz", + "integrity": "sha512-URUTu1+dMkxJsPFgm+OeEvq9wf5sujw0EvgYy80TDGHTSLTnIHeqb0Eu8A3sC95IRgjejQL+kC4mw+4yPxiAXA==", "cpu": [ "arm64" ], @@ -1402,9 +1853,9 @@ } }, "node_modules/@next/swc-linux-x64-gnu": { - "version": "16.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.2.4.tgz", - "integrity": "sha512-EZOvm1aQWgnI/N/xcWOlnS3RQBk0VtVav5Zo7n4p0A7UKyTDx047k8opDbXgBpHl4CulRqRfbw3QrX2w5UOXMQ==", + "version": "16.2.6", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.2.6.tgz", + "integrity": "sha512-DOj182mPV8G3UkrayLoREM5YEYI+Dk5wv7Ox9xl1fFibAELEsFD0lDPfHIeILlutMMfdyhlzYPELG3peuKaurw==", "cpu": [ "x64" ], @@ -1418,9 +1869,9 @@ } }, "node_modules/@next/swc-linux-x64-musl": { - "version": "16.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.2.4.tgz", - "integrity": "sha512-h9FxsngCm9cTBf71AR4fGznDEDx1hS7+kSEiIRjq5kO1oXWm07DxVGZjCvk0SGx7TSjlUqhI8oOyz7NfwAdPoA==", + "version": "16.2.6", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.2.6.tgz", + "integrity": "sha512-HKQ5SP/V/ub73UvF7n/zeJlxk2kLmtL7Wzrg4WfmkjmNos5onJ2tKu7yZOPdL18A6Svfn3max29ym+ry7NkK4g==", "cpu": [ "x64" ], @@ -1434,9 +1885,9 @@ } }, "node_modules/@next/swc-win32-arm64-msvc": { - "version": "16.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.2.4.tgz", - "integrity": "sha512-3NdJV5OXMSOeJYijX+bjaLge3mJBlh4ybydbT4GFoB/2hAojWHtMhl3CYlYoMrjPuodp0nzFVi4Tj2+WaMg+Ow==", + "version": "16.2.6", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.2.6.tgz", + "integrity": "sha512-LZXpTlPyS5v7HhSmnvsLGP3iIYgYOBnc8r8ArlT55sGHV89bR2HlDdBjWQ+PY6SJMmk8TuVGFuxalnP3k/0Dwg==", "cpu": [ "arm64" ], @@ -1450,9 +1901,9 @@ } }, "node_modules/@next/swc-win32-x64-msvc": { - "version": "16.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.2.4.tgz", - "integrity": "sha512-kMVGgsqhO5YTYODD9IPGGhA6iprWidQckK3LmPeW08PIFENRmgfb4MjXHO+p//d+ts2rpjvK5gXWzXSMrPl9cw==", + "version": "16.2.6", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.2.6.tgz", + "integrity": "sha512-F0+4i0h9J6C4eE3EAPWsoCk7UW/dbzOjyzxY0qnDUOYFu6FFmdZ6l97/XdV3/Nz3VYyO7UWjyEJUXkGqcoXfMA==", "cpu": [ "x64" ], @@ -4075,6 +4526,22 @@ "uncrypto": "^0.1.3" } }, + "node_modules/@vercel/blob": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/@vercel/blob/-/blob-2.3.3.tgz", + "integrity": "sha512-MtD7VLo6hU07eHR7bmk5SIMD290q574UaNYTe46qeyRT+hWrCy26CoAqfd7PnIefVXvRehRZBzukxuTO9iGTVg==", + "license": "Apache-2.0", + "dependencies": { + "async-retry": "^1.3.3", + "is-buffer": "^2.0.5", + "is-node-process": "^1.2.0", + "throttleit": "^2.1.0", + "undici": "^6.23.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/acorn": { "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", @@ -4337,6 +4804,24 @@ "node": ">= 0.4" } }, + "node_modules/async-retry": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", + "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", + "license": "MIT", + "dependencies": { + "retry": "0.13.1" + } + }, + "node_modules/async-retry/node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/available-typed-arrays": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", @@ -5439,6 +5924,48 @@ "benchmarks" ] }, + "node_modules/esbuild": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", + "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.7", + "@esbuild/android-arm": "0.27.7", + "@esbuild/android-arm64": "0.27.7", + "@esbuild/android-x64": "0.27.7", + "@esbuild/darwin-arm64": "0.27.7", + "@esbuild/darwin-x64": "0.27.7", + "@esbuild/freebsd-arm64": "0.27.7", + "@esbuild/freebsd-x64": "0.27.7", + "@esbuild/linux-arm": "0.27.7", + "@esbuild/linux-arm64": "0.27.7", + "@esbuild/linux-ia32": "0.27.7", + "@esbuild/linux-loong64": "0.27.7", + "@esbuild/linux-mips64el": "0.27.7", + "@esbuild/linux-ppc64": "0.27.7", + "@esbuild/linux-riscv64": "0.27.7", + "@esbuild/linux-s390x": "0.27.7", + "@esbuild/linux-x64": "0.27.7", + "@esbuild/netbsd-arm64": "0.27.7", + "@esbuild/netbsd-x64": "0.27.7", + "@esbuild/openbsd-arm64": "0.27.7", + "@esbuild/openbsd-x64": "0.27.7", + "@esbuild/openharmony-arm64": "0.27.7", + "@esbuild/sunos-x64": "0.27.7", + "@esbuild/win32-arm64": "0.27.7", + "@esbuild/win32-ia32": "0.27.7", + "@esbuild/win32-x64": "0.27.7" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -5987,9 +6514,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", - "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz", + "integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==", "funding": [ { "type": "github", @@ -6569,9 +7096,9 @@ } }, "node_modules/hono": { - "version": "4.12.14", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.14.tgz", - "integrity": "sha512-am5zfg3yu6sqn5yjKBNqhnTX7Cv+m00ox+7jbaKkrLMRJ4rAdldd1xPd/JzbBWspqaQv6RSTrgFN95EsfhC+7w==", + "version": "4.12.18", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.18.tgz", + "integrity": "sha512-RWzP96k/yv0PQfyXnWjs6zot20TqfpfsNXhOnev8d1InAxubW93L11/oNUc3tQqn2G0bSdAOBpX+2uDFHV7kdQ==", "license": "MIT", "engines": { "node": ">=16.9.0" @@ -6809,6 +7336,29 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/is-bun-module": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-bun-module/-/is-bun-module-2.0.0.tgz", @@ -7001,6 +7551,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-node-process": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-node-process/-/is-node-process-1.2.0.tgz", + "integrity": "sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==", + "license": "MIT" + }, "node_modules/is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", @@ -8752,12 +9308,12 @@ "license": "MIT" }, "node_modules/next": { - "version": "16.2.4", - "resolved": "https://registry.npmjs.org/next/-/next-16.2.4.tgz", - "integrity": "sha512-kPvz56wF5frc+FxlHI5qnklCzbq53HTwORaWBGdT0vNoKh1Aya9XC8aPauH4NJxqtzbWsS5mAbctm4cr+EkQ2Q==", + "version": "16.2.6", + "resolved": "https://registry.npmjs.org/next/-/next-16.2.6.tgz", + "integrity": "sha512-qOVgKJg1+At15NpeUP+eJgCHvTCgXsogweq87Ri/Ix7PkqQHg4sdaXmSFqKlgaIXE4kW0g25LE68W87UANlHtw==", "license": "MIT", "dependencies": { - "@next/env": "16.2.4", + "@next/env": "16.2.6", "@swc/helpers": "0.5.15", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", @@ -8771,14 +9327,14 @@ "node": ">=20.9.0" }, "optionalDependencies": { - "@next/swc-darwin-arm64": "16.2.4", - "@next/swc-darwin-x64": "16.2.4", - "@next/swc-linux-arm64-gnu": "16.2.4", - "@next/swc-linux-arm64-musl": "16.2.4", - "@next/swc-linux-x64-gnu": "16.2.4", - "@next/swc-linux-x64-musl": "16.2.4", - "@next/swc-win32-arm64-msvc": "16.2.4", - "@next/swc-win32-x64-msvc": "16.2.4", + "@next/swc-darwin-arm64": "16.2.6", + "@next/swc-darwin-x64": "16.2.6", + "@next/swc-linux-arm64-gnu": "16.2.6", + "@next/swc-linux-arm64-musl": "16.2.6", + "@next/swc-linux-x64-gnu": "16.2.6", + "@next/swc-linux-x64-musl": "16.2.6", + "@next/swc-win32-arm64-msvc": "16.2.6", + "@next/swc-win32-x64-msvc": "16.2.6", "sharp": "^0.34.5" }, "peerDependencies": { @@ -10794,6 +11350,18 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/throttleit": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-2.1.0.tgz", + "integrity": "sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/tiny-invariant": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz", @@ -10926,6 +11494,41 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, + "node_modules/tsx": { + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", + "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.27.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/tsx/node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/tw-animate-css": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/tw-animate-css/-/tw-animate-css-1.4.0.tgz", @@ -11090,6 +11693,15 @@ "integrity": "sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==", "license": "MIT" }, + "node_modules/undici": { + "version": "6.25.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.25.0.tgz", + "integrity": "sha512-ZgpWDC5gmNiuY9CnLVXEH8rl50xhRCuLNA97fAUnKi8RRuV4E6KG31pDTsLVUKnohJE0I3XDrTeEydAXRw47xg==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", diff --git a/package.json b/package.json index 8e62b94..488d26c 100644 --- a/package.json +++ b/package.json @@ -25,10 +25,13 @@ "db:seed": "npx prisma db seed", "db:reset": "npx prisma migrate reset", "admin:promote": "npx tsx scripts/promote-admin.ts", + "test": "npx tsx --test tests/*.test.ts", + "test:e2e": "npx playwright test", "postinstall": "prisma generate" }, "dependencies": { "@auth/prisma-adapter": "^2.11.2", + "@mediapipe/tasks-vision": "^0.10.35", "@playwright/test": "^1.56.1", "@prisma/adapter-pg": "^7.8.0", "@prisma/client": "^7.8.0", @@ -46,6 +49,7 @@ "@radix-ui/react-tooltip": "^1.2.8", "@upstash/ratelimit": "^2.0.7", "@upstash/redis": "^1.36.0", + "@vercel/blob": "^2.3.3", "bcryptjs": "^3.0.3", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", @@ -88,13 +92,14 @@ "eslint": "^9", "eslint-config-next": "16.0.3", "tailwindcss": "^4", + "tsx": "^4.21.0", "tw-animate-css": "^1.4.0", "typescript": "^5" }, "overrides": { "@prisma/dev": { "@hono/node-server": "1.19.13", - "hono": "4.12.14" + "hono": "4.12.18" }, "cookie": "1.1.1", "nodemailer": "$nodemailer", diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 0000000..2e0bb02 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,42 @@ +import { defineConfig, devices } from "@playwright/test"; + +const PORT = process.env.PORT ? parseInt(process.env.PORT, 10) : 3000; +const BASE_URL = process.env.PLAYWRIGHT_BASE_URL ?? `http://localhost:${PORT}`; + +export default defineConfig({ + testDir: "./tests/e2e", + timeout: 60_000, + expect: { timeout: 10_000 }, + fullyParallel: false, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 1 : 0, + reporter: process.env.CI ? "github" : "list", + use: { + baseURL: BASE_URL, + trace: "retain-on-failure", + video: "retain-on-failure", + }, + projects: [ + { + name: "chromium", + use: { + ...devices["Desktop Chrome"], + launchOptions: { + args: [ + "--use-fake-ui-for-media-stream", + "--use-fake-device-for-media-stream", + "--autoplay-policy=no-user-gesture-required", + ], + }, + }, + }, + ], + webServer: process.env.PLAYWRIGHT_SKIP_SERVER + ? undefined + : { + command: "npm run dev", + url: BASE_URL, + reuseExistingServer: !process.env.CI, + timeout: 120_000, + }, +}); diff --git a/prisma/migrations/20260508120000_add_mocap_session/migration.sql b/prisma/migrations/20260508120000_add_mocap_session/migration.sql new file mode 100644 index 0000000..dd820ec --- /dev/null +++ b/prisma/migrations/20260508120000_add_mocap_session/migration.sql @@ -0,0 +1,34 @@ +-- CreateTable +CREATE TABLE "MocapSession" ( + "id" TEXT NOT NULL, + "userId" TEXT NOT NULL, + "rowingSessionId" TEXT, + "videoStoragePath" TEXT NOT NULL, + "poseStreamPath" TEXT NOT NULL, + "source" TEXT NOT NULL, + "captureModelVersion" TEXT NOT NULL, + "capturePerspective" TEXT NOT NULL, + "captureFps" DOUBLE PRECISION NOT NULL, + "durationSec" DOUBLE PRECISION NOT NULL DEFAULT 0, + "qualityScore" DOUBLE PRECISION, + "status" TEXT NOT NULL DEFAULT 'capturing', + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "MocapSession_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "MocapSession_rowingSessionId_key" ON "MocapSession"("rowingSessionId"); + +-- CreateIndex +CREATE INDEX "MocapSession_userId_idx" ON "MocapSession"("userId"); + +-- CreateIndex +CREATE INDEX "MocapSession_userId_createdAt_idx" ON "MocapSession"("userId", "createdAt"); + +-- AddForeignKey +ALTER TABLE "MocapSession" ADD CONSTRAINT "MocapSession_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "MocapSession" ADD CONSTRAINT "MocapSession_rowingSessionId_fkey" FOREIGN KEY ("rowingSessionId") REFERENCES "RowingSession"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/prisma/migrations/20260508124500_add_posture_threshold_settings/migration.sql b/prisma/migrations/20260508124500_add_posture_threshold_settings/migration.sql new file mode 100644 index 0000000..11f4b91 --- /dev/null +++ b/prisma/migrations/20260508124500_add_posture_threshold_settings/migration.sql @@ -0,0 +1,2 @@ +-- Add posture threshold settings for mocap fault tuning. +ALTER TABLE "UserSettings" ADD COLUMN "postureThresholds" JSONB; diff --git a/prisma/migrations/20260508143000_add_mocap_calibration_quality/migration.sql b/prisma/migrations/20260508143000_add_mocap_calibration_quality/migration.sql new file mode 100644 index 0000000..c877543 --- /dev/null +++ b/prisma/migrations/20260508143000_add_mocap_calibration_quality/migration.sql @@ -0,0 +1,4 @@ +ALTER TABLE "MocapSession" +ADD COLUMN "calibrationCatchFrame" JSONB, +ADD COLUMN "calibrationFinishFrame" JSONB, +ADD COLUMN "qualityFlags" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[]; diff --git a/prisma/migrations/20260508150000_add_mocap_derived_rows/migration.sql b/prisma/migrations/20260508150000_add_mocap_derived_rows/migration.sql new file mode 100644 index 0000000..45297e2 --- /dev/null +++ b/prisma/migrations/20260508150000_add_mocap_derived_rows/migration.sql @@ -0,0 +1,39 @@ +-- Persist post-session mocap analysis outputs. + +CREATE TABLE "StrokePostureMetric" ( + "id" TEXT NOT NULL, + "mocapSessionId" TEXT NOT NULL, + "strokeIndex" INTEGER NOT NULL, + "phaseBoundariesJson" JSONB NOT NULL, + "metricsJson" JSONB NOT NULL, + "segmentationSource" TEXT NOT NULL, + "strokeDataId" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "StrokePostureMetric_pkey" PRIMARY KEY ("id") +); + +CREATE TABLE "PostureFault" ( + "id" TEXT NOT NULL, + "mocapSessionId" TEXT NOT NULL, + "strokeIndex" INTEGER NOT NULL, + "faultType" TEXT NOT NULL, + "severity" TEXT NOT NULL, + "phase" TEXT NOT NULL, + "evidenceJson" JSONB NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "PostureFault_pkey" PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "StrokePostureMetric_mocapSessionId_strokeIndex_segmentationSource_key" ON "StrokePostureMetric"("mocapSessionId", "strokeIndex", "segmentationSource"); +CREATE INDEX "StrokePostureMetric_mocapSessionId_idx" ON "StrokePostureMetric"("mocapSessionId"); +CREATE INDEX "StrokePostureMetric_strokeDataId_idx" ON "StrokePostureMetric"("strokeDataId"); +CREATE INDEX "PostureFault_mocapSessionId_idx" ON "PostureFault"("mocapSessionId"); +CREATE INDEX "PostureFault_mocapSessionId_strokeIndex_idx" ON "PostureFault"("mocapSessionId", "strokeIndex"); +CREATE INDEX "PostureFault_faultType_severity_idx" ON "PostureFault"("faultType", "severity"); + +ALTER TABLE "StrokePostureMetric" ADD CONSTRAINT "StrokePostureMetric_mocapSessionId_fkey" FOREIGN KEY ("mocapSessionId") REFERENCES "MocapSession"("id") ON DELETE CASCADE ON UPDATE CASCADE; +ALTER TABLE "StrokePostureMetric" ADD CONSTRAINT "StrokePostureMetric_strokeDataId_fkey" FOREIGN KEY ("strokeDataId") REFERENCES "StrokeData"("id") ON DELETE SET NULL ON UPDATE CASCADE; +ALTER TABLE "PostureFault" ADD CONSTRAINT "PostureFault_mocapSessionId_fkey" FOREIGN KEY ("mocapSessionId") REFERENCES "MocapSession"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/prisma/migrations/20260508160000_add_mocap_detailed_ai_share/migration.sql b/prisma/migrations/20260508160000_add_mocap_detailed_ai_share/migration.sql new file mode 100644 index 0000000..7163065 --- /dev/null +++ b/prisma/migrations/20260508160000_add_mocap_detailed_ai_share/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "UserSettings" ADD COLUMN "mocapDetailedAIShare" BOOLEAN NOT NULL DEFAULT false; diff --git a/prisma/migrations/20260508170000_add_mocap_preferences_settings/migration.sql b/prisma/migrations/20260508170000_add_mocap_preferences_settings/migration.sql new file mode 100644 index 0000000..965108e --- /dev/null +++ b/prisma/migrations/20260508170000_add_mocap_preferences_settings/migration.sql @@ -0,0 +1,2 @@ +-- Add mocap preferences JSON column missing from previous migrations. +ALTER TABLE "UserSettings" ADD COLUMN "mocapPreferences" JSONB; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 70c5ecd..8b1113c 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -31,6 +31,7 @@ model User { accounts Account[] sessions AuthSession[] rowingSessions RowingSession[] + mocapSessions MocapSession[] personalRecords PersonalRecord[] earnedAwards EarnedAward[] aiAwards AIAwardSuggestion[] @@ -101,30 +102,31 @@ model PasswordResetToken { // ============================================================================ model RowingSession { - id String @id @default(cuid()) - userId String - timestamp DateTime - distance Int - duration Int - energy Int - strokeCount Int - avgPower Float - maxPower Float - wattPerKg Float - avgSplit Float - minSplit Float - avgWork Float - avgStrokeLength Float - avgStrokeRate Float - maxStrokeRate Float - consistencyScore Float? // Pre-computed consistency score (0-100) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - importedAt DateTime @default(now()) - sourceFile String? + id String @id @default(cuid()) + userId String + timestamp DateTime + distance Int + duration Int + energy Int + strokeCount Int + avgPower Float + maxPower Float + wattPerKg Float + avgSplit Float + minSplit Float + avgWork Float + avgStrokeLength Float + avgStrokeRate Float + maxStrokeRate Float + consistencyScore Float? // Pre-computed consistency score (0-100) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + importedAt DateTime @default(now()) + sourceFile String? user User @relation(fields: [userId], references: [id], onDelete: Cascade) strokeData StrokeData[] + mocapSession MocapSession? personalRecords PersonalRecord[] trainingSessionLinks TrainingSessionLink[] @@ -135,7 +137,7 @@ model RowingSession { } model StrokeData { - id String @id @default(cuid()) + id String @id @default(cuid()) sessionId String strokeIndex Int time Float @@ -151,6 +153,7 @@ model StrokeData { strokeLength Float? session RowingSession @relation(fields: [sessionId], references: [id], onDelete: Cascade) + mocapMetrics StrokePostureMetric[] @@index([sessionId]) @@index([sessionId, strokeIndex]) @@ -175,6 +178,76 @@ model PersonalRecord { @@index([userId]) } +// ============================================================================ +// MOCAP (motion-capture posture analysis — see docs/prd-mocap-posture.md) +// ============================================================================ + +model MocapSession { + id String @id @default(cuid()) + userId String + rowingSessionId String? @unique + videoStoragePath String + poseStreamPath String + source String + captureModelVersion String + capturePerspective String + captureFps Float + calibrationCatchFrame Json? + calibrationFinishFrame Json? + calibrationId String? + cameraCount Int? + durationSec Float @default(0) + qualityScore Float? + qualityFlags String[] @default([]) + status String @default("capturing") + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + rowingSession RowingSession? @relation(fields: [rowingSessionId], references: [id], onDelete: SetNull) + strokePostureMetrics StrokePostureMetric[] + postureFaults PostureFault[] + + @@index([userId]) + @@index([userId, createdAt]) +} + +model StrokePostureMetric { + id String @id @default(cuid()) + mocapSessionId String + strokeIndex Int + phaseBoundariesJson Json + metricsJson Json + segmentationSource String + strokeDataId String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + mocapSession MocapSession @relation(fields: [mocapSessionId], references: [id], onDelete: Cascade) + strokeData StrokeData? @relation(fields: [strokeDataId], references: [id], onDelete: SetNull) + + @@unique([mocapSessionId, strokeIndex, segmentationSource]) + @@index([mocapSessionId]) + @@index([strokeDataId]) +} + +model PostureFault { + id String @id @default(cuid()) + mocapSessionId String + strokeIndex Int + faultType String + severity String + phase String + evidenceJson Json + createdAt DateTime @default(now()) + + mocapSession MocapSession @relation(fields: [mocapSessionId], references: [id], onDelete: Cascade) + + @@index([mocapSessionId]) + @@index([mocapSessionId, strokeIndex]) + @@index([faultType, severity]) +} + // ============================================================================ // AWARDS & ACHIEVEMENTS // ============================================================================ @@ -215,15 +288,15 @@ model AIAwardSuggestion { } model GeneratedAchievement { - id String @id @default(cuid()) - userId String - awardId String - story String? @db.Text - imageUrl String? - hasImage Boolean @default(false) + id String @id @default(cuid()) + userId String + awardId String + story String? @db.Text + imageUrl String? + hasImage Boolean @default(false) colorPalette String? @default("classic") - earnedAt DateTime? - generatedAt DateTime @default(now()) + earnedAt DateTime? + generatedAt DateTime @default(now()) user User @relation(fields: [userId], references: [id], onDelete: Cascade) @@ -253,13 +326,27 @@ model TrainingPlan { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt - user User @relation(fields: [userId], references: [id], onDelete: Cascade) - weeks TrainingWeek[] + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + weeks TrainingWeek[] + postureGoal PlanPostureGoal? @@index([userId]) @@index([userId, status]) } +model PlanPostureGoal { + id String @id @default(cuid()) + planId String @unique + faultType String + targetRate Float + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + plan TrainingPlan @relation(fields: [planId], references: [id], onDelete: Cascade) + + @@index([planId]) +} + model TrainingWeek { id String @id @default(cuid()) planId String @@ -372,7 +459,7 @@ model AIInsight { archived Boolean @default(false) dateGenerated DateTime @default(now()) archivedAt DateTime? - feedback String? // 'helpful' | 'not_helpful' | 'action_taken' + feedback String? // 'helpful' | 'not_helpful' | 'action_taken' feedbackAt DateTime? user User @relation(fields: [userId], references: [id], onDelete: Cascade) @@ -402,7 +489,7 @@ model MemoryDocument { status String? uploadedAt DateTime @default(now()) - user User @relation(fields: [userId], references: [id], onDelete: Cascade) + user User @relation(fields: [userId], references: [id], onDelete: Cascade) @@index([userId]) @@index([userId, type]) @@ -413,44 +500,48 @@ model MemoryDocument { // ============================================================================ model UserSettings { - id String @id @default(cuid()) - userId String @unique - theme String @default("system") - units String @default("metric") - dateFormat String @default("MM/DD/YYYY") - timeFormat String @default("24h") - language String @default("en") - timeZone String? @default("UTC") - defaultChartType String @default("line") - animationsEnabled Boolean @default(true) - showPromptSuggestions Boolean @default(true) - customPrompts String[] - trainingZones Json? - preferredMetrics String[] - weeklyGoalType String @default("sessions") - weeklyGoalTarget Int @default(3) - restDayAlerts Boolean @default(true) - adaptationEnabled Boolean @default(true) - sessionReminders Boolean @default(false) - weeklyProgress Boolean @default(true) - achievementAlerts Boolean @default(true) - planReminders Boolean @default(true) - adherenceAlerts Boolean @default(true) - cloudAIEnabled Boolean @default(false) - maxTokens Int @default(1500) - aiConfig Json? - customPromptsAi Json? - userProfileContext String? @db.Text - userProfileRawInput String? @db.Text - dashboardSettings Json? - sessionsViewSettings Json? + id String @id @default(cuid()) + userId String @unique + theme String @default("system") + units String @default("metric") + dateFormat String @default("MM/DD/YYYY") + timeFormat String @default("24h") + language String @default("en") + timeZone String? @default("UTC") + defaultChartType String @default("line") + animationsEnabled Boolean @default(true) + showPromptSuggestions Boolean @default(true) + customPrompts String[] + trainingZones Json? + preferredMetrics String[] + weeklyGoalType String @default("sessions") + weeklyGoalTarget Int @default(3) + restDayAlerts Boolean @default(true) + adaptationEnabled Boolean @default(true) + sessionReminders Boolean @default(false) + weeklyProgress Boolean @default(true) + achievementAlerts Boolean @default(true) + planReminders Boolean @default(true) + adherenceAlerts Boolean @default(true) + cloudAIEnabled Boolean @default(false) + mocapDetailedAIShare Boolean @default(false) + maxTokens Int @default(1500) + aiConfig Json? + customPromptsAi Json? + userProfileContext String? @db.Text + userProfileRawInput String? @db.Text + postureThresholds Json? + mocapPreferences Json? + sidecarPort Int? + dashboardSettings Json? + sessionsViewSettings Json? sessionAnalysisSettings Json? - chartSettings Json? - analyticsSettings Json? - sessionsRevision Int @default(0) - insightsRevision Int @default(0) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt + chartSettings Json? + analyticsSettings Json? + sessionsRevision Int @default(0) + insightsRevision Int @default(0) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt user User @relation(fields: [userId], references: [id], onDelete: Cascade) } diff --git a/src/app/api/chat/posture-context/route.ts b/src/app/api/chat/posture-context/route.ts new file mode 100644 index 0000000..c65e2a6 --- /dev/null +++ b/src/app/api/chat/posture-context/route.ts @@ -0,0 +1,113 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { buildAndValidatePosturePayload } from "@/lib/aiAnalysis"; + +/** + * GET /api/chat/posture-context + * + * Returns a cloud-safe PostureAIPayload built from the authenticated user's + * most recent linked (rowing-session-associated) ready MocapSessions. + * + * Tier policy (mirrors insight generation): + * cloudAIEnabled = false → { payload: null } (Tier 1 hard-wall) + * cloudAIEnabled = true → Tier 3 fault summary + * + mocapDetailedAIShare → Tier 2 adds per-stroke scalar metrics + * + * Raw keypoints, landmarks, pose-stream blobs, and video bytes are never + * included; the hard guard in buildAndValidatePosturePayload enforces this. + */ +export async function GET() { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const userId = session.user.id; + + const userSettings = await prisma.userSettings.findUnique({ + where: { userId }, + select: { cloudAIEnabled: true, mocapDetailedAIShare: true }, + }); + + if (!userSettings?.cloudAIEnabled) { + return NextResponse.json({ payload: null, sessionCount: 0 }); + } + + const mocapSessions = await prisma.mocapSession.findMany({ + where: { + userId, + status: "ready", + rowingSessionId: { not: null }, + }, + select: { + qualityScore: true, + qualityFlags: true, + postureFaults: { + select: { faultType: true, severity: true }, + }, + strokePostureMetrics: { + select: { + strokeIndex: true, + segmentationSource: true, + metricsJson: true, + }, + }, + }, + orderBy: { createdAt: "desc" }, + take: 5, + }); + + if (mocapSessions.length === 0) { + return NextResponse.json({ payload: null, sessionCount: 0 }); + } + + const faults: { faultType: string; severity: string }[] = []; + const metrics: { + strokeIndex: number; + segmentationSource: string; + metricsJson: unknown; + }[] = []; + const qualityFlagSet = new Set(); + let qualityScoreSum = 0; + let qualityScoreCount = 0; + + for (const s of mocapSessions) { + for (const f of s.postureFaults) { + faults.push({ faultType: f.faultType, severity: f.severity }); + } + for (const m of s.strokePostureMetrics) { + metrics.push({ + strokeIndex: m.strokeIndex, + segmentationSource: m.segmentationSource, + metricsJson: m.metricsJson, + }); + } + for (const flag of s.qualityFlags) { + qualityFlagSet.add(flag); + } + if (s.qualityScore !== null) { + qualityScoreSum += s.qualityScore; + qualityScoreCount++; + } + } + + const qualityScore = + qualityScoreCount > 0 ? qualityScoreSum / qualityScoreCount : null; + + const payload = buildAndValidatePosturePayload( + { + faults, + metrics, + qualityFlags: Array.from(qualityFlagSet), + qualityScore, + }, + { + cloudAIEnabled: userSettings.cloudAIEnabled, + mocapDetailedAIShare: userSettings.mocapDetailedAIShare ?? false, + }, + ); + + return NextResponse.json({ payload, sessionCount: mocapSessions.length }); +} diff --git a/src/app/api/mocap/posture-summary/route.ts b/src/app/api/mocap/posture-summary/route.ts new file mode 100644 index 0000000..c217bc1 --- /dev/null +++ b/src/app/api/mocap/posture-summary/route.ts @@ -0,0 +1,79 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; + +/** + * GET /api/mocap/posture-summary + * + * Returns aggregated posture faults and stroke metrics for all of the + * authenticated user's ready mocap sessions. Used by AI insight generation + * to build a tiered PostureAIPayload without touching raw keypoint data. + */ +export async function GET() { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const userId = session.user.id; + + // Fetch all ready mocap sessions for this user including their faults, + // per-stroke metrics, and quality signals. + const mocapSessions = await prisma.mocapSession.findMany({ + where: { userId, status: "ready" }, + select: { + qualityScore: true, + qualityFlags: true, + postureFaults: { + select: { + faultType: true, + severity: true, + }, + }, + strokePostureMetrics: { + select: { + strokeIndex: true, + segmentationSource: true, + metricsJson: true, + }, + }, + }, + }); + + const faults: { faultType: string; severity: string }[] = []; + const metrics: { + strokeIndex: number; + segmentationSource: string; + metricsJson: unknown; + }[] = []; + const qualityFlags: string[] = []; + let qualityScore: number | null = null; + let qualityScoreCount = 0; + + for (const s of mocapSessions) { + for (const f of s.postureFaults) { + faults.push({ faultType: f.faultType, severity: f.severity }); + } + for (const m of s.strokePostureMetrics) { + metrics.push({ + strokeIndex: m.strokeIndex, + segmentationSource: m.segmentationSource, + metricsJson: m.metricsJson, + }); + } + for (const flag of s.qualityFlags) { + if (!qualityFlags.includes(flag)) qualityFlags.push(flag); + } + if (s.qualityScore !== null) { + qualityScore = (qualityScore ?? 0) + s.qualityScore; + qualityScoreCount++; + } + } + + if (qualityScoreCount > 0 && qualityScore !== null) { + qualityScore = qualityScore / qualityScoreCount; + } + + return NextResponse.json({ faults, metrics, qualityFlags, qualityScore }); +} diff --git a/src/app/api/mocap/posture-trend/route.ts b/src/app/api/mocap/posture-trend/route.ts new file mode 100644 index 0000000..7aeead3 --- /dev/null +++ b/src/app/api/mocap/posture-trend/route.ts @@ -0,0 +1,52 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { + aggregatePostureTrend, + type SessionFaultInput, +} from "@/lib/mocap/postureTrendAggregation"; + +/** + * GET /api/mocap/posture-trend + * + * Returns per-fault-type frequency trend across all ready mocap sessions + * linked to the authenticated user. Sessions with low quality scores or + * quality flags are included but marked so the UI can surface them. + */ +export async function GET() { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const userId = session.user.id; + + const mocapSessions = await prisma.mocapSession.findMany({ + where: { userId, status: "ready" }, + select: { + id: true, + createdAt: true, + qualityScore: true, + qualityFlags: true, + postureFaults: { + select: { faultType: true, severity: true }, + }, + _count: { select: { strokePostureMetrics: true } }, + }, + orderBy: { createdAt: "asc" }, + }); + + const inputs: SessionFaultInput[] = mocapSessions.map((s) => ({ + sessionId: s.id, + sessionDate: s.createdAt, + qualityScore: s.qualityScore, + qualityFlags: s.qualityFlags, + faults: s.postureFaults, + strokeCount: s._count.strokePostureMetrics, + })); + + const result = aggregatePostureTrend(inputs); + + return NextResponse.json({ ...result, sessions: inputs }); +} diff --git a/src/app/api/mocap/sessions/[id]/finalize/route.ts b/src/app/api/mocap/sessions/[id]/finalize/route.ts new file mode 100644 index 0000000..4ccdb5b --- /dev/null +++ b/src/app/api/mocap/sessions/[id]/finalize/route.ts @@ -0,0 +1,144 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { z } from "zod"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { getMocapStorage } from "@/lib/mocap/storage"; +import { finalizePoseStreamBlob } from "@/lib/mocap/capturePersistence"; +import { analyzeAndPersistMocapSession } from "@/lib/mocap/sessionAnalysis"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +const Body = z.object({ + durationSec: z.number().nonnegative().max(60 * 60 * 8), + qualityScore: z.number().min(0).max(1).optional(), + qualityFlags: z.array(z.string().min(1).max(80)).max(20).optional(), + skipAnalysis: z.boolean().optional(), +}); + +export async function POST( + req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const row = await prisma.mocapSession.findFirst({ + where: { id, userId: session.user.id }, + select: { + id: true, + userId: true, + status: true, + poseStreamPath: true, + videoStoragePath: true, + capturePerspective: true, + calibrationCatchFrame: true, + calibrationFinishFrame: true, + }, + }); + if (!row) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + if (row.status !== "capturing") { + return NextResponse.json( + { error: `Session not capturing (status=${row.status})` }, + { status: 409 }, + ); + } + + let body: z.infer; + try { + body = Body.parse(await req.json()); + } catch (err) { + return NextResponse.json( + { + error: "Invalid request body", + details: err instanceof Error ? err.message : String(err), + }, + { status: 400 }, + ); + } + + const storage = getMocapStorage(); + + if (body.skipAnalysis) { + let finalized = { frameCount: 0, poseStreamBytes: 0 }; + if (await storage.exists(row.poseStreamPath)) { + try { + finalized = await finalizePoseStreamBlob(storage, row.poseStreamPath); + } catch (err) { + return NextResponse.json( + { error: err instanceof Error ? err.message : String(err) }, + { status: 500 }, + ); + } + } + const updated = await prisma.mocapSession.update({ + where: { id: row.id }, + data: { + status: "ready", + durationSec: body.durationSec, + qualityScore: body.qualityScore ?? null, + qualityFlags: [...new Set([...(body.qualityFlags ?? []), "record-only"])], + }, + }); + return NextResponse.json({ + id: updated.id, + status: updated.status, + durationSec: updated.durationSec, + frameCount: finalized.frameCount, + poseStreamBytes: finalized.poseStreamBytes, + strokeMetricCount: 0, + faultCount: 0, + }); + } + + let finalized: Awaited>; + try { + finalized = await finalizePoseStreamBlob(storage, row.poseStreamPath); + } catch (err) { + return NextResponse.json( + { error: err instanceof Error ? err.message : String(err) }, + { status: 500 }, + ); + } + + const analyzing = await prisma.mocapSession.update({ + where: { id: row.id }, + data: { + status: "analyzing", + durationSec: body.durationSec, + qualityScore: body.qualityScore ?? null, + qualityFlags: body.qualityFlags ?? [], + }, + }); + + let analysis: Awaited>; + try { + analysis = await analyzeAndPersistMocapSession(storage, analyzing); + } catch (err) { + return NextResponse.json( + { error: err instanceof Error ? err.message : String(err) }, + { status: 500 }, + ); + } + + const updated = await prisma.mocapSession.update({ + where: { id: row.id }, + data: { status: "ready" }, + }); + + return NextResponse.json({ + id: updated.id, + status: updated.status, + durationSec: updated.durationSec, + frameCount: finalized.frameCount, + poseStreamBytes: finalized.poseStreamBytes, + strokeMetricCount: analysis.strokeMetricCount, + faultCount: analysis.faultCount, + }); +} diff --git a/src/app/api/mocap/sessions/[id]/link/[rowingSessionId]/route.ts b/src/app/api/mocap/sessions/[id]/link/[rowingSessionId]/route.ts new file mode 100644 index 0000000..3e42665 --- /dev/null +++ b/src/app/api/mocap/sessions/[id]/link/[rowingSessionId]/route.ts @@ -0,0 +1,147 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { getMocapStorage } from "@/lib/mocap/storage"; +import { analyzeAndPersistMocapSessionLinked } from "@/lib/mocap/sessionAnalysis"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +export async function POST( + _req: Request, + { params }: { params: Promise<{ id: string; rowingSessionId: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id, rowingSessionId } = await params; + const userId = session.user.id; + + // Validate: MocapSession belongs to user and is "ready" + const mocapSession = await prisma.mocapSession.findFirst({ + where: { id, userId }, + select: { + id: true, + userId: true, + status: true, + rowingSessionId: true, + poseStreamPath: true, + capturePerspective: true, + calibrationCatchFrame: true, + calibrationFinishFrame: true, + }, + }); + + if (!mocapSession) { + return NextResponse.json({ error: "Mocap session not found" }, { status: 404 }); + } + + if (mocapSession.status !== "ready") { + return NextResponse.json( + { error: `Mocap session not ready (status=${mocapSession.status})` }, + { status: 409 }, + ); + } + + // Enforce 1:1 — reject if mocap session already linked + if (mocapSession.rowingSessionId !== null) { + return NextResponse.json( + { error: "Mocap session is already linked to a rowing session. Unlink first." }, + { status: 409 }, + ); + } + + // Validate: RowingSession belongs to user + const rowingSession = await prisma.rowingSession.findFirst({ + where: { id: rowingSessionId, userId }, + select: { + id: true, + mocapSession: { select: { id: true } }, + }, + }); + + if (!rowingSession) { + return NextResponse.json({ error: "Rowing session not found" }, { status: 404 }); + } + + // Enforce 1:1 — reject if rowing session already linked to another MocapSession + if (rowingSession.mocapSession !== null) { + return NextResponse.json( + { error: "Rowing session is already linked to another mocap session." }, + { status: 409 }, + ); + } + + try { + const linkUpdate = await prisma.mocapSession.updateMany({ + where: { id, userId, rowingSessionId: null, status: "ready" }, + data: { rowingSessionId, status: "analyzing" }, + }); + + if (linkUpdate.count !== 1) { + return NextResponse.json( + { error: "Mocap session is already linked to a rowing session. Unlink first." }, + { status: 409 }, + ); + } + } catch (err) { + if ((err as { code?: string })?.code === "P2002") { + return NextResponse.json( + { error: "Rowing session is already linked to another mocap session." }, + { status: 409 }, + ); + } + throw err; + } + + const storage = getMocapStorage(); + + try { + await analyzeAndPersistMocapSessionLinked(storage, mocapSession, rowingSessionId); + } catch (err) { + // Roll back: clear the link and revert status + await prisma.mocapSession.update({ + where: { id }, + data: { rowingSessionId: null, status: "ready" }, + }); + return NextResponse.json( + { error: err instanceof Error ? err.message : String(err) }, + { status: 500 }, + ); + } + + const updated = await prisma.mocapSession.update({ + where: { id }, + data: { status: "ready" }, + select: { id: true, rowingSessionId: true, status: true }, + }); + + await prisma.userSettings.upsert({ + where: { userId }, + update: { + sessionsRevision: { increment: 1 }, + }, + create: { + userId, + theme: "system", + units: "metric", + dateFormat: "MM/DD/YYYY", + timeFormat: "24h", + language: "en", + defaultChartType: "line", + animationsEnabled: true, + cloudAIEnabled: false, + maxTokens: 4000, + sessionsRevision: 1, + }, + }); + + return NextResponse.json({ + id: updated.id, + rowingSessionId: updated.rowingSessionId, + status: updated.status, + }); +} diff --git a/src/app/api/mocap/sessions/[id]/pose-stream/route.ts b/src/app/api/mocap/sessions/[id]/pose-stream/route.ts new file mode 100644 index 0000000..2e25402 --- /dev/null +++ b/src/app/api/mocap/sessions/[id]/pose-stream/route.ts @@ -0,0 +1,141 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { appendPoseFrames } from "@/lib/mocap/capturePersistence"; +import { getMocapStorage } from "@/lib/mocap/storage"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +export async function POST( + req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const row = await prisma.mocapSession.findFirst({ + where: { id, userId: session.user.id }, + select: { id: true, status: true, poseStreamPath: true }, + }); + if (!row) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + if (row.status !== "capturing") { + return NextResponse.json( + { error: `Session not capturing (status=${row.status})` }, + { status: 409 }, + ); + } + + const buf = new Uint8Array(await req.arrayBuffer()); + const storage = getMocapStorage(); + try { + const framesAppended = await appendPoseFrames( + storage, + row.poseStreamPath, + buf, + ); + return NextResponse.json({ appended: framesAppended }); + } catch (err) { + return NextResponse.json( + { + error: err instanceof Error ? err.message : String(err), + }, + { status: 400 }, + ); + } +} + +function parseRange( + header: string | null, + totalSize: number, +): { start: number; end: number } | null { + if (!header) return null; + const m = /^bytes=(\d*)-(\d*)$/.exec(header.trim()); + if (!m) return null; + const startStr = m[1]; + const endStr = m[2]; + if (startStr === "" && endStr === "") return null; + let start: number; + let end: number; + if (startStr === "") { + const suffix = parseInt(endStr, 10); + if (!Number.isFinite(suffix) || suffix <= 0) return null; + start = Math.max(0, totalSize - suffix); + end = totalSize - 1; + } else { + start = parseInt(startStr, 10); + end = endStr === "" ? totalSize - 1 : parseInt(endStr, 10); + } + if ( + !Number.isFinite(start) || + !Number.isFinite(end) || + start < 0 || + end < start || + start >= totalSize + ) { + return null; + } + end = Math.min(end, totalSize - 1); + return { start, end }; +} + +export async function GET( + req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const row = await prisma.mocapSession.findFirst({ + where: { id, userId: session.user.id }, + select: { poseStreamPath: true }, + }); + if (!row) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + + const storage = getMocapStorage(); + if (!(await storage.exists(row.poseStreamPath))) { + return NextResponse.json({ error: "Pose stream unavailable" }, { status: 404 }); + } + const totalSize = await storage.size(row.poseStreamPath); + const range = parseRange(req.headers.get("range"), totalSize); + + if (!range) { + const bytes = await storage.read(row.poseStreamPath); + const u8 = new Uint8Array(bytes); + return new Response(u8 as BodyInit, { + status: 200, + headers: { + "Content-Type": "application/octet-stream", + "Content-Length": String(totalSize), + "Accept-Ranges": "bytes", + "Cache-Control": "private, no-store", + }, + }); + } + + const slice = await storage.read(row.poseStreamPath, { + start: range.start, + end: range.end + 1, + }); + return new Response(new Uint8Array(slice) as BodyInit, { + status: 206, + headers: { + "Content-Type": "application/octet-stream", + "Content-Length": String(slice.byteLength), + "Content-Range": `bytes ${range.start}-${range.end}/${totalSize}`, + "Accept-Ranges": "bytes", + "Cache-Control": "private, no-store", + }, + }); +} diff --git a/src/app/api/mocap/sessions/[id]/pose/route.ts b/src/app/api/mocap/sessions/[id]/pose/route.ts new file mode 100644 index 0000000..80a8bd7 --- /dev/null +++ b/src/app/api/mocap/sessions/[id]/pose/route.ts @@ -0,0 +1,6 @@ +// Compatibility alias for the original upload route. New clients should use +// /api/mocap/sessions/:id/pose-stream. +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +export { POST } from "../pose-stream/route"; diff --git a/src/app/api/mocap/sessions/[id]/reanalyze/route.ts b/src/app/api/mocap/sessions/[id]/reanalyze/route.ts new file mode 100644 index 0000000..5b1ef9f --- /dev/null +++ b/src/app/api/mocap/sessions/[id]/reanalyze/route.ts @@ -0,0 +1,83 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { getMocapStorage } from "@/lib/mocap/storage"; +import { analyzeAndPersistMocapSession } from "@/lib/mocap/sessionAnalysis"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +export async function POST( + _req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const row = await prisma.mocapSession.findFirst({ + where: { id, userId: session.user.id }, + select: { + id: true, + userId: true, + status: true, + poseStreamPath: true, + videoStoragePath: true, + capturePerspective: true, + calibrationCatchFrame: true, + calibrationFinishFrame: true, + }, + }); + if (!row) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + if (row.status !== "ready") { + return NextResponse.json( + { error: `Session not ready (status=${row.status})` }, + { status: 409 }, + ); + } + + const storage = getMocapStorage(); + if (!(await storage.exists(row.poseStreamPath))) { + return NextResponse.json( + { error: "Cannot re-analyze a record-only session without a pose stream" }, + { status: 409 }, + ); + } + + await prisma.mocapSession.update({ + where: { id: row.id }, + data: { status: "analyzing" }, + }); + + let analysis: Awaited>; + try { + analysis = await analyzeAndPersistMocapSession(storage, row); + } catch (err) { + // Revert status so the session stays usable + await prisma.mocapSession.update({ + where: { id: row.id }, + data: { status: "ready" }, + }); + return NextResponse.json( + { error: err instanceof Error ? err.message : String(err) }, + { status: 500 }, + ); + } + + const updated = await prisma.mocapSession.update({ + where: { id: row.id }, + data: { status: "ready" }, + }); + + return NextResponse.json({ + id: updated.id, + status: updated.status, + strokeMetricCount: analysis.strokeMetricCount, + faultCount: analysis.faultCount, + }); +} diff --git a/src/app/api/mocap/sessions/[id]/route.ts b/src/app/api/mocap/sessions/[id]/route.ts new file mode 100644 index 0000000..cc41a5e --- /dev/null +++ b/src/app/api/mocap/sessions/[id]/route.ts @@ -0,0 +1,59 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { getMocapStorage } from "@/lib/mocap/storage"; + +export async function GET( + _req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const row = await prisma.mocapSession.findFirst({ + where: { id, userId: session.user.id }, + include: { + strokePostureMetrics: { + orderBy: { strokeIndex: "asc" }, + }, + postureFaults: { + orderBy: [{ strokeIndex: "asc" }, { createdAt: "asc" }], + }, + }, + }); + if (!row) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + return NextResponse.json({ session: row }); +} + +export async function DELETE( + _req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const row = await prisma.mocapSession.findFirst({ + where: { id, userId: session.user.id }, + }); + if (!row) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + + const storage = getMocapStorage(); + await Promise.allSettled([ + storage.delete(row.videoStoragePath), + storage.delete(row.poseStreamPath), + ]); + await prisma.mocapSession.delete({ where: { id } }); + + return NextResponse.json({ success: true, id }); +} diff --git a/src/app/api/mocap/sessions/[id]/sidecar/connect/route.ts b/src/app/api/mocap/sessions/[id]/sidecar/connect/route.ts new file mode 100644 index 0000000..ae645b4 --- /dev/null +++ b/src/app/api/mocap/sessions/[id]/sidecar/connect/route.ts @@ -0,0 +1,60 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { z } from "zod"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { checkSidecarHealth, SIDECAR_DEFAULT_PORT } from "@/lib/mocap/sidecarClient"; + +const ConnectBody = z.object({ + port: z.number().int().positive().optional(), +}); + +export async function POST( + req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const mocapSession = await prisma.mocapSession.findFirst({ + where: { id, userId: session.user.id }, + select: { id: true, status: true }, + }); + if (!mocapSession) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + if (mocapSession.status !== "capturing") { + return NextResponse.json( + { error: `Session is ${mocapSession.status}; expected capturing` }, + { status: 409 }, + ); + } + + let body: z.infer; + try { + body = ConnectBody.parse(await req.json().catch(() => ({}))); + } catch (err) { + return NextResponse.json( + { error: "Invalid request body", details: err instanceof Error ? err.message : String(err) }, + { status: 400 }, + ); + } + + const port = body.port ?? SIDECAR_DEFAULT_PORT; + + try { + const health = await checkSidecarHealth(port); + return NextResponse.json({ + status: "connected", + fps: health.fps, + cameras: health.cameras, + schemaVersion: health.schemaVersion, + port, + }); + } catch { + return NextResponse.json({ status: "unreachable", port }, { status: 503 }); + } +} diff --git a/src/app/api/mocap/sessions/[id]/sidecar/status/route.ts b/src/app/api/mocap/sessions/[id]/sidecar/status/route.ts new file mode 100644 index 0000000..a71ca9f --- /dev/null +++ b/src/app/api/mocap/sessions/[id]/sidecar/status/route.ts @@ -0,0 +1,34 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { checkSidecarHealth, SIDECAR_DEFAULT_PORT } from "@/lib/mocap/sidecarClient"; + +export async function GET( + req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const mocapSession = await prisma.mocapSession.findFirst({ + where: { id, userId: session.user.id }, + select: { id: true }, + }); + if (!mocapSession) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + + const url = new URL(req.url); + const port = parseInt(url.searchParams.get("port") ?? String(SIDECAR_DEFAULT_PORT), 10); + + try { + const health = await checkSidecarHealth(port); + return NextResponse.json({ ...health, port }); + } catch { + return NextResponse.json({ status: "unreachable", port }, { status: 503 }); + } +} diff --git a/src/app/api/mocap/sessions/[id]/unlink/route.ts b/src/app/api/mocap/sessions/[id]/unlink/route.ts new file mode 100644 index 0000000..8c58e42 --- /dev/null +++ b/src/app/api/mocap/sessions/[id]/unlink/route.ts @@ -0,0 +1,87 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { getMocapStorage } from "@/lib/mocap/storage"; +import { analyzeAndPersistMocapSession } from "@/lib/mocap/sessionAnalysis"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +export async function POST( + _req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const userId = session.user.id; + + const mocapSession = await prisma.mocapSession.findFirst({ + where: { id, userId }, + select: { + id: true, + userId: true, + status: true, + rowingSessionId: true, + poseStreamPath: true, + capturePerspective: true, + calibrationCatchFrame: true, + calibrationFinishFrame: true, + }, + }); + + if (!mocapSession) { + return NextResponse.json({ error: "Mocap session not found" }, { status: 404 }); + } + + if (mocapSession.status !== "ready") { + return NextResponse.json( + { error: `Mocap session not ready (status=${mocapSession.status})` }, + { status: 409 }, + ); + } + + const previousRowingSessionId = mocapSession.rowingSessionId; + + // Clear the link and set status to "analyzing" + await prisma.mocapSession.update({ + where: { id }, + data: { rowingSessionId: null, status: "analyzing" }, + }); + + const storage = getMocapStorage(); + + try { + // Re-run pose-segmented analysis, which sets segmentationSource = "pose-segmented" + // and clears strokeDataId (analyzeAndPersistMocapSession does not set strokeDataId) + await analyzeAndPersistMocapSession(storage, mocapSession); + } catch (err) { + // Roll back: restore the previous link and revert status + await prisma.mocapSession.update({ + where: { id }, + data: { + rowingSessionId: previousRowingSessionId ?? null, + status: "ready", + }, + }); + return NextResponse.json( + { error: err instanceof Error ? err.message : String(err) }, + { status: 500 }, + ); + } + + const updated = await prisma.mocapSession.update({ + where: { id }, + data: { status: "ready" }, + select: { id: true, status: true }, + }); + + return NextResponse.json({ + id: updated.id, + status: updated.status, + }); +} diff --git a/src/app/api/mocap/sessions/[id]/video/route.ts b/src/app/api/mocap/sessions/[id]/video/route.ts new file mode 100644 index 0000000..fa61714 --- /dev/null +++ b/src/app/api/mocap/sessions/[id]/video/route.ts @@ -0,0 +1,122 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { getMocapStorage } from "@/lib/mocap/storage"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +function parseRange( + header: string | null, + totalSize: number, +): { start: number; end: number } | null { + if (!header) return null; + const m = /^bytes=(\d*)-(\d*)$/.exec(header.trim()); + if (!m) return null; + const startStr = m[1]; + const endStr = m[2]; + if (startStr === "" && endStr === "") return null; + let start: number; + let end: number; + if (startStr === "") { + const suffix = parseInt(endStr, 10); + if (!Number.isFinite(suffix) || suffix <= 0) return null; + start = Math.max(0, totalSize - suffix); + end = totalSize - 1; + } else { + start = parseInt(startStr, 10); + end = endStr === "" ? totalSize - 1 : parseInt(endStr, 10); + } + if (!Number.isFinite(start) || !Number.isFinite(end) || start < 0 || end < start || start >= totalSize) { + return null; + } + end = Math.min(end, totalSize - 1); + return { start, end }; +} + +export async function GET( + req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const row = await prisma.mocapSession.findFirst({ + where: { id, userId: session.user.id }, + select: { videoStoragePath: true, status: true }, + }); + if (!row || row.status === "capturing") { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + + const storage = getMocapStorage(); + const totalSize = await storage.size(row.videoStoragePath); + const range = parseRange(req.headers.get("range"), totalSize); + + if (!range) { + const bytes = await storage.read(row.videoStoragePath); + return new Response(new Uint8Array(bytes) as BodyInit, { + status: 200, + headers: { + "Content-Type": "video/webm", + "Content-Length": String(totalSize), + "Accept-Ranges": "bytes", + "Cache-Control": "private, no-store", + }, + }); + } + + const slice = await storage.read(row.videoStoragePath, { + start: range.start, + end: range.end + 1, + }); + return new Response(new Uint8Array(slice) as BodyInit, { + status: 206, + headers: { + "Content-Type": "video/webm", + "Content-Length": String(slice.byteLength), + "Content-Range": `bytes ${range.start}-${range.end}/${totalSize}`, + "Accept-Ranges": "bytes", + "Cache-Control": "private, no-store", + }, + }); +} + +export async function POST( + req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const row = await prisma.mocapSession.findFirst({ + where: { id, userId: session.user.id }, + select: { id: true, status: true, videoStoragePath: true }, + }); + if (!row) { + return NextResponse.json({ error: "Not found" }, { status: 404 }); + } + if (row.status !== "capturing") { + return NextResponse.json( + { error: `Session not capturing (status=${row.status})` }, + { status: 409 }, + ); + } + + const buf = new Uint8Array(await req.arrayBuffer()); + if (buf.byteLength === 0) { + return NextResponse.json({ appended: 0 }); + } + + const storage = getMocapStorage(); + await storage.appendBytes(row.videoStoragePath, buf); + + return NextResponse.json({ appended: buf.byteLength }); +} diff --git a/src/app/api/mocap/sessions/overlap-check/route.ts b/src/app/api/mocap/sessions/overlap-check/route.ts new file mode 100644 index 0000000..76a07d6 --- /dev/null +++ b/src/app/api/mocap/sessions/overlap-check/route.ts @@ -0,0 +1,79 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; + +export const dynamic = "force-dynamic"; +export const runtime = "nodejs"; + +/** + * POST /api/mocap/sessions/overlap-check + * Given a list of newly-imported RowingSession ids, find any existing + * MocapSessions (unlinked) whose capture window overlaps the rowing + * session's timestamp by ±2 minutes. + * + * Body: { rowingSessionIds: string[] } + * Response: { overlaps: Array<{ rowingSessionId: string; mocapSessionId: string }> } + */ +export async function POST(req: Request) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const userId = session.user.id; + const OVERLAP_MARGIN_MS = 2 * 60 * 1000; // ±2 minutes + + let rowingSessionIds: string[]; + try { + const body = await req.json(); + if (!Array.isArray(body?.rowingSessionIds) || body.rowingSessionIds.length === 0) { + return NextResponse.json({ overlaps: [] }); + } + rowingSessionIds = body.rowingSessionIds as string[]; + } catch { + return NextResponse.json({ overlaps: [] }); + } + + // Fetch the rowing sessions' timestamps + const rowingSessions = await prisma.rowingSession.findMany({ + where: { id: { in: rowingSessionIds }, userId }, + select: { id: true, timestamp: true }, + }); + + if (rowingSessions.length === 0) { + return NextResponse.json({ overlaps: [] }); + } + + // Fetch unlinked mocap sessions for this user (ready status, no rowingSessionId) + const mocapSessions = await prisma.mocapSession.findMany({ + where: { userId, rowingSessionId: null, status: "ready" }, + select: { id: true, createdAt: true, durationSec: true }, + }); + + if (mocapSessions.length === 0) { + return NextResponse.json({ overlaps: [] }); + } + + const overlaps: Array<{ rowingSessionId: string; mocapSessionId: string }> = []; + + for (const rs of rowingSessions) { + const rsTime = rs.timestamp.getTime(); + const rsStart = rsTime - OVERLAP_MARGIN_MS; + const rsEnd = rsTime + OVERLAP_MARGIN_MS; + + for (const ms of mocapSessions) { + const msStart = ms.createdAt.getTime(); + const msEnd = msStart + ms.durationSec * 1000; + + // Overlap check: mocap window [msStart, msEnd] overlaps [rsStart, rsEnd] + if (msEnd >= rsStart && msStart <= rsEnd) { + overlaps.push({ rowingSessionId: rs.id, mocapSessionId: ms.id }); + // One mocap session can only link to one rowing session — stop after first match + break; + } + } + } + + return NextResponse.json({ overlaps }); +} diff --git a/src/app/api/mocap/sessions/route.ts b/src/app/api/mocap/sessions/route.ts new file mode 100644 index 0000000..f29f1ad --- /dev/null +++ b/src/app/api/mocap/sessions/route.ts @@ -0,0 +1,151 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { z } from "zod"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { getMocapStorage } from "@/lib/mocap/storage"; +import { initializePoseStreamBlob } from "@/lib/mocap/capturePersistence"; + +export async function GET() { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const rows = await prisma.mocapSession.findMany({ + where: { userId: session.user.id }, + orderBy: { createdAt: "desc" }, + select: { + id: true, + status: true, + durationSec: true, + createdAt: true, + capturePerspective: true, + qualityScore: true, + qualityFlags: true, + _count: { + select: { + strokePostureMetrics: true, + postureFaults: true, + }, + }, + }, + }); + + return NextResponse.json({ sessions: rows }); +} + +const CalibrationFrame = z.object({ + pose: z.enum(["catch", "finish"]), + capturedAt: z.string().datetime(), + capturePerspective: z.enum(["side-left", "side-right"]), + videoWidth: z.number().int().nonnegative(), + videoHeight: z.number().int().nonnegative(), + meanKeypointConfidence: z.number().min(0).max(1), + trackedKeypointCount: z.number().int().nonnegative().max(33), + qualityFlags: z.number().int().nonnegative(), + poseFrameBase64: z.string().min(1), +}); + +const CreateBody = z + .object({ + source: z.enum(["browser", "sidecar"]), + captureModelVersion: z.string().min(1).max(120), + capturePerspective: z.enum(["side-left", "side-right", "sidecar-3d"]), + captureFps: z.number().positive().max(240), + recordOnly: z.boolean().optional(), + calibrationId: z.string().uuid().optional(), + cameraCount: z.number().int().positive().max(16).optional(), + calibrationCatchFrame: CalibrationFrame.extend({ + pose: z.literal("catch"), + }).optional(), + calibrationFinishFrame: CalibrationFrame.extend({ + pose: z.literal("finish"), + }).optional(), + }) + .superRefine((body, ctx) => { + if (body.source === "sidecar") return; // sidecar sessions have no browser calibration frames + for (const field of ["calibrationCatchFrame", "calibrationFinishFrame"] as const) { + if (body.recordOnly && body[field] === undefined) continue; + if (!body[field]) { + ctx.addIssue({ + code: "custom", + path: [field], + message: "Calibration frame is required unless recordOnly is true", + }); + continue; + } + if (body[field].capturePerspective !== body.capturePerspective) { + ctx.addIssue({ + code: "custom", + path: [field, "capturePerspective"], + message: "Calibration perspective must match capturePerspective", + }); + } + } + }); + +export async function POST(req: Request) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + let body: z.infer; + try { + body = CreateBody.parse(await req.json()); + } catch (err) { + return NextResponse.json( + { error: "Invalid request body", details: err instanceof Error ? err.message : String(err) }, + { status: 400 }, + ); + } + + const userId = session.user.id; + const storage = getMocapStorage(); + + const created = await prisma.$transaction(async (tx) => { + const row = await tx.mocapSession.create({ + data: { + userId, + source: body.source, + captureModelVersion: body.captureModelVersion, + capturePerspective: body.capturePerspective, + captureFps: body.captureFps, + calibrationCatchFrame: body.calibrationCatchFrame, + calibrationFinishFrame: body.calibrationFinishFrame, + calibrationId: body.calibrationId, + cameraCount: body.cameraCount, + videoStoragePath: "pending", + poseStreamPath: "pending", + status: "capturing", + }, + }); + const videoStoragePath = storage.videoPath(userId, row.id); + const poseStreamPath = storage.poseStreamPath(userId, row.id); + return tx.mocapSession.update({ + where: { id: row.id }, + data: { videoStoragePath, poseStreamPath }, + }); + }); + + if (!body.recordOnly) { + try { + await initializePoseStreamBlob(storage, created.poseStreamPath, body.captureFps); + } catch (err) { + await prisma.mocapSession.delete({ where: { id: created.id } }).catch(() => {}); + return NextResponse.json( + { error: "Failed to initialize pose stream blob", details: err instanceof Error ? err.message : String(err) }, + { status: 500 }, + ); + } + } + + return NextResponse.json({ + id: created.id, + videoStoragePath: created.videoStoragePath, + poseStreamPath: created.poseStreamPath, + status: created.status, + createdAt: created.createdAt, + }); +} diff --git a/src/app/api/sessions/list/route.ts b/src/app/api/sessions/list/route.ts index 44e7e53..f02321c 100644 --- a/src/app/api/sessions/list/route.ts +++ b/src/app/api/sessions/list/route.ts @@ -51,6 +51,9 @@ export async function GET() { updatedAt: true, importedAt: true, sourceFile: true, + mocapSession: { + select: { id: true }, + }, }, orderBy: { timestamp: 'desc', diff --git a/src/app/api/settings/route.ts b/src/app/api/settings/route.ts index 0337863..5a08837 100644 --- a/src/app/api/settings/route.ts +++ b/src/app/api/settings/route.ts @@ -42,6 +42,7 @@ export async function GET() { insightsRevision: 0, userProfileContext: null, userProfileRawInput: null, + postureThresholds: null, aiConfig: null, customPromptsAi: null, } @@ -123,6 +124,7 @@ export async function POST(req: Request) { // AI settings if (settingsData.cloudAIEnabled !== undefined) updateData.cloudAIEnabled = settingsData.cloudAIEnabled; + if (settingsData.mocapDetailedAIShare !== undefined) updateData.mocapDetailedAIShare = settingsData.mocapDetailedAIShare; if (settingsData.maxTokens !== undefined) updateData.maxTokens = settingsData.maxTokens; if (settingsData.aiConfig !== undefined) updateData.aiConfig = settingsData.aiConfig; if (settingsData.customPromptsAi !== undefined) updateData.customPromptsAi = settingsData.customPromptsAi; @@ -134,6 +136,8 @@ export async function POST(req: Request) { // User profile context if (settingsData.userProfileContext !== undefined) updateData.userProfileContext = settingsData.userProfileContext; if (settingsData.userProfileRawInput !== undefined) updateData.userProfileRawInput = settingsData.userProfileRawInput; + if (settingsData.postureThresholds !== undefined) updateData.postureThresholds = settingsData.postureThresholds; + if (settingsData.mocapPreferences !== undefined) updateData.mocapPreferences = settingsData.mocapPreferences; // Dashboard and view settings if (settingsData.dashboardSettings !== undefined) updateData.dashboardSettings = settingsData.dashboardSettings; diff --git a/src/app/api/training-plans/[id]/posture-goal/route.ts b/src/app/api/training-plans/[id]/posture-goal/route.ts new file mode 100644 index 0000000..b84559c --- /dev/null +++ b/src/app/api/training-plans/[id]/posture-goal/route.ts @@ -0,0 +1,140 @@ +import { NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { authOptions } from "@/lib/auth"; +import { prisma } from "@/lib/db/prisma"; +import { POSTURE_FAULT_CATALOG_V1 } from "@/lib/mocap/analysis/postureThresholds"; +import { computePostureGoalProgress } from "@/lib/postureGoalProgress"; +import type { PostureFaultType } from "@/lib/mocap/analysis/types"; +import type { SessionFaultInput } from "@/lib/mocap/postureTrendAggregation"; + +async function getAuthedPlan(planId: string, userId: string) { + return prisma.trainingPlan.findFirst({ + where: { id: planId, userId }, + }); +} + +/** + * GET /api/training-plans/[id]/posture-goal + * Returns the plan's posture goal and current progress derived from linked mocap sessions. + */ +export async function GET( + _req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id: planId } = await params; + const plan = await getAuthedPlan(planId, session.user.id); + if (!plan) { + return NextResponse.json({ error: "Plan not found" }, { status: 404 }); + } + + const goal = await prisma.planPostureGoal.findUnique({ where: { planId } }); + if (!goal) { + return NextResponse.json({ goal: null, progress: null }); + } + + // Gather linked mocap sessions through the plan's training sessions + const links = await prisma.trainingSessionLink.findMany({ + where: { trainingSession: { week: { planId } } }, + select: { rowingSessionId: true }, + }); + const rowingSessionIds = links.map((l) => l.rowingSessionId); + + const mocapSessions = await prisma.mocapSession.findMany({ + where: { rowingSessionId: { in: rowingSessionIds }, status: "ready" }, + include: { + postureFaults: { select: { faultType: true, severity: true } }, + strokePostureMetrics: { select: { id: true } }, + }, + }); + + const sessionInputs: SessionFaultInput[] = mocapSessions.map((ms) => ({ + sessionId: ms.id, + sessionDate: ms.createdAt, + qualityScore: ms.qualityScore, + qualityFlags: ms.qualityFlags, + faults: ms.postureFaults.map((f) => ({ + faultType: f.faultType, + severity: f.severity, + })), + strokeCount: ms.strokePostureMetrics.length, + })); + + const progress = computePostureGoalProgress( + sessionInputs, + goal.faultType as PostureFaultType, + goal.targetRate, + ); + + return NextResponse.json({ goal, progress }); +} + +/** + * PUT /api/training-plans/[id]/posture-goal + * Create or replace the posture goal for a plan. + * Body: { faultType: string, targetRate: number } + */ +export async function PUT( + req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id: planId } = await params; + const plan = await getAuthedPlan(planId, session.user.id); + if (!plan) { + return NextResponse.json({ error: "Plan not found" }, { status: 404 }); + } + + const body = await req.json(); + const { faultType, targetRate } = body; + + if (!POSTURE_FAULT_CATALOG_V1.includes(faultType as PostureFaultType)) { + return NextResponse.json({ error: "Invalid faultType" }, { status: 400 }); + } + if (typeof targetRate !== "number" || targetRate < 0 || targetRate > 1) { + return NextResponse.json( + { error: "targetRate must be a number between 0 and 1" }, + { status: 400 }, + ); + } + + const goal = await prisma.planPostureGoal.upsert({ + where: { planId }, + update: { faultType, targetRate }, + create: { planId, faultType, targetRate }, + }); + + return NextResponse.json({ goal }); +} + +/** + * DELETE /api/training-plans/[id]/posture-goal + * Remove the posture goal from a plan. + */ +export async function DELETE( + _req: Request, + { params }: { params: Promise<{ id: string }> }, +) { + const session = await getServerSession(authOptions); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const { id: planId } = await params; + const plan = await getAuthedPlan(planId, session.user.id); + if (!plan) { + return NextResponse.json({ error: "Plan not found" }, { status: 404 }); + } + + await prisma.planPostureGoal.deleteMany({ where: { planId } }); + + return NextResponse.json({ success: true }); +} diff --git a/src/app/dashboard/page.tsx b/src/app/dashboard/page.tsx index 55c38c9..1c76bee 100644 --- a/src/app/dashboard/page.tsx +++ b/src/app/dashboard/page.tsx @@ -21,6 +21,7 @@ import { MigrationPrompt } from '@/components/MigrationPrompt'; import { MetricComparisonWidget } from '@/components/MetricComparisonWidget'; import { PeriodComparisonStats } from '@/components/PeriodComparisonStats'; +import { PostureFaultTrendCard } from '@/components/PostureFaultTrendCard'; import { TimeRangeSelector, defaultTimeRangeOptions, type TimeRange } from '@/components/ui/time-range-selector'; // Chart type options @@ -633,6 +634,9 @@ const Dashboard = () => { {/* Monthly Comparison Header Cards */} + {/* Posture Fault Frequency Trend */} + + {/* AI Insights Section */} {isAnalyzable && (
diff --git a/src/app/mocap/page.tsx b/src/app/mocap/page.tsx new file mode 100644 index 0000000..810b3fe --- /dev/null +++ b/src/app/mocap/page.tsx @@ -0,0 +1,1340 @@ +"use client"; + +import Link from "next/link"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { + BrowserPoseSource, + type PoseSourceStatus, +} from "@/lib/mocap/browserPoseSource"; +import { + cameraQualityFlagLabel, + evaluateCameraReadiness, + type CameraReadinessFrame, + type CameraReadinessResult, +} from "@/lib/mocap/cameraReadiness"; +import { + evaluateMocapCaptureSupport, + hasSustainedLowEffectiveFps, + lowFpsRecordOnlySupport, + readBrowserMocapCapabilities, + recordOnlyQualityFlag, + type EffectiveFpsSample, + type MocapCaptureSupport, + type RecordOnlyReason, +} from "@/lib/mocap/degradedMode"; +import { + BYTES_PER_FRAME_V1, + decodeFrame, +} from "@/lib/mocap/poseFrameStream"; +import { VideoUploader } from "@/lib/mocap/videoUploader"; +import { + getCoachingCues, + type CoachingCue, +} from "@/lib/mocap/coaching/coachingAdvisor"; +import { LiveCoachingEngine } from "@/lib/mocap/coaching/liveCoachingEngine"; +import { + cancelSpokenCues, + speakCue, +} from "@/lib/mocap/coaching/cueAudio"; +import { keypointTripletsToPosePoints } from "@/lib/mocap/analysis/poseFrameStreamAdapter"; +import type { + Calibration, + PoseAnalysisFrame, + PostureFault, +} from "@/lib/mocap/analysis/types"; +import { settings } from "@/lib/settings"; +import { checkSidecarHealth, SIDECAR_DEFAULT_PORT, type SidecarHealth } from "@/lib/mocap/sidecarClient"; + +const CAPTURE_FPS = 30; +const CAPTURE_MODEL_VERSION = "mediapipe-pose-landmarker-lite@0.10.35"; +const VIDEO_TIMESLICE_MS = 1000; +const QUALITY_HISTORY_MS = 5000; + +const SEVERITY_WEIGHT: Record = { + critical: 3, + warning: 2, + info: 1, +}; + +type CaptureState = + | { kind: "idle" } + | { kind: "starting" } + | { + kind: "capturing"; + sessionId: string; + startedAt: number; + } + | { + kind: "stopping"; + sessionId: string; + } + | { + kind: "done"; + sessionId: string; + durationSec: number; + frameCount: number; + recordOnly: boolean; + } + | { kind: "error"; message: string }; + +type CalibrationPose = "catch" | "finish"; + +type CalibrationFrame = { + pose: CalibrationPose; + capturedAt: string; + capturePerspective: "side-left" | "side-right"; + videoWidth: number; + videoHeight: number; + meanKeypointConfidence: number; + trackedKeypointCount: number; + qualityFlags: number; + poseFrameBase64: string; +}; + +type CalibrationState = + | { kind: "idle"; hint?: string } + | { + kind: "starting"; + catchFrame?: CalibrationFrame; + finishFrame?: CalibrationFrame; + hint?: string; + } + | { + kind: "ready"; + catchFrame?: CalibrationFrame; + finishFrame?: CalibrationFrame; + hint?: string; + }; + +type PoseQuality = { + trackedKeypointCount: number; + meanConfidence: number; + qualityFlags: number; + landmarkCount: number; + poseFrameBase64: string; +}; + +const EMPTY_QUALITY: PoseQuality = { + trackedKeypointCount: 0, + meanConfidence: 0, + qualityFlags: 0, + landmarkCount: 0, + poseFrameBase64: "", +}; + +export default function MocapCapturePage() { + const videoRef = useRef(null); + const streamRef = useRef(null); + const recorderRef = useRef(null); + const uploaderRef = useRef(null); + const sourceRef = useRef(null); + const calibrationSourceRef = useRef(null); + const startedAtRef = useRef(0); + const latestPoseFrameRef = useRef(EMPTY_QUALITY); + const qualityHistoryRef = useRef([]); + const effectiveFpsSamplesRef = useRef([]); + const latestCameraReadinessRef = useRef(null); + const engineRef = useRef(null); + const cueDismissTimerRef = useRef | null>(null); + const audioEnabledRef = useRef(false); + const recordOnlyRef = useRef(false); + + const [state, setState] = useState({ kind: "idle" }); + const [calibration, setCalibration] = useState({ + kind: "idle", + }); + const [framesEncoded, setFramesEncoded] = useState(0); + const [poseStatus, setPoseStatus] = useState("idle"); + const [perspective, setPerspective] = useState<"side-left" | "side-right">( + "side-right", + ); + const [elapsedSec, setElapsedSec] = useState(0); + const [quality, setQuality] = useState(EMPTY_QUALITY); + const [cameraReadiness, setCameraReadiness] = + useState(null); + const [framingDegraded, setFramingDegraded] = useState(false); + const [sessionQualityFlags, setSessionQualityFlags] = useState([]); + const [recordOnly, setRecordOnly] = useState(false); + const [useSidecar, setUseSidecar] = useState(false); + const [sidecarHealth, setSidecarHealth] = useState(null); + const [sidecarError, setSidecarError] = useState(null); + const [recordOnlyReason, setRecordOnlyReason] = + useState(null); + const [captureSupport, setCaptureSupport] = + useState(null); + const [activeCue, setActiveCue] = useState(null); + const [sessionFaults, setSessionFaults] = useState([]); + const [audioEnabled, setAudioEnabled] = useState(false); + const [verbosity, setVerbosity] = useState<"quiet" | "verbose">("quiet"); + + // Hydrate live-cue prefs from persisted settings on mount. + useEffect(() => { + const prefs = settings.getMocapSettings().mocapPreferences; + setAudioEnabled(prefs.audioEnabled); + setVerbosity(prefs.verbosity); + audioEnabledRef.current = prefs.audioEnabled; + + const support = evaluateMocapCaptureSupport(readBrowserMocapCapabilities()); + setCaptureSupport(support); + if (support.recordOnlyRecommended) { + setRecordOnly(true); + setRecordOnlyReason(support.reason); + } + }, []); + + useEffect(() => { + audioEnabledRef.current = audioEnabled; + if (!audioEnabled) cancelSpokenCues(); + }, [audioEnabled]); + + useEffect(() => { + recordOnlyRef.current = + recordOnly || + Boolean( + captureSupport?.recordOnlyRecommended && !captureSupport.livePoseSupported, + ); + }, [recordOnly, captureSupport]); + + const updateAudioEnabled = useCallback((next: boolean) => { + setAudioEnabled(next); + const current = settings.getMocapSettings().mocapPreferences; + settings.updateMocapSettings({ + mocapPreferences: { ...current, audioEnabled: next }, + }); + }, []); + + const updateVerbosity = useCallback((next: "quiet" | "verbose") => { + setVerbosity(next); + const current = settings.getMocapSettings().mocapPreferences; + settings.updateMocapSettings({ + mocapPreferences: { ...current, verbosity: next }, + }); + }, []); + + const clearCueDismissTimer = useCallback(() => { + if (cueDismissTimerRef.current) { + clearTimeout(cueDismissTimerRef.current); + cueDismissTimerRef.current = null; + } + }, []); + + const dismissCue = useCallback(() => { + clearCueDismissTimer(); + cancelSpokenCues(); + setActiveCue(null); + }, [clearCueDismissTimer]); + + useEffect(() => { + if (state.kind !== "capturing") return; + const t = setInterval(() => { + setElapsedSec((Date.now() - startedAtRef.current) / 1000); + }, 250); + return () => clearInterval(t); + }, [state.kind]); + + const handlePoseFrame = useCallback( + ( + info: PoseQuality & { + framesEncoded: number; + }, + monitorDegradedFraming: boolean, + ) => { + const nextQuality: PoseQuality = { + trackedKeypointCount: info.trackedKeypointCount, + meanConfidence: info.meanConfidence, + qualityFlags: info.qualityFlags, + landmarkCount: info.landmarkCount, + poseFrameBase64: info.poseFrameBase64, + }; + latestPoseFrameRef.current = nextQuality; + setFramesEncoded(info.framesEncoded); + setQuality(nextQuality); + + const nowMs = Date.now(); + const decodedFrame = decodeBase64PoseFrame(info.poseFrameBase64); + qualityHistoryRef.current = [ + ...qualityHistoryRef.current.filter( + (frame) => frame.timestampMs >= nowMs - QUALITY_HISTORY_MS, + ), + { + timestampMs: nowMs, + trackedKeypointCount: info.trackedKeypointCount, + meanConfidence: info.meanConfidence, + qualityFlags: info.qualityFlags, + keypoints: Array.isArray(decodedFrame?.keypoints) + ? decodedFrame.keypoints + : undefined, + }, + ]; + const readiness = evaluateCameraReadiness(qualityHistoryRef.current, { + capturePerspective: perspective, + nowMs, + }); + latestCameraReadinessRef.current = readiness; + setCameraReadiness(readiness); + + effectiveFpsSamplesRef.current = [ + ...effectiveFpsSamplesRef.current.filter( + (sample) => sample.timestampMs >= nowMs - 5000, + ), + { timestampMs: nowMs, effectiveFps: readiness.effectiveFps }, + ]; + if ( + !monitorDegradedFraming && + !recordOnlyRef.current && + hasSustainedLowEffectiveFps(effectiveFpsSamplesRef.current, { nowMs }) + ) { + const support = lowFpsRecordOnlySupport(); + setCaptureSupport(support); + setRecordOnly(true); + setRecordOnlyReason(support.reason); + } + + // Feed the live coaching engine when active. + if (engineRef.current) { + if (decodedFrame) engineRef.current.pushFrame(decodedFrame); + } + + if (!monitorDegradedFraming) return; + + if (!readiness.sustainedDegraded) { + setFramingDegraded(false); + return; + } + + setFramingDegraded(true); + setSessionQualityFlags((flags) => + appendUniqueFlags(flags, [ + "camera-readiness-degraded", + ...readiness.qualityFlags.filter((flag) => flag !== "ok"), + ]), + ); + }, + [perspective], + ); + + const teardown = useCallback(async () => { + calibrationSourceRef.current = null; + sourceRef.current = null; + recorderRef.current = null; + uploaderRef.current = null; + engineRef.current = null; + clearCueDismissTimer(); + cancelSpokenCues(); + if (streamRef.current) { + for (const track of streamRef.current.getTracks()) track.stop(); + streamRef.current = null; + } + if (videoRef.current) { + videoRef.current.srcObject = null; + } + }, [clearCueDismissTimer]); + + const handleError = useCallback( + async (err: unknown, sessionId?: string) => { + const message = err instanceof Error ? err.message : String(err); + setState({ kind: "error", message }); + try { + await sourceRef.current?.stop(); + } catch { + // ignore + } + try { + await calibrationSourceRef.current?.stop(); + } catch { + // ignore + } + try { + recorderRef.current?.stop(); + } catch { + // ignore + } + await uploaderRef.current?.drain(); + await teardown(); + if (sessionId) { + // Best-effort delete of the abandoned row + fetch(`/api/mocap/sessions/${sessionId}`, { method: "DELETE" }).catch( + () => {}, + ); + } + }, + [teardown], + ); + + const startCalibration = useCallback(async () => { + setCalibration({ kind: "starting" }); + setPoseStatus("idle"); + setFramesEncoded(0); + setQuality(EMPTY_QUALITY); + setCameraReadiness(null); + qualityHistoryRef.current = []; + effectiveFpsSamplesRef.current = []; + latestCameraReadinessRef.current = null; + latestPoseFrameRef.current = EMPTY_QUALITY; + try { + if (!streamRef.current) { + const stream = await navigator.mediaDevices.getUserMedia({ + video: { width: 1280, height: 720, frameRate: CAPTURE_FPS }, + audio: false, + }); + streamRef.current = stream; + } + const video = videoRef.current!; + video.srcObject = streamRef.current; + await video.play(); + + await calibrationSourceRef.current?.stop().catch(() => {}); + const source = new BrowserPoseSource({ + videoEl: video, + uploadPoseStream: false, + onStatus: (s) => setPoseStatus(s), + onFrame: (info) => handlePoseFrame(info, false), + onError: (err) => { + setCalibration({ kind: "idle", hint: err.message }); + }, + }); + calibrationSourceRef.current = source; + await source.init(); + source.start(); + setCalibration({ kind: "ready" }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + setCalibration({ kind: "idle", hint: message }); + await teardown(); + } + }, [handlePoseFrame, teardown]); + + const captureCalibrationFrame = useCallback( + (pose: CalibrationPose) => { + const latest = latestPoseFrameRef.current; + if (!latestCameraReadinessRef.current?.ready) { + setCalibration((current) => ({ + ...current, + hint: + "Move the rower and erg fully into the side view, then hold still for a second.", + })); + return; + } + + const video = videoRef.current; + const frame: CalibrationFrame = { + pose, + capturedAt: new Date().toISOString(), + capturePerspective: perspective, + videoWidth: video?.videoWidth ?? 0, + videoHeight: video?.videoHeight ?? 0, + meanKeypointConfidence: latest.meanConfidence, + trackedKeypointCount: latest.trackedKeypointCount, + qualityFlags: latest.qualityFlags, + poseFrameBase64: latest.poseFrameBase64, + }; + + setCalibration((current) => ({ + kind: "ready", + catchFrame: + pose === "catch" + ? frame + : "catchFrame" in current + ? current.catchFrame + : undefined, + finishFrame: + pose === "finish" + ? frame + : "finishFrame" in current + ? current.finishFrame + : undefined, + })); + }, + [perspective], + ); + + const start = useCallback(async () => { + // Sidecar path: skip browser camera/calibration, use sidecar-3d perspective + if (useSidecar) { + setState({ kind: "starting" }); + try { + const health = await checkSidecarHealth(SIDECAR_DEFAULT_PORT); + setSidecarHealth(health); + setSidecarError(null); + if (health.status !== "ready") { + setState({ kind: "error", message: `Sidecar not ready: ${health.status}` }); + return; + } + const createRes = await fetch("/api/mocap/sessions", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + source: "sidecar", + captureModelVersion: `freemocap-sidecar@schemaV${health.schemaVersion}`, + capturePerspective: "sidecar-3d", + captureFps: health.fps, + cameraCount: health.cameras, + }), + }); + if (!createRes.ok) throw new Error(`Create session failed: ${createRes.status}`); + const created: { id: string } = await createRes.json(); + // Arm the sidecar via the connect endpoint + await fetch(`/api/mocap/sessions/${created.id}/sidecar/connect`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + }); + setState({ kind: "capturing", sessionId: created.id, startedAt: Date.now() }); + } catch (err) { + const msg = err instanceof Error ? err.message : "Sidecar error"; + setSidecarError(msg); + setState({ kind: "error", message: msg }); + } + return; + } + + const captureRecordOnly = + recordOnly || + Boolean( + captureSupport?.recordOnlyRecommended && !captureSupport.livePoseSupported, + ); + if (captureSupport && !captureSupport.videoCaptureSupported) { + setState({ kind: "error", message: captureSupport.message }); + return; + } + + const calibrationFrames = getCalibrationFrames(calibration); + if ( + !captureRecordOnly && + (!calibrationFrames || !latestCameraReadinessRef.current?.ready) + ) { + setCalibration((current) => ({ + ...current, + hint: + "Complete catch and finish calibration with the rower and erg fully in frame before recording.", + })); + return; + } + + setState({ kind: "starting" }); + let sessionId: string | undefined; + try { + const stream = + streamRef.current ?? + (await navigator.mediaDevices.getUserMedia({ + video: { width: 1280, height: 720, frameRate: CAPTURE_FPS }, + audio: false, + })); + streamRef.current = stream; + const video = videoRef.current!; + video.srcObject = stream; + await video.play(); + await calibrationSourceRef.current?.stop(); + calibrationSourceRef.current = null; + + // Spin up the live coaching engine (skipped when in record-only mode). + const calibrationFramesForEngine = buildEngineCalibration( + perspective, + calibrationFrames?.catchFrame, + calibrationFrames?.finishFrame, + ); + const mocapPrefs = settings.getMocapSettings(); + if (!captureRecordOnly) { + engineRef.current = new LiveCoachingEngine({ + fps: CAPTURE_FPS, + capturePerspective: perspective, + calibration: calibrationFramesForEngine, + thresholds: mocapPrefs.postureThresholds.thresholds, + minSeverity: + mocapPrefs.mocapPreferences.verbosity === "verbose" + ? "info" + : "warning", + onCue: (cue) => { + clearCueDismissTimer(); + setActiveCue(cue); + cueDismissTimerRef.current = setTimeout(() => { + setActiveCue(null); + cueDismissTimerRef.current = null; + }, 4000); + if (audioEnabledRef.current) { + speakCue(cue.audioHint); + } + }, + }); + } else { + engineRef.current = null; + } + + const createRes = await fetch("/api/mocap/sessions", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + source: "browser", + captureModelVersion: CAPTURE_MODEL_VERSION, + capturePerspective: perspective, + captureFps: CAPTURE_FPS, + recordOnly: captureRecordOnly, + calibrationCatchFrame: calibrationFrames?.catchFrame, + calibrationFinishFrame: calibrationFrames?.finishFrame, + }), + }); + if (!createRes.ok) { + throw new Error(`Create session failed: ${createRes.status}`); + } + const created: { id: string } = await createRes.json(); + sessionId = created.id; + + uploaderRef.current = new VideoUploader(sessionId, (err) => + handleError(err, sessionId), + ); + + const mimeType = pickRecorderMime(); + const recorder = new MediaRecorder(stream, { + mimeType, + videoBitsPerSecond: 2_500_000, + }); + recorder.ondataavailable = (event) => { + if (event.data && event.data.size > 0) { + uploaderRef.current?.enqueue(event.data); + } + }; + recorder.onerror = (event) => { + handleError( + (event as ErrorEvent).error ?? new Error("MediaRecorder error"), + sessionId, + ); + }; + recorderRef.current = recorder; + + if (!captureRecordOnly) { + const source = new BrowserPoseSource({ + sessionId, + videoEl: video, + onStatus: (s) => setPoseStatus(s), + onFrame: (info) => handlePoseFrame(info, true), + onError: (err) => handleError(err, sessionId), + }); + sourceRef.current = source; + await source.init(); + } else { + sourceRef.current = null; + setPoseStatus("stopped"); + } + + recorder.start(VIDEO_TIMESLICE_MS); + sourceRef.current?.start(); + startedAtRef.current = Date.now(); + setElapsedSec(0); + setFramesEncoded(0); + setFramingDegraded(false); + setSessionQualityFlags( + captureRecordOnly + ? recordOnlySessionFlags([], recordOnlyReason) + : [], + ); + qualityHistoryRef.current = []; + effectiveFpsSamplesRef.current = []; + latestCameraReadinessRef.current = null; + setCameraReadiness(null); + setState({ + kind: "capturing", + sessionId, + startedAt: startedAtRef.current, + }); + } catch (err) { + await handleError(err, sessionId); + } + }, [ + calibration, + captureSupport, + handleError, + handlePoseFrame, + perspective, + recordOnly, + recordOnlyReason, + useSidecar, + clearCueDismissTimer, + ]); + + const stop = useCallback(async () => { + if (state.kind !== "capturing") return; + const sessionId = state.sessionId; + const captureWasRecordOnly = recordOnlyRef.current; + setState({ kind: "stopping", sessionId }); + try { + const recorder = recorderRef.current; + if (recorder && recorder.state !== "inactive") { + await new Promise((resolve) => { + recorder.onstop = () => resolve(); + recorder.stop(); + }); + } + await sourceRef.current?.stop(); + // Drain any pending strokes from the live engine before tearing it down. + try { + engineRef.current?.flush(); + } catch { + // non-fatal + } + engineRef.current = null; + clearCueDismissTimer(); + cancelSpokenCues(); + await uploaderRef.current?.drain(); + + const durationSec = (Date.now() - startedAtRef.current) / 1000; + const finalizeQualityFlags = captureWasRecordOnly + ? recordOnlySessionFlags(sessionQualityFlags, recordOnlyReason) + : sessionQualityFlags; + const finalizeRes = await fetch( + `/api/mocap/sessions/${sessionId}/finalize`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + durationSec, + qualityScore: captureWasRecordOnly + ? undefined + : qualityScoreFor(latestPoseFrameRef.current), + qualityFlags: finalizeQualityFlags, + skipAnalysis: captureWasRecordOnly, + }), + }, + ); + if (!finalizeRes.ok) { + throw new Error(`Finalize failed: ${finalizeRes.status}`); + } + const finalized: { + id: string; + durationSec: number; + frameCount: number; + } = await finalizeRes.json(); + await teardown(); + setCalibration({ kind: "idle" }); + + // Fetch faults for end-of-session summary + let faults: PostureFault[] = []; + try { + const sessionRes = await fetch(`/api/mocap/sessions/${finalized.id}`); + if (sessionRes.ok) { + const sessionData = await sessionRes.json(); + faults = (sessionData.session?.postureFaults ?? []) as PostureFault[]; + setSessionFaults(faults); + } + } catch { + // non-fatal — summary just won't show + } + + setState({ + kind: "done", + sessionId: finalized.id, + durationSec: finalized.durationSec, + frameCount: finalized.frameCount, + recordOnly: captureWasRecordOnly, + }); + } catch (err) { + await handleError(err, sessionId); + } + }, [ + state, + handleError, + sessionQualityFlags, + teardown, + recordOnlyReason, + clearCueDismissTimer, + ]); + + useEffect(() => { + return () => { + sourceRef.current?.stop().catch(() => {}); + calibrationSourceRef.current?.stop().catch(() => {}); + recorderRef.current?.stop(); + teardown(); + }; + }, [teardown]); + + useEffect(() => { + if (state.kind !== "capturing") return; + const sessionId = state.sessionId; + const onPageHide = () => { + try { + recorderRef.current?.requestData?.(); + recorderRef.current?.stop(); + } catch { + // ignore + } + try { + const durationSec = (Date.now() - startedAtRef.current) / 1000; + const captureWasRecordOnly = recordOnlyRef.current; + navigator.sendBeacon?.( + `/api/mocap/sessions/${sessionId}/finalize`, + new Blob( + [ + JSON.stringify({ + durationSec, + qualityScore: captureWasRecordOnly + ? undefined + : qualityScoreFor(latestPoseFrameRef.current), + qualityFlags: captureWasRecordOnly + ? recordOnlySessionFlags(sessionQualityFlags, recordOnlyReason) + : sessionQualityFlags, + skipAnalysis: captureWasRecordOnly, + }), + ], + { + type: "application/json", + }, + ), + ); + } catch { + // ignore + } + }; + window.addEventListener("pagehide", onPageHide); + return () => window.removeEventListener("pagehide", onPageHide); + }, [state, sessionQualityFlags, recordOnlyReason]); + + const calibrationFrames = getCalibrationFrames(calibration); + const nextCalibrationPose: CalibrationPose | null = !( + "catchFrame" in calibration && calibration.catchFrame + ) + ? "catch" + : !("finishFrame" in calibration && calibration.finishFrame) + ? "finish" + : null; + const cameraReady = cameraReadiness?.ready ?? false; + const videoCaptureSupported = captureSupport?.videoCaptureSupported ?? true; + const livePoseSupported = captureSupport?.livePoseSupported ?? true; + const recordOnlyForced = Boolean( + captureSupport?.recordOnlyRecommended && !captureSupport.livePoseSupported, + ); + const recordOnlyActive = recordOnly || recordOnlyForced; + const captureBusy = + state.kind === "capturing" || + state.kind === "starting" || + state.kind === "stopping"; + const canRecord = + !captureBusy && + videoCaptureSupported && + (recordOnlyActive || + (livePoseSupported && + calibration.kind === "ready" && + Boolean(calibrationFrames) && + cameraReady)); + + return ( +
+ + + Motion capture session + + Single-webcam pose capture. Camera permission is requested only when + you click Start. + + + +
+ + + + + {useSidecar && sidecarHealth && ( + + Sidecar ready — {sidecarHealth.cameras} camera{sidecarHealth.cameras !== 1 ? "s" : ""}, {sidecarHealth.fps} fps + + )} + {useSidecar && sidecarError && ( + + {sidecarError} + + )} + + {calibration.kind === "idle" && + (state.kind === "idle" || state.kind === "done") && + livePoseSupported && + !recordOnlyActive ? ( + + ) : null} + {calibration.kind === "starting" ? ( + + ) : null} + {calibration.kind === "ready" && nextCalibrationPose ? ( + + ) : null} + {calibration.kind === "ready" && + (state.kind === "idle" || state.kind === "done") && + livePoseSupported && + !recordOnlyActive ? ( + + ) : null} + {state.kind === "idle" || state.kind === "done" ? ( + + ) : null} + {state.kind === "starting" ? ( + + ) : null} + {state.kind === "capturing" ? ( + + ) : null} + {state.kind === "stopping" ? ( + + ) : null} +
+ + {captureSupport && !captureSupport.videoCaptureSupported ? ( +
+ {captureSupport.message} +
+ ) : null} + + {recordOnlyActive && videoCaptureSupported ? ( +
+ {captureSupport?.recordOnlyRecommended + ? captureSupport.message + : "Record-only mode saves video without live posture analysis. You can review the video later, but no posture rows are created during capture."} +
+ ) : null} + +
+ + + +
+ + {cameraReadiness && !cameraReady && !recordOnlyActive ? ( +
+ {cameraReadiness.message} +
+ ) : null} + + {calibration.hint ? ( +
+ {calibration.hint} +
+ ) : null} + +
+
+ + {framingDegraded ? ( +
+ Camera readiness degraded for several seconds. Check lighting, + tracking, and side-view framing before continuing. +
+ ) : null} + +
+ + + + + + + +
+ + {state.kind === "done" ? ( +
+
+ Session {state.sessionId} stored. +
+
+ {state.recordOnly + ? "Video-only recording" + : `${state.frameCount} pose frames`}{" "} + · {state.durationSec.toFixed(1)}s duration +
+ {!state.recordOnly ? ( + + ) : null} +
+ + View replay → + + · + + All sessions + +
+
+ ) : null} + + {activeCue ? ( +
+
+ {activeCue.severity === "critical" ? "⚠ " : "ℹ "} + {activeCue.message} +
+ {activeCue.drills.length > 0 ? ( +
+ Drills: {activeCue.drills.join(" · ")} +
+ ) : null} + +
+ ) : null} + + {state.kind === "error" ? ( +
+ Error: {state.message} +
+ ) : null} + + +
+ ); +} + +function Stat({ label, value }: { label: string; value: string }) { + return ( +
+
{label}
+
{value}
+
+ ); +} + +function CalibrationStep({ label, done }: { label: string; done: boolean }) { + return ( +
+
{label}
+
+ {done ? "Ready" : "Needed"} +
+
+ ); +} + +function getCalibrationFrames( + calibration: CalibrationState, +): { catchFrame: CalibrationFrame; finishFrame: CalibrationFrame } | null { + if ( + "catchFrame" in calibration && + calibration.catchFrame && + "finishFrame" in calibration && + calibration.finishFrame + ) { + return { + catchFrame: calibration.catchFrame, + finishFrame: calibration.finishFrame, + }; + } + return null; +} + +function qualityScoreFor(quality: PoseQuality): number { + const trackedRatio = quality.trackedKeypointCount / 33; + return Math.max(0, Math.min(1, quality.meanConfidence * trackedRatio)); +} + +function recordOnlySessionFlags( + current: readonly string[], + reason: RecordOnlyReason | null, +): string[] { + return appendUniqueFlags(current, ["record-only", recordOnlyQualityFlag(reason)]); +} + +function appendUniqueFlags( + current: readonly string[], + next: readonly string[], +): string[] { + return [...new Set([...current, ...next])]; +} + +function SessionCoachingSummary({ faults }: { faults: PostureFault[] }) { + if (faults.length === 0) return null; + + // Aggregate by fault type: total weight = count × severity weight + const typeMap = new Map(); + for (const f of faults) { + const existing = typeMap.get(f.faultType); + const w = SEVERITY_WEIGHT[f.severity] ?? 1; + if (!existing) { + typeMap.set(f.faultType, { count: 1, weight: w, severity: f.severity }); + } else { + existing.count++; + existing.weight += w; + if ((SEVERITY_WEIGHT[f.severity] ?? 1) > (SEVERITY_WEIGHT[existing.severity] ?? 1)) { + existing.severity = f.severity; + } + } + } + + const top3 = [...typeMap.entries()] + .sort((a, b) => b[1].weight - a[1].weight) + .slice(0, 3); + + if (top3.length === 0) return null; + + const cues = getCoachingCues( + faults.filter((f) => top3.some(([t]) => t === f.faultType)), + { strokeCount: faults.length }, + { minSeverity: "info" }, + ); + + return ( +
+
Session summary
+ {top3.map(([faultType, info]) => { + const cue = cues.find((c) => c.faultType === faultType); + const label = faultType.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()); + return ( +
+
{label} × {info.count}
+ {cue?.drills.map((d) => ( +
→ {d}
+ ))} +
+ ); + })} +
+ ); +} + +function decodeBase64PoseFrame( + base64: string, +): PoseAnalysisFrame | null { + if (!base64) return null; + try { + const binary = atob(base64); + if (binary.length < BYTES_PER_FRAME_V1) return null; + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); + const decoded = decodeFrame(bytes, 0); + return { + timestampMs: decoded.timestampMs, + keypoints: keypointTripletsToPosePoints(decoded.keypoints), + qualityFlags: decoded.qualityFlags, + }; + } catch { + return null; + } +} + +function buildEngineCalibration( + capturePerspective: "side-left" | "side-right", + catchFrameBase64: { poseFrameBase64: string } | undefined, + finishFrameBase64: { poseFrameBase64: string } | undefined, +): Calibration | undefined { + const catchFrame = catchFrameBase64?.poseFrameBase64 + ? decodeBase64PoseFrame(catchFrameBase64.poseFrameBase64) ?? undefined + : undefined; + const finishFrame = finishFrameBase64?.poseFrameBase64 + ? decodeBase64PoseFrame(finishFrameBase64.poseFrameBase64) ?? undefined + : undefined; + if (!catchFrame && !finishFrame) return undefined; + return { + capturePerspective, + catchFrame, + finishFrame, + }; +} + +function pickRecorderMime(): string { + const candidates = [ + "video/webm;codecs=vp9", + "video/webm;codecs=vp8", + "video/webm", + "video/mp4", + ]; + for (const c of candidates) { + if ( + typeof MediaRecorder !== "undefined" && + typeof MediaRecorder.isTypeSupported === "function" && + MediaRecorder.isTypeSupported(c) + ) { + return c; + } + } + return "video/webm"; +} diff --git a/src/app/mocap/sessions/[id]/page.tsx b/src/app/mocap/sessions/[id]/page.tsx new file mode 100644 index 0000000..defe3df --- /dev/null +++ b/src/app/mocap/sessions/[id]/page.tsx @@ -0,0 +1,1095 @@ +"use client"; + +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { useParams } from "next/navigation"; +import Link from "next/link"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { + HEADER_SIZE, + BYTES_PER_FRAME_V1, + KEYPOINTS_PER_FRAME_V1, + decodeHeader, + decodeFrame, + frameByteOffset, + type PoseStreamHeader, +} from "@/lib/mocap/poseFrameStream"; +import { + buildReplayComparisonOptions, + countFaultsForStroke, +} from "@/lib/mocap/replayComparison"; + +// MediaPipe 33-keypoint skeleton connections for side-view rowing +const SKELETON_CONNECTIONS: [number, number][] = [ + [11, 12], // shoulders + [11, 13], [13, 15], // left arm + [12, 14], [14, 16], // right arm + [11, 23], [12, 24], // torso sides + [23, 24], // hips + [23, 25], [25, 27], // left leg + [24, 26], [26, 28], // right leg + [27, 29], [27, 31], // left foot + [28, 30], [28, 32], // right foot +]; + +interface PhaseBoundaries { + catchFrameIndex: number; + driveStartFrameIndex: number; + finishFrameIndex: number; + recoveryStartFrameIndex: number; + nextCatchFrameIndex: number; + confidence: number; + csvMatchOffsetMs?: number | null; +} + +interface SessionStrokeMetric { + id: string; + strokeIndex: number; + phaseBoundariesJson: PhaseBoundaries; + metricsJson: SessionPostureMetrics; + segmentationSource: string; +} + +interface SessionPostureMetrics { + backAngleAtCatchDeg?: number; + backAngleAtFinishDeg?: number; + laybackAngleDeg?: number; + hipKneeOpeningOffsetFrames?: number | null; + armBendBeforeLegsCompleteFrames?: number | null; + recoveryDriveRatio?: number; +} + +interface FaultEvidence { + metric: string; + value: number; + threshold: number; + frameIndex?: number; +} + +interface SessionFault { + id: string; + strokeIndex: number; + faultType: string; + severity: string; + phase: string; + evidenceJson: FaultEvidence; +} + +interface MocapSessionDetail { + id: string; + status: string; + capturePerspective: string; + captureFps: number; + durationSec: number; + qualityScore: number | null; + qualityFlags: string[]; + createdAt: string; + strokePostureMetrics: SessionStrokeMetric[]; + postureFaults: SessionFault[]; +} + +async function fetchPoseHeader(id: string): Promise { + try { + const res = await fetch(`/api/mocap/sessions/${id}/pose-stream`, { + headers: { Range: `bytes=0-${HEADER_SIZE - 1}` }, + }); + if (!res.ok && res.status !== 206) return null; + const buf = new Uint8Array(await res.arrayBuffer()); + return decodeHeader(buf); + } catch { + return null; + } +} + +async function fetchPoseFrameAtIndex( + id: string, + frameIndex: number, +): Promise { + try { + const start = frameByteOffset(frameIndex); + const end = start + BYTES_PER_FRAME_V1 - 1; + const res = await fetch(`/api/mocap/sessions/${id}/pose-stream`, { + headers: { Range: `bytes=${start}-${end}` }, + }); + if (!res.ok && res.status !== 206) return null; + const buf = new Uint8Array(await res.arrayBuffer()); + const frame = decodeFrame(buf, 0); + return frame.keypoints; + } catch { + return null; + } +} + +function drawSkeleton( + ctx: CanvasRenderingContext2D, + keypoints: Float32Array, + canvasW: number, + canvasH: number, + videoW: number, + videoH: number, +) { + // Compute letterbox bounds (object-contain) + const videoAspect = videoW / videoH; + const canvasAspect = canvasW / canvasH; + let drawW: number, drawH: number, drawX: number, drawY: number; + if (videoAspect > canvasAspect) { + drawW = canvasW; + drawH = canvasW / videoAspect; + drawX = 0; + drawY = (canvasH - drawH) / 2; + } else { + drawH = canvasH; + drawW = canvasH * videoAspect; + drawX = (canvasW - drawW) / 2; + drawY = 0; + } + + ctx.clearRect(0, 0, canvasW, canvasH); + + // Connections + ctx.strokeStyle = "rgba(0, 220, 120, 0.85)"; + ctx.lineWidth = 2; + for (const [a, b] of SKELETON_CONNECTIONS) { + const confA = keypoints[a * 3 + 2]; + const confB = keypoints[b * 3 + 2]; + if (confA < 0.3 || confB < 0.3) continue; + const ax = drawX + keypoints[a * 3] * drawW; + const ay = drawY + keypoints[a * 3 + 1] * drawH; + const bx = drawX + keypoints[b * 3] * drawW; + const by = drawY + keypoints[b * 3 + 1] * drawH; + ctx.beginPath(); + ctx.moveTo(ax, ay); + ctx.lineTo(bx, by); + ctx.stroke(); + } + + // Keypoints + ctx.fillStyle = "rgba(255, 255, 255, 0.9)"; + for (let i = 0; i < KEYPOINTS_PER_FRAME_V1; i++) { + const conf = keypoints[i * 3 + 2]; + if (conf < 0.3) continue; + const x = drawX + keypoints[i * 3] * drawW; + const y = drawY + keypoints[i * 3 + 1] * drawH; + ctx.beginPath(); + ctx.arc(x, y, 3, 0, 2 * Math.PI); + ctx.fill(); + } +} + +function severityColor(severity: string) { + if (severity === "critical") return "bg-red-500"; + if (severity === "warning") return "bg-yellow-500"; + return "bg-blue-400"; +} + +function faultLabel(faultType: string): string { + return faultType.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()); +} + +function fmtTime(sec: number): string { + const m = Math.floor(sec / 60); + const s = Math.floor(sec % 60); + return `${m}:${String(s).padStart(2, "0")}`; +} + +function fmtDate(iso: string): string { + return new Date(iso).toLocaleString(undefined, { + month: "short", + day: "numeric", + year: "numeric", + hour: "2-digit", + minute: "2-digit", + }); +} + +type CompareRole = "fault" | "comparison"; +type ComparePhase = "catch" | "finish"; + +function comparisonFrameKey(role: CompareRole, phase: ComparePhase): string { + return `${role}-${phase}`; +} + +export default function MocapReplayPage() { + const { id } = useParams<{ id: string }>(); + + const videoRef = useRef(null); + const canvasRef = useRef(null); + const animRef = useRef(null); + const fetchingRef = useRef(false); + const lastFrameIndexRef = useRef(-1); + + const [session, setSession] = useState(null); + const [poseHeader, setPoseHeader] = useState(null); + const [poseHeaderChecked, setPoseHeaderChecked] = useState(false); + const [loadError, setLoadError] = useState(null); + const [currentTime, setCurrentTime] = useState(0); + const [selectedFault, setSelectedFault] = useState(null); + const [selectedStroke, setSelectedStroke] = useState(null); + const [reanalyzing, setReanalyzing] = useState(false); + const [reanalyzeError, setReanalyzeError] = useState(null); + const [compareFaultStroke, setCompareFaultStroke] = useState(null); + const [compareStroke, setCompareStroke] = useState(null); + const [comparisonFrames, setComparisonFrames] = useState< + Record + >({}); + const [comparisonLoading, setComparisonLoading] = useState(false); + + const comparisonOptions = useMemo( + () => + session + ? buildReplayComparisonOptions( + session.strokePostureMetrics, + session.postureFaults, + compareFaultStroke, + ) + : { + faultStrokeOptions: [], + cleanStrokeOptions: [], + defaultFaultStrokeIndex: null, + defaultComparisonStrokeIndex: null, + }, + [compareFaultStroke, session], + ); + + const metricsByStroke = useMemo(() => { + const map = new Map(); + for (const metric of session?.strokePostureMetrics ?? []) { + map.set(metric.strokeIndex, metric); + } + return map; + }, [session]); + + const faultsByStroke = useMemo(() => { + const map = new Map(); + for (const fault of session?.postureFaults ?? []) { + if (!map.has(fault.strokeIndex)) map.set(fault.strokeIndex, []); + map.get(fault.strokeIndex)!.push(fault); + } + return map; + }, [session]); + + // Load session data + useEffect(() => { + fetch(`/api/mocap/sessions/${id}`) + .then((r) => { + if (!r.ok) throw new Error(`${r.status}`); + return r.json(); + }) + .then((data) => setSession(data.session)) + .catch((e) => setLoadError(e.message)); + }, [id]); + + // Load pose stream header + useEffect(() => { + if (!session || session.status !== "ready") return; + setPoseHeaderChecked(false); + fetchPoseHeader(id).then((header) => { + setPoseHeader(header); + setPoseHeaderChecked(true); + }); + }, [id, session]); + + useEffect(() => { + if (!session || session.strokePostureMetrics.length === 0) return; + const isValidFaultStroke = comparisonOptions.faultStrokeOptions.some( + (option) => option.strokeIndex === compareFaultStroke, + ); + if (!isValidFaultStroke) { + setCompareFaultStroke(comparisonOptions.defaultFaultStrokeIndex); + } + }, [compareFaultStroke, comparisonOptions, session]); + + useEffect(() => { + if (!session || session.strokePostureMetrics.length === 0) return; + const isValidComparisonStroke = + compareStroke !== null && + compareStroke !== compareFaultStroke && + comparisonOptions.cleanStrokeOptions.includes(compareStroke); + + if (!isValidComparisonStroke) { + setCompareStroke(comparisonOptions.defaultComparisonStrokeIndex); + } + }, [compareFaultStroke, compareStroke, comparisonOptions, session]); + + useEffect(() => { + if ( + !session || + !poseHeader || + compareFaultStroke === null || + compareStroke === null + ) { + setComparisonFrames({}); + return; + } + + const faultMetric = metricsByStroke.get(compareFaultStroke); + const comparisonMetric = metricsByStroke.get(compareStroke); + if (!faultMetric || !comparisonMetric) { + setComparisonFrames({}); + return; + } + + let cancelled = false; + setComparisonLoading(true); + + const frameRequests: Array<[string, number]> = [ + [ + comparisonFrameKey("fault", "catch"), + faultMetric.phaseBoundariesJson.catchFrameIndex, + ], + [ + comparisonFrameKey("fault", "finish"), + faultMetric.phaseBoundariesJson.finishFrameIndex, + ], + [ + comparisonFrameKey("comparison", "catch"), + comparisonMetric.phaseBoundariesJson.catchFrameIndex, + ], + [ + comparisonFrameKey("comparison", "finish"), + comparisonMetric.phaseBoundariesJson.finishFrameIndex, + ], + ]; + + Promise.all( + frameRequests.map(async ([key, frameIndex]) => [ + key, + await fetchPoseFrameAtIndex(id, frameIndex), + ] as const), + ) + .then((entries) => { + if (cancelled) return; + setComparisonFrames(Object.fromEntries(entries)); + }) + .finally(() => { + if (!cancelled) setComparisonLoading(false); + }); + + return () => { + cancelled = true; + }; + }, [compareFaultStroke, compareStroke, id, metricsByStroke, poseHeader, session]); + + // Resize canvas to match video display dimensions + useEffect(() => { + const video = videoRef.current; + const canvas = canvasRef.current; + if (!video || !canvas) return; + const onMeta = () => { + canvas.width = video.clientWidth || 1280; + canvas.height = video.clientHeight || 720; + }; + video.addEventListener("loadedmetadata", onMeta); + return () => video.removeEventListener("loadedmetadata", onMeta); + }, []); + + const renderFrame = useCallback( + async (frameIndex: number) => { + const video = videoRef.current; + const canvas = canvasRef.current; + if (!video || !canvas || fetchingRef.current) return; + if (frameIndex === lastFrameIndexRef.current) return; + + fetchingRef.current = true; + try { + const keypoints = await fetchPoseFrameAtIndex(id, frameIndex); + if (keypoints && canvas) { + lastFrameIndexRef.current = frameIndex; + const ctx = canvas.getContext("2d"); + if (ctx) { + drawSkeleton( + ctx, + keypoints, + canvas.width, + canvas.height, + video.videoWidth || 1280, + video.videoHeight || 720, + ); + } + } + } finally { + fetchingRef.current = false; + } + }, + [id], + ); + + // rAF loop during playback + seeked handler + useEffect(() => { + const video = videoRef.current; + if (!video || !poseHeader) return; + + const fps = poseHeader.fps; + + const loop = () => { + if (!video.paused) { + const fi = Math.floor(video.currentTime * fps); + renderFrame(fi); + animRef.current = requestAnimationFrame(loop); + } + }; + + const onPlay = () => { + animRef.current = requestAnimationFrame(loop); + }; + const onPause = () => { + if (animRef.current) cancelAnimationFrame(animRef.current); + animRef.current = null; + }; + const onSeeked = () => { + if (video.paused) { + renderFrame(Math.floor(video.currentTime * fps)); + } + }; + const onTimeUpdate = () => setCurrentTime(video.currentTime); + + video.addEventListener("play", onPlay); + video.addEventListener("pause", onPause); + video.addEventListener("seeked", onSeeked); + video.addEventListener("timeupdate", onTimeUpdate); + + return () => { + if (animRef.current) cancelAnimationFrame(animRef.current); + video.removeEventListener("play", onPlay); + video.removeEventListener("pause", onPause); + video.removeEventListener("seeked", onSeeked); + video.removeEventListener("timeupdate", onTimeUpdate); + }; + }, [poseHeader, renderFrame]); + + const seekToFrame = useCallback( + (frameIndex: number) => { + const video = videoRef.current; + if (!video || !poseHeader) return; + video.currentTime = frameIndex / poseHeader.fps; + }, + [poseHeader], + ); + + const seekToTime = useCallback((time: number) => { + const video = videoRef.current; + if (!video) return; + video.currentTime = Math.max(0, Math.min(time, video.duration || 0)); + }, []); + + const runAnalysis = useCallback(async () => { + setReanalyzing(true); + setReanalyzeError(null); + try { + const res = await fetch(`/api/mocap/sessions/${id}/reanalyze`, { + method: "POST", + }); + if (!res.ok) throw new Error(`${res.status}`); + // Refetch full session to get the new derived rows + const dataRes = await fetch(`/api/mocap/sessions/${id}`); + if (!dataRes.ok) throw new Error(`${dataRes.status}`); + const data = await dataRes.json(); + setSession(data.session); + } catch (e) { + setReanalyzeError(e instanceof Error ? e.message : String(e)); + } finally { + setReanalyzing(false); + } + }, [id]); + + const freezeAtCatch = useCallback(() => { + if (selectedStroke === null || !session || !poseHeader) return; + const metric = session.strokePostureMetrics.find( + (m) => m.strokeIndex === selectedStroke, + ); + if (metric) seekToFrame(metric.phaseBoundariesJson.catchFrameIndex); + }, [selectedStroke, session, poseHeader, seekToFrame]); + + const freezeAtFinish = useCallback(() => { + if (selectedStroke === null || !session || !poseHeader) return; + const metric = session.strokePostureMetrics.find( + (m) => m.strokeIndex === selectedStroke, + ); + if (metric) seekToFrame(metric.phaseBoundariesJson.finishFrameIndex); + }, [selectedStroke, session, poseHeader, seekToFrame]); + + if (loadError) { + return ( +
+

Failed to load session: {loadError}

+ +
+ ); + } + + if (!session) { + return ( +
+ Loading… +
+ ); + } + + const duration = session.durationSec; + const fps = poseHeader?.fps ?? session.captureFps; + const hasMetrics = session.strokePostureMetrics.length > 0; + const hasPoseStream = Boolean(poseHeader); + const segmentationSource = session.strokePostureMetrics[0]?.segmentationSource ?? null; + const isRecordOnly = session.qualityFlags.includes("record-only"); + const compareFaultMetric = + compareFaultStroke === null ? null : metricsByStroke.get(compareFaultStroke) ?? null; + const compareMetric = + compareStroke === null ? null : metricsByStroke.get(compareStroke) ?? null; + + return ( +
+ {/* Header */} +
+
+ +
+

{fmtDate(session.createdAt)}

+

+ {session.capturePerspective} · {fmtTime(duration)} + {session.qualityScore !== null + ? ` · quality ${Math.round(session.qualityScore * 100)}%` + : ""} + {segmentationSource + ? ` · ${segmentationSource === "csv-aligned" ? "CSV-aligned" : "pose-segmented"}` + : ""} +

+
+
+ + {session.status} + +
+ + {/* Video + skeleton overlay */} +
+
+ + {/* Freeze controls + stroke selector */} + {hasMetrics ? ( +
+ + Stroke:{" "} + + + + + + {fmtTime(currentTime)} / {fmtTime(duration)} + +
+ ) : null} + + {/* Timeline */} + {hasMetrics && duration > 0 ? ( +
{ + const rect = e.currentTarget.getBoundingClientRect(); + seekToTime(((e.clientX - rect.left) / rect.width) * duration); + }} + > + {/* Stroke markers */} + {session.strokePostureMetrics.map((m) => { + const t = m.phaseBoundariesJson.catchFrameIndex / fps; + const pct = Math.min(100, (t / duration) * 100); + return ( +
+ ); + })} + {/* Fault dots */} + {session.postureFaults.map((f, i) => { + const metric = session.strokePostureMetrics.find( + (m) => m.strokeIndex === f.strokeIndex, + ); + if (!metric) return null; + const t = metric.phaseBoundariesJson.catchFrameIndex / fps; + const pct = Math.min(100, (t / duration) * 100); + return ( +
{ + e.stopPropagation(); + setSelectedFault(f); + setSelectedStroke(f.strokeIndex); + const frameIndex = metric.phaseBoundariesJson.catchFrameIndex; + seekToFrame(frameIndex); + }} + /> + ); + })} + {/* Playhead */} + {duration > 0 ? ( +
+ ) : null} +
+ ) : null} + + {/* Not-yet-analyzed state */} + {!hasMetrics && session.status === "ready" && poseHeaderChecked ? ( + + +

+ {hasPoseStream + ? "No posture analysis for this session." + : isRecordOnly + ? "This is a record-only video. Live pose analysis was unavailable during capture, so there is no pose stream to re-analyze." + : "Posture analysis is unavailable because this session has no pose stream."} +

+ {reanalyzeError ? ( +

Analysis failed: {reanalyzeError}

+ ) : null} + {hasPoseStream ? ( + + ) : null} + + + ) : null} + + {/* Stats */} + {hasMetrics ? ( +
+ + + f.severity === "critical").length, + )} + /> + +
+ ) : null} + + {hasMetrics ? ( + + + + Side-by-side stroke compare + + + + {comparisonOptions.faultStrokeOptions.length === 0 ? ( +

+ No fault-heavy strokes yet. Once analysis detects a posture fault, + comparison mode can pair that stroke with a clean stroke from this + mocap session. +

+ ) : comparisonOptions.cleanStrokeOptions.length === 0 ? ( +

+ No clean comparison stroke exists in this mocap session. Every + analyzed stroke currently has at least one detected fault. +

+ ) : ( + <> +
+ + +
+ + {compareFaultMetric && compareMetric ? ( +
+ + +
+ ) : null} + + )} + + + ) : null} + + {/* Fault detail panel */} + {selectedFault ? ( + + + + {faultLabel(selectedFault.faultType)} + + {selectedFault.severity} + + + phase: {selectedFault.phase} · stroke {selectedFault.strokeIndex + 1} + + + + +
+ Metric: {selectedFault.evidenceJson.metric} +
+
+ Value: {selectedFault.evidenceJson.value.toFixed(2)} · + Threshold: {selectedFault.evidenceJson.threshold.toFixed(2)} +
+ {selectedFault.evidenceJson.frameIndex !== undefined ? ( +
Frame: {selectedFault.evidenceJson.frameIndex}
+ ) : null} + + + + ) : null} + + {/* All faults list */} + {hasMetrics && session.postureFaults.length > 0 ? ( +
+

All faults

+
+ {session.postureFaults.map((f, i) => ( + + ))} +
+
+ ) : null} +
+ ); +} + +function StatBox({ label, value }: { label: string; value: string }) { + return ( +
+
{label}
+
{value}
+
+ ); +} + +function CompareStrokePanel({ + title, + metric, + faults, + catchFrame, + finishFrame, + loading, +}: { + title: string; + metric: SessionStrokeMetric; + faults: SessionFault[]; + catchFrame: Float32Array | null; + finishFrame: Float32Array | null; + loading: boolean; +}) { + const faultCount = countFaultsForStroke(faults, metric.strokeIndex); + + return ( +
+
+
+
{title}
+
+ Stroke {metric.strokeIndex + 1} +
+
+ 0 ? "destructive" : "secondary"}> + {faultCount > 0 + ? `${faultCount} ${faultCount === 1 ? "fault" : "faults"}` + : "clean"} + +
+ +
+ + +
+ +
+ {metricRows(metric.metricsJson).map((row) => ( +
+ {row.label} + {row.value} +
+ ))} +
+ +
+
Fault summary
+ {faults.length > 0 ? ( +
+ {faults.map((fault) => ( + + {faultLabel(fault.faultType)} + + ))} +
+ ) : ( +
No detected faults.
+ )} +
+
+ ); +} + +function PhaseSkeletonCanvas({ + label, + keypoints, + loading, +}: { + label: string; + keypoints: Float32Array | null; + loading: boolean; +}) { + const canvasRef = useRef(null); + + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas) return; + canvas.width = canvas.clientWidth || 320; + canvas.height = canvas.clientHeight || 180; + const ctx = canvas.getContext("2d"); + if (!ctx) return; + ctx.clearRect(0, 0, canvas.width, canvas.height); + if (keypoints) { + drawSkeleton(ctx, keypoints, canvas.width, canvas.height, 1280, 720); + } + }, [keypoints]); + + return ( +
+
{label}
+
+ + {loading ? ( +
+ Loading +
+ ) : !keypoints ? ( +
+ Frame unavailable +
+ ) : null} +
+
+ ); +} + +function metricRows(metrics: SessionPostureMetrics): Array<{ + label: string; + value: string; +}> { + return [ + { + label: "Back angle at catch", + value: formatMetric(metrics.backAngleAtCatchDeg, "deg"), + }, + { + label: "Back angle at finish", + value: formatMetric(metrics.backAngleAtFinishDeg, "deg"), + }, + { + label: "Layback angle", + value: formatMetric(metrics.laybackAngleDeg, "deg"), + }, + { + label: "Hip-knee timing offset", + value: formatMetric(metrics.hipKneeOpeningOffsetFrames, "frames"), + }, + { + label: "Arm bend before legs", + value: formatMetric(metrics.armBendBeforeLegsCompleteFrames, "frames"), + }, + { + label: "Recovery / drive ratio", + value: formatMetric(metrics.recoveryDriveRatio, "ratio"), + }, + ]; +} + +function formatMetric( + value: number | null | undefined, + unit: "deg" | "frames" | "ratio", +): string { + if (typeof value !== "number" || !Number.isFinite(value)) return "n/a"; + if (unit === "ratio") return value.toFixed(2); + if (unit === "frames") return `${value.toFixed(0)} fr`; + return `${value.toFixed(1)} deg`; +} diff --git a/src/app/mocap/sessions/page.tsx b/src/app/mocap/sessions/page.tsx new file mode 100644 index 0000000..6ed7f31 --- /dev/null +++ b/src/app/mocap/sessions/page.tsx @@ -0,0 +1,211 @@ +"use client"; + +import { useEffect, useState } from "react"; +import Link from "next/link"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; + +interface MocapSessionSummary { + id: string; + status: string; + durationSec: number; + createdAt: string; + capturePerspective: string; + qualityScore: number | null; + qualityFlags: string[]; + _count: { + strokePostureMetrics: number; + postureFaults: number; + }; +} + +function statusBadge(status: string) { + if (status === "ready") return Ready; + if (status === "analyzing") return Analyzing…; + if (status === "capturing") return Capturing; + return {status}; +} + +function fmtDuration(sec: number): string { + if (sec < 60) return `${sec.toFixed(0)}s`; + const m = Math.floor(sec / 60); + const s = Math.round(sec % 60); + return `${m}m ${s}s`; +} + +function fmtDate(iso: string): string { + return new Date(iso).toLocaleString(undefined, { + month: "short", + day: "numeric", + year: "numeric", + hour: "2-digit", + minute: "2-digit", + }); +} + +export default function MocapSessionsPage() { + const [sessions, setSessions] = useState(null); + const [error, setError] = useState(null); + const [deletingId, setDeletingId] = useState(null); + const [reanalyzingId, setReanalyzingId] = useState(null); + + useEffect(() => { + fetch("/api/mocap/sessions") + .then((r) => { + if (!r.ok) throw new Error(`${r.status}`); + return r.json(); + }) + .then((data) => setSessions(data.sessions)) + .catch((e) => setError(e.message)); + }, []); + + async function handleDelete(id: string) { + if (!window.confirm("Delete this session? This cannot be undone.")) return; + setDeletingId(id); + try { + const res = await fetch(`/api/mocap/sessions/${id}`, { method: "DELETE" }); + if (!res.ok) throw new Error(`${res.status}`); + setSessions((prev) => prev?.filter((s) => s.id !== id) ?? null); + } catch (e) { + alert(`Delete failed: ${e instanceof Error ? e.message : String(e)}`); + } finally { + setDeletingId(null); + } + } + + async function handleReanalyze(id: string) { + setReanalyzingId(id); + try { + const res = await fetch(`/api/mocap/sessions/${id}/reanalyze`, { + method: "POST", + }); + if (!res.ok) throw new Error(`${res.status}`); + const data: { strokeMetricCount: number; faultCount: number } = await res.json(); + setSessions((prev) => + prev?.map((s) => + s.id === id + ? { + ...s, + _count: { + strokePostureMetrics: data.strokeMetricCount, + postureFaults: data.faultCount, + }, + } + : s, + ) ?? null, + ); + } catch (e) { + alert(`Reanalyze failed: ${e instanceof Error ? e.message : String(e)}`); + } finally { + setReanalyzingId(null); + } + } + + return ( +
+
+
+

Mocap sessions

+

+ All recorded motion capture sessions +

+
+ +
+ + {error ? ( + + + Failed to load sessions: {error} + + + ) : sessions === null ? ( + + + Loading… + + + ) : sessions.length === 0 ? ( + + + No sessions yet.{" "} + + Record your first session. + + + + ) : ( +
+ {sessions.map((s) => ( + + +
+ + {fmtDate(s.createdAt)} + + {statusBadge(s.status)} +
+ + {s.capturePerspective} · {fmtDuration(s.durationSec)} + {s.qualityScore !== null + ? ` · quality ${Math.round(s.qualityScore * 100)}%` + : ""} + + + +
+
+ {s._count.strokePostureMetrics} strokes + {s._count.postureFaults} faults + {s.qualityFlags.length > 0 && ( + + ⚠ {s.qualityFlags.join(", ")} + + )} +
+
+ {s.status === "ready" ? ( + <> + + + + ) : null} + +
+
+ + + ))} +
+ )} +
+ ); +} diff --git a/src/app/plans/page.tsx b/src/app/plans/page.tsx index 481de07..9ff7951 100644 --- a/src/app/plans/page.tsx +++ b/src/app/plans/page.tsx @@ -10,6 +10,9 @@ import { Input } from '@/components/ui/input'; import { Label } from '@/components/ui/label'; import { Switch } from '@/components/ui/switch'; import { trainingPlans, TrainingPlan, TrainingWeek, TrainingSession } from '@/lib/trainingPlans'; +import { POSTURE_FAULT_CATALOG_V1 } from '@/lib/mocap/analysis/postureThresholds'; +import type { PostureFaultType } from '@/lib/mocap/analysis/types'; +import type { PostureGoalProgress } from '@/lib/postureGoalProgress'; import { cloudAI } from '@/lib/cloudAI'; import { initializeCloudAIFromSettings, isAIAvailable, getAIConfigurationErrorMessage } from '@/lib/aiConfig'; import { useRowingStore } from '@/lib/store'; @@ -38,10 +41,61 @@ import { Upload, History, RotateCcw, + Activity, } from 'lucide-react'; import { ConfirmDialog } from '@/components/ConfirmDialog'; import { PlanAnalysisArchiveModal } from '@/components/PlanAnalysisArchiveModal'; +function PostureGoalSetter({ + planId, + onSave, +}: { + planId: string; + onSave: (planId: string, faultType: PostureFaultType, targetRate: number) => void; +}) { + const [faultType, setFaultType] = useState(''); + const [targetRate, setTargetRate] = useState(0.1); + + return ( +
+
+ + +
+ {faultType && ( +
+ Target ≤ + setTargetRate(parseFloat(e.target.value))} + className="flex-1" + /> + {(targetRate * 100).toFixed(0)}%/stroke +
+ )} +
+ ); +} + export default function PlansPage() { const router = useRouter(); const { getSessions, setPendingPlanAnalysis } = useRowingStore(); @@ -64,15 +118,32 @@ export default function PlansPage() { goals: '', level: 'intermediate' as 'beginner' | 'intermediate' | 'advanced', focus: 'general_fitness' as 'general_fitness' | 'endurance' | 'speed' | 'strength' | 'competition', - duration: 8 + duration: 8, + postureFaultType: '' as PostureFaultType | '', + postureTargetRate: 0.1, }); + const [postureGoalProgress, setPostureGoalProgress] = useState<{ + goal: { faultType: string; targetRate: number } | null; + progress: PostureGoalProgress | null; + } | null>(null); + useEffect(() => { loadPlans(); - // Keep AI availability consistent across reloads/navigation. initializeCloudAIFromSettings(); }, []); + useEffect(() => { + if (!activePlan) { + setPostureGoalProgress(null); + return; + } + fetch(`/api/training-plans/${activePlan.id}/posture-goal`) + .then(r => r.ok ? r.json() : null) + .then(data => data && setPostureGoalProgress(data)) + .catch(() => {}); + }, [activePlan?.id]); + // Auto-select current week when active plan changes useEffect(() => { const selectWeek = async () => { @@ -194,6 +265,18 @@ export default function PlansPage() { }); } + // Attach posture goal if specified + if (planForm.postureFaultType) { + await fetch(`/api/training-plans/${newPlan.id}/posture-goal`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + faultType: planForm.postureFaultType, + targetRate: planForm.postureTargetRate, + }), + }); + } + setPlans([newPlan, ...plans]); setShowCreateForm(false); resetForm(); @@ -346,7 +429,9 @@ export default function PlansPage() { goals: plan.goals.join(', '), level: plan.level, focus: plan.focus, - duration: plan.duration + duration: plan.duration, + postureFaultType: '', + postureTargetRate: 0.1, }); }; @@ -435,6 +520,36 @@ Please provide: router.push('/chat?fromPlanAnalysis=true'); }, [activePlan, getSessions, setPendingPlanAnalysis, router]); + const handleSavePostureGoal = async (planId: string, faultType: PostureFaultType, targetRate: number) => { + try { + const res = await fetch(`/api/training-plans/${planId}/posture-goal`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ faultType, targetRate }), + }); + if (res.ok) { + const data = await res.json(); + setPostureGoalProgress({ goal: data.goal, progress: null }); + // Reload progress + fetch(`/api/training-plans/${planId}/posture-goal`) + .then(r => r.ok ? r.json() : null) + .then(d => d && setPostureGoalProgress(d)) + .catch(() => {}); + } + } catch { + setError('Failed to save posture goal'); + } + }; + + const handleRemovePostureGoal = async (planId: string) => { + try { + await fetch(`/api/training-plans/${planId}/posture-goal`, { method: 'DELETE' }); + setPostureGoalProgress({ goal: null, progress: null }); + } catch { + setError('Failed to remove posture goal'); + } + }; + const resetForm = () => { setPlanForm({ title: '', @@ -442,7 +557,9 @@ Please provide: goals: '', level: 'intermediate', focus: 'general_fitness', - duration: 8 + duration: 8, + postureFaultType: '', + postureTargetRate: 0.1, }); }; @@ -602,6 +719,50 @@ Please provide:
+
+ +

+ Track a technique target using linked mocap sessions. +

+
+
+ + +
+ {planForm.postureFaultType && ( +
+ + setPlanForm(prev => ({ ...prev, postureTargetRate: parseFloat(e.target.value) }))} + className="w-full" + /> +
+ 0 (none) + 1 (every stroke) +
+
+ )} +
+
+
+ {/* Posture Goal Progress */} + {postureGoalProgress !== null && ( +
+
+
+ + Posture Goal +
+
+ {postureGoalProgress.goal && ( + + )} + {!postureGoalProgress.goal && ( + No goal set + )} +
+
+ + {postureGoalProgress.goal ? ( + <> +
+ Reduce {postureGoalProgress.goal.faultType.replace(/_/g, ' ')} to{' '} + ≤ {(postureGoalProgress.goal.targetRate * 100).toFixed(0)}% fault rate +
+ {postureGoalProgress.progress && postureGoalProgress.progress.linkedMocapSessionCount > 0 ? ( +
+
+
+ {(postureGoalProgress.progress.currentRate * 100).toFixed(1)}% +
+
Current Rate
+
+
+
+ {(postureGoalProgress.progress.targetRate * 100).toFixed(0)}% +
+
Target Rate
+
+
+
+ {postureGoalProgress.progress.linkedMocapSessionCount} +
+
Mocap Sessions
+
+
+ ) : ( +
+ No linked mocap sessions yet. Link a mocap session to a training session to track posture progress. +
+ )} + {postureGoalProgress.progress?.achieved && ( + Goal Achieved + )} + + ) : ( +
+
+ Add a posture goal to track technique improvement across mocap sessions. +
+ +
+ )} +
+ )} + {/* Week Navigation */}
diff --git a/src/app/sessions/page.tsx b/src/app/sessions/page.tsx index 3a286f7..9207b6d 100644 --- a/src/app/sessions/page.tsx +++ b/src/app/sessions/page.tsx @@ -15,7 +15,7 @@ import { TableRow, } from '@/components/ui/table'; import { Badge } from '@/components/ui/badge'; -import { ArrowUpDown, Calendar, TrendingUp, Clock, Zap, Target, ArrowUp, ArrowDown, Filter, X, Trophy, Sparkles } from 'lucide-react'; +import { ArrowUpDown, Calendar, TrendingUp, Clock, Zap, Target, ArrowUp, ArrowDown, Filter, X, Trophy, Sparkles, Video } from 'lucide-react'; import { formatSessionDate } from '@/lib/dateTimeUtils'; import { TimeRangeSelector, defaultTimeRangeOptions, type TimeRange } from '@/components/ui/time-range-selector'; @@ -362,6 +362,12 @@ export default function SessionsPage() { Stroke Data
+ +
+
+ @@ -428,6 +434,16 @@ export default function SessionsPage() { No stroke file )} + e.stopPropagation()}> + {session.mocapSession ? ( + + + + + ) : null} + ); })} diff --git a/src/app/settings/page.tsx b/src/app/settings/page.tsx index 8e5f74c..bb0ab20 100644 --- a/src/app/settings/page.tsx +++ b/src/app/settings/page.tsx @@ -1,6 +1,6 @@ 'use client'; -import { useState, useEffect } from 'react'; +import { useState, useEffect, type ReactNode } from 'react'; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'; import { Button } from '@/components/ui/button'; import { Badge } from '@/components/ui/badge'; @@ -19,6 +19,13 @@ import { DialogTitle, } from '@/components/ui/dialog'; import { settings, Settings, UserPreferences, DataManagement, TrainingSettings, NotificationSettings, PrivacySettings, AISettings, SmartRowSettings } from '@/lib/settings'; +import { + defaultPostureThresholdSettings, + postureThresholdsV1, + thresholdBandsEqual, + validatePostureThresholdBands, + type PostureThresholdBands, +} from '@/lib/mocap/analysis/postureThresholds'; import { cloudAI } from '@/lib/cloudAI'; import { deleteAllInsightsFromDB } from '@/lib/dataSync'; import { useAIInsights } from '@/hooks/useAIInsights'; @@ -85,6 +92,7 @@ type SettingsCategory = | 'notificationSettings' | 'privacySettings' | 'smartRowSettings' + | 'mocapSettings' | 'aiSettings'; export default function SettingsPage() { @@ -350,6 +358,9 @@ export default function SettingsPage() { case 'smartRowSettings': settings.updateSmartRowSettings(updates); break; + case 'mocapSettings': + settings.updateMocapSettings(updates); + break; case 'aiSettings': settings.updateAISettings(updates); break; @@ -458,6 +469,7 @@ export default function SettingsPage() { { id: 'trainingSettings', name: 'Training Settings', icon: Target, description: 'Training zones, goals, and preferences' }, { id: 'notificationSettings', name: 'Notifications', icon: Bell, description: 'Alerts and reminders' }, { id: 'privacySettings', name: 'Privacy', icon: Shield, description: 'Data sharing and privacy controls' }, + { id: 'mocapSettings', name: 'Mocap', icon: Target, description: 'Posture analysis thresholds' }, { id: 'aiSettings', name: 'AI Settings', icon: Brain, description: 'Configure AI assistant, training plans, achievement generation, etc.' } ]; @@ -1077,6 +1089,294 @@ export default function SettingsPage() {
); + const savePostureThresholds = (thresholds: PostureThresholdBands) => { + const validation = validatePostureThresholdBands(thresholds); + if (!validation.valid) { + setErrorMessage(validation.errors[0] || 'Invalid posture threshold'); + return; + } + + saveSettings('mocapSettings', { + postureThresholds: { + version: postureThresholdsV1.version, + thresholds, + userOverridden: !thresholdBandsEqual( + thresholds, + postureThresholdsV1.thresholds + ), + }, + postureThresholdWarning: null, + }); + }; + + const updatePostureThreshold = ( + faultKey: keyof PostureThresholdBands, + fieldKey: string, + value: number + ) => { + if (!settingsData?.mocapSettings) return; + const thresholds = settingsData.mocapSettings.postureThresholds.thresholds; + const next = { + ...thresholds, + [faultKey]: { + ...thresholds[faultKey], + [fieldKey]: value, + }, + } as PostureThresholdBands; + savePostureThresholds(next); + }; + + const resetPostureFault = (faultKey: keyof PostureThresholdBands) => { + if (!settingsData?.mocapSettings) return; + const thresholds = settingsData.mocapSettings.postureThresholds.thresholds; + savePostureThresholds({ + ...thresholds, + [faultKey]: { ...postureThresholdsV1.thresholds[faultKey] }, + } as PostureThresholdBands); + }; + + const resetAllPostureThresholds = () => { + saveSettings('mocapSettings', { + postureThresholds: defaultPostureThresholdSettings(), + postureThresholdWarning: null, + }); + }; + + const renderMocapSettings = () => { + const active = settingsData.mocapSettings.postureThresholds; + const thresholds = active.thresholds; + const validation = validatePostureThresholdBands(thresholds); + + const field = ( + faultKey: keyof PostureThresholdBands, + fieldKey: string, + label: string, + value: number, + step = 1 + ) => ( +
+ + + updatePostureThreshold( + faultKey, + fieldKey, + Number(e.target.value) + ) + } + className="mt-1" + /> +
+ ); + + return ( +
+ {settingsData.mocapSettings.postureThresholdWarning && ( + + + + {settingsData.mocapSettings.postureThresholdWarning} + + + )} + + {!validation.valid && ( + + + {validation.errors.join(' ')} + + )} + + + +
+
+ Posture Fault Thresholds + + Tune the five v1 mocap fault rules used by analysis. + +
+
+ + {active.userOverridden ? 'Custom' : 'Defaults'} + + +
+
+ + + resetPostureFault('rounded_back_at_catch')} + > +
+ {field( + 'rounded_back_at_catch', + 'warningBelowDeg', + 'Warning below degrees', + thresholds.rounded_back_at_catch.warningBelowDeg + )} + {field( + 'rounded_back_at_catch', + 'criticalBelowDeg', + 'Critical below degrees', + thresholds.rounded_back_at_catch.criticalBelowDeg + )} +
+ + + resetPostureFault('early_arm_bend')} + > +
+ {field( + 'early_arm_bend', + 'infoBeforeLegsCompleteFrames', + 'Info at frames early', + thresholds.early_arm_bend.infoBeforeLegsCompleteFrames + )} + {field( + 'early_arm_bend', + 'warningBeforeLegsCompleteFrames', + 'Warning at frames early', + thresholds.early_arm_bend.warningBeforeLegsCompleteFrames + )} +
+ + + resetPostureFault('back_opens_before_legs_drive')} + > + {field( + 'back_opens_before_legs_drive', + 'warningTorsoOpensBeforeLegsFrames', + 'Warning at frames early', + thresholds.back_opens_before_legs_drive + .warningTorsoOpensBeforeLegsFrames + )} + + + resetPostureFault('excessive_layback')} + > +
+ {field( + 'excessive_layback', + 'infoAboveDeg', + 'Info above degrees', + thresholds.excessive_layback.infoAboveDeg + )} + {field( + 'excessive_layback', + 'warningAboveDeg', + 'Warning above degrees', + thresholds.excessive_layback.warningAboveDeg + )} +
+ + + resetPostureFault('slow_recovery_ratio')} + > +
+ {field( + 'slow_recovery_ratio', + 'warningAboveRatio', + 'Warning above ratio', + thresholds.slow_recovery_ratio.warningAboveRatio, + 0.1 + )} + {field( + 'slow_recovery_ratio', + 'criticalAboveRatio', + 'Critical above ratio', + thresholds.slow_recovery_ratio.criticalAboveRatio, + 0.1 + )} +
+ + + + + + + Live Coaching Cues + + Controls visual and audio feedback during a mocap session. + Cues fire post-stroke (≤ 1s after each stroke completes). + + + +
+
+ +

+ Quiet (default): only warning + critical faults.{' '} + Verbose: also surfaces info-severity cues. +

+
+ +
+ +
+
+ +

+ Speak the short audio hint for each cue using the + browser's speech synthesis. Visual cues remain on regardless. +

+
+ + saveSettings('mocapSettings', { + mocapPreferences: { + ...settingsData.mocapSettings.mocapPreferences, + audioEnabled: checked, + }, + }) + } + data-testid="mocap-cue-audio" + /> +
+ + +
+ ); + }; + const renderAISettings = () => { // ✅ No hooks here - moved to component level // Conditional logic is safe now @@ -1208,6 +1508,23 @@ export default function SettingsPage() { />
+ {/* Posture Data Sharing Tier Toggle */} +
+
+ +

+ Tier 3 (default): Sends fault counts and severity totals only — no body geometry.{' '} + Tier 2 (opt-in): Also sends per-stroke back angle, layback, and recovery ratio. + Raw pose keypoints are never sent to cloud AI. +

+
+ saveSettings('aiSettings', { mocapDetailedAIShare: checked })} + disabled={!settingsData.aiSettings.cloudAIEnabled} + /> +
+ {settingsData.aiSettings.cloudAIEnabled && ( <> {/* API Key Input - render the input ONLY when the user clicks "Edit". @@ -2424,6 +2741,8 @@ export default function SettingsPage() { return renderNotificationSettings(); case 'privacySettings': return renderPrivacySettings(); + case 'mocapSettings': + return renderMocapSettings(); case 'aiSettings': return renderAISettings(); default: @@ -2573,3 +2892,31 @@ export default function SettingsPage() { ); } + +function ThresholdCard({ + title, + description, + children, + onReset, +}: { + title: string; + description: string; + children: ReactNode; + onReset: () => void; +}) { + return ( +
+
+
+

{title}

+

{description}

+
+ +
+ {children} +
+ ); +} diff --git a/src/app/sync/page.tsx b/src/app/sync/page.tsx index bdc9351..71a6181 100644 --- a/src/app/sync/page.tsx +++ b/src/app/sync/page.tsx @@ -7,13 +7,24 @@ import { processZipFile, ZipImportResult, ZipProcessProgress } from '@/lib/zipPa import { formatValidationErrors, hasCriticalErrors } from '@/lib/validation'; import { ImportResult, Session } from '@/types/session'; import { saveSessionsToDBChunked, UploadProgress } from '@/lib/dataSync'; +import { clearSessionsCache } from '@/lib/services/sessionsCache'; +import { confirmMocapSessionLink } from '@/lib/mocap/linking'; import { useSettings } from '@/hooks/useSettings'; import { Button } from '@/components/ui/button'; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'; import { Progress } from '@/components/ui/progress'; -import { Upload, FileText, AlertCircle, CheckCircle, ArrowRight, FileArchive, Database, RefreshCw, Settings } from 'lucide-react'; +import { Upload, FileText, AlertCircle, CheckCircle, ArrowRight, FileArchive, Database, RefreshCw, Settings, Video } from 'lucide-react'; import Link from 'next/link'; +type MocapOverlapStatus = 'idle' | 'linking' | 'linked' | 'conflict' | 'error'; + +interface MocapOverlap { + rowingSessionId: string; + mocapSessionId: string; + status?: MocapOverlapStatus; + message?: string; +} + type UploadState = 'idle' | 'dragging' | 'validating' | 'processing' | 'saving' | 'syncing' | 'success' | 'error'; export default function UploadPage() { @@ -27,6 +38,8 @@ export default function UploadPage() { const [uploadProgress, setUploadProgress] = useState(null); const [zipProgress, setZipProgress] = useState(null); const [syncMessage, setSyncMessage] = useState(''); + const [mocapOverlaps, setMocapOverlaps] = useState([]); + const [dismissedOverlaps, setDismissedOverlaps] = useState(false); const handleDragOver = useCallback((e: React.DragEvent) => { e.preventDefault(); @@ -68,6 +81,85 @@ export default function UploadPage() { processFile(file); }, []); + const checkMocapOverlap = useCallback(async (savedSessions: Session[]) => { + if (savedSessions.length === 0) return; + try { + const ids = savedSessions.map((s) => s.id).filter(Boolean); + const res = await fetch('/api/mocap/sessions/overlap-check', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ rowingSessionIds: ids }), + }); + if (!res.ok) return; + const data = await res.json(); + if (Array.isArray(data.overlaps) && data.overlaps.length > 0) { + setMocapOverlaps(data.overlaps.map((overlap: MocapOverlap) => ({ + ...overlap, + status: 'idle', + }))); + setDismissedOverlaps(false); + } + } catch { + // Non-critical — silently ignore overlap check errors + } + }, []); + + const handleConfirmMocapLink = useCallback(async (overlap: MocapOverlap) => { + const matchesOverlap = (candidate: MocapOverlap) => + candidate.mocapSessionId === overlap.mocapSessionId && + candidate.rowingSessionId === overlap.rowingSessionId; + + setMocapOverlaps((current) => current.map((candidate) => + matchesOverlap(candidate) + ? { ...candidate, status: 'linking', message: undefined } + : candidate + )); + + try { + const result = await confirmMocapSessionLink(overlap); + + if (result.ok) { + const linkedSession = getSessions().find((session) => session.id === result.rowingSessionId); + if (linkedSession) { + updateSessionsInStore([ + { + ...linkedSession, + mocapSession: { id: result.mocapSessionId }, + }, + ]); + } + clearSessionsCache(); + + setMocapOverlaps((current) => current.map((candidate) => + matchesOverlap(candidate) + ? { ...candidate, status: 'linked', message: 'Linked and re-analyzed with csv-aligned posture data.' } + : candidate + )); + return; + } + + setMocapOverlaps((current) => current.map((candidate) => + matchesOverlap(candidate) + ? { + ...candidate, + status: result.reason === 'conflict' ? 'conflict' : 'error', + message: result.message, + } + : candidate + )); + } catch (err) { + setMocapOverlaps((current) => current.map((candidate) => + matchesOverlap(candidate) + ? { + ...candidate, + status: 'error', + message: err instanceof Error ? err.message : 'Failed to link mocap session.', + } + : candidate + )); + } + }, [getSessions, updateSessionsInStore]); + const processFile = async (file: File) => { setSelectedFile(file); setError(''); @@ -76,6 +168,8 @@ export default function UploadPage() { setZipResult(null); setUploadProgress(null); setZipProgress(null); + setMocapOverlaps([]); + setDismissedOverlaps(false); try { // Check if it's a ZIP file @@ -118,6 +212,7 @@ export default function UploadPage() { // skip DB save since we already saved with chunked upload if (saveResult.sessions && saveResult.sessions.length > 0) { updateSessionsInStore(saveResult.sessions); + await checkMocapOverlap(saveResult.sessions); } } @@ -173,6 +268,7 @@ export default function UploadPage() { // skip DB save since we already saved with chunked upload if (saveResult.sessions && saveResult.sessions.length > 0) { addSessions(saveResult.sessions, { skipDbSave: true }); + await checkMocapOverlap(saveResult.sessions); } } @@ -195,6 +291,8 @@ export default function UploadPage() { setSyncMessage(''); setUploadProgress(null); setZipProgress(null); + setMocapOverlaps([]); + setDismissedOverlaps(false); }, []); const formatDuration = (seconds: number): string => { @@ -307,6 +405,9 @@ export default function UploadPage() { if (saveResult.success) { addSessions(sessions, { skipDbSave: true }); + if (saveResult.sessions && saveResult.sessions.length > 0) { + await checkMocapOverlap(saveResult.sessions); + } totalImported += result.importedSessions; totalDistance += result.totalDistance; totalTime += result.totalTime; @@ -642,16 +743,79 @@ export default function UploadPage() { )} + {/* Mocap Overlap Prompt */} + {mocapOverlaps.length > 0 && !dismissedOverlaps && ( +
+
+
+
+ )} + {/* Action Buttons */}
diff --git a/src/components/PostureFaultTrendCard.tsx b/src/components/PostureFaultTrendCard.tsx new file mode 100644 index 0000000..eef8581 --- /dev/null +++ b/src/components/PostureFaultTrendCard.tsx @@ -0,0 +1,224 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { useRowingStore } from '@/lib/store'; +import { + LineChart, + Line, + XAxis, + YAxis, + CartesianGrid, + Tooltip, + Legend, + ResponsiveContainer, +} from 'recharts'; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'; +import { Activity, AlertTriangle } from 'lucide-react'; +import type { PostureFaultType } from '@/lib/mocap/analysis/types'; +import type { PostureTrendResult, FaultTrendPoint, SessionFaultInput } from '@/lib/mocap/postureTrendAggregation'; + +const FAULT_LABELS: Record = { + rounded_back_at_catch: 'Rounded Back', + early_arm_bend: 'Early Arm Bend', + back_opens_before_legs_drive: 'Back Opens Early', + excessive_layback: 'Excessive Layback', + slow_recovery_ratio: 'Slow Recovery', + left_right_asymmetry: 'L/R Asymmetry', + knee_track_deviation: 'Knee Track', + shin_not_vertical_at_catch: 'Shin Angle', +}; + +const FAULT_COLORS: Record = { + rounded_back_at_catch: '#ef4444', + early_arm_bend: '#f97316', + back_opens_before_legs_drive: '#eab308', + excessive_layback: '#8b5cf6', + slow_recovery_ratio: '#06b6d4', + left_right_asymmetry: '#10b981', + knee_track_deviation: '#3b82f6', + shin_not_vertical_at_catch: '#f59e0b', +}; + +interface ChartPoint { + date: string; + lowQuality?: boolean; + [faultType: string]: number | string | boolean | undefined; +} + +function buildChartData(data: PostureTrendResult): ChartPoint[] { + const dateMap = new Map(); + + for (const trend of data.trends) { + for (const point of trend.points) { + if (!dateMap.has(point.date)) { + dateMap.set(point.date, { date: point.date }); + } + const row = dateMap.get(point.date)!; + row[trend.faultType] = point.count; + if (point.lowQuality) row.lowQuality = true; + } + } + + return Array.from(dateMap.values()).sort((a, b) => a.date.localeCompare(b.date)); +} + +function lowQualitySessionDates(data: PostureTrendResult): Set { + const dates = new Set(); + for (const trend of data.trends) { + for (const point of trend.points) { + if (point.lowQuality) dates.add(point.date); + } + } + return dates; +} + +function CustomDot(props: { + cx?: number; + cy?: number; + payload?: ChartPoint; +}) { + const { cx, cy, payload } = props; + if (!cx || !cy || !payload?.lowQuality) return null; + return ( + + ); +} + +type PostureTrendApiResponse = PostureTrendResult & { sessions?: Array }; + +function deserializeSessions(raw: PostureTrendApiResponse['sessions']): SessionFaultInput[] { + if (!raw) return []; + return raw.map((s) => ({ ...s, sessionDate: new Date(s.sessionDate) })); +} + +export function PostureFaultTrendCard() { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const evaluatePostureAwards = useRowingStore((s) => s.evaluatePostureAwards); + + useEffect(() => { + fetch('/api/mocap/posture-trend') + .then((r) => { + if (!r.ok) throw new Error('Failed to load posture trend data'); + return r.json() as Promise; + }) + .then((response) => { + const { sessions: rawSessions, ...trendResult } = response; + setData(trendResult); + const sessions = deserializeSessions(rawSessions); + if (sessions.length > 0) evaluatePostureAwards(sessions); + }) + .catch((e: unknown) => setError(e instanceof Error ? e.message : 'Unknown error')) + .finally(() => setLoading(false)); + }, [evaluatePostureAwards]); + + const lowQualityDates = data ? lowQualitySessionDates(data) : new Set(); + const chartData = data ? buildChartData(data) : []; + const activeFaultTypes = data?.trends.map((t) => t.faultType) ?? []; + + return ( +
+
+ +

Posture Fault Frequency

+ {lowQualityDates.size > 0 && ( + + + {lowQualityDates.size} low-quality session{lowQualityDates.size > 1 ? 's' : ''} + + )} +
+ +
+ {loading && ( +
+ Loading... +
+ )} + + {error && ( +
+ {error} +
+ )} + + {!loading && !error && data?.totalSessions === 0 && ( +
+ +

No linked mocap sessions found.

+

Link a motion capture session to a training session to track posture trends.

+
+ )} + + {!loading && !error && data && data.totalSessions > 0 && data.trends.length === 0 && ( +
+ +

No posture faults recorded yet.

+

Faults will appear here as mocap sessions are analyzed.

+
+ )} + + {!loading && !error && data && data.trends.length > 0 && ( + <> + + + + v.slice(5)} + /> + + [ + value, + FAULT_LABELS[name as PostureFaultType] ?? name, + ]} + labelFormatter={(label: string) => { + const isLQ = lowQualityDates.has(label); + return `${label}${isLQ ? ' ⚠ low quality' : ''}`; + }} + /> + + FAULT_LABELS[value as PostureFaultType] ?? value + } + wrapperStyle={{ fontSize: 11 }} + /> + {activeFaultTypes.map((ft) => ( + } + activeDot={{ r: 4 }} + connectNulls + /> + ))} + + +

+ Fault counts per session · {data.linkedSessionsWithFaults} of {data.totalSessions} session{data.totalSessions !== 1 ? 's' : ''} have recorded faults + {lowQualityDates.size > 0 && ' · Dashed circles = low capture quality'} +

+ + )} +
+
+ ); +} diff --git a/src/components/navigation.tsx b/src/components/navigation.tsx index 05b45e6..e2fcf63 100644 --- a/src/components/navigation.tsx +++ b/src/components/navigation.tsx @@ -16,7 +16,7 @@ import { } from '@/components/ui/dropdown-menu'; import { cn } from '@/lib/utils'; import { settings } from '@/lib/settings'; -import { Upload, BarChart3, List, Trophy, MessageCircle, Target, Settings as SettingsIcon, Gauge, Brain, User, LogOut, UserCircle, Sun, Moon, Monitor, RefreshCw } from 'lucide-react'; +import { Upload, BarChart3, List, Trophy, MessageCircle, Target, Settings as SettingsIcon, Gauge, Brain, User, LogOut, UserCircle, Sun, Moon, Monitor, RefreshCw, Video } from 'lucide-react'; const navigation = [ { name: 'Dashboard', href: '/dashboard', icon: BarChart3 }, @@ -26,6 +26,7 @@ const navigation = [ { name: 'Personal Records', href: '/prs', icon: Trophy }, { name: 'Training Plans', href: '/plans', icon: Target }, { name: 'AI Coach', href: '/chat', icon: MessageCircle }, + { name: 'Mocap', href: '/mocap', icon: Video }, { name: 'Sync', href: '/sync', icon: RefreshCw }, ]; @@ -116,7 +117,7 @@ export function Navigation() {
- + Rowing Tracker diff --git a/src/hooks/useAIInsights.ts b/src/hooks/useAIInsights.ts index 478cd68..68148e7 100644 --- a/src/hooks/useAIInsights.ts +++ b/src/hooks/useAIInsights.ts @@ -6,6 +6,31 @@ import { cloudAI, CloudInsight } from '@/lib/cloudAI'; import { initializeCloudAIFromSettings, isAIAvailable, getAIConfigurationErrorMessage } from '@/lib/aiConfig'; import { memoryStorage } from '@/lib/memoryStorage'; import { saveInsightsToDB, fetchInsightsFromDB } from '@/lib/dataSync'; +import { SettingsService } from '@/lib/settings'; +import { buildPostureAIPayload, assertNoKeypointsInPayload, PostureAIPayload } from '@/lib/mocap/aiPayload'; + +async function fetchPosturePayload(): Promise { + const settings = SettingsService.getInstance().getSettings(); + const { cloudAIEnabled, mocapDetailedAIShare } = settings.aiSettings; + if (!cloudAIEnabled) return null; + + try { + const res = await fetch('/api/mocap/posture-summary'); + if (!res.ok) return null; + const { faults, metrics, qualityFlags, qualityScore } = await res.json(); + + if (!faults?.length && !metrics?.length) return null; + + const payload = buildPostureAIPayload(faults, metrics, qualityFlags ?? [], qualityScore ?? null, { + cloudAIEnabled, + mocapDetailedAIShare, + }); + if (payload) assertNoKeypointsInPayload(payload); + return payload; + } catch { + return null; + } +} export interface AIInsightData { insights: (Insight | CloudInsight)[]; @@ -483,8 +508,11 @@ export function useAIInsights(forceRefresh: boolean = false): AIInsightData { // Initialize cloud AI with latest settings initializeCloudAI(); + // Build tiered posture payload (null when cloud AI disabled or no data) + const posturePayload = await fetchPosturePayload(); + // Generate insights using cloud AI - const cloudInsights = await cloudAI.generateInsights(sessions); + const cloudInsights = await cloudAI.generateInsights(sessions, posturePayload); // Generate local trends and other data (cloud AI only handles insights) const trends = [ diff --git a/src/hooks/useChat.ts b/src/hooks/useChat.ts index e6967bc..2a3e980 100644 --- a/src/hooks/useChat.ts +++ b/src/hooks/useChat.ts @@ -271,6 +271,18 @@ export function useChat() { .pop(); const previousResponseId = lastAIMessage?.responseId; + // Fetch posture context from server (non-blocking; omit on error) + let posturePayload = null; + try { + const ctxRes = await fetch('/api/chat/posture-context'); + if (ctxRes.ok) { + const ctxData = await ctxRes.json(); + posturePayload = ctxData.payload ?? null; + } + } catch { + // proceed without posture context + } + const aiResponse = await cloudAI.sendChatMessage( content.trim(), conversationHistory, @@ -296,7 +308,8 @@ export function useChat() { }; }); }, - attachments + attachments, + posturePayload, ); // Add AI response (only if not empty) diff --git a/src/lib/aiAnalysis.ts b/src/lib/aiAnalysis.ts index 5ea2c46..4bc9bf5 100644 --- a/src/lib/aiAnalysis.ts +++ b/src/lib/aiAnalysis.ts @@ -1,5 +1,10 @@ import { Session } from '@/types/session'; import { cloudAI } from '@/lib/cloudAI'; +import { + buildPostureAIPayload, + assertNoKeypointsInPayload, + type PostureAIPayload, +} from '@/lib/mocap/aiPayload'; // Types for AI analysis results export interface TrendData { @@ -401,3 +406,65 @@ export class AIAnalysisService { // Export singleton instance export const aiAnalysis = AIAnalysisService.getInstance(); + +// ============================================================================ +// Posture AI Payload Integration +// ============================================================================ + +/** + * Input type for posture data fetched server-side (from DB records). + * No keypoints — only aggregated fault/metric rows. + */ +export interface MocapSessionPostureData { + faults: Array<{ faultType: string; severity: string }>; + metrics: Array<{ + strokeIndex: number; + segmentationSource: string; + metricsJson: unknown; + }>; + qualityFlags: string[]; + qualityScore: number | null; +} + +/** + * Build a cloud-safe posture AI payload from DB-fetched mocap data and user + * settings, then verify the hard keypoint guard. + * + * Returns null when cloudAI is disabled (no posture data leaves the device). + * Call this server-side (e.g. in an API route) where DB access is available; + * pass the returned payload into the cloud AI prompt builder. + */ +export function buildAndValidatePosturePayload( + postureData: MocapSessionPostureData, + opts: { cloudAIEnabled: boolean; mocapDetailedAIShare: boolean }, +): PostureAIPayload | null { + const payload = buildPostureAIPayload( + postureData.faults, + postureData.metrics, + postureData.qualityFlags, + postureData.qualityScore, + opts, + ); + + if (payload !== null) { + // Hard guard: must never contain keypoint arrays. + assertNoKeypointsInPayload(payload); + } + + return payload; +} + +/** + * Serialise a validated PostureAIPayload as a JSON context block suitable + * for appending to an AI prompt string. + */ +export function formatPosturePayloadForPrompt( + payload: PostureAIPayload, +): string { + const tierLabel = + payload.tier === 3 + ? 'Tier 3 – Fault Summary (no body geometry)' + : 'Tier 2 – Fault Summary + Per-Stroke Metrics (no keypoints)'; + + return `\n\n---\nPOSTURE ANALYSIS CONTEXT [${tierLabel}]:\n${JSON.stringify(payload, null, 2)}\n---`; +} diff --git a/src/lib/awards.ts b/src/lib/awards.ts index aa0fb73..2e076f2 100644 --- a/src/lib/awards.ts +++ b/src/lib/awards.ts @@ -1,16 +1,19 @@ import { Session } from '@/types/session'; -import { - Trophy, - Timer, - Flame, - Medal, - Award as AwardIcon, +import { + Trophy, + Timer, + Flame, + Medal, + Award as AwardIcon, Zap, Activity, TrendingUp, Target, - Crown + Crown, + ShieldCheck } from 'lucide-react'; +import { cleanCatchQualifies } from './postureAchievements'; +import type { SessionFaultInput } from './mocap/postureTrendAggregation'; export interface Award { id: string; @@ -501,5 +504,18 @@ export const AWARDS: Award[] = [ const date = new Date(s.timestamp); return date.getHours() >= 21; }) + }, + + // Posture Achievements + { + id: 'posture-clean-catch', + title: 'Clean Catch', + description: 'Row a mocap-linked session with ≤10% rounded-back-at-catch faults (min. 20 strokes, quality capture required)', + icon: ShieldCheck, + color: 'text-cyan-500', + condition: (_sessions: Session[], stats?: { postureSessions?: SessionFaultInput[] }): boolean => { + const postureSessions = stats?.postureSessions; + return postureSessions ? postureSessions.some(cleanCatchQualifies) : false; + } } ]; diff --git a/src/lib/cloudAI.ts b/src/lib/cloudAI.ts index 380a3aa..e0b1976 100644 --- a/src/lib/cloudAI.ts +++ b/src/lib/cloudAI.ts @@ -1,5 +1,6 @@ import { Session } from '@/types/session'; import { TrainingPlan, TrainingWeek, TrainingSession } from '@/lib/trainingPlans'; +import type { PostureAIPayload } from '@/lib/mocap/aiPayload'; import { SettingsService } from '@/lib/settings'; import { DEFAULT_PLAN_GENERATION_PROMPT, @@ -101,6 +102,7 @@ export interface CloudInsight { confidence: number; // 0-1 from AI evidence: string[]; // Supporting data points dateGenerated: Date; + category?: 'posture'; } export class CloudAIService { @@ -160,7 +162,8 @@ export class CloudAIService { userSessions?: Session[], previousResponseId?: string, onToken?: (token: string) => void, - attachments?: FileAttachment[] + attachments?: FileAttachment[], + posturePayload?: PostureAIPayload | null, ): Promise<{ content: string; responseId: string }> { if (!this.config) { throw new Error('Cloud AI service not configured'); @@ -272,7 +275,7 @@ export class CloudAIService { // Prepare initial input messages const messages: Array<{ role: string; content: string | Array<{ type: string; text?: string; image_url?: string; file?: { file_data: string; filename: string } }> }> = [ - { role: 'system', content: this.getChatSystemPrompt() }, + { role: 'system', content: this.getChatSystemPrompt(posturePayload) }, ...conversationHistory.slice(-10).map(msg => ({ role: msg.role, content: msg.content @@ -811,7 +814,7 @@ export class CloudAIService { // Get system prompt for chat AI trainer - private getChatSystemPrompt(_sessions?: Session[]): string { + private getChatSystemPrompt(posturePayload?: PostureAIPayload | null, _sessions?: Session[]): string { return `You are a personal AI rowing coach and trainer. You specialize in indoor rowing performance, technique, and training optimization. CRITICAL FORMATTING RULES - READ CAREFULLY: @@ -928,7 +931,7 @@ COMMUNICATION STYLE: - Keep responses focused and practical - Structure responses with clear headers for easy scanning -Remember: You're building a long-term coaching relationship. Be supportive, knowledgeable, and genuinely helpful in their rowing journey.${this.getUserProfileContext()}`; +Remember: You're building a long-term coaching relationship. Be supportive, knowledgeable, and genuinely helpful in their rowing journey.${this.getUserProfileContext()}${this.buildPostureContextBlock(posturePayload)}`; } // Get user's session context for personalized coaching @@ -982,7 +985,10 @@ Remember: You're building a long-term coaching relationship. Be supportive, know // Generate rowing-specific insights using OpenAI - async generateInsights(sessions: Session[]): Promise { + async generateInsights( + sessions: Session[], + posturePayload?: PostureAIPayload | null, + ): Promise { if (!this.config) { throw new Error('Cloud AI service not configured'); } @@ -997,7 +1003,7 @@ Remember: You're building a long-term coaching relationship. Be supportive, know try { const anonymizedData = this.anonymizeSessions(sessions); - const prompt = this.buildInsightPrompt(anonymizedData); + const prompt = this.buildInsightPrompt(anonymizedData, posturePayload); const useCaseConfig = this.aiSettings.insights; const config: ApiRequestConfig = { @@ -1058,6 +1064,18 @@ Remember: You're building a long-term coaching relationship. Be supportive, know return ''; } + // Append posture context block to system prompt when a cloud-safe payload exists. + // Raw keypoints are never present here — the hard guard ran server-side before + // the payload reached the client. + private buildPostureContextBlock(payload?: PostureAIPayload | null): string { + if (!payload) return ''; + const tierLabel = + payload.tier === 3 + ? 'Tier 3 – Fault Summary (no body geometry)' + : 'Tier 2 – Fault Summary + Per-Stroke Metrics (no keypoints)'; + return `\n\n---\nPOSTURE ANALYSIS CONTEXT [${tierLabel}]:\n${JSON.stringify(payload, null, 2)}\n\nUse this posture data to answer questions about the user's technique and form. Do not speculate about raw pose coordinates — only the summarised fault counts and scalar metrics above are available.\n---`; + } + // Get system prompt for rowing performance analysis private getSystemPrompt(): string { const userContext = this.getUserProfileContext(); @@ -1068,7 +1086,10 @@ Remember: You're building a long-term coaching relationship. Be supportive, know } // Build user prompt with session data using configurable prompt - private buildInsightPrompt(sessions: Record[]): string { + private buildInsightPrompt( + sessions: Record[], + posturePayload?: PostureAIPayload | null, + ): string { // Include more sessions for better progress analysis const recentSessions = sessions.slice(-20); // Last 20 sessions for better context const sessionSummary = this.createSessionSummary(recentSessions); @@ -1114,6 +1135,17 @@ JSON structure: CRITICAL: Your response must be ONLY the JSON array of insights. Do not include any explanations, markdown, or the training data itself.`; + // Append posture context block when available (tier 2 or tier 3 payload). + // Raw keypoint data is never included — the hard guard in aiPayload.ts + // enforces this before the payload reaches this point. + if (posturePayload) { + const tierLabel = + posturePayload.tier === 3 + ? 'Tier 3 – Fault Summary (no body geometry)' + : 'Tier 2 – Fault Summary + Per-Stroke Metrics (no keypoints)'; + return `${finalPrompt}\n\n---\nPOSTURE ANALYSIS CONTEXT [${tierLabel}]:\n${JSON.stringify(posturePayload, null, 2)}\n---`; + } + return finalPrompt; } @@ -1257,17 +1289,26 @@ Average sessions per week: ${(totalSessions / Math.max(1, Math.ceil((dates[dates // Ensure we have an array const insightsArray = Array.isArray(insightsData) ? insightsData : [insightsData]; - return insightsArray.map((insight: Record, index: number) => ({ - id: `cloud-insight-${Date.now()}-${index}`, - type: (insight.type as string) || 'recommendation', - title: (insight.title as string) || 'Performance Insight', - description: (insight.description as string) || 'No description provided', - actionable: Boolean(insight.actionable), - priority: (insight.priority as string) || 'medium', - confidence: Math.max(0, Math.min(1, Number(insight.confidence) || 0.5)), - evidence: Array.isArray(insight.evidence) ? insight.evidence as string[] : [], - dateGenerated: new Date() - })) as CloudInsight[]; + const POSTURE_TERMS = /posture|back angle|rounded back|layback|arm bend|drive ratio|recovery ratio|catch position|finish position|spine|trunk lean/i; + + return insightsArray.map((insight: Record, index: number) => { + const title = (insight.title as string) || 'Performance Insight'; + const description = (insight.description as string) || 'No description provided'; + const isPosture = POSTURE_TERMS.test(title) || POSTURE_TERMS.test(description); + + return { + id: `cloud-insight-${Date.now()}-${index}`, + type: (insight.type as string) || 'recommendation', + title, + description, + actionable: Boolean(insight.actionable), + priority: (insight.priority as string) || 'medium', + confidence: Math.max(0, Math.min(1, Number(insight.confidence) || 0.5)), + evidence: Array.isArray(insight.evidence) ? insight.evidence as string[] : [], + dateGenerated: new Date(), + ...(isPosture ? { category: 'posture' as const } : {}), + }; + }) as CloudInsight[]; } catch (error) { console.error('Failed to parse AI response:', error); console.error('Response content:', response); diff --git a/src/lib/mocap/__tests__/fixtures/generate-v2-blob.mjs b/src/lib/mocap/__tests__/fixtures/generate-v2-blob.mjs new file mode 100644 index 0000000..be13930 --- /dev/null +++ b/src/lib/mocap/__tests__/fixtures/generate-v2-blob.mjs @@ -0,0 +1,72 @@ +/** + * Generates a synthetic v2 (sidecar-3d) PoseFrameStream blob for tests. + * Run: node generate-v2-blob.mjs + * Writes: v2-blob-3d.bin + */ +import { writeFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +const __dir = dirname(fileURLToPath(import.meta.url)); + +const MAGIC = new Uint8Array([0x4d, 0x4f, 0x50, 0x53]); +const HEADER_SIZE = 32; +const FORMAT_VERSION = 1; +const KEYPOINT_SCHEMA_V2 = 2; +const KEYPOINTS_PER_FRAME_V2 = 33; +const BYTES_PER_FRAME_V2 = 4 + KEYPOINTS_PER_FRAME_V2 * 4 * 4 + 4; +const COORDINATE_SPACE_WORLD_MM_3D = 1; +const FPS = 30; +const FRAME_COUNT = 100; +const CAMERA_COUNT = 3; + +// Build header +const header = new Uint8Array(HEADER_SIZE); +const hv = new DataView(header.buffer); +header.set(MAGIC, 0); +hv.setUint16(4, FORMAT_VERSION, true); +hv.setUint16(6, KEYPOINT_SCHEMA_V2, true); +hv.setFloat32(8, FPS, true); +hv.setUint16(12, KEYPOINTS_PER_FRAME_V2, true); +hv.setUint16(14, BYTES_PER_FRAME_V2, true); +hv.setUint32(16, FRAME_COUNT, true); +hv.setUint8(20, COORDINATE_SPACE_WORLD_MM_3D); +hv.setUint8(21, CAMERA_COUNT); + +// Build 100 frames simulating one rowing stroke in world-mm-3d +// Rowing motion: catch at frame 0, finish at frame 35, recovery frames 36-99 +const frames = new Uint8Array(FRAME_COUNT * BYTES_PER_FRAME_V2); + +for (let f = 0; f < FRAME_COUNT; f++) { + const offset = f * BYTES_PER_FRAME_V2; + const fv = new DataView(frames.buffer, offset, BYTES_PER_FRAME_V2); + const t = f / FPS; + const phase = f < 35 ? f / 35 : (f - 35) / 65; // 0→1 in drive, 0→1 in recovery + + // timestampMs + fv.setFloat32(0, 1700000000000 + t * 1000, true); + + // Write 33 keypoints as [x, y, z, confidence] + for (let k = 0; k < KEYPOINTS_PER_FRAME_V2; k++) { + const base = 4 + k * 16; + // Approximate world-mm-3d positions (simplified rowing skeleton) + const x = 50 + Math.sin(k * 0.5) * 200; // lateral position mm + const y = 500 + Math.cos(k * 0.3) * 300 + phase * 100; // vertical mm + const z = 1000 + Math.sin(f * 0.1 + k) * 50; // forward/back mm + const conf = 0.8 + 0.15 * Math.sin(k + f * 0.05); + fv.setFloat32(base, x, true); + fv.setFloat32(base + 4, y, true); + fv.setFloat32(base + 8, z, true); + fv.setFloat32(base + 12, conf, true); + } + + // qualityFlags + fv.setUint32(BYTES_PER_FRAME_V2 - 4, 0, true); +} + +const blob = new Uint8Array(HEADER_SIZE + FRAME_COUNT * BYTES_PER_FRAME_V2); +blob.set(header, 0); +blob.set(frames, HEADER_SIZE); + +writeFileSync(join(__dir, "v2-blob-3d.bin"), blob); +console.log(`Written ${blob.byteLength} bytes → v2-blob-3d.bin`); diff --git a/src/lib/mocap/__tests__/fixtures/v2-blob-3d.bin b/src/lib/mocap/__tests__/fixtures/v2-blob-3d.bin new file mode 100644 index 0000000..62f62cd Binary files /dev/null and b/src/lib/mocap/__tests__/fixtures/v2-blob-3d.bin differ diff --git a/src/lib/mocap/aiPayload.ts b/src/lib/mocap/aiPayload.ts new file mode 100644 index 0000000..e565b07 --- /dev/null +++ b/src/lib/mocap/aiPayload.ts @@ -0,0 +1,107 @@ +/** + * Posture AI payload builder — 3-tier cloud data policy. + * + * Tier 1 (raw PoseFrameStream): HARD WALL — never sent to cloud AI. + * Tier 3 (fault summary): Default when cloudAIEnabled = true. + * Tier 2 (per-stroke metrics): Opt-in; requires mocapDetailedAIShare = true. + * + * This module has NO Prisma imports: it is a pure data-transformation layer. + */ + +export interface PostureFaultSummary { + totalFaults: number; + faultCounts: Partial>; + severityCounts: { info: number; warning: number; critical: number }; + qualityFlags: string[]; + sessionQualityScore: number | null; +} + +export interface PostureMetricSummary { + strokeIndex: number; + segmentationSource: string; + backAngleAtCatchDeg: number; + laybackAngleDeg: number; + recoveryDriveRatio: number; +} + +export interface PostureAIPayload { + tier: 2 | 3; + faultSummary: PostureFaultSummary; + strokeMetrics?: PostureMetricSummary[]; // only present on tier 2 +} + +/** + * Build a cloud-safe posture AI payload respecting the 3-tier policy. + * + * Returns null when cloudAIEnabled is false (Tier 1 hard-wall). + * Returns a Tier 3 payload (fault summary only) by default. + * Returns a Tier 2 payload (fault summary + per-stroke metrics) when + * mocapDetailedAIShare is also true. + */ +export function buildPostureAIPayload( + faults: Array<{ faultType: string; severity: string }>, + metrics: Array<{ + strokeIndex: number; + segmentationSource: string; + metricsJson: unknown; + }>, + qualityFlags: string[], + qualityScore: number | null, + opts: { cloudAIEnabled: boolean; mocapDetailedAIShare: boolean }, +): PostureAIPayload | null { + // Tier 1 hard-wall: no posture data leaves the device. + if (!opts.cloudAIEnabled) return null; + + const faultCounts: Partial> = {}; + const severityCounts = { info: 0, warning: 0, critical: 0 }; + + for (const f of faults) { + faultCounts[f.faultType] = (faultCounts[f.faultType] ?? 0) + 1; + if (f.severity === "info") severityCounts.info++; + else if (f.severity === "warning") severityCounts.warning++; + else if (f.severity === "critical") severityCounts.critical++; + } + + const faultSummary: PostureFaultSummary = { + totalFaults: faults.length, + faultCounts, + severityCounts, + qualityFlags, + sessionQualityScore: qualityScore, + }; + + // Tier 3 (default): fault summary only — no body geometry, no keypoints. + if (!opts.mocapDetailedAIShare) { + return { tier: 3, faultSummary }; + } + + // Tier 2 (opt-in): fault summary + per-stroke scalar metrics — NO keypoints. + const strokeMetrics: PostureMetricSummary[] = metrics.map((m) => { + const mj = m.metricsJson as Record; + return { + strokeIndex: m.strokeIndex, + segmentationSource: m.segmentationSource, + backAngleAtCatchDeg: + typeof mj.backAngleAtCatchDeg === "number" ? mj.backAngleAtCatchDeg : 0, + laybackAngleDeg: + typeof mj.laybackAngleDeg === "number" ? mj.laybackAngleDeg : 0, + recoveryDriveRatio: + typeof mj.recoveryDriveRatio === "number" ? mj.recoveryDriveRatio : 0, + }; + }); + + return { tier: 2, faultSummary, strokeMetrics }; +} + +/** + * Hard guard: throws if the serialised payload contains keypoint arrays. + * Call this before appending any posture payload to a cloud-bound prompt. + */ +export function assertNoKeypointsInPayload(payload: unknown): void { + const str = JSON.stringify(payload); + if (str.includes('"keypoints"') || str.includes('"landmarks"')) { + throw new Error( + "HARD GUARD VIOLATION: cloud-bound payload contains keypoint data", + ); + } +} diff --git a/src/lib/mocap/analysis/index.ts b/src/lib/mocap/analysis/index.ts new file mode 100644 index 0000000..1883435 --- /dev/null +++ b/src/lib/mocap/analysis/index.ts @@ -0,0 +1,8 @@ +export * from "./types"; +export * from "./strokePhaseSegmenter"; +export * from "./postureMetrics"; +export * from "./postureFaultDetector"; +export * from "./postureThresholds"; +export * from "./poseFrameStreamAdapter"; +export * from "./postSessionAnalysis"; +export * from "./strokeAlignment"; diff --git a/src/lib/mocap/analysis/poseFrameStreamAdapter.ts b/src/lib/mocap/analysis/poseFrameStreamAdapter.ts new file mode 100644 index 0000000..6872c18 --- /dev/null +++ b/src/lib/mocap/analysis/poseFrameStreamAdapter.ts @@ -0,0 +1,115 @@ +import { + BYTES_PER_FRAME_V1, + BYTES_PER_FRAME_V2, + HEADER_SIZE, + KEYPOINT_SCHEMA_V1, + KEYPOINT_SCHEMA_V2, + KEYPOINTS_PER_FRAME_V1, + KEYPOINTS_PER_FRAME_V2, + OPEN_FRAME_COUNT, + PoseStreamFormatError, + decodeFrame, + decodeHeader, +} from "../poseFrameStream"; +import type { + CapturePerspective, + PoseAnalysisFrame, + PoseFrameStream, + PosePoint, +} from "./types"; + +export function adaptPoseFrameStreamBlob( + blob: Uint8Array, + capturePerspective: CapturePerspective, +): PoseFrameStream { + return adaptPoseFrameStreamBytes( + blob.subarray(0, HEADER_SIZE), + blob.subarray(HEADER_SIZE), + capturePerspective, + ); +} + +export function adaptPoseFrameStreamBytes( + headerBytes: Uint8Array, + packedFrames: Uint8Array, + capturePerspective: CapturePerspective, +): PoseFrameStream { + const header = decodeHeader(headerBytes); + const isV2 = header.keypointSchemaVersion === KEYPOINT_SCHEMA_V2; + const expectedKeypoints = isV2 ? KEYPOINTS_PER_FRAME_V2 : KEYPOINTS_PER_FRAME_V1; + const expectedBytesPerFrame = isV2 ? BYTES_PER_FRAME_V2 : BYTES_PER_FRAME_V1; + + if ( + header.keypointsPerFrame !== expectedKeypoints || + header.bytesPerFrame !== expectedBytesPerFrame + ) { + throw new PoseStreamFormatError( + `PoseFrameStream header does not match v${header.keypointSchemaVersion} frame layout`, + ); + } + + if (packedFrames.byteLength % header.bytesPerFrame !== 0) { + throw new PoseStreamFormatError( + `Packed frames length ${packedFrames.byteLength} is not a multiple of frame size ${header.bytesPerFrame}`, + ); + } + + const frameCount = packedFrames.byteLength / header.bytesPerFrame; + if (header.frameCount !== OPEN_FRAME_COUNT && header.frameCount !== frameCount) { + throw new PoseStreamFormatError( + `Header frameCount ${header.frameCount} does not match packed frame count ${frameCount}`, + ); + } + + const schema = header.keypointSchemaVersion; + return { + fps: header.fps, + capturePerspective, + coordinateSpace: header.coordinateSpace, + cameraCount: header.cameraCount, + frames: Array.from({ length: frameCount }, (_, frameIndex) => + adaptFrame(packedFrames, frameIndex * header.bytesPerFrame, schema), + ), + }; +} + +function adaptFrame( + bytes: Uint8Array, + offset: number, + schema: number, +): PoseAnalysisFrame { + const frame = decodeFrame(bytes, offset, schema); + return { + timestampMs: frame.timestampMs, + keypoints: + schema === KEYPOINT_SCHEMA_V2 + ? keypointQuadsToPosePoints(frame.keypoints) + : keypointTripletsToPosePoints(frame.keypoints), + qualityFlags: frame.qualityFlags, + }; +} + +export function keypointTripletsToPosePoints(keypoints: Float32Array): PosePoint[] { + const points: PosePoint[] = []; + for (let i = 0; i < keypoints.length; i += 3) { + points.push({ + x: keypoints[i], + y: keypoints[i + 1], + confidence: keypoints[i + 2], + }); + } + return points; +} + +export function keypointQuadsToPosePoints(keypoints: Float32Array): PosePoint[] { + const points: PosePoint[] = []; + for (let i = 0; i < keypoints.length; i += 4) { + points.push({ + x: keypoints[i], + y: keypoints[i + 1], + z: keypoints[i + 2], + confidence: keypoints[i + 3], + }); + } + return points; +} diff --git a/src/lib/mocap/analysis/postSessionAnalysis.ts b/src/lib/mocap/analysis/postSessionAnalysis.ts new file mode 100644 index 0000000..1fdb2f8 --- /dev/null +++ b/src/lib/mocap/analysis/postSessionAnalysis.ts @@ -0,0 +1,107 @@ +import { PostureFaultDetector } from "./postureFaultDetector"; +import { PostureMetricsCalculator } from "./postureMetrics"; +import { StrokePhaseSegmenter } from "./strokePhaseSegmenter"; +import type { + Calibration, + PoseFrameStream, + PostureFault, + PostureMetrics, + Stroke, +} from "./types"; +import { postureThresholdsV1, type PostureThresholdBands } from "./postureThresholds"; + +export interface DerivedStrokePostureMetric { + strokeIndex: number; + phaseBoundariesJson: StrokePhaseBoundariesJson; + metricsJson: PostureMetricsJson; + segmentationSource: string; +} + +export interface DerivedPostureFault { + strokeIndex: number; + faultType: string; + severity: string; + phase: string; + evidenceJson: PostureFault["evidence"]; +} + +export interface PostSessionAnalysisResult { + metrics: DerivedStrokePostureMetric[]; + faults: DerivedPostureFault[]; +} + +export interface StrokePhaseBoundariesJson { + catchFrameIndex: number; + driveStartFrameIndex: number; + finishFrameIndex: number; + recoveryStartFrameIndex: number; + nextCatchFrameIndex: number; + confidence: number; + csvMatchOffsetMs?: number | null; // present when segmentationSource === "csv-aligned"; null = unmatched +} + +export type PostureMetricsJson = Omit< + PostureMetrics, + "strokeIndex" | "segmentationSource" +>; + +export function analyzePoseFrameStream( + stream: PoseFrameStream, + opts: { + calibration?: Calibration; + thresholds?: PostureThresholdBands; + } = {}, +): PostSessionAnalysisResult { + const thresholds = opts.thresholds ?? postureThresholdsV1.thresholds; + const strokes = StrokePhaseSegmenter(stream); + const metrics: DerivedStrokePostureMetric[] = []; + const faults: DerivedPostureFault[] = []; + + for (const stroke of strokes) { + const postureMetrics = PostureMetricsCalculator( + stream, + stroke, + opts.calibration, + ); + metrics.push(metricToDerivedRow(stroke, postureMetrics)); + for (const fault of PostureFaultDetector(postureMetrics, thresholds)) { + faults.push(faultToDerivedRow(fault)); + } + } + + return { metrics, faults }; +} + +function metricToDerivedRow( + stroke: Stroke, + metrics: PostureMetrics, +): DerivedStrokePostureMetric { + const { + strokeIndex, + segmentationSource, + ...metricsJson + } = metrics; + return { + strokeIndex, + segmentationSource, + phaseBoundariesJson: { + catchFrameIndex: stroke.catchFrameIndex, + driveStartFrameIndex: stroke.driveStartFrameIndex, + finishFrameIndex: stroke.finishFrameIndex, + recoveryStartFrameIndex: stroke.recoveryStartFrameIndex, + nextCatchFrameIndex: stroke.nextCatchFrameIndex, + confidence: stroke.confidence, + }, + metricsJson, + }; +} + +function faultToDerivedRow(fault: PostureFault): DerivedPostureFault { + return { + strokeIndex: fault.strokeIndex, + faultType: fault.faultType, + severity: fault.severity, + phase: fault.phase, + evidenceJson: fault.evidence, + }; +} diff --git a/src/lib/mocap/analysis/postureFaultDetector.ts b/src/lib/mocap/analysis/postureFaultDetector.ts new file mode 100644 index 0000000..e448476 --- /dev/null +++ b/src/lib/mocap/analysis/postureFaultDetector.ts @@ -0,0 +1,225 @@ +import { postureThresholdsV1, type PostureThresholdBands } from "./postureThresholds"; +import type { PostureFault, PostureMetrics, Sidecar3DMetrics } from "./types"; + +export function PostureFaultDetector( + metrics: PostureMetrics, + thresholds: PostureThresholdBands = postureThresholdsV1.thresholds, +): PostureFault[] { + const faults: PostureFault[] = []; + + if ( + metrics.backAngleAtCatchDeg < + thresholds.rounded_back_at_catch.criticalBelowDeg + ) { + faults.push({ + strokeIndex: metrics.strokeIndex, + faultType: "rounded_back_at_catch", + severity: "critical", + phase: "catch", + evidence: { + metric: "backAngleAtCatchDeg", + value: metrics.backAngleAtCatchDeg, + threshold: thresholds.rounded_back_at_catch.criticalBelowDeg, + }, + }); + } else if ( + metrics.backAngleAtCatchDeg < + thresholds.rounded_back_at_catch.warningBelowDeg + ) { + faults.push({ + strokeIndex: metrics.strokeIndex, + faultType: "rounded_back_at_catch", + severity: "warning", + phase: "catch", + evidence: { + metric: "backAngleAtCatchDeg", + value: metrics.backAngleAtCatchDeg, + threshold: thresholds.rounded_back_at_catch.warningBelowDeg, + }, + }); + } + + const armLead = metrics.armBendBeforeLegsCompleteFrames; + if ( + armLead !== null && + armLead >= thresholds.early_arm_bend.warningBeforeLegsCompleteFrames + ) { + faults.push({ + strokeIndex: metrics.strokeIndex, + faultType: "early_arm_bend", + severity: "warning", + phase: "drive", + evidence: { + metric: "armBendBeforeLegsCompleteFrames", + value: armLead, + threshold: thresholds.early_arm_bend.warningBeforeLegsCompleteFrames, + frameIndex: metrics.armBendOnsetFrameIndex ?? undefined, + }, + }); + } else if ( + armLead !== null && + armLead >= thresholds.early_arm_bend.infoBeforeLegsCompleteFrames + ) { + faults.push({ + strokeIndex: metrics.strokeIndex, + faultType: "early_arm_bend", + severity: "info", + phase: "drive", + evidence: { + metric: "armBendBeforeLegsCompleteFrames", + value: armLead, + threshold: thresholds.early_arm_bend.infoBeforeLegsCompleteFrames, + frameIndex: metrics.armBendOnsetFrameIndex ?? undefined, + }, + }); + } + + const openingOffset = metrics.hipKneeOpeningOffsetFrames; + if ( + openingOffset !== null && + openingOffset <= + -thresholds.back_opens_before_legs_drive + .warningTorsoOpensBeforeLegsFrames + ) { + faults.push({ + strokeIndex: metrics.strokeIndex, + faultType: "back_opens_before_legs_drive", + severity: "warning", + phase: "drive", + evidence: { + metric: "hipKneeOpeningOffsetFrames", + value: openingOffset, + threshold: + -thresholds.back_opens_before_legs_drive + .warningTorsoOpensBeforeLegsFrames, + }, + }); + } + + if (metrics.laybackAngleDeg > thresholds.excessive_layback.warningAboveDeg) { + faults.push({ + strokeIndex: metrics.strokeIndex, + faultType: "excessive_layback", + severity: "warning", + phase: "finish", + evidence: { + metric: "laybackAngleDeg", + value: metrics.laybackAngleDeg, + threshold: thresholds.excessive_layback.warningAboveDeg, + }, + }); + } else if ( + metrics.laybackAngleDeg > thresholds.excessive_layback.infoAboveDeg + ) { + faults.push({ + strokeIndex: metrics.strokeIndex, + faultType: "excessive_layback", + severity: "info", + phase: "finish", + evidence: { + metric: "laybackAngleDeg", + value: metrics.laybackAngleDeg, + threshold: thresholds.excessive_layback.infoAboveDeg, + }, + }); + } + + if ( + metrics.recoveryDriveRatio > + thresholds.slow_recovery_ratio.criticalAboveRatio + ) { + faults.push({ + strokeIndex: metrics.strokeIndex, + faultType: "slow_recovery_ratio", + severity: "critical", + phase: "recovery", + evidence: { + metric: "recoveryDriveRatio", + value: metrics.recoveryDriveRatio, + threshold: thresholds.slow_recovery_ratio.criticalAboveRatio, + }, + }); + } else if ( + metrics.recoveryDriveRatio > + thresholds.slow_recovery_ratio.warningAboveRatio + ) { + faults.push({ + strokeIndex: metrics.strokeIndex, + faultType: "slow_recovery_ratio", + severity: "warning", + phase: "recovery", + evidence: { + metric: "recoveryDriveRatio", + value: metrics.recoveryDriveRatio, + threshold: thresholds.slow_recovery_ratio.warningAboveRatio, + }, + }); + } + + // Sidecar-3D fault stubs — thresholds defined in follow-up issues + if (metrics.sidecar3D) { + faults.push(...detectSidecar3DFaults(metrics.strokeIndex, metrics.sidecar3D)); + } + + return faults; +} + +function detectSidecar3DFaults( + strokeIndex: number, + sidecar3D: Sidecar3DMetrics, +): PostureFault[] { + const faults: PostureFault[] = []; + + if (sidecar3D.lateralShoulderSymmetryMm !== undefined) { + // Threshold TBD — emit pending stub so UI can show "detection coming soon" + faults.push({ + strokeIndex, + faultType: "left_right_asymmetry", + severity: "pending", + phase: "drive", + evidence: { + metric: "leftRightAsymmetry", + value: sidecar3D.lateralShoulderSymmetryMm, + threshold: 0, + }, + }); + } + + if ( + sidecar3D.leftKneeTrackDeviationMm !== undefined || + sidecar3D.rightKneeTrackDeviationMm !== undefined + ) { + const value = + Math.max( + sidecar3D.leftKneeTrackDeviationMm ?? 0, + sidecar3D.rightKneeTrackDeviationMm ?? 0, + ); + faults.push({ + strokeIndex, + faultType: "knee_track_deviation", + severity: "pending", + phase: "drive", + evidence: { + metric: "kneeTrackDeviation", + value, + threshold: 0, + }, + }); + } + + if (sidecar3D.nearShinAngleDeg !== undefined) { + faults.push({ + strokeIndex, + faultType: "shin_not_vertical_at_catch", + severity: "pending", + phase: "catch", + evidence: { + metric: "shinVerticalAtCatchDeg", + value: sidecar3D.nearShinAngleDeg, + threshold: 0, + }, + }); + } + + return faults; +} diff --git a/src/lib/mocap/analysis/postureMetrics.ts b/src/lib/mocap/analysis/postureMetrics.ts new file mode 100644 index 0000000..e5e2b24 --- /dev/null +++ b/src/lib/mocap/analysis/postureMetrics.ts @@ -0,0 +1,371 @@ +import { + getPosePoint, + landmarkSide, + type Calibration, + type PoseAnalysisFrame, + type PoseFrameStream, + type PoseLandmarkName, + type PosePoint, + type PostureMetrics, + type Sidecar3DMetrics, + type Stroke, +} from "./types"; + +const MIN_CONFIDENCE = 0.25; + +export function PostureMetricsCalculator( + stream: PoseFrameStream, + stroke: Stroke, + calibration?: Calibration, +): PostureMetrics { + void calibration; + const projectedStream = toProjectedStream(stream); + const catchFrame = frameAt(projectedStream, stroke.catchFrameIndex); + const finishFrame = frameAt(projectedStream, stroke.finishFrameIndex); + const backAngleAtCatchDeg = torsoBackAngleDeg(projectedStream, catchFrame); + const backAngleAtFinishDeg = torsoBackAngleDeg(projectedStream, finishFrame); + const laybackAngleDeg = Math.max(0, 90 - backAngleAtFinishDeg); + + const legSignal = legExtensionSignal(projectedStream, stroke); + const catchLeg = legSignal[0]?.value ?? 0; + const finishLeg = legSignal[stroke.finishFrameIndex - stroke.catchFrameIndex] + ?.value ?? catchLeg; + const legRange = Math.max(0.0001, finishLeg - catchLeg); + + const legExtensionStartFrameIndex = firstSignalFrameAtLeast( + legSignal, + catchLeg + legRange * 0.2, + ); + const legExtensionCompleteFrameIndex = firstSignalFrameAtLeast( + legSignal, + catchLeg + legRange * 0.8, + ); + const torsoOpenFrameIndex = firstTorsoChangeFrame( + projectedStream, + stroke, + backAngleAtCatchDeg, + ); + const armBendOnsetFrameIndex = firstArmBendFrame(projectedStream, stroke); + + return { + strokeIndex: stroke.strokeIndex, + segmentationSource: stroke.segmentationSource, + backAngleAtCatchDeg, + backAngleAtFinishDeg, + laybackAngleDeg, + hipKneeOpeningOffsetFrames: + legExtensionStartFrameIndex === null || torsoOpenFrameIndex === null + ? null + : torsoOpenFrameIndex - legExtensionStartFrameIndex, + armBendOnsetFrameIndex, + legExtensionCompleteFrameIndex, + armBendBeforeLegsCompleteFrames: + armBendOnsetFrameIndex === null || legExtensionCompleteFrameIndex === null + ? null + : legExtensionCompleteFrameIndex - armBendOnsetFrameIndex, + recoveryDriveRatio: recoveryDriveRatio(stroke), + leftRightAsymmetry: + stream.capturePerspective === "sidecar-3d" + ? { available: false, reason: "insufficient-tracking" } + : { available: false, reason: "requires-sidecar-3d" }, + shinVerticalAtCatchDeg: + stream.capturePerspective === "sidecar-3d" + ? { available: false, reason: "insufficient-tracking" } + : { available: false, reason: "requires-sidecar-3d" }, + kneeTrackDeviation: + stream.capturePerspective === "sidecar-3d" + ? { available: false, reason: "insufficient-tracking" } + : { available: false, reason: "requires-sidecar-3d" }, + sidecar3D: + stream.capturePerspective === "sidecar-3d" + ? computeSidecar3DMetrics(stream, stroke) + : undefined, + }; +} + +function frameAt(stream: PoseFrameStream, frameIndex: number): PoseAnalysisFrame { + const frame = stream.frames[frameIndex]; + if (!frame) { + throw new Error(`Frame ${frameIndex} is outside the PoseFrameStream`); + } + return frame; +} + +function sideNames(stream: PoseFrameStream): { + shoulder: PoseLandmarkName; + elbow: PoseLandmarkName; + wrist: PoseLandmarkName; + hip: PoseLandmarkName; + knee: PoseLandmarkName; +} { + const side = landmarkSide(stream.capturePerspective); + return { + shoulder: `${side}Shoulder` as PoseLandmarkName, + elbow: `${side}Elbow` as PoseLandmarkName, + wrist: `${side}Wrist` as PoseLandmarkName, + hip: `${side}Hip` as PoseLandmarkName, + knee: `${side}Knee` as PoseLandmarkName, + }; +} + +function requiredPoint( + frame: PoseAnalysisFrame, + name: PoseLandmarkName, +): PosePoint { + const point = getPosePoint(frame, name); + if (!point || point.confidence < MIN_CONFIDENCE) { + throw new Error(`Missing tracked landmark ${name}`); + } + return point; +} + +function torsoBackAngleDeg( + stream: PoseFrameStream, + frame: PoseAnalysisFrame, +): number { + const names = sideNames(stream); + const hip = requiredPoint(frame, names.hip); + const shoulder = requiredPoint(frame, names.shoulder); + const dx = shoulder.x - hip.x; + const dyUp = hip.y - shoulder.y; + const raw = radiansToDegrees(Math.atan2(dyUp, dx)); + const normalized = raw < 0 ? raw + 180 : raw; + return normalized > 90 ? 180 - normalized : normalized; +} + +function legExtensionSignal( + stream: PoseFrameStream, + stroke: Stroke, +): Array<{ frameIndex: number; value: number }> { + const names = sideNames(stream); + const signal: Array<{ frameIndex: number; value: number }> = []; + for ( + let frameIndex = stroke.catchFrameIndex; + frameIndex <= stroke.finishFrameIndex; + frameIndex++ + ) { + const frame = frameAt(stream, frameIndex); + const hip = requiredPoint(frame, names.hip); + const knee = requiredPoint(frame, names.knee); + signal.push({ + frameIndex, + value: Math.hypot(hip.x - knee.x, hip.y - knee.y), + }); + } + return signal; +} + +function firstSignalFrameAtLeast( + signal: Array<{ frameIndex: number; value: number }>, + threshold: number, +): number | null { + for (const point of signal) { + if (point.value >= threshold) return point.frameIndex; + } + return null; +} + +function firstTorsoChangeFrame( + stream: PoseFrameStream, + stroke: Stroke, + catchAngleDeg: number, +): number | null { + for ( + let frameIndex = stroke.catchFrameIndex + 1; + frameIndex <= stroke.finishFrameIndex; + frameIndex++ + ) { + const frame = frameAt(stream, frameIndex); + if (Math.abs(torsoBackAngleDeg(stream, frame) - catchAngleDeg) >= 5) { + return frameIndex; + } + } + return null; +} + +function firstArmBendFrame( + stream: PoseFrameStream, + stroke: Stroke, +): number | null { + const initialAngle = elbowAngleDeg(stream, frameAt(stream, stroke.catchFrameIndex)); + const threshold = Math.min(160, initialAngle - 15); + for ( + let frameIndex = stroke.catchFrameIndex + 1; + frameIndex <= stroke.finishFrameIndex; + frameIndex++ + ) { + const angle = elbowAngleDeg(stream, frameAt(stream, frameIndex)); + if (angle <= threshold) return frameIndex; + } + return null; +} + +function elbowAngleDeg( + stream: PoseFrameStream, + frame: PoseAnalysisFrame, +): number { + const names = sideNames(stream); + const shoulder = requiredPoint(frame, names.shoulder); + const elbow = requiredPoint(frame, names.elbow); + const wrist = requiredPoint(frame, names.wrist); + return angleAtPointDeg(shoulder, elbow, wrist); +} + +function angleAtPointDeg(a: PosePoint, vertex: PosePoint, b: PosePoint): number { + const ax = a.x - vertex.x; + const ay = a.y - vertex.y; + const bx = b.x - vertex.x; + const by = b.y - vertex.y; + const denom = Math.hypot(ax, ay) * Math.hypot(bx, by); + if (denom === 0) return 0; + const cos = Math.max(-1, Math.min(1, (ax * bx + ay * by) / denom)); + return radiansToDegrees(Math.acos(cos)); +} + +function recoveryDriveRatio(stroke: Stroke): number { + const driveFrames = Math.max( + 1, + stroke.finishFrameIndex - stroke.driveStartFrameIndex, + ); + const recoveryFrames = Math.max( + 1, + stroke.nextCatchFrameIndex - stroke.recoveryStartFrameIndex, + ); + return recoveryFrames / driveFrames; +} + +function radiansToDegrees(radians: number): number { + return (radians * 180) / Math.PI; +} + +// --- Coordinate-space adapter (ADR-0005 §3) --- + +interface SessionBounds { + yMin: number; + yMax: number; + zMin: number; + zMax: number; +} + +function computeSessionBounds(stream: PoseFrameStream): SessionBounds { + let yMin = Infinity, yMax = -Infinity, zMin = Infinity, zMax = -Infinity; + for (const frame of stream.frames) { + const kps = Array.isArray(frame.keypoints) ? frame.keypoints : Object.values(frame.keypoints); + for (const kp of kps) { + if (!kp || kp.confidence < MIN_CONFIDENCE) continue; + if (kp.y < yMin) yMin = kp.y; + if (kp.y > yMax) yMax = kp.y; + if (kp.z !== undefined) { + if (kp.z < zMin) zMin = kp.z; + if (kp.z > zMax) zMax = kp.z; + } + } + } + return { yMin, yMax, zMin, zMax }; +} + +function projectToNormalized(point: PosePoint, bounds: SessionBounds): PosePoint { + if (point.z === undefined) return point; + const yRange = bounds.yMax - bounds.yMin; + const zRange = bounds.zMax - bounds.zMin; + return { + x: zRange > 0 ? (point.z - bounds.zMin) / zRange : 0.5, + y: yRange > 0 ? (point.y - bounds.yMin) / yRange : 0.5, + confidence: point.confidence, + }; +} + +function toProjectedStream(stream: PoseFrameStream): PoseFrameStream { + if (stream.coordinateSpace !== "world-mm-3d") return stream; + const bounds = computeSessionBounds(stream); + const projectedFrames = stream.frames.map((frame) => { + const kps = frame.keypoints; + const projected = Array.isArray(kps) + ? (kps as PosePoint[]).map((kp) => projectToNormalized(kp, bounds)) + : Object.fromEntries( + Object.entries(kps as Record).map(([k, v]) => [ + k, + projectToNormalized(v, bounds), + ]), + ); + return { ...frame, keypoints: projected }; + }); + return { ...stream, frames: projectedFrames as PoseFrameStream["frames"] }; +} + +// --- Sidecar-3D specific metrics --- + +function computeSidecar3DMetrics( + stream: PoseFrameStream, + stroke: Stroke, +): Sidecar3DMetrics { + const metrics: Sidecar3DMetrics = {}; + const frames = stream.frames.slice(stroke.catchFrameIndex, stroke.finishFrameIndex + 1); + if (frames.length === 0) return metrics; + + // lateralShoulderSymmetryMm: mean absolute x-displacement left vs right shoulder + const shoulderDeltas: number[] = []; + const hipDeltas: number[] = []; + for (const frame of frames) { + const ls = getPosePoint(frame, "leftShoulder"); + const rs = getPosePoint(frame, "rightShoulder"); + const lh = getPosePoint(frame, "leftHip"); + const rh = getPosePoint(frame, "rightHip"); + if (ls && rs && ls.confidence >= MIN_CONFIDENCE && rs.confidence >= MIN_CONFIDENCE) { + shoulderDeltas.push(Math.abs(ls.x - rs.x)); + } + if (lh && rh && lh.confidence >= MIN_CONFIDENCE && rh.confidence >= MIN_CONFIDENCE) { + hipDeltas.push(Math.abs(lh.x - rh.x)); + } + } + if (shoulderDeltas.length > 0) { + metrics.lateralShoulderSymmetryMm = + shoulderDeltas.reduce((a, b) => a + b, 0) / shoulderDeltas.length; + } + if (hipDeltas.length > 0) { + metrics.lateralHipSymmetryMm = + hipDeltas.reduce((a, b) => a + b, 0) / hipDeltas.length; + } + + // knee track deviation: peak |knee.x - ankle.x| during drive + let leftKneePeak = 0; + let rightKneePeak = 0; + for (const frame of frames) { + const lk = getPosePoint(frame, "leftKnee"); + const la = getPosePoint(frame, "leftAnkle"); + const rk = getPosePoint(frame, "rightKnee"); + const ra = getPosePoint(frame, "rightAnkle"); + if (lk && la && lk.confidence >= MIN_CONFIDENCE && la.confidence >= MIN_CONFIDENCE) { + leftKneePeak = Math.max(leftKneePeak, Math.abs(lk.x - la.x)); + } + if (rk && ra && rk.confidence >= MIN_CONFIDENCE && ra.confidence >= MIN_CONFIDENCE) { + rightKneePeak = Math.max(rightKneePeak, Math.abs(rk.x - ra.x)); + } + } + if (leftKneePeak > 0) metrics.leftKneeTrackDeviationMm = leftKneePeak; + if (rightKneePeak > 0) metrics.rightKneeTrackDeviationMm = rightKneePeak; + + // nearShinAngleDeg: shin angle from nearer (lower |z|) ankle/knee pair at catch + const catchFrame = stream.frames[stroke.catchFrameIndex]; + if (catchFrame) { + const lk = getPosePoint(catchFrame, "leftKnee"); + const la = getPosePoint(catchFrame, "leftAnkle"); + const rk = getPosePoint(catchFrame, "rightKnee"); + const ra = getPosePoint(catchFrame, "rightAnkle"); + const leftZ = lk?.z ?? Infinity; + const rightZ = rk?.z ?? Infinity; + const [knee, ankle] = Math.abs(leftZ) <= Math.abs(rightZ) + ? [lk, la] + : [rk, ra]; + if ( + knee && ankle && + knee.confidence >= MIN_CONFIDENCE && ankle.confidence >= MIN_CONFIDENCE + ) { + const dx = knee.x - ankle.x; + const dy = knee.y - ankle.y; + metrics.nearShinAngleDeg = radiansToDegrees(Math.atan2(Math.abs(dx), Math.abs(dy))); + } + } + + return metrics; +} diff --git a/src/lib/mocap/analysis/postureThresholds.ts b/src/lib/mocap/analysis/postureThresholds.ts new file mode 100644 index 0000000..b72a757 --- /dev/null +++ b/src/lib/mocap/analysis/postureThresholds.ts @@ -0,0 +1,263 @@ +import type { PostureFaultType } from "./types"; + +export type PostureThresholdVersion = "V1" | `V${number}`; + +export interface PostureThresholdBands { + rounded_back_at_catch: { + warningBelowDeg: number; + criticalBelowDeg: number; + }; + early_arm_bend: { + infoBeforeLegsCompleteFrames: number; + warningBeforeLegsCompleteFrames: number; + }; + back_opens_before_legs_drive: { + warningTorsoOpensBeforeLegsFrames: number; + }; + excessive_layback: { + infoAboveDeg: number; + warningAboveDeg: number; + }; + slow_recovery_ratio: { + warningAboveRatio: number; + criticalAboveRatio: number; + }; +} + +export interface VersionedPostureThresholds { + version: PostureThresholdVersion; + thresholds: PostureThresholdBands; +} + +export interface UserPostureThresholdSettings extends VersionedPostureThresholds { + userOverridden: boolean; +} + +export interface ResolvedPostureThresholdSettings { + settings: UserPostureThresholdSettings; + warning: string | null; +} + +export const POSTURE_FAULT_CATALOG_V1: readonly PostureFaultType[] = [ + "rounded_back_at_catch", + "early_arm_bend", + "back_opens_before_legs_drive", + "excessive_layback", + "slow_recovery_ratio", +]; + +export const postureThresholdsV1: VersionedPostureThresholds = { + version: "V1", + thresholds: { + // British Rowing Technique: catch/drive keeps the back straight and leaning + // forward; CONTEXT.md fixes the v1 warning/critical bands at 30/20 deg. + rounded_back_at_catch: { + warningBelowDeg: 30, + criticalBelowDeg: 20, + }, + // Concept2 Indoor Rowing Technique: drive sequence is legs, body, then arms; + // early elbow flexion before leg extension is therefore treated conservatively. + early_arm_bend: { + infoBeforeLegsCompleteFrames: 1, + warningBeforeLegsCompleteFrames: 4, + }, + // British Rowing and Concept2 both teach the drive as legs before body swing; + // any torso opening before the leg signal starts is a v1 warning. + back_opens_before_legs_drive: { + warningTorsoOpensBeforeLegsFrames: 1, + }, + // Concept2 finish guidance says the upper body leans back slightly; British + // Rowing calls leaning too far back a recovery-delaying fault. + excessive_layback: { + infoAboveDeg: 30, + warningAboveDeg: 45, + }, + // Concept2 frames recovery as the rest/preparation phase; ratios beyond 2.5x + // are deliberately conservative v1 flags for very slow recoveries. + slow_recovery_ratio: { + warningAboveRatio: 2.5, + criticalAboveRatio: 3.5, + }, + }, +}; + +export function defaultPostureThresholdSettings( + defaults: VersionedPostureThresholds = postureThresholdsV1, +): UserPostureThresholdSettings { + return { + version: defaults.version, + thresholds: cloneThresholds(defaults.thresholds), + userOverridden: false, + }; +} + +export function migratePostureThresholdSettings( + stored: unknown, + defaults: VersionedPostureThresholds = postureThresholdsV1, +): UserPostureThresholdSettings { + return resolvePostureThresholdSettings(stored, defaults).settings; +} + +export function resolvePostureThresholdSettings( + stored: unknown, + defaults: VersionedPostureThresholds = postureThresholdsV1, +): ResolvedPostureThresholdSettings { + if (!isUserPostureThresholdSettings(stored)) { + return { + settings: defaultPostureThresholdSettings(defaults), + warning: + stored === null || stored === undefined + ? null + : "Saved posture thresholds were malformed and defaults are active.", + }; + } + + if (stored.version !== defaults.version && !stored.userOverridden) { + return { + settings: defaultPostureThresholdSettings(defaults), + warning: null, + }; + } + + const validation = validatePostureThresholdBands(stored.thresholds); + if (!validation.valid) { + return { + settings: defaultPostureThresholdSettings(defaults), + warning: + "Saved posture thresholds were invalid and defaults are active.", + }; + } + + return { + settings: { + version: stored.version, + thresholds: cloneThresholds(stored.thresholds), + userOverridden: stored.userOverridden, + }, + warning: null, + }; +} + +export function validatePostureThresholdBands( + thresholds: PostureThresholdBands, +): { valid: true } | { valid: false; errors: string[] } { + const errors: string[] = []; + + if ( + thresholds.rounded_back_at_catch.criticalBelowDeg >= + thresholds.rounded_back_at_catch.warningBelowDeg + ) { + errors.push("Rounded-back critical angle must be below warning angle."); + } + if ( + thresholds.early_arm_bend.infoBeforeLegsCompleteFrames > + thresholds.early_arm_bend.warningBeforeLegsCompleteFrames + ) { + errors.push("Early-arm-bend info frame count must be at or below warning."); + } + if ( + thresholds.excessive_layback.infoAboveDeg > + thresholds.excessive_layback.warningAboveDeg + ) { + errors.push("Excessive-layback info angle must be at or below warning."); + } + if ( + thresholds.slow_recovery_ratio.warningAboveRatio >= + thresholds.slow_recovery_ratio.criticalAboveRatio + ) { + errors.push("Slow-recovery warning ratio must be below critical ratio."); + } + + for (const [key, value] of Object.entries(flattenThresholds(thresholds))) { + if (!Number.isFinite(value) || value < 0) { + errors.push(`${key} must be a non-negative number.`); + } + } + + return errors.length === 0 ? { valid: true } : { valid: false, errors }; +} + +export function thresholdBandsEqual( + a: PostureThresholdBands, + b: PostureThresholdBands, +): boolean { + const flatA = flattenThresholds(a); + const flatB = flattenThresholds(b); + return Object.keys(flatA).every((key) => flatA[key] === flatB[key]); +} + +export function cloneThresholds( + thresholds: PostureThresholdBands, +): PostureThresholdBands { + return { + rounded_back_at_catch: { ...thresholds.rounded_back_at_catch }, + early_arm_bend: { ...thresholds.early_arm_bend }, + back_opens_before_legs_drive: { + ...thresholds.back_opens_before_legs_drive, + }, + excessive_layback: { ...thresholds.excessive_layback }, + slow_recovery_ratio: { ...thresholds.slow_recovery_ratio }, + }; +} + +export function isUserPostureThresholdSettings( + value: unknown, +): value is UserPostureThresholdSettings { + if (!value || typeof value !== "object") return false; + const candidate = value as Partial; + return ( + typeof candidate.version === "string" && + typeof candidate.userOverridden === "boolean" && + isPostureThresholdBands(candidate.thresholds) && + validatePostureThresholdBands(candidate.thresholds).valid + ); +} + +function isPostureThresholdBands( + value: unknown, +): value is PostureThresholdBands { + if (!value || typeof value !== "object") return false; + const t = value as PostureThresholdBands; + return ( + isFiniteNumber(t.rounded_back_at_catch?.warningBelowDeg) && + isFiniteNumber(t.rounded_back_at_catch?.criticalBelowDeg) && + isFiniteNumber(t.early_arm_bend?.infoBeforeLegsCompleteFrames) && + isFiniteNumber(t.early_arm_bend?.warningBeforeLegsCompleteFrames) && + isFiniteNumber( + t.back_opens_before_legs_drive?.warningTorsoOpensBeforeLegsFrames, + ) && + isFiniteNumber(t.excessive_layback?.infoAboveDeg) && + isFiniteNumber(t.excessive_layback?.warningAboveDeg) && + isFiniteNumber(t.slow_recovery_ratio?.warningAboveRatio) && + isFiniteNumber(t.slow_recovery_ratio?.criticalAboveRatio) + ); +} + +function isFiniteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value); +} + +function flattenThresholds( + thresholds: PostureThresholdBands, +): Record { + return { + "rounded_back_at_catch.warningBelowDeg": + thresholds.rounded_back_at_catch.warningBelowDeg, + "rounded_back_at_catch.criticalBelowDeg": + thresholds.rounded_back_at_catch.criticalBelowDeg, + "early_arm_bend.infoBeforeLegsCompleteFrames": + thresholds.early_arm_bend.infoBeforeLegsCompleteFrames, + "early_arm_bend.warningBeforeLegsCompleteFrames": + thresholds.early_arm_bend.warningBeforeLegsCompleteFrames, + "back_opens_before_legs_drive.warningTorsoOpensBeforeLegsFrames": + thresholds.back_opens_before_legs_drive + .warningTorsoOpensBeforeLegsFrames, + "excessive_layback.infoAboveDeg": thresholds.excessive_layback.infoAboveDeg, + "excessive_layback.warningAboveDeg": + thresholds.excessive_layback.warningAboveDeg, + "slow_recovery_ratio.warningAboveRatio": + thresholds.slow_recovery_ratio.warningAboveRatio, + "slow_recovery_ratio.criticalAboveRatio": + thresholds.slow_recovery_ratio.criticalAboveRatio, + }; +} diff --git a/src/lib/mocap/analysis/strokeAlignment.ts b/src/lib/mocap/analysis/strokeAlignment.ts new file mode 100644 index 0000000..ef740e9 --- /dev/null +++ b/src/lib/mocap/analysis/strokeAlignment.ts @@ -0,0 +1,145 @@ +export interface CsvStrokeTarget { + id: string; + strokeIndex: number; + timeMs: number; // elapsed ms from CSV session start (StrokeData.time * 1000) +} + +export interface StrokeMatch { + csvStrokeDataId: string; + csvStrokeIndex: number; + offsetMs: number; // signed: (poseCatchTime + delta) - csvTime +} + +export interface AlignmentResult { + matches: Map; // pose strokeIndex → match + deltaMs: number; + matchedCount: number; + unmatchedPoseStrokes: number[]; + unmatchedCsvStrokes: number[]; +} + +export const ALIGNMENT_TOLERANCE_MS = 2500; + +// Candidate delta rounding bucket (ms). Coarser = faster, finer = more accurate. +const DELTA_BUCKET_MS = 100; + +export function alignStrokesToCsv( + poseCatchTimesMs: number[], + csvStrokes: CsvStrokeTarget[], + toleranceMs = ALIGNMENT_TOLERANCE_MS, +): AlignmentResult { + if (poseCatchTimesMs.length === 0 || csvStrokes.length === 0) { + return { + matches: new Map(), + deltaMs: 0, + matchedCount: 0, + unmatchedPoseStrokes: poseCatchTimesMs.map((_, i) => i), + unmatchedCsvStrokes: csvStrokes.map((s) => s.strokeIndex), + }; + } + + const deltaMs = estimateDelta(poseCatchTimesMs, csvStrokes, toleranceMs); + return applyDelta(poseCatchTimesMs, csvStrokes, deltaMs, toleranceMs); +} + +function estimateDelta( + poseTimes: number[], + csvStrokes: CsvStrokeTarget[], + tolerance: number, +): number { + // Score each candidate delta (csv[j] - pose[i], rounded to nearest bucket). + // Limit brute-force to at most 50 pose × all csv pairs. + const csvTimes = csvStrokes.map((s) => s.timeMs); + const candidates = new Set(); + const poseLimit = Math.min(poseTimes.length, 50); + + for (let i = 0; i < poseLimit; i++) { + for (const csvTime of csvTimes) { + const raw = csvTime - poseTimes[i]; + candidates.add(Math.round(raw / DELTA_BUCKET_MS) * DELTA_BUCKET_MS); + } + } + + let bestDelta = 0; + let bestScore = -1; + for (const delta of candidates) { + const score = scoreWithDelta(poseTimes, csvTimes, delta, tolerance); + if (score > bestScore) { + bestScore = score; + bestDelta = delta; + } + } + return bestDelta; +} + +function scoreWithDelta( + poseTimes: number[], + csvTimes: number[], + delta: number, + tolerance: number, +): number { + let matched = 0; + const usedCsv = new Set(); + for (const poseTime of poseTimes) { + const adjusted = poseTime + delta; + let bestDist = tolerance; + let bestIdx = -1; + for (let j = 0; j < csvTimes.length; j++) { + if (usedCsv.has(j)) continue; + const dist = Math.abs(adjusted - csvTimes[j]); + if (dist < bestDist) { + bestDist = dist; + bestIdx = j; + } + } + if (bestIdx !== -1) { + matched++; + usedCsv.add(bestIdx); + } + } + return matched; +} + +function applyDelta( + poseCatchTimesMs: number[], + csvStrokes: CsvStrokeTarget[], + deltaMs: number, + toleranceMs: number, +): AlignmentResult { + const matches = new Map(); + const usedCsvIndices = new Set(); + + // Collect all pairs within tolerance and sort by proximity (greedy optimal for 1:1 matching). + const pairs: Array<{ poseIdx: number; csvIdx: number; dist: number }> = []; + for (let p = 0; p < poseCatchTimesMs.length; p++) { + const adjusted = poseCatchTimesMs[p] + deltaMs; + for (let c = 0; c < csvStrokes.length; c++) { + const dist = Math.abs(adjusted - csvStrokes[c].timeMs); + if (dist <= toleranceMs) { + pairs.push({ poseIdx: p, csvIdx: c, dist }); + } + } + } + pairs.sort((a, b) => a.dist - b.dist); + + const matchedPoseIndices = new Set(); + for (const { poseIdx, csvIdx, dist: _ } of pairs) { + if (matchedPoseIndices.has(poseIdx) || usedCsvIndices.has(csvIdx)) continue; + const csv = csvStrokes[csvIdx]; + matches.set(poseIdx, { + csvStrokeDataId: csv.id, + csvStrokeIndex: csv.strokeIndex, + offsetMs: poseCatchTimesMs[poseIdx] + deltaMs - csv.timeMs, + }); + matchedPoseIndices.add(poseIdx); + usedCsvIndices.add(csvIdx); + } + + return { + matches, + deltaMs, + matchedCount: matchedPoseIndices.size, + unmatchedPoseStrokes: poseCatchTimesMs.map((_, i) => i).filter((i) => !matchedPoseIndices.has(i)), + unmatchedCsvStrokes: csvStrokes.map((s) => s.strokeIndex).filter((_, c) => !usedCsvIndices.has(c)), + }; +} diff --git a/src/lib/mocap/analysis/strokePhaseSegmenter.ts b/src/lib/mocap/analysis/strokePhaseSegmenter.ts new file mode 100644 index 0000000..0385e63 --- /dev/null +++ b/src/lib/mocap/analysis/strokePhaseSegmenter.ts @@ -0,0 +1,160 @@ +import { + getPosePoint, + landmarkSide, + type PoseFrameStream, + type PoseLandmarkName, + type Stroke, +} from "./types"; + +interface SignalPoint { + frameIndex: number; + value: number; + tracked: boolean; +} + +export function StrokePhaseSegmenter(stream: PoseFrameStream): Stroke[] { + if (stream.frames.length < 3) return []; + + const signal = smoothSignal(buildHipKneeDistanceSignal(stream), stream.fps); + const catches = findCatchCandidates(signal, stream.fps); + const strokes: Stroke[] = []; + + for (let i = 0; i < catches.length - 1; i++) { + const catchFrameIndex = catches[i]; + const nextCatchFrameIndex = catches[i + 1]; + if (nextCatchFrameIndex - catchFrameIndex < 3) continue; + + const finishFrameIndex = maxSignalFrame( + signal, + catchFrameIndex, + nextCatchFrameIndex, + ); + if (finishFrameIndex <= catchFrameIndex) continue; + + const recoveryStartFrameIndex = Math.min( + finishFrameIndex + 1, + nextCatchFrameIndex, + ); + const trackedFrames = signal + .slice(catchFrameIndex, nextCatchFrameIndex + 1) + .filter((p) => p.tracked).length; + + strokes.push({ + strokeIndex: strokes.length, + segmentationSource: "pose-segmented", + catchFrameIndex, + driveStartFrameIndex: catchFrameIndex, + finishFrameIndex, + recoveryStartFrameIndex, + nextCatchFrameIndex, + confidence: trackedFrames / (nextCatchFrameIndex - catchFrameIndex + 1), + }); + } + + return strokes; +} + +function buildHipKneeDistanceSignal(stream: PoseFrameStream): SignalPoint[] { + const side = landmarkSide(stream.capturePerspective); + const hipName = `${side}Hip` as PoseLandmarkName; + const kneeName = `${side}Knee` as PoseLandmarkName; + + return stream.frames.map((frame, frameIndex) => { + const hip = getPosePoint(frame, hipName); + const knee = getPosePoint(frame, kneeName); + if (!hip || !knee || hip.confidence < 0.25 || knee.confidence < 0.25) { + return { frameIndex, value: Number.NaN, tracked: false }; + } + const dx = hip.x - knee.x; + const dy = hip.y - knee.y; + return { + frameIndex, + value: Math.hypot(dx, dy), + tracked: true, + }; + }); +} + +function smoothSignal(signal: SignalPoint[], fps: number): SignalPoint[] { + const radius = Math.max(1, Math.round(fps * 0.06)); + return signal.map((point, i) => { + let total = 0; + let count = 0; + for ( + let j = Math.max(0, i - radius); + j <= Math.min(signal.length - 1, i + radius); + j++ + ) { + const value = signal[j].value; + if (Number.isFinite(value)) { + total += value; + count++; + } + } + return { + frameIndex: point.frameIndex, + value: count > 0 ? total / count : point.value, + tracked: point.tracked, + }; + }); +} + +function findCatchCandidates(signal: SignalPoint[], fps: number): number[] { + const values = signal + .map((p) => p.value) + .filter((value) => Number.isFinite(value)); + if (values.length < 3) return []; + + const min = Math.min(...values); + const max = Math.max(...values); + const range = Math.max(0.0001, max - min); + const lowThreshold = min + range * 0.35; + const minGap = Math.max(4, Math.round(fps * 0.4)); + const catches: number[] = []; + + for (let i = 0; i < signal.length; i++) { + const prev = signal[Math.max(0, i - 1)]?.value; + const cur = signal[i].value; + const next = signal[Math.min(signal.length - 1, i + 1)]?.value; + if (!Number.isFinite(cur) || cur > lowThreshold) continue; + + const isEndpointMinimum = + (i === 0 && Number.isFinite(next) && cur <= next) || + (i === signal.length - 1 && Number.isFinite(prev) && cur <= prev); + const isInteriorMinimum = + i > 0 && + i < signal.length - 1 && + Number.isFinite(prev) && + Number.isFinite(next) && + cur <= prev && + cur <= next && + (cur < prev || cur < next); + + if (!isEndpointMinimum && !isInteriorMinimum) continue; + const last = catches[catches.length - 1]; + if (last === undefined || i - last >= minGap) { + catches.push(i); + } else if (cur < signal[last].value) { + catches[catches.length - 1] = i; + } + } + + return catches; +} + +function maxSignalFrame( + signal: SignalPoint[], + startFrame: number, + endFrame: number, +): number { + let maxFrame = startFrame; + let maxValue = -Infinity; + for (let i = startFrame; i <= endFrame; i++) { + const value = signal[i]?.value; + if (Number.isFinite(value) && value > maxValue) { + maxValue = value; + maxFrame = i; + } + } + return maxFrame; +} diff --git a/src/lib/mocap/analysis/types.ts b/src/lib/mocap/analysis/types.ts new file mode 100644 index 0000000..c6b79b7 --- /dev/null +++ b/src/lib/mocap/analysis/types.ts @@ -0,0 +1,160 @@ +export type CapturePerspective = "side-left" | "side-right" | "sidecar-3d"; + +export type StrokeSegmentationSource = "pose-segmented" | "csv-aligned"; + +export type PostureFaultType = + | "rounded_back_at_catch" + | "early_arm_bend" + | "back_opens_before_legs_drive" + | "excessive_layback" + | "slow_recovery_ratio" + | "left_right_asymmetry" + | "knee_track_deviation" + | "shin_not_vertical_at_catch"; + +export type FaultSeverity = "info" | "warning" | "critical" | "pending"; + +export type PoseLandmarkName = + | "leftShoulder" + | "rightShoulder" + | "leftElbow" + | "rightElbow" + | "leftWrist" + | "rightWrist" + | "leftHip" + | "rightHip" + | "leftKnee" + | "rightKnee" + | "leftAnkle" + | "rightAnkle"; + +export const POSE_LANDMARK_INDEX: Record = { + leftShoulder: 11, + rightShoulder: 12, + leftElbow: 13, + rightElbow: 14, + leftWrist: 15, + rightWrist: 16, + leftHip: 23, + rightHip: 24, + leftKnee: 25, + rightKnee: 26, + leftAnkle: 27, + rightAnkle: 28, +}; + +export interface PosePoint { + x: number; + y: number; + z?: number; // world-mm-3d only (sidecar-3d, keypointSchemaVersion 2) + confidence: number; +} + +export type PoseKeypoints = + | readonly PosePoint[] + | Partial>; + +export interface PoseAnalysisFrame { + timestampMs: number; + keypoints: PoseKeypoints; + qualityFlags?: number; +} + +export interface PoseFrameStream { + fps: number; + capturePerspective: CapturePerspective; + frames: readonly PoseAnalysisFrame[]; + coordinateSpace?: "normalized-2d" | "world-mm-3d"; + cameraCount?: number; +} + +export interface Stroke { + strokeIndex: number; + segmentationSource: StrokeSegmentationSource; + catchFrameIndex: number; + driveStartFrameIndex: number; + finishFrameIndex: number; + recoveryStartFrameIndex: number; + nextCatchFrameIndex: number; + confidence: number; +} + +export interface Calibration { + capturePerspective: CapturePerspective; + catchFrame?: PoseAnalysisFrame; + finishFrame?: PoseAnalysisFrame; +} + +export interface UnavailableMetric { + available: false; + reason: "requires-sidecar-3d" | "insufficient-tracking"; +} + +export interface AvailableMetric { + available: true; + value: T; +} + +export type MaybeMetric = AvailableMetric | UnavailableMetric; + +export interface Sidecar3DMetrics { + lateralShoulderSymmetryMm?: number; + lateralHipSymmetryMm?: number; + leftKneeTrackDeviationMm?: number; + rightKneeTrackDeviationMm?: number; + nearShinAngleDeg?: number; +} + +export interface PostureMetrics { + strokeIndex: number; + segmentationSource: StrokeSegmentationSource; + backAngleAtCatchDeg: number; + backAngleAtFinishDeg: number; + laybackAngleDeg: number; + hipKneeOpeningOffsetFrames: number | null; + armBendOnsetFrameIndex: number | null; + legExtensionCompleteFrameIndex: number | null; + armBendBeforeLegsCompleteFrames: number | null; + recoveryDriveRatio: number; + leftRightAsymmetry: MaybeMetric; + shinVerticalAtCatchDeg: MaybeMetric; + kneeTrackDeviation: MaybeMetric; + sidecar3D?: Sidecar3DMetrics; +} + +export interface PostureFault { + strokeIndex: number; + faultType: PostureFaultType; + severity: FaultSeverity; + phase: "catch" | "drive" | "finish" | "recovery"; + evidence: { + metric: keyof PostureMetrics | "armBendBeforeLegsCompleteFrames"; + value: number; + threshold: number; + frameIndex?: number; + }; +} + +export function getPosePoint( + frame: PoseAnalysisFrame, + name: PoseLandmarkName, +): PosePoint | null { + const keypoints = frame.keypoints; + if (isPosePointArray(keypoints)) { + const point = keypoints[POSE_LANDMARK_INDEX[name]]; + return point?.confidence > 0 ? point : null; + } + const point = keypoints[name]; + return point?.confidence && point.confidence > 0 ? point : null; +} + +function isPosePointArray(keypoints: PoseKeypoints): keypoints is readonly PosePoint[] { + return Array.isArray(keypoints); +} + +export function landmarkSide( + perspective: CapturePerspective, +): "left" | "right" { + // sidecar-3d uses right-side projection for v1-compatible fault rules + return perspective === "side-left" ? "left" : "right"; +} diff --git a/src/lib/mocap/browserPoseSource.ts b/src/lib/mocap/browserPoseSource.ts new file mode 100644 index 0000000..168f475 --- /dev/null +++ b/src/lib/mocap/browserPoseSource.ts @@ -0,0 +1,253 @@ +/** + * BrowserPoseSource: main-thread orchestration for the pose worker. + * + * Owns the worker, the upload queue, and the per-frame ImageBitmap pipeline. + * Frames flow: