From 8f6e31e2932dde2cd60854f342885a1d33390299 Mon Sep 17 00:00:00 2001 From: Ayush Raj Chourasia Date: Wed, 13 May 2026 18:46:39 +0000 Subject: [PATCH 1/3] Add gameplay review workflows --- .../GameplayReviewWorkspace.test.tsx | 182 +++++ .../intelligence/GameplayReviewWorkspace.tsx | 573 ++++++++++++++++ app/src/pages/Intelligence.tsx | 6 +- app/src/services/gameplayReviewService.ts | 224 +++++++ src/core/all.rs | 5 + src/openhuman/about_app/catalog.rs | 50 ++ src/openhuman/gameplay_review/mod.rs | 14 + src/openhuman/gameplay_review/ops.rs | 630 ++++++++++++++++++ src/openhuman/gameplay_review/schemas.rs | 178 +++++ src/openhuman/gameplay_review/store.rs | 175 +++++ src/openhuman/gameplay_review/types.rs | 173 +++++ src/openhuman/mod.rs | 1 + 12 files changed, 2210 insertions(+), 1 deletion(-) create mode 100644 app/src/components/intelligence/GameplayReviewWorkspace.test.tsx create mode 100644 app/src/components/intelligence/GameplayReviewWorkspace.tsx create mode 100644 app/src/services/gameplayReviewService.ts create mode 100644 src/openhuman/gameplay_review/mod.rs create mode 100644 src/openhuman/gameplay_review/ops.rs create mode 100644 src/openhuman/gameplay_review/schemas.rs create mode 100644 src/openhuman/gameplay_review/store.rs create mode 100644 src/openhuman/gameplay_review/types.rs diff --git a/app/src/components/intelligence/GameplayReviewWorkspace.test.tsx b/app/src/components/intelligence/GameplayReviewWorkspace.test.tsx new file mode 100644 index 0000000000..8419d66683 --- /dev/null +++ b/app/src/components/intelligence/GameplayReviewWorkspace.test.tsx @@ -0,0 +1,182 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + analyzeGameplaySession, + askGameplaySession, + draftGameplayClipMetadata, + listGameplaySessions, + prepareGameplayFrames, + registerGameplaySession, + saveGameplayPreset, +} from '../../services/gameplayReviewService'; +import { GameplayReviewWorkspace } from './GameplayReviewWorkspace'; + +vi.mock('../../services/gameplayReviewService', () => ({ + analyzeGameplaySession: vi.fn(), + askGameplaySession: vi.fn(), + draftGameplayClipMetadata: vi.fn(), + flattenClipCandidates: vi.fn((session: any) => session?.analysis?.clip_candidates ?? []), + flattenDrafts: vi.fn((session: any) => session?.analysis?.draft_metadata ?? []), + flattenHighlights: vi.fn((session: any) => session?.analysis?.highlights ?? []), + formatSpoilerMode: vi.fn((mode: string) => (mode === 'full' ? 'Full spoilers' : 'Light spoilers')), + listGameplaySessions: vi.fn(), + normalizeGameplayError: vi.fn((error: unknown) => (error instanceof Error ? error.message : String(error))), + prepareGameplayFrames: vi.fn(), + registerGameplaySession: vi.fn(), + saveGameplayPreset: vi.fn(), +})); + +const mockedListGameplaySessions = vi.mocked(listGameplaySessions); +const mockedPrepareGameplayFrames = vi.mocked(prepareGameplayFrames); +const mockedRegisterGameplaySession = vi.mocked(registerGameplaySession); +const mockedAnalyzeGameplaySession = vi.mocked(analyzeGameplaySession); +const mockedAskGameplaySession = vi.mocked(askGameplaySession); +const mockedDraftGameplayClipMetadata = vi.mocked(draftGameplayClipMetadata); +const mockedSaveGameplayPreset = vi.mocked(saveGameplayPreset); + +describe('GameplayReviewWorkspace', () => { + beforeEach(() => { + mockedListGameplaySessions.mockResolvedValue([]); + mockedPrepareGameplayFrames.mockReset(); + mockedRegisterGameplaySession.mockReset(); + mockedAnalyzeGameplaySession.mockReset(); + mockedAskGameplaySession.mockReset(); + mockedDraftGameplayClipMetadata.mockReset(); + mockedSaveGameplayPreset.mockReset(); + }); + + it('imports selected frames, analyzes the session, and asks follow-up questions', async () => { + mockedPrepareGameplayFrames.mockResolvedValue([ + { + source_name: 'frame-1.png', + file_name: 'frame-1.png', + image_ref: 'data:image/png;base64,AAA', + captured_at_ms: 123, + }, + ]); + mockedRegisterGameplaySession.mockResolvedValue({ + session_id: 'gameplay-apex-1', + game_id: 'Apex Legends', + session_title: 'Ranked climb', + source_label: '/recordings/apex', + spoiler_mode: 'light', + preset_id: 'Apex preset', + imported_at_ms: 1000, + analyzed_at_ms: null, + frames: [ + { file_name: 'frame-1.png', image_ref: 'data:image/png;base64,AAA', captured_at_ms: 123 }, + ], + analysis: null, + }); + mockedAnalyzeGameplaySession.mockResolvedValue({ + session_id: 'gameplay-apex-1', + game_id: 'Apex Legends', + session_title: 'Ranked climb', + source_label: '/recordings/apex', + spoiler_mode: 'light', + preset_id: 'Apex preset', + imported_at_ms: 1000, + analyzed_at_ms: 2000, + frames: [ + { file_name: 'frame-1.png', image_ref: 'data:image/png;base64,AAA', captured_at_ms: 123 }, + ], + analysis: { + recap: 'Gameplay recap', + highlights: [ + { + id: 'h1', + frame_index: 0, + captured_at_ms: 123, + title: 'Highlight: clutch fight', + rationale: 'Clean finish', + confidence: 0.93, + kind: 'highlight', + }, + ], + clip_candidates: [ + { + id: 'c1', + frame_index: 0, + start_label: 'frame-1.png', + end_label: 'frame-1.png', + rationale: 'Clean finish', + confidence: 0.93, + }, + ], + draft_metadata: [ + { + platform: 'twitch', + title: 'Apex Legends — Highlight: clutch fight (twitch)', + description: 'Session: Ranked climb', + tags: ['apex-legends'], + }, + ], + follow_up_questions: ['What was the turning point?'], + spoiler_note: null, + }, + }); + mockedAskGameplaySession.mockResolvedValue({ + answer: 'Best clip candidate for Ranked climb: Highlight: clutch fight — Clean finish', + matched_highlights: [], + suggested_follow_up: ['What was the turning point?'], + }); + mockedDraftGameplayClipMetadata.mockResolvedValue([ + { + platform: 'twitch', + title: 'Apex Legends — Highlight: clutch fight (twitch)', + description: 'Session: Ranked climb', + tags: ['apex-legends'], + }, + ]); + mockedSaveGameplayPreset.mockResolvedValue({}); + + const { container } = render(); + + await waitFor(() => expect(screen.getByText('No saved sessions yet.')).toBeInTheDocument()); + + fireEvent.change(screen.getByPlaceholderText('Apex Legends'), { + target: { value: 'Apex Legends' }, + }); + fireEvent.change(screen.getByPlaceholderText('Ranked climb on Friday night'), { + target: { value: 'Ranked climb' }, + }); + fireEvent.change(screen.getByPlaceholderText('/recordings/apex/night-01'), { + target: { value: '/recordings/apex' }, + }); + fireEvent.change(screen.getByPlaceholderText('Apex coaching preset'), { + target: { value: 'Apex preset' }, + }); + fireEvent.change(screen.getByPlaceholderText('aim, positioning, fight selection'), { + target: { value: 'aim, positioning' }, + }); + fireEvent.change(screen.getByPlaceholderText('What should the reviewer pay attention to?'), { + target: { value: 'Play clean and keep it spoiler-safe.' }, + }); + fireEvent.change(screen.getByPlaceholderText('twitch,kick,youtube'), { + target: { value: 'twitch,kick' }, + }); + + const fileInput = container.querySelector('input[type="file"]') as HTMLInputElement | null; + expect(fileInput).not.toBeNull(); + const file = new File([new Uint8Array([1, 2, 3])], 'frame-1.png', { type: 'image/png' }); + Object.defineProperty(file, 'webkitRelativePath', { value: 'session/frame-1.png' }); + fireEvent.change(fileInput as HTMLInputElement, { + target: { files: [file] }, + }); + + fireEvent.click(screen.getByRole('button', { name: /import and analyze/i })); + + await waitFor(() => expect(mockedPrepareGameplayFrames).toHaveBeenCalled()); + await waitFor(() => expect(screen.getByText(/Gameplay recap/)).toBeInTheDocument()); + expect( + screen.getByText('Highlight: clutch fight', { + selector: 'div.font-medium.text-stone-900', + }) + ).toBeInTheDocument(); + + fireEvent.click(screen.getByRole('button', { name: /ask session/i })); + await waitFor(() => expect(mockedAskGameplaySession).toHaveBeenCalled()); + expect(screen.getByText(/Best clip candidate/)).toBeInTheDocument(); + }); +}); diff --git a/app/src/components/intelligence/GameplayReviewWorkspace.tsx b/app/src/components/intelligence/GameplayReviewWorkspace.tsx new file mode 100644 index 0000000000..8e9a462818 --- /dev/null +++ b/app/src/components/intelligence/GameplayReviewWorkspace.tsx @@ -0,0 +1,573 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; + +import type { ToastNotification } from '../../types/intelligence'; +import { + analyzeGameplaySession, + askGameplaySession, + draftGameplayClipMetadata, + flattenClipCandidates, + flattenDrafts, + flattenHighlights, + formatSpoilerMode, + listGameplaySessions, + normalizeGameplayError, + prepareGameplayFrames, + registerGameplaySession, + saveGameplayPreset, + type GameplayFrameInput, + type GameplayReviewSession, + type SpoilerMode, +} from '../../services/gameplayReviewService'; + +interface GameplayReviewWorkspaceProps { + onToast?: (toast: Omit) => void; +} + +interface ImportState { + gameId: string; + sessionTitle: string; + sourceLabel: string; + spoilerMode: SpoilerMode; + presetName: string; + coachingFocus: string; + notes: string; + audioFeedback: boolean; + platforms: string; +} + +const INITIAL_IMPORT_STATE: ImportState = { + gameId: '', + sessionTitle: '', + sourceLabel: '', + spoilerMode: 'light', + presetName: '', + coachingFocus: '', + notes: '', + audioFeedback: false, + platforms: 'twitch,kick,youtube', +}; + +const DIRECTORY_INPUT_PROPS: Record = { + webkitdirectory: '', +}; + +function formatTimestamp(ms?: number | null): string { + if (!ms) return 'n/a'; + return new Date(ms).toLocaleString(); +} + +function splitPlatforms(value: string): string[] { + return value + .split(',') + .map(item => item.trim()) + .filter(Boolean); +} + +export function GameplayReviewWorkspace({ onToast }: GameplayReviewWorkspaceProps) { + const [form, setForm] = useState(INITIAL_IMPORT_STATE); + const [selectedFiles, setSelectedFiles] = useState([]); + const [recentSessions, setRecentSessions] = useState([]); + const [activeSession, setActiveSession] = useState(null); + const [question, setQuestion] = useState('What were my best moments?'); + const [questionAnswer, setQuestionAnswer] = useState(''); + const [questionBusy, setQuestionBusy] = useState(false); + const [importBusy, setImportBusy] = useState(false); + const [analysisBusy, setAnalysisBusy] = useState(false); + const [draftBusy, setDraftBusy] = useState(false); + const [error, setError] = useState(null); + const fileInputRef = useRef(null); + + const refreshSessions = useCallback(async () => { + try { + const sessions = await listGameplaySessions(); + setRecentSessions(sessions); + if (!activeSession && sessions.length > 0) { + setActiveSession(sessions[0]); + } + } catch (err) { + setError(normalizeGameplayError(err)); + } + }, [activeSession]); + + useEffect(() => { + void refreshSessions(); + }, [refreshSessions]); + + const activeHighlights = useMemo(() => flattenHighlights(activeSession), [activeSession]); + const activeDrafts = useMemo(() => flattenDrafts(activeSession), [activeSession]); + const activeClips = useMemo(() => flattenClipCandidates(activeSession), [activeSession]); + + const handleSelectFiles = useCallback(() => { + fileInputRef.current?.click(); + }, []); + + const handleFilesChanged = useCallback((event: React.ChangeEvent) => { + const files = Array.from(event.target.files ?? []); + setSelectedFiles(files); + }, []); + + const handleImportSession = useCallback(async () => { + setError(null); + if (!form.gameId.trim()) { + setError('Game name is required.'); + return; + } + if (!form.sessionTitle.trim()) { + setError('Session title is required.'); + return; + } + if (selectedFiles.length === 0) { + setError('Choose a folder or a set of keyframe images first.'); + return; + } + + setImportBusy(true); + try { + const frames = await prepareGameplayFrames(selectedFiles); + if (frames.length === 0) { + throw new Error('No image frames were found in that folder.'); + } + + if (form.presetName.trim()) { + await saveGameplayPreset({ + game_id: form.gameId.trim(), + display_name: form.presetName.trim(), + coaching_focus: form.coachingFocus + .split(',') + .map(item => item.trim()) + .filter(Boolean), + audio_feedback: form.audioFeedback, + spoiler_mode: form.spoilerMode, + notes: form.notes.trim() || null, + }); + } + + const session = await registerGameplaySession({ + game_id: form.gameId.trim(), + session_title: form.sessionTitle.trim(), + source_label: form.sourceLabel.trim() || null, + spoiler_mode: form.spoilerMode, + preset_id: form.presetName.trim() || null, + frames: frames.map(frame => ({ + file_name: frame.file_name, + image_ref: frame.image_ref, + captured_at_ms: frame.captured_at_ms, + } satisfies GameplayFrameInput)), + }); + + setAnalysisBusy(true); + const analyzed = await analyzeGameplaySession({ + session_id: session.session_id, + max_highlights: 5, + platforms: splitPlatforms(form.platforms), + }); + setActiveSession(analyzed); + setQuestionAnswer(''); + onToast?.({ + type: 'success', + title: 'Gameplay session analyzed', + message: `Reviewed ${analyzed.frames.length} frame(s) for ${analyzed.game_id}.`, + }); + void refreshSessions(); + } catch (err) { + const message = normalizeGameplayError(err); + setError(message); + onToast?.({ + type: 'error', + title: 'Gameplay review failed', + message, + }); + } finally { + setAnalysisBusy(false); + setImportBusy(false); + } + }, [form, onToast, refreshSessions, selectedFiles]); + + const handleQuestion = useCallback(async () => { + if (!activeSession) { + setError('Select or analyze a session first.'); + return; + } + setQuestionBusy(true); + try { + const response = await askGameplaySession({ session_id: activeSession.session_id, question }); + setQuestionAnswer(response.answer); + } catch (err) { + setError(normalizeGameplayError(err)); + } finally { + setQuestionBusy(false); + } + }, [activeSession, question]); + + const handleDraftClips = useCallback(async () => { + if (!activeSession) return; + setDraftBusy(true); + try { + const drafts = await draftGameplayClipMetadata({ + session_id: activeSession.session_id, + platform: splitPlatforms(form.platforms)[0] || 'twitch', + highlight_id: activeHighlights[0]?.id ?? null, + }); + setActiveSession(prev => + prev + ? { + ...prev, + analysis: prev.analysis + ? { ...prev.analysis, draft_metadata: drafts } + : prev.analysis, + } + : prev + ); + } catch (err) { + setError(normalizeGameplayError(err)); + } finally { + setDraftBusy(false); + } + }, [activeHighlights, activeSession, form.platforms]); + + return ( +
+
+
+
+

+ Gameplay review +

+

Import a session, find the clips, draft the post

+

+ Load a folder of keyframes from a long session, generate a concise recap, ask follow-up questions, + and turn the strongest moments into platform-ready clip metadata. +

+
+
+
Selected files
+
{selectedFiles.length} file(s)
+
{formatSpoilerMode(form.spoilerMode)}
+
+
+ +
+ + + + + + +