diff --git a/README.md b/README.md index 90f3bce..39ccc29 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,8 @@ taskmaster/ │ ├── detection_loop.py # camera -> detectors -> events loop │ ├── phone_detector.py # phone-in-frame detection │ └── gaze_detector.py # gaze/face detection (planned) -├── setup.sh # one-shot install for Python + Electron +├── setup.sh # one-shot install for Python + Electron (macOS/Linux) +├── setup.ps1 # same, for Windows PowerShell ├── PLAN.md └── README.md ``` @@ -81,11 +82,16 @@ taskmaster/ One command installs both the Python CV worker and the Electron app: ```bash +# macOS / Linux ./setup.sh + +# Windows (PowerShell) +./setup.ps1 ``` It creates the Python venv at `python/.venv` (Python 3.11), installs -`requirements.txt`, and runs `npm install` in `electron/`. +`requirements.txt`, and runs `npm install` in `electron/`. The Electron app +expects the worker at that venv path, so run setup before `npm run dev`.
Manual setup (if you prefer) diff --git a/electron/src/main/index.ts b/electron/src/main/index.ts index 15dbdd8..8516abf 100644 --- a/electron/src/main/index.ts +++ b/electron/src/main/index.ts @@ -10,6 +10,7 @@ import { stopBrowserActivityBridge, } from './browser-activity-bridge.ts' import { registerIpcHandlers } from './ipc-handlers.ts' +import { stopPythonWorker } from './python-bridge.ts' @@ -176,10 +177,15 @@ app.whenReady().then(() => { }) registerIpcHandlers() registerMiniTimerIpcHandlers() + // The CV worker is started on demand (see python-bridge.ts) when the renderer + // requests detection, not here — nothing runs while no session is active. createWindow() createTray() }) +// Safety net: force the worker down when the app quits, even if a consumer +// never released it, so no Python process outlives the app. app.on('before-quit', () => { + stopPythonWorker() stopBrowserActivityBridge() }) diff --git a/electron/src/main/ipc-handlers.ts b/electron/src/main/ipc-handlers.ts index 3cbc800..93e2af3 100644 --- a/electron/src/main/ipc-handlers.ts +++ b/electron/src/main/ipc-handlers.ts @@ -1,7 +1,8 @@ // Registers all ipcMain.handle() and ipcMain.on() listeners. -// This is the entry point for every message the renderer sends. +// This is the entry point for every message the renderer sends — start session, save settings, get history, etc. import { ipcMain } from 'electron' import { detectCommonWindowsApps } from './appDetection/detectCommonWindowsApps.ts' +import { requestPythonWorker, releasePythonWorker } from './python-bridge.ts' import { getLatestBrowserActivity, setBrowserMonitoringActive, @@ -15,11 +16,19 @@ export function registerIpcHandlers() { const detectedApps = detectCommonWindowsApps() console.log('[Taskmaster] Detected common apps:') - console.log(JSON.stringify(detectedApps, null, 2)) + console.log(JSON.stringify(detectedApps, null, 2)) return detectedApps }) + // On-demand CV worker control. The renderer fires these (fire-and-forget) when + // it starts/stops wanting detection; the worker is reference-counted in + // python-bridge.ts. removeAllListeners guards against double-registration. + ipcMain.removeAllListeners('taskmaster:cv-request') + ipcMain.removeAllListeners('taskmaster:cv-release') + ipcMain.on('taskmaster:cv-request', () => requestPythonWorker()) + ipcMain.on('taskmaster:cv-release', () => releasePythonWorker()) + /* Renderer enables tab reporting only while a focus session is active. */ ipcMain.on('taskmaster:browser-monitoring-active', (event, isActive: boolean) => { setBrowserMonitoringActive(isActive) diff --git a/electron/src/main/python-bridge.ts b/electron/src/main/python-bridge.ts index e603eb5..272aaba 100644 --- a/electron/src/main/python-bridge.ts +++ b/electron/src/main/python-bridge.ts @@ -1,2 +1,185 @@ -// Spawns the Python CV worker as a child process, manages its lifecycle (start/restart/kill), -// and connects to its WebSocket. Forwards CV events (gaze, phone) to the rest of the app. \ No newline at end of file +// Manages the Python CV worker process (the WebSocket server in python/main.py). +// +// The worker runs on demand, not for the whole life of the app: the renderer +// calls requestPythonWorker() when it wants detection (e.g. an onboarding check +// or a focus session) and releasePythonWorker() when it's done. The worker is +// started on the first request and stopped shortly after the last release, so +// no Python process runs while nothing is watching the camera. +// +// The renderer connects to the worker's WebSocket directly to stream frames, so +// this module only owns the process lifecycle, not the detection data. + +import { spawn, type ChildProcess } from 'node:child_process' +import { existsSync } from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const currentDir = path.dirname(fileURLToPath(import.meta.url)) + +// The python/ project folder: this file lives in electron/src/main/, so three +// levels up is the repo root, then into python/. +const pythonProjectDir = path.join(currentDir, '../../../python') + +// Port the WebSocket server listens on (must match python/main.py). +const WORKER_PORT = 8765 + +// Path to the Python interpreter inside the project's virtual environment. +// The folder layout differs by OS (Windows uses Scripts/python.exe). +function venvPythonPath(): string { + const isWindows = process.platform === 'win32' + const binDir = isWindows ? 'Scripts' : 'bin' + const executable = isWindows ? 'python.exe' : 'python' + return path.join(pythonProjectDir, '.venv', binDir, executable) +} + +// The running worker, or null when it isn't running. +let worker: ChildProcess | null = null + +// True while we're deliberately shutting the worker down (app quit / stop), so +// the exit handler knows a crash from an intentional kill and doesn't restart. +let stopping = false + +// Crash-restart bookkeeping. If the worker keeps dying immediately (e.g. a +// missing dependency), restarting forever just spins, so we cap consecutive +// fast restarts and back off. A restart that survives a while resets the count. +const RESTART_DELAY_MS = 1000 +const MAX_RAPID_RESTARTS = 5 +// How long the worker must stay up before we treat it as "healthy" again. +const HEALTHY_UPTIME_MS = 10_000 +let rapidRestarts = 0 +let restartTimer: ReturnType | null = null + +// How many consumers (renderer detection sessions) currently want the worker. +// It runs while this is > 0 and stops shortly after it returns to 0. +let activeConsumers = 0 + +// Grace period before stopping after the last consumer leaves, so a quick +// pause/resume, page change, or dev double-mount doesn't kill then respawn it. +const STOP_GRACE_MS = 2000 +let stopTimer: ReturnType | null = null + +// Ask for the worker. Starts it on the first request and cancels any pending +// grace-period shutdown. Call releasePythonWorker() once for each request. +export function requestPythonWorker(): void { + activeConsumers += 1 + if (stopTimer) { + clearTimeout(stopTimer) + stopTimer = null + } + startPythonWorker() +} + +// Drop one request. When the last consumer leaves, stop the worker after a +// short grace period (a new request during that window cancels the stop). +export function releasePythonWorker(): void { + activeConsumers = Math.max(0, activeConsumers - 1) + if (activeConsumers > 0 || stopTimer) { + return + } + stopTimer = setTimeout(() => { + stopTimer = null + stopPythonWorker() + }, STOP_GRACE_MS) +} + +// Start the CV worker. No-ops if it's already running. Internal: callers use +// requestPythonWorker() so the worker is reference-counted. +function startPythonWorker(): void { + if (worker) { + return + } + stopping = false + + // The worker only runs from the project venv (Python 3.11 + the CV deps). If + // it's missing — e.g. setup was never run, or a Windows clone that skipped + // setup.ps1 — spawning would fail with a cryptic ENOENT, so flag it loudly + // and bail. Detection stays off until the user runs setup; the rest of the + // app is unaffected. + const pythonPath = venvPythonPath() + if (!existsSync(pythonPath)) { + console.error( + `[python] CV worker venv not found at ${pythonPath}\n` + + ` Run ./setup.sh (macOS/Linux) or setup.ps1 (Windows) to create it.`, + ) + return + } + + // Equivalent to running, from the python/ folder: + // .venv/bin/python -m uvicorn main:app --port 8765 + worker = spawn( + pythonPath, + ['-m', 'uvicorn', 'main:app', '--port', String(WORKER_PORT)], + { cwd: pythonProjectDir }, + ) + + // Mirror the worker's output into the app console for debugging. uvicorn logs + // to stderr, so both streams are forwarded. + const forwardOutput = (data: Buffer) => { + console.log(`[python] ${data.toString().trim()}`) + } + worker.stdout?.on('data', forwardOutput) + worker.stderr?.on('data', forwardOutput) + + // When the process stops, clear the handle. If it died on its own (not an + // intentional stop), bring it back up so detection keeps working. + const startedAt = Date.now() + worker.on('exit', (code) => { + console.log(`[python] worker exited (code ${code})`) + worker = null + + if (stopping) { + return + } + + // Nobody is asking for detection anymore (e.g. it crashed during the stop + // grace period), so don't resurrect a worker no one wants. + if (activeConsumers === 0) { + return + } + + // A worker that ran long enough before dying is treated as a fresh failure, + // not part of a crash loop, so reset the rapid-restart counter. + if (Date.now() - startedAt >= HEALTHY_UPTIME_MS) { + rapidRestarts = 0 + } + + if (rapidRestarts >= MAX_RAPID_RESTARTS) { + console.error( + `[python] worker crashed ${rapidRestarts} times in a row; giving up`, + ) + return + } + + rapidRestarts += 1 + console.log(`[python] restarting worker (attempt ${rapidRestarts})`) + restartTimer = setTimeout(startPythonWorker, RESTART_DELAY_MS) + }) + + // Fired when the process can't be spawned at all (e.g. the venv is missing). + worker.on('error', (error) => { + console.error('[python] failed to start worker:', error) + worker = null + }) + + console.log(`[python] CV worker starting on port ${WORKER_PORT}`) +} + +// Stop the CV worker immediately. Used by the grace-period timer and as the +// app-quit safety net so the worker never outlives the app, regardless of how +// many consumers think they're still holding it. +export function stopPythonWorker(): void { + // Mark this as intentional so the exit handler won't restart the worker, and + // cancel any restart or grace-period stop already queued. + stopping = true + activeConsumers = 0 + if (restartTimer) { + clearTimeout(restartTimer) + restartTimer = null + } + if (stopTimer) { + clearTimeout(stopTimer) + stopTimer = null + } + worker?.kill() // SIGTERM; uvicorn shuts down cleanly + worker = null +} diff --git a/electron/src/preload/index.js b/electron/src/preload/index.js index 3c28f8b..9b15161 100644 --- a/electron/src/preload/index.js +++ b/electron/src/preload/index.js @@ -8,6 +8,14 @@ console.log('Taskmaster preload loaded') contextBridge.exposeInMainWorld('taskmaster', { detectCommonApps: () => ipcRenderer.invoke('taskmaster:detect-common-apps'), + + // On-demand CV worker control. request() before connecting to the worker's + // WebSocket, release() when done — each request must be paired with a release. + cv: { + request: () => ipcRenderer.send('taskmaster:cv-request'), + release: () => ipcRenderer.send('taskmaster:cv-release'), + }, + openMiniTimer: () => ipcRenderer.invoke('taskmaster:mini-timer-open'), sendMiniTimerState: (state) => ipcRenderer.send('taskmaster:mini-timer-state', state), diff --git a/electron/src/renderer/components/onboarding/BrowserActivitySelectionStep.tsx b/electron/src/renderer/components/onboarding/BrowserActivitySelectionStep.tsx index 5741011..3fb0720 100644 --- a/electron/src/renderer/components/onboarding/BrowserActivitySelectionStep.tsx +++ b/electron/src/renderer/components/onboarding/BrowserActivitySelectionStep.tsx @@ -123,7 +123,7 @@ export default function BrowserActivitySelectionStep({ } return (
-

Step 4

+

Step 6

diff --git a/electron/src/renderer/components/onboarding/OnboardingAdditionalFunctions.tsx b/electron/src/renderer/components/onboarding/OnboardingAdditionalFunctions.tsx index 1a9595b..b91349d 100644 --- a/electron/src/renderer/components/onboarding/OnboardingAdditionalFunctions.tsx +++ b/electron/src/renderer/components/onboarding/OnboardingAdditionalFunctions.tsx @@ -15,7 +15,7 @@ export default function DistractionOptionsStep({ }: DistractionOptionsStepProps) { return (
-

Step 4

+

Step 7

diff --git a/electron/src/renderer/components/onboarding/OnboardingFaceCheck.tsx b/electron/src/renderer/components/onboarding/OnboardingFaceCheck.tsx new file mode 100644 index 0000000..0dbf6b6 --- /dev/null +++ b/electron/src/renderer/components/onboarding/OnboardingFaceCheck.tsx @@ -0,0 +1,97 @@ +/** + * Face check onboarding screen. + * + * Lets the user confirm that the gaze module can actually see their face. It + * streams the camera to the Python worker (via useCvDetection) and lights up a + * green indicator when a face is detected. The check is informational only — + * Continue is never blocked, so the user can move on either way. + */ +import { useEffect, useRef } from "react"; +import { useCvDetection } from "../../hooks/useCvDetection"; + +type FaceCheckStepProps = { + onBack: () => void; + onContinue: () => void; +}; + +export default function FaceCheckStep({ + onBack, + onContinue, +}: FaceCheckStepProps) { + const videoRef = useRef(null); + + // Run detection while this screen is mounted. + const { stream, connected, gaze } = useCvDetection(true); + + // The check passes only when the user is actually LOOKING AT THE SCREEN, not + // just when a face is present. The gaze module reports status "focused" when + // the head is turned toward the screen, "distracted" when turned away or no + // face is in view. + const lookingAtScreen = gaze !== null && gaze.status === "focused"; + + const statusMessage = !connected + ? "Status: connecting to detector…" + : lookingAtScreen + ? "Status: looking at the screen" + : "Status: please look at the screen"; + + // React can't set srcObject through JSX, so attach the stream via the ref. + useEffect(() => { + if (videoRef.current && stream) { + videoRef.current.srcObject = stream; + } + }, [stream]); + + return ( +
+

Step 3

+
+ +
+
+
+ +
+
+ +

+ Camera processing is local and used only for focus detection. +

+
+ +
+
+

Face check

+

+ Make sure Taskmaster can tell when you're looking at the screen. +

+
+

+ Look at your screen — the indicator turns green when you're looking + at it. You can continue even if it doesn't. +

+
+ +
+ + +
+
+
+ ); +} diff --git a/electron/src/renderer/components/onboarding/OnboardingPhoneCheck.tsx b/electron/src/renderer/components/onboarding/OnboardingPhoneCheck.tsx new file mode 100644 index 0000000..5ed7c1e --- /dev/null +++ b/electron/src/renderer/components/onboarding/OnboardingPhoneCheck.tsx @@ -0,0 +1,94 @@ +/** + * Phone check onboarding screen. + * + * Lets the user confirm that the phone module works by holding their phone up + * to the camera. Streams the camera to the Python worker (via useCvDetection) + * and lights up a green indicator when a phone is detected. Informational only + * — Continue is never blocked. + */ +import { useEffect, useRef } from "react"; +import { useCvDetection } from "../../hooks/useCvDetection"; + +type PhoneCheckStepProps = { + onBack: () => void; + onContinue: () => void; +}; + +export default function PhoneCheckStep({ + onBack, + onContinue, +}: PhoneCheckStepProps) { + const videoRef = useRef(null); + + // Run detection while this screen is mounted. + const { stream, connected, phone } = useCvDetection(true); + + // The phone module reports status "detected" when it sees a phone. + const phoneDetected = phone !== null && phone.status === "detected"; + + const statusMessage = !connected + ? "Status: connecting to detector…" + : phoneDetected + ? "Status: phone detected" + : "Status: hold up your phone to test…"; + + // React can't set srcObject through JSX, so attach the stream via the ref. + useEffect(() => { + if (videoRef.current && stream) { + videoRef.current.srcObject = stream; + } + }, [stream]); + + return ( +
+

Step 4

+
+ +
+
+
+ +
+
+ +

+ Camera processing is local and used only for focus detection. +

+
+ +
+
+

Phone check

+

+ Make sure Taskmaster can spot your phone. +

+
+

+ Hold your phone up to the camera — the indicator turns green when + it's detected. You can continue even if it doesn't. +

+
+ +
+ + +
+
+
+ ); +} diff --git a/electron/src/renderer/components/onboarding/WhitelistSelectionStep.tsx b/electron/src/renderer/components/onboarding/WhitelistSelectionStep.tsx index 85ae15f..644ed37 100644 --- a/electron/src/renderer/components/onboarding/WhitelistSelectionStep.tsx +++ b/electron/src/renderer/components/onboarding/WhitelistSelectionStep.tsx @@ -122,7 +122,7 @@ export default function FocusEnvironmentStep({ return (
-

Step 3

+

Step 5

diff --git a/electron/src/renderer/hooks/useCvDetection.ts b/electron/src/renderer/hooks/useCvDetection.ts new file mode 100644 index 0000000..238806d --- /dev/null +++ b/electron/src/renderer/hooks/useCvDetection.ts @@ -0,0 +1,217 @@ +/** + * useCvDetection — streams webcam frames to the Python CV worker and returns + * the latest phone / gaze results. + * + * The browser owns the camera (the one the user picked in onboarding), grabs + * one frame per second, encodes it to JPEG, and sends it over a WebSocket to + * the Python worker. The worker replies with detection events, which we expose + * as React state. + * + * Usage: + * const { connected, phone, gaze } = useCvDetection(isSessionActive) + * // gaze?.status === "focused" | "distracted" + * // phone?.status === "none" | "detected" + */ + +import { useEffect, useState } from "react"; + +// Where the Python worker's WebSocket lives (see python/main.py). +const WS_URL = "ws://127.0.0.1:8765/ws"; + +// Same localStorage key the onboarding camera step writes the chosen camera to. +const SELECTED_CAMERA_KEY = "taskmaster:selectedCameraId"; + +// Send one frame per second. Detection takes ~1s on CPU, so this is the natural +// rate and keeps CPU/bandwidth low. +const CAPTURE_INTERVAL_MS = 1000; + +// Downscale frames to this width before sending — smaller = faster + lighter, +// and the detectors don't need full resolution. +const TARGET_WIDTH = 640; + +// JPEG quality (0..1). 0.7 is a good size/clarity trade-off. +const JPEG_QUALITY = 0.7; + +// One detection event from the worker (matches python/main.py / PLAN.md). +export type DetectionEvent = { + type: "phone" | "gaze"; + status: string; + confidence: number; + timestamp: number; +}; + +// What the hook hands back to the UI. +export type CvStatus = { + connected: boolean; + phone: DetectionEvent | null; + gaze: DetectionEvent | null; + // The live camera stream, so a screen can show a preview of what's being sent. + stream: MediaStream | null; +}; + +export function useCvDetection(enabled: boolean): CvStatus { + const [status, setStatus] = useState({ + connected: false, + phone: null, + gaze: null, + stream: null, + }); + + useEffect(() => { + // Only run detection when the caller turns it on (e.g. during a session). + if (!enabled) { + return; + } + + // Ask the main process to start the Python worker (it's reference-counted + // and started on demand). connectWithRetry below waits out its boot. + window.taskmaster?.cv?.request(); + + // Release the worker exactly once, whether startup fails or we unmount. + // Without the guard, releasing in the catch path AND on unmount would + // decrement the reference count twice for a single request. + let workerReleased = false; + const releaseWorker = () => { + if (workerReleased) { + return; + } + workerReleased = true; + window.taskmaster?.cv?.release(); + }; + + // `cancelled` guards against async work finishing after we've torn down. + let cancelled = false; + let socket: WebSocket | null = null; + let stream: MediaStream | null = null; + let intervalId: number | null = null; + + // Off-screen elements: a