From e863addbe0f6684ef14071719932ef5b0ec1e8ce Mon Sep 17 00:00:00 2001 From: Kresna Date: Sat, 1 Aug 2026 18:15:25 +0700 Subject: [PATCH 1/2] feat(voice-to-text): run transcription in a Web Worker (no UI freeze) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Transcription ran on the main thread, so the whole page froze during inference — which on the larger 'Better' model + a 30s clip on a phone can take minutes and looks hung (it wasn't; a shorter clip completes fine). - Move inference into a dedicated Web Worker (stt.worker + stt.client). The main thread stays responsive: the audio player works, and progress updates flow. A single long-lived worker keeps the in-worker model cache warm across runs. - Show a live elapsed timer while transcribing (now that the thread is free), and warn that the 'Better' model is much slower on phones. 545 tests · lint clean · build green (stt.worker chunk emitted; transformers stays in workbox globIgnores). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/islands/media/VoiceToText.tsx | 30 +++++++++++++++---- src/tools/media/stt.client.ts | 48 +++++++++++++++++++++++++++++++ src/tools/media/stt.worker.ts | 23 +++++++++++++++ 3 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 src/tools/media/stt.client.ts create mode 100644 src/tools/media/stt.worker.ts diff --git a/src/islands/media/VoiceToText.tsx b/src/islands/media/VoiceToText.tsx index d77c749..8c4dc86 100644 --- a/src/islands/media/VoiceToText.tsx +++ b/src/islands/media/VoiceToText.tsx @@ -8,7 +8,8 @@ import { CopyButton } from '@/components/ui/CopyButton'; import { downloadService } from '@/services/download'; import { useAudioRecorder } from '@/hooks/useAudioRecorder'; import { decodeToMono16k } from '@/tools/media/stt-audio.lib'; -import { createTranscriber, type SttModelId } from '@/tools/media/stt.engine'; +import { transcribeInWorker } from '@/tools/media/stt.client'; +import { type SttModelId } from '@/tools/media/stt.engine'; import { segmentsToText, segmentsToSrt, @@ -61,6 +62,7 @@ export default function VoiceToText() { const [language, setLanguage] = useState(''); const [modelProgress, setModelProgress] = useState(null); const [transcribing, setTranscribing] = useState(false); + const [elapsed, setElapsed] = useState(0); const [segments, setSegments] = useState(null); const [editedText, setEditedText] = useState(''); const [tab, setTab] = useState('text'); @@ -77,6 +79,15 @@ export default function VoiceToText() { // Revoke the preview URL on unmount. useEffect(() => () => { if (urlRef.current) URL.revokeObjectURL(urlRef.current); }, []); + // Tick an elapsed counter while transcribing (the worker keeps the UI responsive). + useEffect(() => { + if (!transcribing) return; + setElapsed(0); + const started = Date.now(); + const id = setInterval(() => setElapsed(Math.floor((Date.now() - started) / 1000)), 1000); + return () => clearInterval(id); + }, [transcribing]); + const setAudio = (blob: Blob) => { if (urlRef.current) URL.revokeObjectURL(urlRef.current); const url = URL.createObjectURL(blob); @@ -123,10 +134,14 @@ export default function VoiceToText() { setModelProgress(null); // only shows once real download progress fires (first load) try { const audio = await decodeToMono16k(audioBlob); - const engine = await createTranscriber(model, r => setModelProgress(r)); - setModelProgress(null); // model ready (or cached) — now inference (indeterminate) const isMultilingual = MODELS.find(m => m.value === model)?.multilingual; - const segs = await engine.transcribe(audio, { language: isMultilingual ? language || undefined : undefined }); + const segs = await transcribeInWorker( + audio, + model, + isMultilingual ? language || undefined : undefined, + r => setModelProgress(r), + ); + setModelProgress(null); // model ready (or cached) — now inference (indeterminate) setSegments(segs); setEditedText(segmentsToText(segs)); setTab('text'); @@ -231,7 +246,12 @@ export default function VoiceToText() { )} {busy && modelProgress === null && ( -

Transcribing on your device… this can take a moment.

+

+ Transcribing on your device… ({formatClock(elapsed)}) + {MODELS.find(m => m.value === model)?.value === 'onnx-community/whisper-small' + ? ' — the “Better” model is much slower, especially on phones; a short clip can take a few minutes.' + : ' this can take a moment.'} +

)} {error && {error}} diff --git a/src/tools/media/stt.client.ts b/src/tools/media/stt.client.ts new file mode 100644 index 0000000..111778f --- /dev/null +++ b/src/tools/media/stt.client.ts @@ -0,0 +1,48 @@ +import type { SttModelId } from './stt.engine'; +import type { TranscriptSegment } from './stt.lib'; + +// A single long-lived worker so the model cache inside it persists across runs. +let worker: Worker | null = null; + +function getWorker(): Worker { + if (!worker) { + worker = new Worker(new URL('./stt.worker.ts', import.meta.url), { type: 'module' }); + } + return worker; +} + +interface ProgressMsg { type: 'progress'; ratio: number } +interface ReadyMsg { type: 'ready' } +interface ResultMsg { type: 'result'; segments: TranscriptSegment[] } +interface ErrorMsg { type: 'error'; message: string } +type WorkerMsg = ProgressMsg | ReadyMsg | ResultMsg | ErrorMsg; + +/** + * Transcribe on a background worker so the main thread (UI) stays responsive. + * `onProgress` reports model-download progress (0..1). Resolves with the segments. + */ +export function transcribeInWorker( + audio: Float32Array, + model: SttModelId, + language: string | undefined, + onProgress?: (ratio: number) => void, +): Promise { + return new Promise((resolve, reject) => { + const w = getWorker(); + const onMessage = (e: MessageEvent) => { + const m = e.data; + if (m.type === 'progress') onProgress?.(m.ratio); + else if (m.type === 'result') { cleanup(); resolve(m.segments); } + else if (m.type === 'error') { cleanup(); reject(new Error(m.message)); } + }; + const onError = () => { cleanup(); reject(new Error('The transcription worker crashed.')); }; + const cleanup = () => { + w.removeEventListener('message', onMessage as EventListener); + w.removeEventListener('error', onError); + }; + w.addEventListener('message', onMessage as EventListener); + w.addEventListener('error', onError); + // Transfer the audio buffer to avoid a copy (we don't reuse it on this side). + w.postMessage({ audio, model, language }, [audio.buffer]); + }); +} diff --git a/src/tools/media/stt.worker.ts b/src/tools/media/stt.worker.ts new file mode 100644 index 0000000..63560a7 --- /dev/null +++ b/src/tools/media/stt.worker.ts @@ -0,0 +1,23 @@ +// Runs Whisper transcription off the main thread so the UI never freezes during +// inference (which can take a while for larger models on WASM/CPU). +import { createTranscriber, type SttModelId } from './stt.engine'; + +interface WorkerCtx { + postMessage(msg: unknown): void; + onmessage: ((e: MessageEvent) => void) | null; +} +const ctx = self as unknown as WorkerCtx; + +interface Req { audio: Float32Array; model: SttModelId; language?: string } + +ctx.onmessage = async (e: MessageEvent) => { + const { audio, model, language } = e.data; + try { + const engine = await createTranscriber(model, r => ctx.postMessage({ type: 'progress', ratio: r })); + ctx.postMessage({ type: 'ready' }); + const segments = await engine.transcribe(audio, { language }); + ctx.postMessage({ type: 'result', segments }); + } catch (err) { + ctx.postMessage({ type: 'error', message: err instanceof Error ? err.message : 'Transcription failed' }); + } +}; From 9b121e7c35a188fc5aef18b7c8f6eb503c91f293 Mon Sep 17 00:00:00 2001 From: Kresna Date: Sat, 1 Aug 2026 18:20:48 +0700 Subject: [PATCH 2/2] feat(voice-to-text): request persistent storage to avoid model re-download On mobile the browser can evict the cached Whisper model between sessions under storage pressure, forcing a re-download on refresh. Request persistent storage (navigator.storage.persist) so the origin's cache isn't evicted. --- src/islands/media/VoiceToText.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/islands/media/VoiceToText.tsx b/src/islands/media/VoiceToText.tsx index 8c4dc86..0e9ab34 100644 --- a/src/islands/media/VoiceToText.tsx +++ b/src/islands/media/VoiceToText.tsx @@ -79,6 +79,12 @@ export default function VoiceToText() { // Revoke the preview URL on unmount. useEffect(() => () => { if (urlRef.current) URL.revokeObjectURL(urlRef.current); }, []); + // Ask the browser to keep this origin's storage persistent, so the (potentially + // large) cached Whisper model isn't evicted between sessions and re-downloaded. + useEffect(() => { + navigator.storage?.persist?.().catch(() => {}); + }, []); + // Tick an elapsed counter while transcribing (the worker keeps the UI responsive). useEffect(() => { if (!transcribing) return;