From e62471ad3edaa54a5edbef45c38089a4bb6e11ef Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Fri, 14 Aug 2026 03:17:09 +0000 Subject: [PATCH] feat(record): say what a meeting is being heard in, and let it be changed mid-meeting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A default is right most of the time and wrong exactly when it matters. The call that turned out to be in English, the standup that switched — and until now the only way to correct it was to stop, open settings and start again, which costs the part of the meeting where somebody noticed. So `model_swap` finally does what its comment has promised since it was written. It carries an optional language as well as an optional model, both meaning "leave this as it is", and the daemon rebuilds the decoder under the running session: the file, the timing, the audio archive and everything already transcribed continue. A swap that fails leaves the old pipeline running, because losing a meeting is the worst possible answer to "change the language". Until today it answered `cannot load: this binary was built without recognition support` — in a build with recognition support. The interface gains a line while recording, on every screen, because a recording survives navigation: *"Đang nghe bằng Tiếng Việt · gipformer-65m"*, with a change control beside it. It reads `/status`, not the browser's own preference: a session that named no language resolves to the daemon's setting, and a banner reading `localStorage` would announce "detecting automatically" while the daemon confidently decoded Vietnamese. `SessionStatus::Recording` carries the language for exactly that reason. Multilingual meetings are now a choice rather than a side effect. "Nhiều ngôn ngữ (tự động)" is offered whether or not a multilingual model is installed — that is something a user knows before they own the model for it — and choosing it downloads what it needs, like choosing a language does. Two defects fell out of driving it: * **Installing a second speech model broke recording.** With nothing named in the settings the resolver only handled "exactly one", so adding a model for another language made every recording fail with `session needs a live model`. It now ranks the installed models for the language, which is the same answer the picker gives; with no language at all it prefers a multilingual model, because one that only speaks Vietnamese is the wrong guess for a meeting nobody described. * **The app claimed to be recording when the daemon had refused.** A non-transient error now stops the microphone and clears the recording state, with the daemon's own words in the banner. Before, the timer ran and the button stayed red while nothing was being written. `full-flow.mjs` asserts the change end to end: `(model's own) → vi, segments 3 → 3, still recording`. --- apps/web/e2e/full-flow.mjs | 32 ++++ .../src/components/record/CaptureControls.tsx | 8 +- .../web/src/components/record/ListeningIn.tsx | 142 ++++++++++++++ .../src/components/record/SpokenLanguage.tsx | 46 ++++- apps/web/src/components/shell/RootLayout.tsx | 8 + apps/web/src/i18n/en.json | 9 +- apps/web/src/i18n/ja.json | 9 +- apps/web/src/i18n/vi.json | 9 +- apps/web/src/i18n/zh.json | 9 +- apps/web/src/lib/engine-context.tsx | 2 + apps/web/src/lib/engine-provider.tsx | 12 +- apps/web/src/lib/protocol.ts | 4 +- apps/web/src/lib/session.ts | 29 ++- crates/summo-engine/src/protocol.rs | 27 ++- crates/summo-engine/src/server.rs | 181 ++++++++++++++++-- crates/summo-engine/src/state.rs | 28 +++ 16 files changed, 520 insertions(+), 35 deletions(-) create mode 100644 apps/web/src/components/record/ListeningIn.tsx diff --git a/apps/web/e2e/full-flow.mjs b/apps/web/e2e/full-flow.mjs index 6c51b03..d854d49 100644 --- a/apps/web/e2e/full-flow.mjs +++ b/apps/web/e2e/full-flow.mjs @@ -119,6 +119,38 @@ await page .first() .click(); +// ---- changing the language mid-meeting ----------------------------------- +// +// The part a default cannot do. A preference is right most of the time and wrong exactly when it +// matters — the call that turned out to be in English — and the old answer was to stop, change a +// setting and start again, which costs the part of the meeting where somebody noticed. +{ + const at = (path) => `${appUrl}${path}${path.includes("?") ? "&" : "?"}token=${token}`; + const before = await (await fetch(at("/status"))).json(); + if (before.state !== "recording") + problems.push(`not recording before the change: ${before.state}`); + + await page.getByRole("button", { name: "Đổi", exact: true }).click(); + await page.getByLabel("Ngôn ngữ nói").selectOption("vi"); + await page.waitForTimeout(3000); + + const after = await (await fetch(at("/status"))).json(); + console.log( + `language mid-meeting: ${before.language ?? "(model's own)"} → ${after.language}, ` + + `segments ${before.segments} → ${after.segments}, still ${after.state}`, + ); + if (after.state !== "recording") { + problems.push(`the meeting ended when the language changed: ${JSON.stringify(after)}`); + } + if (after.language !== "vi") { + problems.push(`the daemon did not take the new language: ${JSON.stringify(after)}`); + } + // Nothing already transcribed may be lost: the count only ever goes up. + if (after.segments < before.segments) { + problems.push(`segments went backwards: ${before.segments} → ${after.segments}`); + } +} + console.log("clicking stop…"); await page .getByRole("button", { name: /Dừng ghi/ }) diff --git a/apps/web/src/components/record/CaptureControls.tsx b/apps/web/src/components/record/CaptureControls.tsx index 8ebe60f..b8bef20 100644 --- a/apps/web/src/components/record/CaptureControls.tsx +++ b/apps/web/src/components/record/CaptureControls.tsx @@ -5,6 +5,7 @@ import { useI18n } from "../../i18n/context"; import { useEngine } from "../../lib/engine-context"; import { TARGETS, hearsOthers, load, save, translating, type Capture } from "../../lib/capture"; import type { Lane } from "../../lib/protocol"; +import { ListeningIn } from "./ListeningIn"; import { SpokenLanguage } from "./SpokenLanguage"; /** @@ -45,7 +46,12 @@ export function CaptureControls() { // puts it there. It used to centre itself in a `max-w-xl`, which is why the record screen had // its controls floating in the middle of a pane and its button somewhere else entirely.
-
+ {/* While recording, what it is hearing — and a way to correct it without stopping. Above the + controls, because those are disabled mid-session and this one is the only thing on the + card that can still be acted on. */} + + +
{t("record.audio_source")}
diff --git a/apps/web/src/components/record/ListeningIn.tsx b/apps/web/src/components/record/ListeningIn.tsx new file mode 100644 index 0000000..8502e2f --- /dev/null +++ b/apps/web/src/components/record/ListeningIn.tsx @@ -0,0 +1,142 @@ +import { useCallback, useState } from "react"; + +import { useI18n } from "../../i18n/context"; +import { useEngine } from "../../lib/engine-context"; +import { url } from "../../lib/library"; +import { + AUTO, + autoAvailable, + fetchLanguages, + languageName, + quality, + ready, +} from "../../lib/languages"; +import { useLoad } from "../../lib/use-load"; + +/** + * What the recording is listening for, said out loud while it records. + * + * A default is not enough, and this is the reason: a language preference is right most of the time + * and wrong exactly when it matters — the customer call in English, the standup that switched. The + * old shape asked once, at install, and then never mentioned it again, so a meeting recorded in the + * wrong language looked identical to one recorded in the right one until somebody read the + * transcript. + * + * So while recording, the app says which language it is hearing and offers to change it. Changing + * is not a restart: the daemon rebuilds the decoder under the running session, and the file, the + * timing and everything already transcribed continue. That is what makes this honest to show at + * second three rather than as a question before the first word. + * + * It is deliberately not a modal. The promise is that pressing record records — anything that + * stands between the press and the capture is a bug, including a dialog asking to confirm what the + * user already chose. + */ +export function ListeningIn() { + const { session, retune } = useEngine(); + const { t, locale } = useI18n(); + const [open, setOpen] = useState(false); + const [dismissed, setDismissed] = useState(false); + // Bumped after a change, to re-read what the daemon now says rather than what this component + // asked for — the two differ when a swap fails and the old pipeline keeps running. + const [generation, setGeneration] = useState(0); + const { handshake } = useEngine(); + + const probe = useLoad( + useCallback(async () => fetchLanguages(handshake), [handshake]), + [handshake], + ); + const languages = probe.data?.languages ?? []; + + // What the *daemon* resolved, not what this browser last chose. A session started without naming + // a language falls back to the settings file, and a banner reading the local preference would + // announce "detecting automatically" while the daemon confidently decoded Vietnamese — the exact + // class of quiet mismatch this banner exists to end. Re-read after every change, and while + // recording, because a swap makes the previous answer wrong. + const live = useLoad( + useCallback(async () => { + const response = await fetch(url(handshake, "/status")); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + // Tagged by `state`, flattened — `{"state":"recording","live_model":…}` — not an + // externally tagged `{"Recording":{…}}`. Reading the wrong shape is how this banner spent its + // first run insisting every meeting was being detected automatically. + return (await response.json()) as { + state?: string; + live_model?: string; + language?: string; + }; + }, [handshake]), + [handshake, session.recording, generation], + ); + + if (!session.recording || dismissed) return null; + + const recording = live.data?.state === "recording" ? live.data : undefined; + + // A session that named no language is not "detecting" — it is running whatever the model does. + // For a single-language model that is its language, and saying "automatic" there would be the + // same lie in the other direction: gipformer hears Vietnamese and nothing else, whatever the + // session forgot to specify. + const named = recording?.language; + const covered = languages.filter( + (language) => language.model === recording?.live_model && !language.multilingual_only, + ); + const spoken = named ?? (covered.length === 1 ? (covered[0]?.code ?? AUTO) : AUTO); + const current = languages.find((language) => language.code === spoken); + const auto = spoken === AUTO; + const label = auto ? t("record.spoken_auto") : languageName(spoken, locale); + + // Only languages that can be used right now. Mid-meeting is the wrong moment to start a 153 MB + // download, and offering one would be offering to lose the next four minutes of the call. + const usable = languages.filter((language) => ready(language)); + const canAuto = autoAvailable(languages); + + return ( +
+ + {t("record.listening_in", { language: label })} + {recording?.live_model ? ` · ${recording.live_model}` : ""} + {current && quality(current) === "poor" ? ` · ${t("record.spoken_poor")}` : ""} + + + {!open ? ( + <> + + + + ) : ( + + )} +
+ ); +} diff --git a/apps/web/src/components/record/SpokenLanguage.tsx b/apps/web/src/components/record/SpokenLanguage.tsx index 3fad98e..72e8e67 100644 --- a/apps/web/src/components/record/SpokenLanguage.tsx +++ b/apps/web/src/components/record/SpokenLanguage.tsx @@ -40,6 +40,14 @@ import { Button } from "../ui"; * multilingual model on disk; it also costs accuracy and can flip mid-meeting, so it is never the * default. */ +/** + * The value of the "several languages" entry. + * + * Not a language code, and not the empty string it resolves to: a `