feat(record): say what a meeting is being heard in, and let it be changed mid-meeting

apps/web/e2e/full-flow.mjs

@@ -119,6 +119,38 @@ await page
   .first()
   .click();
 
+// ---- changing the language mid-meeting -----------------------------------
+//
+// The part a default cannot do. A preference is right most of the time and wrong exactly when it
+// matters — the call that turned out to be in English — and the old answer was to stop, change a
+// setting and start again, which costs the part of the meeting where somebody noticed.
+{
+  const at = (path) => `${appUrl}${path}${path.includes("?") ? "&" : "?"}token=${token}`;
+  const before = await (await fetch(at("/status"))).json();
+  if (before.state !== "recording")
+    problems.push(`not recording before the change: ${before.state}`);
+
+  await page.getByRole("button", { name: "Đổi", exact: true }).click();
+  await page.getByLabel("Ngôn ngữ nói").selectOption("vi");
+  await page.waitForTimeout(3000);
+
+  const after = await (await fetch(at("/status"))).json();
+  console.log(
+    `language mid-meeting: ${before.language ?? "(model's own)"} → ${after.language}, ` +
+      `segments ${before.segments} → ${after.segments}, still ${after.state}`,
+  );
+  if (after.state !== "recording") {
+    problems.push(`the meeting ended when the language changed: ${JSON.stringify(after)}`);
+  }
+  if (after.language !== "vi") {
+    problems.push(`the daemon did not take the new language: ${JSON.stringify(after)}`);
+  }
+  // Nothing already transcribed may be lost: the count only ever goes up.
+  if (after.segments < before.segments) {
+    problems.push(`segments went backwards: ${before.segments} → ${after.segments}`);
+  }
+}
+
 console.log("clicking stop…");
 await page
   .getByRole("button", { name: /Dừng ghi/ })

apps/web/src/components/record/CaptureControls.tsx

@@ -5,6 +5,7 @@ import { useI18n } from "../../i18n/context";
 import { useEngine } from "../../lib/engine-context";
 import { TARGETS, hearsOthers, load, save, translating, type Capture } from "../../lib/capture";
 import type { Lane } from "../../lib/protocol";
+import { ListeningIn } from "./ListeningIn";
 import { SpokenLanguage } from "./SpokenLanguage";
 
 /**
@@ -45,7 +46,12 @@ export function CaptureControls() {
     // puts it there. It used to centre itself in a `max-w-xl`, which is why the record screen had
     // its controls floating in the middle of a pane and its button somewhere else entirely.
     <div className="w-full">
-      <fieldset disabled={busy} className="disabled:opacity-60">
+      {/* While recording, what it is hearing — and a way to correct it without stopping. Above the
+          controls, because those are disabled mid-session and this one is the only thing on the
+          card that can still be acted on. */}
+      <ListeningIn />
+
+      <fieldset disabled={busy} className="mt-2 disabled:opacity-60">
         <legend className="sr-only">{t("record.audio_source")}</legend>
 
         <div className="flex flex-wrap items-center gap-2">

apps/web/src/components/record/ListeningIn.tsx

@@ -0,0 +1,142 @@
+import { useCallback, useState } from "react";
+
+import { useI18n } from "../../i18n/context";
+import { useEngine } from "../../lib/engine-context";
+import { url } from "../../lib/library";
+import {
+  AUTO,
+  autoAvailable,
+  fetchLanguages,
+  languageName,
+  quality,
+  ready,
+} from "../../lib/languages";
+import { useLoad } from "../../lib/use-load";
+
+/**
+ * What the recording is listening for, said out loud while it records.
+ *
+ * A default is not enough, and this is the reason: a language preference is right most of the time
+ * and wrong exactly when it matters — the customer call in English, the standup that switched. The
+ * old shape asked once, at install, and then never mentioned it again, so a meeting recorded in the
+ * wrong language looked identical to one recorded in the right one until somebody read the
+ * transcript.
+ *
+ * So while recording, the app says which language it is hearing and offers to change it. Changing
+ * is not a restart: the daemon rebuilds the decoder under the running session, and the file, the
+ * timing and everything already transcribed continue. That is what makes this honest to show at
+ * second three rather than as a question before the first word.
+ *
+ * It is deliberately not a modal. The promise is that pressing record records — anything that
+ * stands between the press and the capture is a bug, including a dialog asking to confirm what the
+ * user already chose.
+ */
+export function ListeningIn() {
+  const { session, retune } = useEngine();
+  const { t, locale } = useI18n();
+  const [open, setOpen] = useState(false);
+  const [dismissed, setDismissed] = useState(false);
+  // Bumped after a change, to re-read what the daemon now says rather than what this component
+  // asked for — the two differ when a swap fails and the old pipeline keeps running.
+  const [generation, setGeneration] = useState(0);
+  const { handshake } = useEngine();
+
+  const probe = useLoad(
+    useCallback(async () => fetchLanguages(handshake), [handshake]),
+    [handshake],
+  );
+  const languages = probe.data?.languages ?? [];
+
+  // What the *daemon* resolved, not what this browser last chose. A session started without naming
+  // a language falls back to the settings file, and a banner reading the local preference would
+  // announce "detecting automatically" while the daemon confidently decoded Vietnamese — the exact
+  // class of quiet mismatch this banner exists to end. Re-read after every change, and while
+  // recording, because a swap makes the previous answer wrong.
+  const live = useLoad(
+    useCallback(async () => {
+      const response = await fetch(url(handshake, "/status"));
+      if (!response.ok) throw new Error(`HTTP ${response.status}`);
+      // Tagged by `state`, flattened — `{"state":"recording","live_model":…}` — not an
+      // externally tagged `{"Recording":{…}}`. Reading the wrong shape is how this banner spent its
+      // first run insisting every meeting was being detected automatically.
+      return (await response.json()) as {
+        state?: string;
+        live_model?: string;
+        language?: string;
+      };
+    }, [handshake]),
+    [handshake, session.recording, generation],
+  );
+
+  if (!session.recording || dismissed) return null;
+
+  const recording = live.data?.state === "recording" ? live.data : undefined;
+
+  // A session that named no language is not "detecting" — it is running whatever the model does.
+  // For a single-language model that is its language, and saying "automatic" there would be the
+  // same lie in the other direction: gipformer hears Vietnamese and nothing else, whatever the
+  // session forgot to specify.
+  const named = recording?.language;
+  const covered = languages.filter(
+    (language) => language.model === recording?.live_model && !language.multilingual_only,
+  );
+  const spoken = named ?? (covered.length === 1 ? (covered[0]?.code ?? AUTO) : AUTO);
+  const current = languages.find((language) => language.code === spoken);
+  const auto = spoken === AUTO;
+  const label = auto ? t("record.spoken_auto") : languageName(spoken, locale);
+
+  // Only languages that can be used right now. Mid-meeting is the wrong moment to start a 153 MB
+  // download, and offering one would be offering to lose the next four minutes of the call.
+  const usable = languages.filter((language) => ready(language));
+  const canAuto = autoAvailable(languages);
+
+  return (
+    <div className="border-accent/30 bg-accent-soft text-meta flex flex-wrap items-center gap-2 rounded-[var(--radius-card)] border px-3 py-2">
+      <span className="text-fg-dim">
+        {t("record.listening_in", { language: label })}
+        {recording?.live_model ? ` · ${recording.live_model}` : ""}
+        {current && quality(current) === "poor" ? ` · ${t("record.spoken_poor")}` : ""}
+      </span>
+
+      {!open ? (
+        <>
+          <button type="button" onClick={() => setOpen(true)} className="font-medium underline">
+            {t("record.listening_change")}
+          </button>
+          <button
+            type="button"
+            onClick={() => setDismissed(true)}
+            className="text-fg-faint ms-auto"
+            aria-label={t("common.dismiss")}
+          >
+            ✕
+          </button>
+        </>
+      ) : (
+        <label className="flex items-center gap-2">
+          <span className="sr-only">{t("record.spoken")}</span>
+          <select
+            aria-label={t("record.spoken")}
+            value={spoken}
+            onChange={(event) => {
+              retune(event.target.value);
+              setOpen(false);
+              // The daemon rebuilds the decoder before it answers; a moment later `/status` is the
+              // truth about whether it worked.
+              window.setTimeout(() => setGeneration((n) => n + 1), 600);
+            }}
+            className="border-line bg-bg-soft text-fg h-7 rounded-[var(--radius-card)] border px-2 text-sm"
+          >
+            {canAuto && <option value={AUTO}>{t("record.spoken_auto")}</option>}
+            {usable.map((language) => (
+              <option key={language.code} value={language.code}>
+                {languageName(language.code, locale)}
+              </option>
+            ))}
+          </select>
+          <span className="text-fg-faint text-micro">{t("record.listening_note")}</span>
+        </label>
+      )}
+    </div>
+  );
+}

apps/web/src/components/record/SpokenLanguage.tsx

@@ -40,6 +40,14 @@ import { Button } from "../ui";
  * multilingual model on disk; it also costs accuracy and can flip mid-meeting, so it is never the
  * default.
  */
+/**
+ * The value of the "several languages" entry.
+ *
+ * Not a language code, and not the empty string it resolves to: a `<select>` needs a value that is
+ * distinct from every option beside it, and the empty string is already taken by "as configured".
+ */
+const MULTI = "__multi__";
+
 export function SpokenLanguage({
   value,
   onChange,
@@ -94,6 +102,12 @@ export function SpokenLanguage({
   // which is right for a wire format and wrong for a list a person scrolls: in Vietnamese, `af`
   // renders as "Tiếng Hà Lan (Nam Phi)" and sits between English and Vietnamese for no reason a
   // reader can see.
+  // The multilingual entry: one model that covers everything, detecting per utterance. Offered
+  // whether or not it is installed, because "this meeting is in two languages" is a thing a user
+  // knows before they own a model for it — and choosing it should start the download, exactly as
+  // choosing a language does.
+  const multilingual = languages.find((language) => language.multilingual_only && language.model);
+
   const options = languages
     .filter((language) => language.model)
     .sort((a, b) => {
@@ -110,7 +124,12 @@ export function SpokenLanguage({
           value={value}
           aria-label={t("record.spoken")}
           onChange={(event) => {
-            const code = event.target.value;
+            const raw = event.target.value;
+            // `MULTI` is not a language, it is a request for one model and no language. It resolves
+            // to the same empty code the daemon and sherpa-onnx already mean by "detect".
+            const code = raw === MULTI ? AUTO : raw;
+            if (raw === MULTI && multilingual && !multilingual.installed)
+              void install(multilingual);
             onChange(code);
             // Written through to the daemon so the choice survives this browser. A failure here is
             // deliberately swallowed: the recording still has the language, and a preference that
@@ -121,13 +140,26 @@ export function SpokenLanguage({
         >
           {/* Detection first when it is possible, because somebody who does not know what will be
               spoken is exactly who needs it. */}
-          {auto && <option value={AUTO}>{t("record.spoken_auto")}</option>}
+          {/* First, always, whichever it is: the option that describes what the control is doing
+              *now*. With detection available that is "automatic"; without it, "as configured" —
+              and without either the browser would show the first language in the list, so a
+              control meaning "whatever the settings say" silently claimed to be recording English,
+              the one wrong answer that never announces itself. */}
+          {auto ? (
+            <option value={AUTO}>{t("record.spoken_auto")}</option>
+          ) : (
+            value === AUTO && <option value={AUTO}>{t("record.spoken_default")}</option>
+          )}
 
-          {/* When nothing has been chosen and detection is not available, the empty value has to be
-              an option of its own. Without it the browser shows the first entry in the list, so a
-              control that means "whatever the settings say" silently claimed to be recording
-              English — the one wrong answer that never announces itself. */}
-          {!auto && value === AUTO && <option value={AUTO}>{t("record.spoken_default")}</option>}
+          {/* Then the multilingual entry: one model that hears everything, detecting per utterance.
+              Offered whether or not it is installed, because "this meeting is in two languages" is
+              something a user knows before they own a model for it. */}
+          {!auto && multilingual && (
+            <option value={MULTI}>
+              {t("record.spoken_multi")}
+              {multilingual.installed ? "" : ` · ${megabytes(multilingual.size_bytes)}`}
+            </option>
+          )}
           {options.map((language) => (
             <option key={language.code} value={language.code}>
               {languageName(language.code, locale)}

apps/web/src/components/shell/RootLayout.tsx

@@ -29,6 +29,7 @@ import { useEngine } from "../../lib/engine-context";
 import { deviceWarning } from "../../lib/session";
 import { useErrorText } from "../../lib/errors";
 import { RecordButton } from "../RecordButton";
+import { ListeningIn } from "../record/ListeningIn";
 import { StatusBar } from "../StatusBar";
 import { Waveform } from "../Waveform";
 import { motion } from "motion/react";
@@ -246,6 +247,13 @@ export function RootLayout({ children }: { children: ReactNode }) {
 
       <NudgeBar />
 
+      {/* What the running recording is hearing, on every screen — because a recording survives
+          navigation, and "this is in English actually" is realised while looking at something
+          else. Renders nothing when idle. */}
+      <div className="px-4 empty:hidden [&:has(>*)]:py-2">
+        <ListeningIn />
+      </div>
+
       {/* A refused microphone is the one failure with a repair path, so it gets a link to the
           panel that repairs it. Every other failure gets the sentence alone: a button that leads
           somewhere unhelpful is worse than no button. */}

apps/web/src/i18n/en.json

@@ -61,7 +61,11 @@
     "spoken_installing": "Downloading… {pct}%",
     "spoken_wait": "Recording works as soon as it finishes.",
     "spoken_missing": "No model for this language yet.",
-    "spoken_default": "As configured"
+    "spoken_default": "As configured",
+    "listening_in": "Listening in {language}",
+    "listening_change": "Change",
+    "listening_note": "Changes take effect immediately; nothing already recorded is lost.",
+    "spoken_multi": "Several languages (detected)"
   },
   "import": {
     "title": "Import a recording",
@@ -361,7 +365,8 @@
     "close": "Close",
     "retry": "Retry",
     "loading": "Loading…",
-    "saving": "Saving…"
+    "saving": "Saving…",
+    "dismiss": "Dismiss"
   },
   "setup": {
     "title": "Welcome to Summo",

apps/web/src/i18n/ja.json

@@ -61,7 +61,11 @@
     "spoken_installing": "ダウンロード中… {pct}%",
     "spoken_wait": "完了したらすぐ録音できます。",
     "spoken_missing": "この言語のモデルはまだありません。",
-    "spoken_default": "設定に従う"
+    "spoken_default": "設定に従う",
+    "listening_in": "{language} で聞き取り中",
+    "listening_change": "変更",
+    "listening_note": "その場で切り替わります。録音済みの内容は失われません。",
+    "spoken_multi": "複数言語（自動判定）"
   },
   "import": {
     "title": "録音ファイルを取り込む",
@@ -361,7 +365,8 @@
     "close": "閉じる",
     "retry": "やり直す",
     "loading": "読み込み中…",
-    "saving": "保存中…"
+    "saving": "保存中…",
+    "dismiss": "閉じる"
   },
   "setup": {
     "title": "Summoへようこそ",

apps/web/src/i18n/vi.json

@@ -61,7 +61,11 @@
     "spoken_installing": "Đang tải… {pct}%",
     "spoken_wait": "Ghi được ngay khi tải xong.",
     "spoken_missing": "Chưa có model cho ngôn ngữ này.",
-    "spoken_default": "Theo cài đặt"
+    "spoken_default": "Theo cài đặt",
+    "listening_in": "Đang nghe bằng {language}",
+    "listening_change": "Đổi",
+    "listening_note": "Đổi ngay giữa buổi, không mất phần đã ghi.",
+    "spoken_multi": "Nhiều ngôn ngữ (tự động)"
   },
   "import": {
     "title": "Nhập bản ghi có sẵn",
@@ -361,7 +365,8 @@
     "close": "Đóng",
     "retry": "Thử lại",
     "loading": "Đang tải…",
-    "saving": "Đang lưu…"
+    "saving": "Đang lưu…",
+    "dismiss": "Đóng"
   },
   "setup": {
     "title": "Chào mừng tới Summo",

apps/web/src/i18n/zh.json

@@ -61,7 +61,11 @@
     "spoken_installing": "下载中… {pct}%",
     "spoken_wait": "下载完成即可开始录音。",
     "spoken_missing": "还没有适用于此语言的模型。",
-    "spoken_default": "按设置"
+    "spoken_default": "按设置",
+    "listening_in": "正在以{language}聆听",
+    "listening_change": "更改",
+    "listening_note": "立即生效，已录制的内容不会丢失。",
+    "spoken_multi": "多种语言（自动识别）"
   },
   "import": {
     "title": "导入录音",
@@ -361,7 +365,8 @@
     "close": "关闭",
     "retry": "重试",
     "loading": "加载中…",
-    "saving": "保存中…"
+    "saving": "保存中…",
+    "dismiss": "关闭"
   },
   "setup": {
     "title": "欢迎使用 Summo",