Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,4 @@ prompts/


sunflower_app_prompt.txt
*.apk
145 changes: 145 additions & 0 deletions lib/audio/audio_prompt_resolver.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
// Pure prompt-routing for the flat-bundle audio path. Pulled out of
// `_sendAudioBackedPrompt` so the four-branch decision tree is a
// testable function rather than nested ifs in an async method.
//
// Decisions captured here (in order, first match wins):
//
// 1. User typed text alongside the audio → use the typed text as the
// user content; pair with the bundle's system prompt override
// (or null → caller falls back to the classroom prompt).
// 2. Bundle declares audioModes → use the mode-based SFT prompt
// (Transcribe / Answer) the user pre-selected. Per-mode override
// from `audioPromptOverrides` takes precedence — empty string is
// honoured (the QA bundle pairs audio with a blank user prompt).
// System message = bundle override OR kTrainedSystemPrompt.
// 3. Bundle has a legacy `audioUserPrompt` but no modes → use that.
// Kept for back-compat / custom installs.
// 4. Fallback → caller's `classroomFallback` user content with no
// system override (caller pairs with the classroom prompt).
//
// All decisions encoded into [AudioPromptPlan] so the caller stays
// declarative.

import '../classroom_prompt.dart';
import '../conversation/cards.dart';
import '../languages.dart';
import '../model_settings_sheet.dart';
import 'audio_types.dart';

class AudioPromptPlan {
/// User content for the messages JSON. Empty string is intentional
/// (the QA bundle's Answer mode trains on a blank user message).
final String userContent;

/// System prompt override. `null` → caller pairs with its default
/// classroom system prompt.
final String? systemPrompt;

/// What kind of card the response should render as.
final CardType cardType;

/// Language the response will be in.
final Language responseLang;

/// Source language for cross-language answer cards. `null` when the
/// response language is also the source (transcribe mode, or when
/// the user typed in the same language they spoke).
final Language? sourceLang;

/// In-progress status string for diagnostics.
final String inProgressStatus;

const AudioPromptPlan({
required this.userContent,
required this.systemPrompt,
required this.cardType,
required this.responseLang,
required this.sourceLang,
required this.inProgressStatus,
});
}

/// Resolve the prompt + card plan for one flat-bundle audio turn.
///
/// `audioModes` is the user-mode-eligible subset for the currently
/// loaded bundle (computed by caller from `bundle.audioModes`). When
/// it is empty, the bundle doesn't declare modes — branches 3/4 apply.
AudioPromptPlan resolveAudioPrompt({
required KnownModel? bundle,
required List<AudioMode> audioModes,
required AudioMode currentMode,
required Language sourceLang,
required Language responseLang,
required String typedText,
required String classroomFallback,
}) {
final hasTyped = typedText.isNotEmpty;

if (hasTyped) {
return AudioPromptPlan(
userContent: typedText,
systemPrompt: bundle?.systemPrompt,
cardType: CardType.answer,
responseLang: responseLang,
sourceLang: sourceLang != responseLang ? sourceLang : null,
inProgressStatus: 'Thinking with audio in ${responseLang.name}...',
);
}

if (audioModes.isNotEmpty) {
final override = bundle?.audioPromptOverrides[currentMode];
final userContent =
override ?? _defaultUserPromptForMode(currentMode, sourceLang);
final systemPrompt = bundle?.systemPrompt ?? kTrainedSystemPrompt;
final isTranscribe = currentMode == AudioMode.transcribe;
final cardType =
isTranscribe ? CardType.voiceTranscript : CardType.answer;
final cardResponseLang = isTranscribe ? sourceLang : responseLang;
// sourceLang is only meaningful on answer cards in the mode-aware
// flat path; transcripts self-label with the spoken language.
final cardSourceLang = isTranscribe ? null : sourceLang;
final inProgress = isTranscribe
? 'Transcribing ${sourceLang.name}...'
: 'Listening in ${responseLang.name}...';
return AudioPromptPlan(
userContent: userContent,
systemPrompt: systemPrompt,
cardType: cardType,
responseLang: cardResponseLang,
sourceLang: cardSourceLang,
inProgressStatus: inProgress,
);
}

final legacyPrompt = bundle?.audioUserPrompt;
if (legacyPrompt != null) {
return AudioPromptPlan(
userContent: legacyPrompt,
systemPrompt: bundle?.systemPrompt,
cardType: CardType.answer,
responseLang: responseLang,
sourceLang: null,
inProgressStatus: 'Listening in ${responseLang.name}...',
);
}

return AudioPromptPlan(
userContent: classroomFallback,
systemPrompt: null,
cardType: CardType.answer,
responseLang: responseLang,
sourceLang: null,
inProgressStatus: 'Listening in ${responseLang.name}...',
);
}

/// Default audio user-prompt template per mode. Verbatim from the SFT
/// corpus — diverging from these strings drops on-distribution quality.
String _defaultUserPromptForMode(AudioMode mode, Language sourceLang) {
switch (mode) {
case AudioMode.transcribe:
return 'Please transcribe this ${sourceLang.name} audio.';
case AudioMode.answer:
return 'Please answer this spoken question.';
}
}
60 changes: 60 additions & 0 deletions lib/audio/audio_types.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
// Shared audio domain types. Previously private to `main.dart`, which
// meant the catalogue in `model_settings_sheet.dart` declared audio
// modes as strings ('answer' / 'transcribe') and the page parsed them
// back to an enum — a typo there silently disabled a bundle's audio.
// Lifting these to a public module lets the catalogue (and tests)
// reference the enum directly.

import '../languages.dart';

/// Lifecycle of the input bar's mic mode.
///
/// idle ──tap mic──> recording
/// recording ──tap stop──> idle (auto-transcribed)
/// recording ──silence end-pointing──> idle (auto-transcribed)
/// recording ──15s cap──> idle (auto-transcribed)
/// recording ──tap cancel──> idle (discarded, no transcribe)
enum RecPhase { idle, recording }

/// What the user wants done with a just-transcribed voice clip.
/// Both map to trained SFT prefixes — see the audio prompt resolver.
enum VoiceIntent { translate, explain }

/// Audio task families the user can pre-select for flat-bundle voice
/// turns (the audio_tower-direct path, not the stitched cascade).
/// Each value maps to a verbatim SFT-trained user prompt. Per-bundle
/// availability is declared in `KnownModel.audioModes`; the page picks
/// the first available mode as the default when a bundle is loaded.
enum AudioMode { transcribe, answer }

String audioModeLabel(AudioMode m) {
switch (m) {
case AudioMode.transcribe:
return 'Transcribe';
case AudioMode.answer:
return 'Answer';
}
}

/// Parses a catalogue string back to an [AudioMode]. Returns null on
/// unknown values — callers should either error out (typo in catalogue)
/// or filter them away with `whereType<AudioMode>()`.
AudioMode? audioModeFromName(String name) {
for (final m in AudioMode.values) {
if (m.name == name) return m;
}
return null;
}

/// In-flight transcript awaiting a user intent. Lives on the page so
/// the input bar can swap to the review UI in place of the recorder /
/// text input — no modal context-switch between hearing the transcript
/// and committing to an action.
class VoiceReviewState {
final String transcript;
final Language target;
const VoiceReviewState({required this.transcript, required this.target});

VoiceReviewState withTarget(Language language) =>
VoiceReviewState(transcript: transcript, target: language);
}
54 changes: 44 additions & 10 deletions lib/completion/cactus_runner.dart
Original file line number Diff line number Diff line change
Expand Up @@ -242,17 +242,51 @@ class CactusRunner {
/// signal stop first so the worker exits `cactusComplete` cleanly —
/// killing the isolate while native code is mid-call is a
/// use-after-free against the model handle.
void disposeForShutdown() {
///
/// Awaits the worker reaching a natural completion (cactusStop is
/// cooperative — the worker exits at the next token boundary). Falls
/// back to a bounded grace period and then `Isolate.immediate` if
/// the worker doesn't acknowledge — at that point we accept the
/// hazard rather than blocking the disposal indefinitely. Callers
/// (typically `State.dispose`) MUST `await` this before destroying
/// the model handle.
Future<void> disposeForShutdown({
Duration grace = const Duration(milliseconds: 750),
}) async {
final model = _activeModel;
if (model != null) {
cactusStop(model);
if (model == null) {
// No active completion — fast path. Still tear down any leftover
// listeners just in case.
await _activeSub?.cancel();
_activePort?.close();
_activeIsolate?.kill(priority: Isolate.immediate);
_activeSub = null;
_activePort = null;
_activeIsolate = null;
return;
}

debugPrint('[runner] shutdown: signalling stop and awaiting worker');
cactusStop(model);

// Wait up to [grace] for the worker to emit `done`/`error` and
// exit. The active sub/port get cleaned up inside [run]'s
// finally block when that happens, so we poll for the active
// isolate going null.
final deadline = DateTime.now().add(grace);
while (_activeIsolate != null && DateTime.now().isBefore(deadline)) {
await Future<void>.delayed(const Duration(milliseconds: 25));
}
if (_activeIsolate != null) {
debugPrint(
'[runner] shutdown: grace elapsed, force-killing worker');
await _activeSub?.cancel();
_activePort?.close();
_activeIsolate?.kill(priority: Isolate.immediate);
_activeSub = null;
_activePort = null;
_activeIsolate = null;
_activeModel = null;
}
_activeSub?.cancel();
_activePort?.close();
_activeIsolate?.kill(priority: Isolate.immediate);
_activeSub = null;
_activePort = null;
_activeIsolate = null;
_activeModel = null;
}
}
57 changes: 47 additions & 10 deletions lib/conversation/cards.dart
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
//
// See `card_spec.md` §6, §7, §8 for the source of truth.

import 'package:flutter/foundation.dart';

import '../languages.dart';
import 'labels.dart';

Expand All @@ -14,6 +16,15 @@ enum ChipKind { translate, explain }
/// A single artifact in the conversation stack. Always produced by the
/// model — the user's input doesn't get its own card; it lives in the
/// input bar until they submit, then disappears.
///
/// **Streaming hot path.** Token appends used to mutate `body` via
/// `body += token` from outside; that was O(n) per token (Dart strings
/// are immutable, each `+=` allocates n+k bytes) and at ~9 tok/s on
/// long responses it stacked up to O(n²) total. Now there's a
/// [StringBuffer] inside the card; [appendToken] writes to it and
/// bumps [bodyVersion]. Widgets subscribe to [bodyVersion] (a
/// `ValueListenable<int>`) so only the active card's body widget
/// repaints per token — not the whole Scaffold.
class ConversationCard {
final String id;
final CardType type;
Expand All @@ -22,15 +33,9 @@ class ConversationCard {
/// Optional source language for cross-language cards. The header
/// renders "{source} → {language} answer" when this is set and
/// differs from [language]; otherwise the source is implicit and we
/// fall back to the shorter "{language} answer" form. Used by audio
/// Answer-mode cards where the spoken language can differ from the
/// reply language picked by the user.
/// fall back to the shorter "{language} answer" form.
final Language? sourceLanguage;

/// Mutable on purpose: streaming completions append tokens to this
/// field and the UI rebuilds via setState on the owning widget.
String body;

final DateTime timestamp;

/// True while tokens are still arriving. Drives the streaming-tint
Expand All @@ -54,25 +59,57 @@ class ConversationCard {
/// v1 leaves this empty for the same reason as [anchoredTerms].
final Map<String, String> glosses;

// Streaming buffer. Reads materialise to a String on access; writes
// go through [appendToken] or the `body` setter.
final StringBuffer _buffer;
final ValueNotifier<int> _bodyVersion = ValueNotifier<int>(0);

ConversationCard({
required this.id,
required this.type,
required this.language,
this.sourceLanguage,
this.body = '',
String body = '',
DateTime? timestamp,
this.isStreaming = true,
this.decodeTps = 0,
this.ttftMs = 0,
this.errorMessage,
this.anchoredTerms = const [],
this.glosses = const {},
}) : timestamp = timestamp ?? DateTime.now();
}) : timestamp = timestamp ?? DateTime.now(),
_buffer = StringBuffer(body);

/// Materialised body. O(buffer.length) on access — read only when the
/// UI is about to paint it, not in tight loops.
String get body => _buffer.toString();

/// Replace the body wholesale. Used by completion finalisation
/// (`c.body = cleanedResponse`) and tests.
set body(String value) {
_buffer
..clear()
..write(value);
_bodyVersion.value++;
}

/// Append a streamed token. Cheap (amortised O(token length));
/// notifies [bodyVersion] subscribers so the active card body can
/// repaint without bouncing through ConversationState's listeners.
void appendToken(String token) {
if (token.isEmpty) return;
_buffer.write(token);
_bodyVersion.value++;
}

/// Per-card change signal. Subscribe to this from the widget that
/// actually renders [body] so the streaming hot path doesn't trigger
/// a rebuild of the whole conversation surface.
ValueListenable<int> get bodyVersion => _bodyVersion;

/// The label shown in the card header. Type-qualified so model
/// outputs aren't all flattened to "Luganda" / "Acholi" — the user
/// needs to tell an answer apart from a translation at a glance.
/// Localised TODO when we move past chip-only localisation.
String get headerLabel {
switch (type) {
case CardType.voiceTranscript:
Expand Down
Loading