Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Fluid.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */; };
A62300000000000000000002 /* AudioBufferConverterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A62300000000000000000001 /* AudioBufferConverterTests.swift */; };
C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */; };
C0A3E0010000000000000002 /* TranscribeCppLongFormProcessorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0A3E0010000000000000001 /* TranscribeCppLongFormProcessorTests.swift */; };
DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */; };
7CDB0A2F2F3C4D5600FB7CAD /* dictation_fixture.wav in Resources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */; };
7CDB0A302F3C4D5600FB7CAD /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */; };
Expand Down Expand Up @@ -55,6 +56,7 @@
980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TemperatureSupportTests.swift; sourceTree = "<group>"; };
A62300000000000000000001 /* AudioBufferConverterTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioBufferConverterTests.swift; sourceTree = "<group>"; };
C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioEngineRetirementDrainTests.swift; sourceTree = "<group>"; };
C0A3E0010000000000000001 /* TranscribeCppLongFormProcessorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TranscribeCppLongFormProcessorTests.swift; sourceTree = "<group>"; };
DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectAudioReliabilityTests.swift; sourceTree = "<group>"; };
7C078D8F2E3B339200FB7CAC /* FluidVoice Debug.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "FluidVoice Debug.app"; sourceTree = BUILT_PRODUCTS_DIR; };
7C91B0022F42AA0100C0DEF0 /* HotkeyShortcutTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HotkeyShortcutTests.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -134,6 +136,7 @@
980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */,
A62300000000000000000001 /* AudioBufferConverterTests.swift */,
C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */,
C0A3E0010000000000000001 /* TranscribeCppLongFormProcessorTests.swift */,
DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */,
);
path = FluidDictationIntegrationTests;
Expand Down Expand Up @@ -294,6 +297,7 @@
272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */,
A62300000000000000000002 /* AudioBufferConverterTests.swift in Sources */,
C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */,
C0A3E0010000000000000002 /* TranscribeCppLongFormProcessorTests.swift in Sources */,
DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
Expand Down
31 changes: 22 additions & 9 deletions Sources/Fluid/Persistence/SettingsStore.swift
Original file line number Diff line number Diff line change
Expand Up @@ -4205,7 +4205,7 @@ final class SettingsStore: ObservableObject {
case .parakeetTDTv2: return "~442.9 MiB"
case .parakeetRealtime: return "~428.4 MiB"
case .qwen3Asr: return "~2.0 GiB"
case .cohereTranscribeSixBit: return "~1.54 GiB"
case .cohereTranscribeSixBit: return "~1.45 GiB"
case .nemotronOffline: return "~530.8 MiB"
case .nemotronStreaming: return "~668.2 MiB"
case .nemotronStreaming320: return "~668.2 MiB"
Expand All @@ -4226,7 +4226,7 @@ final class SettingsStore: ObservableObject {
case .parakeetTDTv2: return 464_421_712
case .parakeetRealtime: return 449_190_189
case .qwen3Asr: return 2000 * 1024 * 1024
case .cohereTranscribeSixBit: return 1_650_748_785
case .cohereTranscribeSixBit: return 1_558_162_944
case .nemotronOffline: return 556_552_620
case .nemotronStreaming, .nemotronStreaming320: return 700_685_415
case .whisperTiny: return 45_981_088
Expand Down Expand Up @@ -4266,6 +4266,16 @@ final class SettingsStore: ObservableObject {
}
}

/// The GGUF filename for any model served by transcribe.cpp.
var transcribeCppModelFile: String? {
switch self {
case .cohereTranscribeSixBit:
return "cohere-transcribe-03-2026-Q4_K_M.gguf"
default:
return self.whisperModelFile
}
}

var legacyWhisperModelFile: String? {
switch self {
case .whisperTiny: return "ggml-tiny.bin"
Expand Down Expand Up @@ -4692,13 +4702,16 @@ final class SettingsStore: ObservableObject {
return false
#endif
case .cohereTranscribeSixBit:
guard
let spec = self.externalCoreMLSpec,
let directory = SettingsStore.shared.externalCoreMLArtifactsDirectory(for: self)
else {
return false
}
return spec.validatesInstalledArtifacts(at: directory)
guard let cacheDirectory = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first,
let modelFile = self.transcribeCppModelFile
else { return false }
let modelURL = cacheDirectory
.appendingPathComponent("CohereTranscribeModels", isDirectory: true)
.appendingPathComponent(modelFile, isDirectory: false)
guard let attributes = try? FileManager.default.attributesOfItem(atPath: modelURL.path),
let size = attributes[.size] as? NSNumber
else { return false }
return size.int64Value == self.expectedDownloadBytes
case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320:
let hint: String
switch self {
Expand Down
47 changes: 45 additions & 2 deletions Sources/Fluid/Services/ASRService.swift
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@ final class ASRService: ObservableObject {
}

@Published var isRunning: Bool = false
@Published private(set) var isFileTranscriptionRunning: Bool = false
@Published var finalText: String = ""
@Published var partialTranscription: String = ""
@Published var wordBoostStatusText: String = "Word boost: off"
Expand All @@ -195,6 +196,20 @@ final class ASRService: ObservableObject {
self.isRunning || self.isStarting
}

var isAnyTranscriptionRunning: Bool {
self.isRunningOrStarting || self.isFileTranscriptionRunning
}

func beginFileTranscription() -> Bool {
guard !self.isAnyTranscriptionRunning else { return false }
self.isFileTranscriptionRunning = true
return true
}

func endFileTranscription() {
self.isFileTranscriptionRunning = false
}

private let audioCaptureReadinessGate = AudioCaptureReadinessGate()
private let firstPCMTimeoutNanoseconds: UInt64 = 2_000_000_000
private var audioCaptureStartGeneration: UInt64 = 0
Expand Down Expand Up @@ -353,7 +368,7 @@ final class ASRService: ObservableObject {
case .parakeetRealtime:
return self.getParakeetRealtimeProvider()
case .cohereTranscribeSixBit:
return self.getExternalCoreMLProvider()
return self.getWhisperProvider()
Comment on lines 370 to +371

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Avoid rejecting Cohere based on eight currently free GB

Routing Cohere through WhisperProvider also applies that provider's availableMemoryGB < requiredMemoryGB guard, while Cohere's requirement is 8 GB and availableMemoryGB() counts only free, inactive, and purgeable pages. An 8-GB Apple Silicon Mac can never report the full 8 GB as available after macOS is running, so users on an otherwise supported configuration now download the GGUF and then always receive an insufficient-memory error; use total physical memory or a realistic free-memory threshold for this model.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Keep Cohere file transcription seam-aware

When Cohere is used to transcribe a meeting file longer than 20 minutes, routing it to WhisperProvider makes it inherit prefersNativeFileTranscription == false. The fallback in MeetingTranscriptionService.transcribeFile divides the file into disjoint 20-minute chunks and directly joins their transcripts, so the new 30-second overlap processor only handles seams within each outer chunk and cannot recover words split across a 20-minute boundary. The previous Cohere provider opted into whole-file transcription; preserve that behavior or add overlap and seam merging to the outer chunks.

Useful? React with 👍 / 👎.

case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320:
return self.getNemotronProvider(mode: model.nemotronProviderMode)
case .qwen3Asr:
Expand Down Expand Up @@ -554,7 +569,7 @@ final class ASRService: ObservableObject {
case .parakeetRealtime:
return ParakeetRealtimeProvider()
case .cohereTranscribeSixBit:
return ExternalCoreMLTranscriptionProvider(modelOverride: model)
return WhisperProvider(modelOverride: model)
case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320:
return NemotronProvider(mode: model.nemotronProviderMode)
case .qwen3Asr:
Expand All @@ -571,6 +586,9 @@ final class ASRService: ObservableObject {
/// - model: The model to download
/// - progressHandler: Optional callback for download progress (0.0 to 1.0)
func downloadModel(_ model: SettingsStore.SpeechModel, progressHandler: ((Double) -> Void)?) async throws {
guard !self.isFileTranscriptionRunning else {
throw self.fileTranscriptionModelOperationError()
}
guard self.modelDownloadTask == nil, self.ensureReadyTask == nil else {
throw NSError(
domain: "ASRService",
Expand Down Expand Up @@ -644,6 +662,13 @@ final class ASRService: ObservableObject {

/// Call this when the transcription provider setting changes to reset state
func resetTranscriptionProvider() {
guard !self.isFileTranscriptionRunning else {
DebugLogger.shared.warning(
"ASRService: Ignoring model reset during file transcription",
source: "ASRService"
)
return
}
let newModel = SettingsStore.shared.selectedSpeechModel
DebugLogger.shared.info("ASRService: Switching to '\(newModel.displayName)', resetting provider state...", source: "ASRService")

Expand Down Expand Up @@ -1386,6 +1411,10 @@ final class ASRService: ObservableObject {
) async -> AudioCaptureStartOutcome {
DebugLogger.shared.info("🎤 START() called - beginning recording session", source: "ASRService")

guard !self.isFileTranscriptionRunning else {
DebugLogger.shared.warning("START() blocked - file transcription is active", source: "ASRService")
return .failed
}
guard self.micStatus == .authorized else {
DebugLogger.shared.error("❌ START() blocked - mic not authorized", source: "ASRService")
return .failed
Expand Down Expand Up @@ -3738,6 +3767,9 @@ final class ASRService: ObservableObject {
// MARK: - Cache management

func clearModelCache() async throws {
guard !self.isFileTranscriptionRunning else {
throw self.fileTranscriptionModelOperationError()
}
DebugLogger.shared.debug("Clearing model cache via transcription provider", source: "ASRService")
await self.transcriptionExecutor.cancelAndAwaitPending()
try await self.transcriptionProvider.clearCache()
Expand All @@ -3746,6 +3778,9 @@ final class ASRService: ObservableObject {
}

func clearModelCache(for model: SettingsStore.SpeechModel) async throws {
guard !self.isFileTranscriptionRunning else {
throw self.fileTranscriptionModelOperationError()
}
DebugLogger.shared.debug("Clearing model cache for \(model.displayName)", source: "ASRService")
if SettingsStore.shared.selectedSpeechModel == model {
await self.transcriptionExecutor.cancelAndAwaitPending()
Expand All @@ -3762,6 +3797,14 @@ final class ASRService: ObservableObject {
await self.checkIfModelsExistAsync()
}

private func fileTranscriptionModelOperationError() -> NSError {
NSError(
domain: "ASRService",
code: -2002,
userInfo: [NSLocalizedDescriptionKey: "Voice model controls are unavailable during meeting transcription."]
)
}

// MARK: - Timer-based Streaming Transcription (No VAD)

private func startStreamingTranscription() {
Expand Down
26 changes: 2 additions & 24 deletions Sources/Fluid/Services/ExternalCoreMLModelRegistry.swift
Original file line number Diff line number Diff line change
Expand Up @@ -206,30 +206,8 @@ struct ExternalCoreMLASRModelSpec {
}

enum ExternalCoreMLModelRegistry {
static func spec(for model: SettingsStore.SpeechModel) -> ExternalCoreMLASRModelSpec? {
switch model {
case .cohereTranscribeSixBit:
return ExternalCoreMLASRModelSpec(
backend: .cohereTranscribe,
artifactFolderHint: "cohere-transcribe-03-2026-CoreML-6bit",
manifestFileName: "coreml_manifest.json",
frontendFileName: "cohere_frontend.mlpackage",
encoderFileName: "cohere_encoder.mlpackage",
crossKVProjectorFileName: "cohere_cross_kv_projector.mlpackage",
decoderFileName: "cohere_decoder_fullseq_masked.mlpackage",
cachedDecoderFileName: "cohere_decoder_cached.mlpackage",
expectedModelID: "CohereLabs/cohere-transcribe-03-2026",
expectedSampleRate: 16_000,
computeConfiguration: .aneSmall,
sourceURL: URL(string: "https://huggingface.co/BarathwajAnandan/cohere-transcribe-03-2026-CoreML-6bit"),
repositoryOwner: "BarathwajAnandan",
repositoryName: "cohere-transcribe-03-2026-CoreML-6bit",
repositoryRevision: "main",
artifactBundleVersion: "2026-04-02-cohere-refresh-1"
)
default:
return nil
}
static func spec(for _: SettingsStore.SpeechModel) -> ExternalCoreMLASRModelSpec? {
nil
}
}

Expand Down
18 changes: 16 additions & 2 deletions Sources/Fluid/Services/MeetingTranscriptionService.swift
Original file line number Diff line number Diff line change
Expand Up @@ -159,14 +159,19 @@ final class MeetingTranscriptionService: ObservableObject {
/// - Parameters:
/// - fileURL: URL to the audio/video file
func transcribeFile(_ fileURL: URL) async throws -> TranscriptionResult {
guard self.asrService.beginFileTranscription() else {
throw TranscriptionError.transcriptionFailed("Another transcription is already in progress.")
}
self.isTranscribing = true
error = nil
self.progress = 0.0
let startTime = Date()
let transcriptionModel = SettingsStore.shared.selectedSpeechModel

defer {
isTranscribing = false
progress = 0.0
self.asrService.endFileTranscription()
}

do {
Expand Down Expand Up @@ -279,6 +284,10 @@ final class MeetingTranscriptionService: ObservableObject {

// Calculate chunk size in source file frames
let sourceFramesPerChunk = AVAudioFrameCount(Double(samplesPerChunk) / resampleRatio)
let usesCohereSeamMerging = transcriptionModel == .cohereTranscribeSixBit
let sourceOverlapFrames = usesCohereSeamMerging
? AVAudioFrameCount(2 * fileSampleRate)
: 0
var currentFrame: AVAudioFramePosition = 0

self.currentStatus = duration > 0 ? "Transcribing audio (\(Int(duration))s)..." : "Transcribing audio..."
Expand Down Expand Up @@ -325,7 +334,10 @@ final class MeetingTranscriptionService: ObservableObject {
chunkCount += 1
}

currentFrame += AVAudioFramePosition(framesToRead)
let overlap = currentFrame + AVAudioFramePosition(framesToRead) < audioFile.length
? min(sourceOverlapFrames, framesToRead - 1)
: 0
currentFrame += AVAudioFramePosition(framesToRead - overlap)

// Update progress
let progressPercent = Double(currentFrame) / Double(audioFile.length)
Expand All @@ -341,7 +353,9 @@ final class MeetingTranscriptionService: ObservableObject {
}

// Combine all chunk transcriptions
let finalText = allTranscriptions.joined(separator: " ")
let finalText = usesCohereSeamMerging
? CohereTranscribeCppLongFormProcessor.merge(allTranscriptions)
: allTranscriptions.joined(separator: " ")
let avgConfidence = chunkCount > 0 ? totalConfidence / Float(chunkCount) : 0

let transcriptionResult = (text: finalText, confidence: avgConfidence)
Expand Down
Loading
Loading