diff --git a/Fluid.xcodeproj/project.pbxproj b/Fluid.xcodeproj/project.pbxproj index d475db0e..93770647 100644 --- a/Fluid.xcodeproj/project.pbxproj +++ b/Fluid.xcodeproj/project.pbxproj @@ -19,6 +19,7 @@ 272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */; }; A62300000000000000000002 /* AudioBufferConverterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A62300000000000000000001 /* AudioBufferConverterTests.swift */; }; C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */; }; + C0A3E0010000000000000002 /* TranscribeCppLongFormProcessorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0A3E0010000000000000001 /* TranscribeCppLongFormProcessorTests.swift */; }; DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */; }; 7CDB0A2F2F3C4D5600FB7CAD /* dictation_fixture.wav in Resources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */; }; 7CDB0A302F3C4D5600FB7CAD /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */; }; @@ -55,6 +56,7 @@ 980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TemperatureSupportTests.swift; sourceTree = ""; }; A62300000000000000000001 /* AudioBufferConverterTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioBufferConverterTests.swift; sourceTree = ""; }; C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioEngineRetirementDrainTests.swift; sourceTree = ""; }; + C0A3E0010000000000000001 /* TranscribeCppLongFormProcessorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TranscribeCppLongFormProcessorTests.swift; sourceTree = ""; }; DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectAudioReliabilityTests.swift; sourceTree = ""; }; 7C078D8F2E3B339200FB7CAC /* FluidVoice Debug.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "FluidVoice Debug.app"; sourceTree = BUILT_PRODUCTS_DIR; }; 7C91B0022F42AA0100C0DEF0 /* HotkeyShortcutTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HotkeyShortcutTests.swift; sourceTree = ""; }; @@ -134,6 +136,7 @@ 980330F3CE464336ADCE3E23 /* TemperatureSupportTests.swift */, A62300000000000000000001 /* AudioBufferConverterTests.swift */, C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */, + C0A3E0010000000000000001 /* TranscribeCppLongFormProcessorTests.swift */, DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */, ); path = FluidDictationIntegrationTests; @@ -294,6 +297,7 @@ 272BFB5CB271489892CAE50C /* TemperatureSupportTests.swift in Sources */, A62300000000000000000002 /* AudioBufferConverterTests.swift in Sources */, C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */, + C0A3E0010000000000000002 /* TranscribeCppLongFormProcessorTests.swift in Sources */, DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift index 8c91c842..5b08b41e 100644 --- a/Sources/Fluid/Persistence/SettingsStore.swift +++ b/Sources/Fluid/Persistence/SettingsStore.swift @@ -4205,7 +4205,7 @@ final class SettingsStore: ObservableObject { case .parakeetTDTv2: return "~442.9 MiB" case .parakeetRealtime: return "~428.4 MiB" case .qwen3Asr: return "~2.0 GiB" - case .cohereTranscribeSixBit: return "~1.54 GiB" + case .cohereTranscribeSixBit: return "~1.45 GiB" case .nemotronOffline: return "~530.8 MiB" case .nemotronStreaming: return "~668.2 MiB" case .nemotronStreaming320: return "~668.2 MiB" @@ -4226,7 +4226,7 @@ final class SettingsStore: ObservableObject { case .parakeetTDTv2: return 464_421_712 case .parakeetRealtime: return 449_190_189 case .qwen3Asr: return 2000 * 1024 * 1024 - case .cohereTranscribeSixBit: return 1_650_748_785 + case .cohereTranscribeSixBit: return 1_558_162_944 case .nemotronOffline: return 556_552_620 case .nemotronStreaming, .nemotronStreaming320: return 700_685_415 case .whisperTiny: return 45_981_088 @@ -4266,6 +4266,16 @@ final class SettingsStore: ObservableObject { } } + /// The GGUF filename for any model served by transcribe.cpp. + var transcribeCppModelFile: String? { + switch self { + case .cohereTranscribeSixBit: + return "cohere-transcribe-03-2026-Q4_K_M.gguf" + default: + return self.whisperModelFile + } + } + var legacyWhisperModelFile: String? { switch self { case .whisperTiny: return "ggml-tiny.bin" @@ -4692,13 +4702,16 @@ final class SettingsStore: ObservableObject { return false #endif case .cohereTranscribeSixBit: - guard - let spec = self.externalCoreMLSpec, - let directory = SettingsStore.shared.externalCoreMLArtifactsDirectory(for: self) - else { - return false - } - return spec.validatesInstalledArtifacts(at: directory) + guard let cacheDirectory = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first, + let modelFile = self.transcribeCppModelFile + else { return false } + let modelURL = cacheDirectory + .appendingPathComponent("CohereTranscribeModels", isDirectory: true) + .appendingPathComponent(modelFile, isDirectory: false) + guard let attributes = try? FileManager.default.attributesOfItem(atPath: modelURL.path), + let size = attributes[.size] as? NSNumber + else { return false } + return size.int64Value == self.expectedDownloadBytes case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: let hint: String switch self { diff --git a/Sources/Fluid/Services/ASRService.swift b/Sources/Fluid/Services/ASRService.swift index c0becd00..e6fd20fa 100644 --- a/Sources/Fluid/Services/ASRService.swift +++ b/Sources/Fluid/Services/ASRService.swift @@ -172,6 +172,7 @@ final class ASRService: ObservableObject { } @Published var isRunning: Bool = false + @Published private(set) var isFileTranscriptionRunning: Bool = false @Published var finalText: String = "" @Published var partialTranscription: String = "" @Published var wordBoostStatusText: String = "Word boost: off" @@ -195,6 +196,20 @@ final class ASRService: ObservableObject { self.isRunning || self.isStarting } + var isAnyTranscriptionRunning: Bool { + self.isRunningOrStarting || self.isFileTranscriptionRunning + } + + func beginFileTranscription() -> Bool { + guard !self.isAnyTranscriptionRunning else { return false } + self.isFileTranscriptionRunning = true + return true + } + + func endFileTranscription() { + self.isFileTranscriptionRunning = false + } + private let audioCaptureReadinessGate = AudioCaptureReadinessGate() private let firstPCMTimeoutNanoseconds: UInt64 = 2_000_000_000 private var audioCaptureStartGeneration: UInt64 = 0 @@ -353,7 +368,7 @@ final class ASRService: ObservableObject { case .parakeetRealtime: return self.getParakeetRealtimeProvider() case .cohereTranscribeSixBit: - return self.getExternalCoreMLProvider() + return self.getWhisperProvider() case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: return self.getNemotronProvider(mode: model.nemotronProviderMode) case .qwen3Asr: @@ -554,7 +569,7 @@ final class ASRService: ObservableObject { case .parakeetRealtime: return ParakeetRealtimeProvider() case .cohereTranscribeSixBit: - return ExternalCoreMLTranscriptionProvider(modelOverride: model) + return WhisperProvider(modelOverride: model) case .nemotronOffline, .nemotronStreaming, .nemotronStreaming320: return NemotronProvider(mode: model.nemotronProviderMode) case .qwen3Asr: @@ -571,6 +586,9 @@ final class ASRService: ObservableObject { /// - model: The model to download /// - progressHandler: Optional callback for download progress (0.0 to 1.0) func downloadModel(_ model: SettingsStore.SpeechModel, progressHandler: ((Double) -> Void)?) async throws { + guard !self.isFileTranscriptionRunning else { + throw self.fileTranscriptionModelOperationError() + } guard self.modelDownloadTask == nil, self.ensureReadyTask == nil else { throw NSError( domain: "ASRService", @@ -644,6 +662,13 @@ final class ASRService: ObservableObject { /// Call this when the transcription provider setting changes to reset state func resetTranscriptionProvider() { + guard !self.isFileTranscriptionRunning else { + DebugLogger.shared.warning( + "ASRService: Ignoring model reset during file transcription", + source: "ASRService" + ) + return + } let newModel = SettingsStore.shared.selectedSpeechModel DebugLogger.shared.info("ASRService: Switching to '\(newModel.displayName)', resetting provider state...", source: "ASRService") @@ -1386,6 +1411,10 @@ final class ASRService: ObservableObject { ) async -> AudioCaptureStartOutcome { DebugLogger.shared.info("🎤 START() called - beginning recording session", source: "ASRService") + guard !self.isFileTranscriptionRunning else { + DebugLogger.shared.warning("START() blocked - file transcription is active", source: "ASRService") + return .failed + } guard self.micStatus == .authorized else { DebugLogger.shared.error("❌ START() blocked - mic not authorized", source: "ASRService") return .failed @@ -3738,6 +3767,9 @@ final class ASRService: ObservableObject { // MARK: - Cache management func clearModelCache() async throws { + guard !self.isFileTranscriptionRunning else { + throw self.fileTranscriptionModelOperationError() + } DebugLogger.shared.debug("Clearing model cache via transcription provider", source: "ASRService") await self.transcriptionExecutor.cancelAndAwaitPending() try await self.transcriptionProvider.clearCache() @@ -3746,6 +3778,9 @@ final class ASRService: ObservableObject { } func clearModelCache(for model: SettingsStore.SpeechModel) async throws { + guard !self.isFileTranscriptionRunning else { + throw self.fileTranscriptionModelOperationError() + } DebugLogger.shared.debug("Clearing model cache for \(model.displayName)", source: "ASRService") if SettingsStore.shared.selectedSpeechModel == model { await self.transcriptionExecutor.cancelAndAwaitPending() @@ -3762,6 +3797,14 @@ final class ASRService: ObservableObject { await self.checkIfModelsExistAsync() } + private func fileTranscriptionModelOperationError() -> NSError { + NSError( + domain: "ASRService", + code: -2002, + userInfo: [NSLocalizedDescriptionKey: "Voice model controls are unavailable during meeting transcription."] + ) + } + // MARK: - Timer-based Streaming Transcription (No VAD) private func startStreamingTranscription() { diff --git a/Sources/Fluid/Services/ExternalCoreMLModelRegistry.swift b/Sources/Fluid/Services/ExternalCoreMLModelRegistry.swift index e2a1cbe1..c0cf3459 100644 --- a/Sources/Fluid/Services/ExternalCoreMLModelRegistry.swift +++ b/Sources/Fluid/Services/ExternalCoreMLModelRegistry.swift @@ -206,30 +206,8 @@ struct ExternalCoreMLASRModelSpec { } enum ExternalCoreMLModelRegistry { - static func spec(for model: SettingsStore.SpeechModel) -> ExternalCoreMLASRModelSpec? { - switch model { - case .cohereTranscribeSixBit: - return ExternalCoreMLASRModelSpec( - backend: .cohereTranscribe, - artifactFolderHint: "cohere-transcribe-03-2026-CoreML-6bit", - manifestFileName: "coreml_manifest.json", - frontendFileName: "cohere_frontend.mlpackage", - encoderFileName: "cohere_encoder.mlpackage", - crossKVProjectorFileName: "cohere_cross_kv_projector.mlpackage", - decoderFileName: "cohere_decoder_fullseq_masked.mlpackage", - cachedDecoderFileName: "cohere_decoder_cached.mlpackage", - expectedModelID: "CohereLabs/cohere-transcribe-03-2026", - expectedSampleRate: 16_000, - computeConfiguration: .aneSmall, - sourceURL: URL(string: "https://huggingface.co/BarathwajAnandan/cohere-transcribe-03-2026-CoreML-6bit"), - repositoryOwner: "BarathwajAnandan", - repositoryName: "cohere-transcribe-03-2026-CoreML-6bit", - repositoryRevision: "main", - artifactBundleVersion: "2026-04-02-cohere-refresh-1" - ) - default: - return nil - } + static func spec(for _: SettingsStore.SpeechModel) -> ExternalCoreMLASRModelSpec? { + nil } } diff --git a/Sources/Fluid/Services/MeetingTranscriptionService.swift b/Sources/Fluid/Services/MeetingTranscriptionService.swift index 67d1eede..f9df9096 100644 --- a/Sources/Fluid/Services/MeetingTranscriptionService.swift +++ b/Sources/Fluid/Services/MeetingTranscriptionService.swift @@ -159,14 +159,19 @@ final class MeetingTranscriptionService: ObservableObject { /// - Parameters: /// - fileURL: URL to the audio/video file func transcribeFile(_ fileURL: URL) async throws -> TranscriptionResult { + guard self.asrService.beginFileTranscription() else { + throw TranscriptionError.transcriptionFailed("Another transcription is already in progress.") + } self.isTranscribing = true error = nil self.progress = 0.0 let startTime = Date() + let transcriptionModel = SettingsStore.shared.selectedSpeechModel defer { isTranscribing = false progress = 0.0 + self.asrService.endFileTranscription() } do { @@ -279,6 +284,10 @@ final class MeetingTranscriptionService: ObservableObject { // Calculate chunk size in source file frames let sourceFramesPerChunk = AVAudioFrameCount(Double(samplesPerChunk) / resampleRatio) + let usesCohereSeamMerging = transcriptionModel == .cohereTranscribeSixBit + let sourceOverlapFrames = usesCohereSeamMerging + ? AVAudioFrameCount(2 * fileSampleRate) + : 0 var currentFrame: AVAudioFramePosition = 0 self.currentStatus = duration > 0 ? "Transcribing audio (\(Int(duration))s)..." : "Transcribing audio..." @@ -325,7 +334,10 @@ final class MeetingTranscriptionService: ObservableObject { chunkCount += 1 } - currentFrame += AVAudioFramePosition(framesToRead) + let overlap = currentFrame + AVAudioFramePosition(framesToRead) < audioFile.length + ? min(sourceOverlapFrames, framesToRead - 1) + : 0 + currentFrame += AVAudioFramePosition(framesToRead - overlap) // Update progress let progressPercent = Double(currentFrame) / Double(audioFile.length) @@ -341,7 +353,9 @@ final class MeetingTranscriptionService: ObservableObject { } // Combine all chunk transcriptions - let finalText = allTranscriptions.joined(separator: " ") + let finalText = usesCohereSeamMerging + ? CohereTranscribeCppLongFormProcessor.merge(allTranscriptions) + : allTranscriptions.joined(separator: " ") let avgConfidence = chunkCount > 0 ? totalConfidence / Float(chunkCount) : 0 let transcriptionResult = (text: finalText, confidence: avgConfidence) diff --git a/Sources/Fluid/Services/WhisperProvider.swift b/Sources/Fluid/Services/WhisperProvider.swift index c2e47a1a..b2480013 100644 --- a/Sources/Fluid/Services/WhisperProvider.swift +++ b/Sources/Fluid/Services/WhisperProvider.swift @@ -1,9 +1,261 @@ import Foundation import TranscribeCpp -/// TranscriptionProvider implementation using transcribe.cpp for Whisper GGUF models. +nonisolated enum CohereTranscribeCppLongFormProcessor { + static func previewSamples( + _ samples: [Float], + sampleRate: Int = 16_000, + maximumSeconds: Double = 30 + ) -> [Float] { + guard sampleRate > 0, maximumSeconds > 0 else { return [] } + let maximumSamples = max(1, Int(Double(sampleRate) * maximumSeconds)) + guard samples.count > maximumSamples else { return samples } + return Array(samples.suffix(maximumSamples)) + } + + static func ranges( + sampleCount: Int, + sampleRate: Int, + maximumChunkSeconds: Double = 30, + overlapSeconds: Double = 2 + ) -> [Range] { + guard sampleCount > 0, sampleRate > 0, maximumChunkSeconds > 0 else { return [] } + + let maximumChunkSamples = max(1, Int(Double(sampleRate) * maximumChunkSeconds)) + guard sampleCount > maximumChunkSamples else { return [0..] = [] + var start = 0 + + while start < sampleCount { + let end = min(start + maximumChunkSamples, sampleCount) + ranges.append(start.. String { + var result = "" + for text in texts { + let next = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !next.isEmpty else { continue } + if result.isEmpty { + result = next + continue + } + + if let merged = self.mergeCJK(left: result, right: next) { + result = merged + continue + } + + let leftTokens = result.split(whereSeparator: \.isWhitespace).map(String.init) + let rightTokens = next.split(whereSeparator: \.isWhitespace).map(String.init) + let overlap = self.wordOverlap(left: leftTokens, right: rightTokens) + if overlap > 0, let leftToken = leftTokens.last { + result = self.preservingRightBoundaryPunctuation( + in: result, + leftToken: leftToken, + rightToken: rightTokens[overlap - 1] + ) + } + let remaining = rightTokens.dropFirst(overlap) + guard !remaining.isEmpty else { continue } + result += " " + remaining.joined(separator: " ") + } + return result + } + + private static func wordOverlap(left: [String], right: [String]) -> Int { + let maximum = min(24, left.count, right.count) + guard maximum > 0 else { return 0 } + let normalizedLeft = left.map(self.normalize) + let normalizedRight = right.map(self.normalize) + + for count in stride(from: maximum, through: 1, by: -1) { + let leftStart = normalizedLeft.count - count + let matches = (0.. 1 || normalizedRight[0].count >= 4 || Int(normalizedRight[0]) != nil { + return count + } + } + + guard maximum >= 2 else { return 0 } + for count in stride(from: maximum, through: 2, by: -1) { + let leftStart = normalizedLeft.count - count + var similarCount = 0 + var similarCharacterCount = 0 + for index in 0..= max(2, count - 1), similarCharacterCount >= 8 { + return count + } + } + + var bestRightCount = 0 + var bestMatchLength = 0 + for leftCount in 2...maximum { + let leftPhrase = self.normalize(left.suffix(leftCount).joined()) + for rightCount in 2...maximum { + let rightPhrase = self.normalize(right.prefix(rightCount).joined()) + let matchLength = min(leftPhrase.count, rightPhrase.count) + guard matchLength >= 12 else { continue } + let distanceLimit = max(2, matchLength / 6) + guard abs(leftPhrase.count - rightPhrase.count) <= distanceLimit, + self.editDistance(leftPhrase, rightPhrase, limit: distanceLimit) <= distanceLimit + else { continue } + if matchLength > bestMatchLength { + bestMatchLength = matchLength + bestRightCount = rightCount + } + } + } + + return bestRightCount + } + + private static func preservingRightBoundaryPunctuation( + in left: String, + leftToken: String, + rightToken: String + ) -> String { + guard self.normalize(leftToken) == self.normalize(rightToken), + self.trailingPunctuation(in: leftToken).isEmpty + else { return left } + return left + self.trailingPunctuation(in: rightToken) + } + + private static func trailingPunctuation(in token: String) -> String { + let suffix = token.unicodeScalars.reversed().prefix { CharacterSet.punctuationCharacters.contains($0) } + return String(String.UnicodeScalarView(suffix.reversed())) + } + + private static func normalize(_ token: String) -> String { + token.lowercased().unicodeScalars + .filter { CharacterSet.alphanumerics.contains($0) } + .map(String.init) + .joined() + } + + private static func tokensAreSimilar(_ left: String, _ right: String) -> Bool { + guard !left.isEmpty, !right.isEmpty else { return false } + if left == right { return true } + if min(left.count, right.count) >= 5, left.hasPrefix(right) || right.hasPrefix(left) { + return true + } + let maximumDistance = max(left.count, right.count) >= 8 ? 2 : 1 + return self.editDistance(left, right, limit: maximumDistance) <= maximumDistance + } + + private static func editDistance(_ left: String, _ right: String, limit: Int) -> Int { + let lhs = Array(left) + let rhs = Array(right) + guard abs(lhs.count - rhs.count) <= limit else { return limit + 1 } + var previous = Array(0...rhs.count) + + for (leftIndex, leftCharacter) in lhs.enumerated() { + var current = [leftIndex + 1] + Array(repeating: 0, count: rhs.count) + var rowMinimum = current[0] + for (rightIndex, rightCharacter) in rhs.enumerated() { + current[rightIndex + 1] = min( + current[rightIndex] + 1, + previous[rightIndex + 1] + 1, + previous[rightIndex] + (leftCharacter == rightCharacter ? 0 : 1) + ) + rowMinimum = min(rowMinimum, current[rightIndex + 1]) + } + if rowMinimum > limit { return limit + 1 } + previous = current + } + + return previous[rhs.count] + } + + private static func mergeCJK(left: String, right: String) -> String? { + guard self.containsCJK(String(left.suffix(32))), + self.containsCJK(String(right.prefix(32))) + else { return nil } + let leftScalars = left.unicodeScalars.filter { CharacterSet.alphanumerics.contains($0) } + let rightScalars = Array(right.unicodeScalars) + let indexedRightScalars = rightScalars.enumerated() + .filter { CharacterSet.alphanumerics.contains($0.element) } + let maximumLeftCount = min(48, leftScalars.count) + let maximumRightCount = min(48, indexedRightScalars.count) + guard maximumLeftCount >= 2, maximumRightCount >= 2 else { + return self.joinCJKWithoutOverlap(left: left, right: right) + } + + for leftCount in stride(from: maximumLeftCount, through: 2, by: -1) { + let leftPhrase = String(String.UnicodeScalarView(leftScalars.suffix(leftCount))) + let distanceLimit = leftCount >= 4 ? max(1, leftCount / 6) : 0 + let minimumRightCount = max(2, leftCount - distanceLimit) + let allowedRightCount = min(maximumRightCount, leftCount + distanceLimit) + guard allowedRightCount >= minimumRightCount else { continue } + for rightCount in stride(from: allowedRightCount, through: minimumRightCount, by: -1) { + let rightPhrase = String(String.UnicodeScalarView(indexedRightScalars.prefix(rightCount).map(\.element))) + guard self.editDistance(leftPhrase, rightPhrase, limit: distanceLimit) <= distanceLimit else { continue } + let endIndex = indexedRightScalars[rightCount - 1].offset + 1 + return left + String(String.UnicodeScalarView(rightScalars.dropFirst(endIndex))) + } + } + return self.joinCJKWithoutOverlap(left: left, right: right) + } + + private static func joinCJKWithoutOverlap(left: String, right: String) -> String { + guard let leftScalar = left.unicodeScalars.last, + let rightScalar = right.unicodeScalars.first, + !CharacterSet.whitespacesAndNewlines.contains(leftScalar), + !CharacterSet.whitespacesAndNewlines.contains(rightScalar) + else { return left + right } + + let needsSpace = self.isHangul(leftScalar) + || self.isHangul(rightScalar) + || self.isLatinAlphanumeric(leftScalar) + || self.isLatinAlphanumeric(rightScalar) + return needsSpace ? left + " " + right : left + right + } + + private static func isHangul(_ scalar: UnicodeScalar) -> Bool { + (0xac00...0xd7af).contains(scalar.value) + } + + private static func isLatinAlphanumeric(_ scalar: UnicodeScalar) -> Bool { + scalar.value <= 0x024f && CharacterSet.alphanumerics.contains(scalar) + } + + private static func containsCJK(_ text: String) -> Bool { + text.unicodeScalars.contains { scalar in + switch scalar.value { + case 0x3040...0x30ff, 0x3400...0x4dbf, 0x4e00...0x9fff, 0xac00...0xd7af: + return true + default: + return false + } + } + } +} + +/// TranscriptionProvider implementation using transcribe.cpp GGUF models. final class WhisperProvider: TranscriptionProvider { - let name = "Whisper (Universal)" + var name: String { + self.selectedModel == .cohereTranscribeSixBit ? "Cohere Transcribe" : "Whisper (Universal)" + } var isAvailable: Bool { guard case .success = Self.backendInitialization else { return false } @@ -22,6 +274,7 @@ final class WhisperProvider: TranscriptionProvider { private var session: Session? private var ready = false private var loadedModelName: String? + private var loadedSpeechModel: SettingsStore.SpeechModel? private let overriddenModelDirectory: URL? private let urlSession: URLSession @@ -49,7 +302,7 @@ final class WhisperProvider: TranscriptionProvider { } private var modelName: String { - self.selectedModel.whisperModelFile ?? "whisper-base-Q8_0.gguf" + self.selectedModel.transcribeCppModelFile ?? "whisper-base-Q8_0.gguf" } private var legacyModelName: String? { @@ -71,10 +324,19 @@ final class WhisperProvider: TranscriptionProvider { guard let cacheDir = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first else { preconditionFailure("Could not find caches directory") } - return cacheDir.appendingPathComponent("WhisperModels") + let directoryName = self.selectedModel == .cohereTranscribeSixBit + ? "CohereTranscribeModels" + : "WhisperModels" + return cacheDir.appendingPathComponent(directoryName) } private var modelDownloadURL: URL? { + if self.selectedModel == .cohereTranscribeSixBit { + return URL( + string: "https://huggingface.co/handy-computer/cohere-transcribe-03-2026-gguf/resolve/main/\(self.modelName)" + ) + } + let modelName = self.modelName let suffix = "-Q8_0.gguf" guard modelName.hasSuffix(suffix) else { return nil } @@ -93,6 +355,7 @@ final class WhisperProvider: TranscriptionProvider { self.model = nil self.ready = false self.loadedModelName = nil + self.loadedSpeechModel = nil } private func currentLoadedModelName() -> String? { @@ -101,19 +364,32 @@ final class WhisperProvider: TranscriptionProvider { return self.loadedModelName } - private func installModel(_ model: Model, session: Session, modelName: String) { + private func installModel( + _ model: Model, + session: Session, + modelName: String, + speechModel: SettingsStore.SpeechModel + ) { self.stateLock.lock() defer { self.stateLock.unlock() } self.model = model self.session = session self.loadedModelName = modelName + self.loadedSpeechModel = speechModel self.ready = true } - private func activeSession() -> Session? { + private func activeRuntime() -> (session: Session, speechModel: SettingsStore.SpeechModel)? { + self.stateLock.lock() + defer { self.stateLock.unlock() } + guard let session, let loadedSpeechModel else { return nil } + return (session, loadedSpeechModel) + } + + private func currentLoadedSpeechModel() -> SettingsStore.SpeechModel? { self.stateLock.lock() defer { self.stateLock.unlock() } - return self.session + return self.loadedSpeechModel } private func removeLegacyModelIfNeeded() { @@ -132,8 +408,33 @@ final class WhisperProvider: TranscriptionProvider { } } + private func removeLegacyCohereCachesIfNeeded(for model: SettingsStore.SpeechModel) { + guard model == .cohereTranscribeSixBit, self.overriddenModelDirectory == nil, + let cacheDirectory = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first + else { return } + + let legacyDirectories = [ + cacheDirectory.appendingPathComponent("cohere-transcribe-03-2026-CoreML-6bit", isDirectory: true), + cacheDirectory.appendingPathComponent("FluidAudio/CompiledCohereModels", isDirectory: true), + ] + for directory in legacyDirectories where FileManager.default.fileExists(atPath: directory.path) { + do { + try FileManager.default.removeItem(at: directory) + DebugLogger.shared.info( + "WhisperProvider: Removed legacy Cohere cache at \(directory.path)", + source: "WhisperProvider" + ) + } catch { + DebugLogger.shared.warning( + "WhisperProvider: Failed to remove legacy Cohere cache at \(directory.path): \(error.localizedDescription)", + source: "WhisperProvider" + ) + } + } + } + private func isModelFileValid(at url: URL, for targetModel: SettingsStore.SpeechModel) -> Bool { - guard let expectedModelFile = targetModel.whisperModelFile, + guard let expectedModelFile = targetModel.transcribeCppModelFile, url.lastPathComponent == expectedModelFile else { return false @@ -151,7 +452,7 @@ final class WhisperProvider: TranscriptionProvider { try Task.checkCancellation() let targetModel = self.selectedModel - let currentModelName = targetModel.whisperModelFile ?? "whisper-base-Q8_0.gguf" + let currentModelName = targetModel.transcribeCppModelFile ?? "whisper-base-Q8_0.gguf" let loadedModelName = self.currentLoadedModelName() if self.isReady, loadedModelName != currentModelName { @@ -195,22 +496,29 @@ final class WhisperProvider: TranscriptionProvider { userInfo: [NSLocalizedDescriptionKey: "Whisper model file is missing or corrupted. Please re-download the model."] ) } - self.removeLegacyModelIfNeeded() + if targetModel.isWhisperModel { + self.removeLegacyModelIfNeeded() + } let requiredMemoryGB = targetModel.requiredMemoryGB - let availableMemoryGB = Self.availableMemoryGB() + let checksSystemMemory = targetModel == .cohereTranscribeSixBit + let measuredMemoryGB = checksSystemMemory ? Self.totalMemoryGB() : Self.availableMemoryGB() + let measuredMemoryLabel = checksSystemMemory ? "System" : "Available" DebugLogger.shared.info( - "WhisperProvider: Memory check - Required: \(String(format: "%.1f", requiredMemoryGB))GB, Available: \(String(format: "%.1f", availableMemoryGB))GB", + "WhisperProvider: Memory check - Required: \(String(format: "%.1f", requiredMemoryGB))GB, \(measuredMemoryLabel): \(String(format: "%.1f", measuredMemoryGB))GB", source: "WhisperProvider" ) - if availableMemoryGB < requiredMemoryGB { + if measuredMemoryGB < requiredMemoryGB { + let recoverySuggestion = checksSystemMemory + ? "Please choose a model with a lower memory requirement." + : "Please try a smaller model or close other applications to free up memory." let errorMessage = """ Insufficient memory for \(targetModel.displayName). Required: \(String(format: "%.1f", requiredMemoryGB)) GB - Available: \(String(format: "%.1f", availableMemoryGB)) GB + \(measuredMemoryLabel): \(String(format: "%.1f", measuredMemoryGB)) GB - Please try a smaller model or close other applications to free up memory. + \(recoverySuggestion) """ DebugLogger.shared.error("WhisperProvider: \(errorMessage)", source: "WhisperProvider") throw NSError( @@ -241,7 +549,13 @@ final class WhisperProvider: TranscriptionProvider { let loadedSession = try loadedModel.session() try Task.checkCancellation() - self.installModel(loadedModel, session: loadedSession, modelName: currentModelName) + self.installModel( + loadedModel, + session: loadedSession, + modelName: currentModelName, + speechModel: targetModel + ) + self.removeLegacyCohereCachesIfNeeded(for: targetModel) DebugLogger.shared.info( "WhisperProvider: Model ready (\(currentModelName), backend=\(loadedModel.backend), arch=\(loadedModel.arch))", source: "WhisperProvider" @@ -266,6 +580,10 @@ final class WhisperProvider: TranscriptionProvider { } } + private static func totalMemoryGB() -> Double { + Double(ProcessInfo.processInfo.physicalMemory) / (1024 * 1024 * 1024) + } + private static func availableMemoryGB() -> Double { var pageSize: vm_size_t = 0 host_page_size(mach_host_self(), &pageSize) @@ -297,31 +615,57 @@ final class WhisperProvider: TranscriptionProvider { throw NSError( domain: "WhisperProvider", code: -2, - userInfo: [NSLocalizedDescriptionKey: "Audio too short for Whisper transcription"] + userInfo: [NSLocalizedDescriptionKey: "Audio too short for transcription"] ) } - guard let session = self.activeSession() else { + guard let runtime = self.activeRuntime() else { throw NSError( domain: "WhisperProvider", code: -1, - userInfo: [NSLocalizedDescriptionKey: "Whisper model not loaded"] + userInfo: [NSLocalizedDescriptionKey: "Speech model not loaded"] ) } - let transcript = try await session.run( - samples, - options: RunOptions(timestamps: .segment) + let isCohere = runtime.speechModel == .cohereTranscribeSixBit + let options = RunOptions( + timestamps: isCohere ? .none : .segment, + language: isCohere ? SettingsStore.shared.selectedCohereLanguage.rawValue : nil ) - let fullText = transcript.text.trimmingCharacters(in: .whitespacesAndNewlines) + let texts: [String] + if isCohere { + let ranges = CohereTranscribeCppLongFormProcessor.ranges( + sampleCount: samples.count, + sampleRate: 16_000 + ) + var chunkTexts: [String] = [] + chunkTexts.reserveCapacity(ranges.count) + for range in ranges { + try Task.checkCancellation() + let transcript = try await runtime.session.run(Array(samples[range]), options: options) + chunkTexts.append(transcript.text) + } + texts = chunkTexts + } else { + texts = try [await runtime.session.run(samples, options: options).text] + } + let fullText = CohereTranscribeCppLongFormProcessor.merge(texts) return ASRTranscriptionResult(text: fullText, confidence: 1.0) } + func transcribeStreaming(_ samples: [Float]) async throws -> ASRTranscriptionResult { + guard self.currentLoadedSpeechModel() == .cohereTranscribeSixBit else { + return try await self.transcribe(samples) + } + return try await self.transcribe(CohereTranscribeCppLongFormProcessor.previewSamples(samples)) + } + func modelsExistOnDisk() -> Bool { return self.isModelFileValid(at: self.modelURL, for: self.selectedModel) } func clearCache() async throws { + let targetModel = self.selectedModel self.unloadModel() if FileManager.default.fileExists(atPath: self.modelURL.path) { @@ -338,6 +682,7 @@ final class WhisperProvider: TranscriptionProvider { try FileManager.default.removeItem(at: self.modelDirectory) } } + self.removeLegacyCohereCachesIfNeeded(for: targetModel) } private func downloadModel(progressHandler: ((Double) -> Void)?) async throws { diff --git a/Sources/Fluid/UI/AISettings/VoiceEngineSettingsViewModel.swift b/Sources/Fluid/UI/AISettings/VoiceEngineSettingsViewModel.swift index bd561f02..ba93b28b 100644 --- a/Sources/Fluid/UI/AISettings/VoiceEngineSettingsViewModel.swift +++ b/Sources/Fluid/UI/AISettings/VoiceEngineSettingsViewModel.swift @@ -11,7 +11,7 @@ final class VoiceEngineSettingsViewModel: ObservableObject { var asr: ASRService { self.appServices.asr } var areSpeechModelActionsBlocked: Bool { - self.asr.isRunning + self.asr.isAnyTranscriptionRunning || self.downloadingModel != nil || self.asr.hasActiveModelDownload || self.asr.hasActiveModelPreparation diff --git a/Tests/FluidDictationIntegrationTests/TranscribeCppLongFormProcessorTests.swift b/Tests/FluidDictationIntegrationTests/TranscribeCppLongFormProcessorTests.swift new file mode 100644 index 00000000..33a1783f --- /dev/null +++ b/Tests/FluidDictationIntegrationTests/TranscribeCppLongFormProcessorTests.swift @@ -0,0 +1,172 @@ +@testable import FluidVoice_Debug +import XCTest + +final class CohereCppLongFormTests: XCTestCase { + @MainActor + func testFileTranscriptionGateBlocksConcurrentSpeechOperations() { + let service = ASRService() + + XCTAssertTrue(service.beginFileTranscription()) + XCTAssertTrue(service.isFileTranscriptionRunning) + XCTAssertTrue(service.isAnyTranscriptionRunning) + XCTAssertFalse(service.beginFileTranscription()) + + service.endFileTranscription() + XCTAssertFalse(service.isFileTranscriptionRunning) + XCTAssertFalse(service.isAnyTranscriptionRunning) + } + + func testCohereEnablesBoundedStreamingPreviews() { + XCTAssertTrue(SettingsStore.SpeechModel.cohereTranscribeSixBit.supportsStreaming) + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.previewSamples( + Array(0..<50).map(Float.init), + sampleRate: 10, + maximumSeconds: 3 + ), + Array(20..<50).map(Float.init) + ) + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.previewSamples( + Array(0..<20).map(Float.init), + sampleRate: 10, + maximumSeconds: 3 + ), + Array(0..<20).map(Float.init) + ) + } + + func testRangesStayBoundedAndOverlap() { + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.ranges( + sampleCount: 70, + sampleRate: 10, + maximumChunkSeconds: 3, + overlapSeconds: 1 + ), + [0..<30, 20..<50, 40..<70] + ) + } + + func testRangesKeepShortAudioWhole() { + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.ranges(sampleCount: 20, sampleRate: 10), + [0..<20] + ) + XCTAssertTrue(CohereTranscribeCppLongFormProcessor.ranges(sampleCount: 0, sampleRate: 10).isEmpty) + } + + func testMergeRemovesRepeatedWordsAcrossPunctuation() { + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.merge([ + "GPU, CPU, memory, networking.", + "memory networking storage and power.", + ]), + "GPU, CPU, memory, networking. storage and power." + ) + } + + func testMergePreservesPunctuationRecognizedOnlyInOverlappedChunk() { + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.merge(["The end", "The end. Next"]), + "The end. Next" + ) + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.merge(["The end", "The end."]), + "The end." + ) + } + + func testMergeToleratesMinorRecognitionDrift() { + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.merge([ + "solving the networking", + "the network storage problem", + ]), + "solving the networking storage problem" + ) + } + + func testMergeAlignsContractionsAndHyphenatedWords() { + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.merge([ + "I'm sure there are trade-offs there.", + "I'm sure there's tradeoffs there. Plus, specialists collaborate.", + ]), + "I'm sure there are trade-offs there. Plus, specialists collaborate." + ) + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.merge([ + "networking chips and scale-up switches and scale-out switches.", + "Scale up switches and scale out switches. Cooling matters.", + ]), + "networking chips and scale-up switches and scale-out switches. Cooling matters." + ) + } + + func testMergeHandlesCJKWithoutAddingSpaces() { + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.merge(["这是一个测试结果", "测试结果非常准确"]), + "这是一个测试结果非常准确" + ) + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.merge(["这是一个测试，结果", "测试结果非常准确"]), + "这是一个测试，结果非常准确" + ) + } + + func testMergeToleratesMinorCJKRecognitionDrift() { + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.merge(["这是一个测试结果", "测试结杲非常准确"]), + "这是一个测试结果非常准确" + ) + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.merge(["测试结果", "测试结结果非常准确"]), + "测试结果非常准确" + ) + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.merge(["测试结果", "试结果非常准确"]), + "测试结果非常准确" + ) + } + + func testMergePreservesCJKAndMixedScriptBoundariesWithoutOverlap() { + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.merge(["한국어 테스트", "새로운 문장"]), + "한국어 테스트 새로운 문장" + ) + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.merge(["中文 hello", "world 中文"]), + "中文 hello world 中文" + ) + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.merge(["第一句", "第二句"]), + "第一句第二句" + ) + } + + func testEarlierCJKDoesNotBreakEnglishBoundarySpacing() { + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.merge(["中文结束. English starts", "starts here"]), + "中文结束. English starts here" + ) + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.merge(["中", "a"]), + "中 a" + ) + } + + func testMergeIgnoresEmptyChunks() { + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.merge(["", " ", "complete transcript"]), + "complete transcript" + ) + } + + func testMergeRejectsUnrelatedShortFuzzyWords() { + XCTAssertEqual( + CohereTranscribeCppLongFormProcessor.merge(["turn on", "burn in slowly"]), + "turn on burn in slowly" + ) + } +}