diff --git a/Info.plist b/Info.plist index 2ce4b98a..10df2a53 100644 --- a/Info.plist +++ b/Info.plist @@ -24,6 +24,10 @@ FluidVoice needs Accessibility permission to listen for global hotkeys and type text into other apps when you stop recording. NSMicrophoneUsageDescription FluidVoice needs microphone access to record your voice for speech-to-text transcription. Your audio is processed locally and never sent to external servers. + NSAudioCaptureUsageDescription + FluidVoice needs system audio access to record the meeting application you explicitly select. Meeting audio stays on this Mac. + NSScreenCaptureUsageDescription + FluidVoice needs Screen & System Audio access to record audio from the meeting application you explicitly select. LSUIElement NSAppleEventsUsageDescription diff --git a/MEETING_TRANSCRIPTION_PRD.md b/MEETING_TRANSCRIPTION_PRD.md index e452c006..e8a86454 100644 --- a/MEETING_TRANSCRIPTION_PRD.md +++ b/MEETING_TRANSCRIPTION_PRD.md @@ -25,7 +25,7 @@ Checkboxes should be checked only when the requirement and its acceptance criter ### Document map -- [Agreed product requirements](#3-agreed-product-requirements--implementation-status) +- [Confirmed decisions](#3-confirmed-product-decisions) - [Goals and non-goals](#4-goals-and-non-goals) - [High-level milestones](#5-high-level-milestones) - [User experience and canvas states](#7-user-experience-and-canvas-states) @@ -128,6 +128,15 @@ The direction below is agreed. Its checkboxes remain open until the correspondin ## 5. High-level milestones +### Milestone delivery policy + +Each milestone is implemented on a stacked branch targeting `1.6.8/meeting`; only the integration branch targets `main`. A milestone is not complete when code merely compiles. + +- [ ] **[MUST · FOUNDATION] DELIVERY-001** Define the milestone's deterministic tests, real-system proof, performance/resource checks, and regression boundaries before implementation is called complete. +- [ ] **[MUST · FOUNDATION] DELIVERY-002** Assign independent architecture/reliability and UX/accessibility reviewers who did not own the implementation. +- [ ] **[MUST · FOUNDATION] DELIVERY-003** Resolve every blocking finding and record deferred non-blockers against a named later milestone. +- [ ] **[MUST · FOUNDATION] DELIVERY-004** Merge the reviewed milestone PR into `1.6.8/meeting`; open one final integration PR from `1.6.8/meeting` to `main` after the intended milestone set is complete. + ### Milestone 0 — Decisions and durable foundations - [ ] **[MUST · FOUNDATION] M0-EXIT-001** Resolve FOUNDATION and V1 open decisions in Section 22; M3/M4 decisions remain gated by their own milestones. @@ -146,6 +155,18 @@ Goal: prove the complete path before polishing secondary automation. - [ ] **[MUST · M1] M1-EXIT-005** The menu-bar menu shows **Stop Meeting Recording** first while capture is active. - [ ] **[MUST · M1] M1-EXIT-006** Window closure and sidebar navigation do not stop the session. +#### Milestone 1 test and review gate + +- [ ] **[MUST · M1] M1-TEST-001** Deterministic tests cover legal state transitions, concurrent/repeated Start and Stop, stale-generation callbacks, and exactly one finalization/processing job. +- [ ] **[MUST · M1] M1-TEST-002** A real online-call capture proves separate non-empty application/system and microphone files with timestamps inside one session manifest. +- [ ] **[MUST · M1] M1-TEST-003** A real in-room capture proves microphone-only recording does not request Screen/System Audio permission solely for that mode. +- [ ] **[MUST · M1] M1-TEST-004** Stop proves the complete offline English path: finalized audio → transcription/diarization → timestamped canvas result. +- [ ] **[MUST · M1] M1-TEST-005** Apple Silicon fixtures cover remote speaker separation, clean confirmed personal-mic **You**, and ambiguous/leaked microphone speech remaining unknown. +- [ ] **[MUST · M1] M1-TEST-006** UI proof covers menu item order and Stop action, menu-bar recording mark, sidebar navigation, main-window close/reopen, and the four canvas states. +- [ ] **[MUST · M1] M1-REVIEW-001** An independent audio/architecture reviewer approves capture ownership, track separation, timestamp handling, provider serialization, and no dictation first-PCM regression. +- [ ] **[MUST · M1] M1-REVIEW-002** An independent UX/accessibility reviewer approves sidebar placement, state clarity, menu-bar behavior, keyboard/VoiceOver labels, and low-resource rendering. +- [ ] **[MUST · M1] M1-REVIEW-003** Every blocking review finding is fixed or explicitly recorded as a later-milestone item before the M1 stacked PR merges. + ### Milestone 2 — Trustworthy offline V1 Goal: make the working slice safe to ship. @@ -719,8 +740,8 @@ flowchart TD - [ ] **[MUST · V1] TEST-REPO-002** Validate implementation builds with `sh build_with_FI_incremental.sh` and inspect the installed `/Applications/FluidVoice.app`. - [ ] **[MUST · V1] TEST-REPO-003** Do not treat Debug/XCTest-host runs as release proof; use focused logic checks plus installed-app, real-meeting evidence. - [ ] **[MUST · V1] TEST-REPO-004** Verify the minimum supported macOS path and low-resource Apple Silicon behavior; describe Intel as reduced capability until separately proven. -- [ ] **[MUST · V1] TEST-REPO-005** Update the requested release-notes version locally, credit contributors as required, and never commit ignored release notes. - [ ] **[MUST · V1] TEST-REPO-006** Run real Intel capture plus plain English transcription before claiming Intel meeting support; universal build slices alone are insufficient. +- [ ] **[MUST · V1] TEST-REPO-005** Update the requested release-notes version locally, credit contributors as required, and never commit ignored release notes. ### 21.7 Measurable budgets to set before implementation diff --git a/Sources/Fluid/AppDelegate.swift b/Sources/Fluid/AppDelegate.swift index b41052b5..c46e88f3 100644 --- a/Sources/Fluid/AppDelegate.swift +++ b/Sources/Fluid/AppDelegate.swift @@ -89,7 +89,9 @@ class AppDelegate: NSObject, NSApplicationDelegate, UNUserNotificationCenterDele didFinishShutdown = true } - let deadline = Date().addingTimeInterval(8) + // Meeting capture can spend up to three seconds stopping its runtime and + // four seconds finalizing audio before the durable session save. + let deadline = Date().addingTimeInterval(12) while !didFinishShutdown, Date() < deadline { RunLoop.current.run(mode: .default, before: Date().addingTimeInterval(0.01)) } diff --git a/Sources/Fluid/ContentView.swift b/Sources/Fluid/ContentView.swift index 353ba102..6bd8e0c6 100644 --- a/Sources/Fluid/ContentView.swift +++ b/Sources/Fluid/ContentView.swift @@ -78,7 +78,8 @@ enum SidebarItem: Hashable { case voiceEngine case aiEnhancements case preferences - case meetingTools + case fileTranscription + case meetingTranscription case customDictionary case stats case history @@ -591,6 +592,9 @@ struct ContentView: View { DispatchQueue.main.asyncAfter(deadline: .now() + 1.5) { DebugLogger.shared.info("🚦 Startup delay complete, signaling UI ready...", source: "ContentView") self.appServices.signalUIReady() + self.menuBarManager.configure( + meetingCoordinator: self.appServices.meetingSessionCoordinator + ) Task { @MainActor in await AudioStartupGate.shared.scheduleOpenAfterInitialUISettled() @@ -958,6 +962,8 @@ struct ContentView: View { switch destination { case .customDictionary: self.selectedSidebarItem = .customDictionary + case .meetingTranscription: + self.selectedSidebarItem = .meetingTranscription case .preferences: self.selectedSidebarItem = .preferences } @@ -1166,7 +1172,12 @@ struct ContentView: View { Section { self.sidebarNavigationLink(.commandMode, title: "Command Mode", systemImage: "terminal.fill") - self.sidebarNavigationLink(.meetingTools, title: "File Transcription", systemImage: "doc.text.fill") + self.sidebarNavigationLink(.fileTranscription, title: "File Transcription", systemImage: "doc.text.fill") + self.sidebarNavigationLink( + .meetingTranscription, + title: "Meeting Transcription", + systemImage: "person.2.wave.2.fill" + ) } header: { self.sidebarSectionHeader("Use") } @@ -1264,8 +1275,10 @@ struct ContentView: View { )) case .preferences: return AnyView(self.preferencesView) - case .meetingTools: - return AnyView(self.meetingToolsView) + case .fileTranscription: + return AnyView(self.fileTranscriptionView) + case .meetingTranscription: + return AnyView(self.meetingTranscriptionView) case .customDictionary: return AnyView(CustomDictionaryView()) case .stats: @@ -1500,10 +1513,21 @@ struct ContentView: View { }) } - // MARK: - Meeting Transcription (Coming Soon) + // MARK: - File and Meeting Transcription + + private var fileTranscriptionView: some View { + FileTranscriptionView( + asrService: self.asr, + transcriptionService: self.appServices.fileTranscriptionService + ) + } - private var meetingToolsView: some View { - MeetingTranscriptionView(asrService: self.asr) + private var meetingTranscriptionView: some View { + MeetingTranscriptionView( + coordinator: self.appServices.meetingSessionCoordinator, + asrService: self.asr, + onOpenVoiceEngine: { self.selectedSidebarItem = .voiceEngine } + ) } // MARK: - Stats View @@ -3068,6 +3092,7 @@ struct ContentView: View { /// Capture app context at start to avoid mismatches if the user switches apps mid-session private func startRecording() { + guard !self.presentExclusiveActivityBlockIfNeeded() else { return } let model = SettingsStore.shared.selectedSpeechModel DebugLogger.shared.info( "ContentView: startRecording() for model=\(model.displayName), supportsStreaming=\(model.supportsStreaming)", @@ -3077,7 +3102,6 @@ struct ContentView: View { DebugLogger.shared.debug("ContentView: start ignored because capture is already active", source: "ContentView") return } - self.advanceOverlayLifecycle() self.setActiveRecordingMode(.dictate) let shouldShowDictationOverlay = !self.isRecordingForCommand @@ -3128,6 +3152,20 @@ struct ContentView: View { } } + private func presentExclusiveActivityBlockIfNeeded() -> Bool { + guard let activity = self.asr.activeExclusiveActivity else { return false } + let message = "Wait for the active \(activity.displayName) to finish." + self.asr.errorTitle = "Dictation Unavailable" + self.asr.errorMessage = message + self.asr.showError = true + AccessibilityNotification.Announcement("Dictation unavailable. \(message)").post() + DebugLogger.shared.info( + "ContentView: dictation start blocked by \(activity.rawValue)", + source: "ContentView" + ) + return true + } + private func prewarmPrivateAIDictationIfNeeded(for slot: SettingsStore.DictationShortcutSlot) { let appBundleID = self.recordingAppInfo?.bundleId guard PrivateAIProviderPromptFormat.isAvailable(settings: SettingsStore.shared), @@ -3337,6 +3375,7 @@ struct ContentView: View { self.beginDictationRecording(for: selection, mode: .promptMode) }, commandModeCallback: { + guard !self.presentExclusiveActivityBlockIfNeeded() else { return } DebugLogger.shared.info("Command mode triggered", source: "ContentView") self.captureRecordingContext() @@ -3368,6 +3407,7 @@ struct ContentView: View { } }, rewriteModeCallback: { + guard !self.presentExclusiveActivityBlockIfNeeded() else { return } guard !self.showPrivateAIEditModeUnavailableIfNeeded() else { return } self.captureRecordingContext() @@ -3721,6 +3761,7 @@ extension ContentView { } private func beginDictationRecording(for slot: SettingsStore.DictationShortcutSlot, mode: ActiveRecordingMode) { + guard !self.presentExclusiveActivityBlockIfNeeded() else { return } DebugLogger.shared.debug("Begin dictation recording for slot \(slot.rawValue)", source: "ContentView") self.appBench("begin_recording slot=\(slot.rawValue) mode=\(mode.rawValue)") if self.isOnboardingVoicePlaygroundStepActive { diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift index d4b6e60b..fa9bc3d9 100644 --- a/Sources/Fluid/Persistence/SettingsStore.swift +++ b/Sources/Fluid/Persistence/SettingsStore.swift @@ -1813,6 +1813,23 @@ final class SettingsStore: ObservableObject { set { self.defaults.set(newValue, forKey: Keys.preferredInputDeviceUID) } } + var meetingRecordingDefaults: MeetingRecordingDefaults { + get { + guard let data = self.defaults.data(forKey: Keys.meetingRecordingDefaults), + let saved = try? JSONDecoder().decode(MeetingRecordingDefaults.self, from: data) + else { + return .unconfigured + } + return saved + } + set { + guard let data = try? JSONEncoder().encode(newValue) else { + return + } + self.defaults.set(data, forKey: Keys.meetingRecordingDefaults) + } + } + var preferredOutputDeviceUID: String? { get { self.defaults.string(forKey: Keys.preferredOutputDeviceUID) } set { self.defaults.set(newValue, forKey: Keys.preferredOutputDeviceUID) } @@ -4885,6 +4902,7 @@ private extension SettingsStore { static let primaryDictationShortcutsKey = "PrimaryDictationShortcuts" static let preferredInputDeviceUID = "PreferredInputDeviceUID" static let preferredOutputDeviceUID = "PreferredOutputDeviceUID" + static let meetingRecordingDefaults = "MeetingRecordingDefaults" static let microphoneSelectionMode = "MicrophoneSelectionMode" static let appMicSelectionMigrationVersion = "AppOnlyMicrophoneSelectionMigrationVersion" static let visualizerNoiseThreshold = "VisualizerNoiseThreshold" diff --git a/Sources/Fluid/Services/ASRService.swift b/Sources/Fluid/Services/ASRService.swift index 81f6223c..38a1cdca 100644 --- a/Sources/Fluid/Services/ASRService.swift +++ b/Sources/Fluid/Services/ASRService.swift @@ -54,6 +54,45 @@ enum AudioCaptureStartOutcome: Equatable { case failed } +enum ASRExclusiveActivity: String, Equatable { + case dictation + case fileTranscription + case localAPI + case meeting + case modelMaintenance + + var displayName: String { + switch self { + case .dictation: + return "dictation" + case .fileTranscription: + return "file transcription" + case .localAPI: + return "local API transcription" + case .meeting: + return "meeting transcription" + case .modelMaintenance: + return "voice model maintenance" + } + } +} + +struct ASRActivityLease: Equatable { + let id: UUID + let activity: ASRExclusiveActivity +} + +enum ASRActivityError: LocalizedError { + case activityInProgress(ASRExclusiveActivity) + + var errorDescription: String? { + switch self { + case let .activityInProgress(activity): + return "Wait for the active \(activity.displayName) to finish." + } + } +} + // swiftlint:disable file_length type_body_length /// A comprehensive speech recognition service that handles real-time audio transcription. /// @@ -190,11 +229,41 @@ final class ASRService: ObservableObject { private(set) var dictionaryTrainingAudioGeneration = 0 @Published private(set) var isStarting: Bool = false // Guard against re-entrant start() calls + @Published private(set) var activeExclusiveActivity: ASRExclusiveActivity? private var audioCaptureStartWaiters: [CheckedContinuation] = [] + private var activeActivityLease: ASRActivityLease? + private var dictationActivityLease: ASRActivityLease? + private var providerResetPending = false var isRunningOrStarting: Bool { self.isRunning || self.isStarting } + func acquireExclusiveActivity(_ activity: ASRExclusiveActivity) throws -> ASRActivityLease { + guard let activeActivityLease = self.activeActivityLease else { + let lease = ASRActivityLease(id: UUID(), activity: activity) + self.activeActivityLease = lease + self.activeExclusiveActivity = activity + return lease + } + throw ASRActivityError.activityInProgress(activeActivityLease.activity) + } + + func releaseExclusiveActivity(_ lease: ASRActivityLease) { + guard self.activeActivityLease == lease else { return } + self.activeActivityLease = nil + self.activeExclusiveActivity = nil + + guard self.providerResetPending else { return } + self.providerResetPending = false + self.resetTranscriptionProvider() + } + + private func releaseDictationActivityIfNeeded() { + guard let lease = self.dictationActivityLease else { return } + self.dictationActivityLease = nil + self.releaseExclusiveActivity(lease) + } + private let audioCaptureReadinessGate = AudioCaptureReadinessGate() private let firstPCMTimeoutNanoseconds: UInt64 = 2_000_000_000 private var audioCaptureStartGeneration: UInt64 = 0 @@ -645,6 +714,19 @@ final class ASRService: ObservableObject { /// Call this when the transcription provider setting changes to reset state func resetTranscriptionProvider() { + guard self.activeActivityLease == nil else { + self.providerResetPending = true + DebugLogger.shared.info( + "ASRService: Deferring provider reset until \(self.activeExclusiveActivity?.displayName ?? "active work") finishes", + source: "ASRService" + ) + return + } + + guard let resetLease = try? self.acquireExclusiveActivity(.modelMaintenance) else { + self.providerResetPending = true + return + } let newModel = SettingsStore.shared.selectedSpeechModel DebugLogger.shared.info("ASRService: Switching to '\(newModel.displayName)', resetting provider state...", source: "ASRService") @@ -686,6 +768,7 @@ final class ASRService: ObservableObject { // Use Task for async check to support providers like AppleSpeechAnalyzerProvider Task { [weak self] in guard let self = self else { return } + defer { self.releaseExclusiveActivity(resetLease) } _ = await retiringTask?.result guard SettingsStore.shared.selectedSpeechModel == newModel else { return } await self.checkIfModelsExistAsync() @@ -1430,6 +1513,16 @@ final class ASRService: ObservableObject { DebugLogger.shared.warning("START() blocked - app is terminating", source: "ASRService") return .failed } + do { + self.dictationActivityLease = try self.acquireExclusiveActivity(.dictation) + } catch { + let message = error.localizedDescription + self.errorTitle = "Dictation Unavailable" + self.errorMessage = message + self.showError = true + DebugLogger.shared.warning("START() blocked - \(message)", source: "ASRService") + return .failed + } self.audioCaptureStartGeneration &+= 1 let startGeneration = self.audioCaptureStartGeneration self.isStarting = true @@ -1447,6 +1540,7 @@ final class ASRService: ObservableObject { "Audio capture start cancelled during route handoff generation=\(startGeneration)", source: "ASRService" ) + self.releaseDictationActivityIfNeeded() return .failed } @@ -1628,6 +1722,7 @@ final class ASRService: ObservableObject { DebugLogger.shared.info("✅ START() completed successfully", source: "ASRService") return .started } catch { + self.releaseDictationActivityIfNeeded() await self.audioCaptureReadinessGate.cancel( sessionID: captureSessionID, attemptID: readinessAttemptID @@ -1820,6 +1915,7 @@ final class ASRService: ObservableObject { } guard self.isRunning else { self.isDictionaryTrainingCaptureActive = false + self.releaseDictationActivityIfNeeded() DebugLogger.shared.warning("⚠️ STOP() - not running, returning empty string", source: "ASRService") return "" } @@ -1827,6 +1923,7 @@ final class ASRService: ObservableObject { defer { self.applyPendingParakeetVocabularyReloadIfNeeded() self.isDictionaryTrainingCaptureActive = false + self.releaseDictationActivityIfNeeded() } await self.cancelAudioRouteRecoveryAndWait() @@ -2109,6 +2206,12 @@ final class ASRService: ObservableObject { } func transcribeSamplesForAPI(_ inputSamples: [Float]) async throws -> ASRTranscriptionResult { + let lease = try self.acquireExclusiveActivity(.localAPI) + defer { self.releaseExclusiveActivity(lease) } + return try await self.transcribeSamplesForAPIWithLease(inputSamples) + } + + private func transcribeSamplesForAPIWithLease(_ inputSamples: [Float]) async throws -> ASRTranscriptionResult { var samples = inputSamples guard !samples.isEmpty else { return ASRTranscriptionResult(text: "", confidence: 0) @@ -2146,6 +2249,8 @@ final class ASRService: ObservableObject { } func transcribeFileForAPI(_ fileURL: URL) async throws -> (result: ASRTranscriptionResult, sampleCount: Int) { + let lease = try self.acquireExclusiveActivity(.localAPI) + defer { self.releaseExclusiveActivity(lease) } guard FileManager.default.isReadableFile(atPath: fileURL.path) else { throw NSError( domain: "ASRService", @@ -2168,7 +2273,7 @@ final class ASRService: ObservableObject { guard provider.prefersNativeFileTranscription else { let samples = try LocalAPIAudioDecoder.samples(from: fileURL) - let result = try await self.transcribeSamplesForAPI(samples) + let result = try await self.transcribeSamplesForAPIWithLease(samples) return (result, samples.count) } @@ -2189,14 +2294,46 @@ final class ASRService: ObservableObject { return (ASRTranscriptionResult(text: cleanedText, confidence: result.confidence), estimatedSamples) } + func transcribeMeetingSamples( + _ samples: [Float], + provider: any TranscriptionProvider, + languageCode: String + ) async throws -> ASRTranscriptionResult { + guard self.activeExclusiveActivity == .meeting else { + throw ASRActivityError.activityInProgress(self.activeExclusiveActivity ?? .meeting) + } + return try await self.transcriptionExecutor.run { [provider] in + if let whisperProvider = provider as? WhisperProvider { + return try await whisperProvider.transcribe(samples, languageCode: languageCode) + } + return try await provider.transcribe(samples) + } + } + + func transcribeMeetingFile( + at fileURL: URL, + provider: any TranscriptionProvider + ) async throws -> ASRTranscriptionResult { + guard self.activeExclusiveActivity == .meeting else { + throw ASRActivityError.activityInProgress(self.activeExclusiveActivity ?? .meeting) + } + return try await self.transcriptionExecutor.run { [provider] in + try await provider.transcribeFile(at: fileURL) + } + } + func stopWithoutTranscription() async { if self.isStarting, self.isRunning == false { await self.cancelPendingAudioCaptureStart(reason: "stop_without_transcription") } - guard self.isRunning else { return } + guard self.isRunning else { + self.releaseDictationActivityIfNeeded() + return + } defer { self.applyPendingParakeetVocabularyReloadIfNeeded() self.isDictionaryTrainingCaptureActive = false + self.releaseDictationActivityIfNeeded() } await self.cancelAudioRouteRecoveryAndWait() @@ -3820,6 +3957,8 @@ final class ASRService: ObservableObject { // MARK: - Cache management func clearModelCache() async throws { + let activityLease = try self.acquireExclusiveActivity(.modelMaintenance) + defer { self.releaseExclusiveActivity(activityLease) } DebugLogger.shared.debug("Clearing model cache via transcription provider", source: "ASRService") await self.transcriptionExecutor.cancelAndAwaitPending() try await self.transcriptionProvider.clearCache() @@ -3828,6 +3967,8 @@ final class ASRService: ObservableObject { } func clearModelCache(for model: SettingsStore.SpeechModel) async throws { + let activityLease = try self.acquireExclusiveActivity(.modelMaintenance) + defer { self.releaseExclusiveActivity(activityLease) } DebugLogger.shared.debug("Clearing model cache for \(model.displayName)", source: "ASRService") if SettingsStore.shared.selectedSpeechModel == model { await self.transcriptionExecutor.cancelAndAwaitPending() @@ -3840,8 +3981,7 @@ final class ASRService: ObservableObject { } guard SettingsStore.shared.selectedSpeechModel == model else { return } - self.resetTranscriptionProvider() - await self.checkIfModelsExistAsync() + self.providerResetPending = true } // MARK: - Timer-based Streaming Transcription (No VAD) diff --git a/Sources/Fluid/Services/AppServices.swift b/Sources/Fluid/Services/AppServices.swift index bbcc7a08..44b750da 100644 --- a/Sources/Fluid/Services/AppServices.swift +++ b/Sources/Fluid/Services/AppServices.swift @@ -68,6 +68,16 @@ final class AppServices: ObservableObject { return service } + private var _fileTranscriptionService: FileTranscriptionService? + var fileTranscriptionService: FileTranscriptionService { + if let existing = self._fileTranscriptionService { + return existing + } + let service = FileTranscriptionService(asrService: self.asr) + self._fileTranscriptionService = service + return service + } + private var _microphonePreferenceCoordinator: MicrophonePreferenceCoordinator? var microphonePreferenceCoordinator: MicrophonePreferenceCoordinator { if let existing = self._microphonePreferenceCoordinator { @@ -78,6 +88,64 @@ final class AppServices: ObservableObject { return coordinator } + private var _meetingAudioActivityArbiter: AudioActivityArbiter? + var meetingAudioActivityArbiter: AudioActivityArbiter { + if let existing = self._meetingAudioActivityArbiter { + return existing + } + let arbiter = AudioActivityArbiter { [weak self] in + guard let self else { throw MeetingCoordinatorError.activityInProgress } + return self.asr + } + self._meetingAudioActivityArbiter = arbiter + return arbiter + } + + private var _meetingSessionCoordinator: MeetingSessionCoordinator? + var meetingSessionCoordinator: MeetingSessionCoordinator { + if let existing = self._meetingSessionCoordinator { + return existing + } + DebugLogger.shared.info("Lazily creating MeetingSessionCoordinator", source: "AppServices") + let coordinator = MeetingSessionCoordinator( + store: MeetingSessionStore.shared, + capture: MeetingCaptureEngine(), + processing: MeetingProcessingPipeline( + asrServiceProvider: { AppServices.shared.asr } + ), + audioArbiter: self.meetingAudioActivityArbiter, + preferredMicrophoneUID: { [weak self] in + self?.microphonePreferenceCoordinator.inputDeviceForCapture()?.uid + } + ) + self._meetingSessionCoordinator = coordinator + self.setupMeetingCoordinatorForwarding() + Task { @MainActor [weak coordinator] in + await coordinator?.restoreRecoverableSessionIfNeeded() + } + return coordinator + } + + /// Read-only meeting check that does not initialize the meeting stack. + var hasActiveMeetingSessionActivity: Bool { + guard let coordinator = self._meetingSessionCoordinator else { return false } + switch coordinator.state { + case .preparing, .recording, .recordingDegraded, .stopping, .processing: + return true + case .idle, .completed, .interrupted, .failed: + return false + } + } + + var hasActiveFileTranscriptionActivity: Bool { + self._asr?.activeExclusiveActivity == .fileTranscription + } + + /// Legacy UI guard; ASRService is the atomic source of truth for acquisition. + var hasActiveMeetingActivity: Bool { + self.hasActiveMeetingSessionActivity || self._asr?.activeExclusiveActivity == .meeting + } + private var cancellables = Set() private init() { @@ -106,6 +174,13 @@ final class AppServices: ObservableObject { .store(in: &self.cancellables) } + private func setupMeetingCoordinatorForwarding() { + guard let coordinator = self._meetingSessionCoordinator else { return } + coordinator.objectWillChange + .sink { [weak self] _ in self?.objectWillChange.send() } + .store(in: &self.cancellables) + } + // MARK: - Safe Initialization /// Safely initialize all services after the UI is ready. @@ -125,6 +200,10 @@ final class AppServices: ObservableObject { } func shutdownForTermination() async { + if let meetingSessionCoordinator = self._meetingSessionCoordinator { + await meetingSessionCoordinator.shutdownForTermination() + self._meetingSessionCoordinator = nil + } if let asr = self._asr { await asr.shutdownForTermination() self._asr = nil diff --git a/Sources/Fluid/Services/MeetingTranscriptionService.swift b/Sources/Fluid/Services/FileTranscriptionService.swift similarity index 93% rename from Sources/Fluid/Services/MeetingTranscriptionService.swift rename to Sources/Fluid/Services/FileTranscriptionService.swift index 905af5fd..fb8cc598 100644 --- a/Sources/Fluid/Services/MeetingTranscriptionService.swift +++ b/Sources/Fluid/Services/FileTranscriptionService.swift @@ -100,13 +100,14 @@ struct TranscriptionResult: Identifiable, Sendable, Codable { /// Service for transcribing complete audio/video files with optional speaker diarization /// NOTE: This service shares the ASR models with ASRService to avoid duplicate memory usage @MainActor -final class MeetingTranscriptionService: ObservableObject { +final class FileTranscriptionService: ObservableObject { @Published var isTranscribing: Bool = false @Published var progress: Double = 0.0 @Published var currentStatus: String = "" @Published var error: String? @Published var result: TranscriptionResult? @Published var fallbackNotice: String? + @Published private(set) var currentFileURL: URL? // MARK: - Supported Formats @@ -139,6 +140,7 @@ final class MeetingTranscriptionService: ObservableObject { } enum TranscriptionError: LocalizedError { + case activityInProgress(String) case modelLoadFailed(String) case audioConversionFailed(String) case transcriptionFailed(String) @@ -146,6 +148,8 @@ final class MeetingTranscriptionService: ObservableObject { var errorDescription: String? { switch self { + case let .activityInProgress(msg): + return msg case let .modelLoadFailed(msg): return "Failed to load ASR models: \(msg)" case let .audioConversionFailed(msg): @@ -167,6 +171,8 @@ final class MeetingTranscriptionService: ObservableObject { private func errorCategory(for error: TranscriptionError) -> String { switch error { + case .activityInProgress: + return "activityInProgress" case .modelLoadFailed: return "modelLoadFailed" case .audioConversionFailed: @@ -199,15 +205,30 @@ final class MeetingTranscriptionService: ObservableObject { /// - Parameters: /// - fileURL: URL to the audio/video file func transcribeFile(_ fileURL: URL) async throws -> TranscriptionResult { + guard !self.isTranscribing else { + throw TranscriptionError.activityInProgress("This file is already being transcribed.") + } + + let activityLease: ASRActivityLease + do { + activityLease = try self.asrService.acquireExclusiveActivity(.fileTranscription) + } catch { + let wrappedError = TranscriptionError.activityInProgress(error.localizedDescription) + self.error = wrappedError.localizedDescription + throw wrappedError + } + self.isTranscribing = true + self.currentFileURL = fileURL error = nil self.fallbackNotice = nil self.progress = 0.0 let startTime = Date() defer { - isTranscribing = false - progress = 0.0 + self.isTranscribing = false + self.progress = 0.0 + self.asrService.releaseExclusiveActivity(activityLease) } do { @@ -242,7 +263,7 @@ final class MeetingTranscriptionService: ObservableObject { } catch { // Fall back to 0 if we can't determine duration duration = 0 - DebugLogger.shared.warning("Could not determine audio duration: \(error.localizedDescription)", source: "MeetingTranscriptionService") + DebugLogger.shared.warning("Could not determine audio duration: \(error.localizedDescription)", source: "FileTranscriptionService") } let isVideoContainer = UTType(filenameExtension: fileExtension) @@ -264,14 +285,14 @@ final class MeetingTranscriptionService: ObservableObject { } DebugLogger.shared.warning( "Speaker labeling unavailable for this file; falling back to standard transcription", - source: "MeetingTranscriptionService" + source: "FileTranscriptionService" ) self.fallbackNotice = "Speaker labeling was unavailable for this file. The transcript was completed without speaker labels." self.progress = 0.3 } else if SettingsStore.shared.fileTranscriptionSpeakerLabelsEnabled, isVideoContainer { DebugLogger.shared.info( "Speaker labeling skipped for video container; using standard transcription", - source: "MeetingTranscriptionService" + source: "FileTranscriptionService" ) } @@ -280,8 +301,8 @@ final class MeetingTranscriptionService: ObservableObject { self.progress = 0.3 DebugLogger.shared.info( - "MeetingTranscriptionService: using native file transcription path for provider=\(provider.name)", - source: "MeetingTranscriptionService" + "FileTranscriptionService: using native file transcription path for provider=\(provider.name)", + source: "FileTranscriptionService" ) let nativeResult = try await provider.transcribeFile(at: fileURL) @@ -314,8 +335,8 @@ final class MeetingTranscriptionService: ObservableObject { if provider.prefersNativeFileTranscription && isVideoContainer { DebugLogger.shared.info( - "MeetingTranscriptionService: using buffered transcription path for video container [provider=\(provider.name), extension=\(fileExtension)]", - source: "MeetingTranscriptionService" + "FileTranscriptionService: using buffered transcription path for video container [provider=\(provider.name), extension=\(fileExtension)]", + source: "FileTranscriptionService" ) } @@ -404,7 +425,7 @@ final class MeetingTranscriptionService: ObservableObject { if allTranscriptions.isEmpty { DebugLogger.shared.warning( "No audio chunks were long enough to transcribe (minimum 1 second required)", - source: "MeetingTranscriptionService" + source: "FileTranscriptionService" ) } @@ -496,9 +517,11 @@ final class MeetingTranscriptionService: ObservableObject { /// Reset the service state func reset() { + guard !self.isTranscribing else { return } self.result = nil self.error = nil self.fallbackNotice = nil + self.currentFileURL = nil self.currentStatus = "" self.progress = 0.0 } @@ -528,7 +551,7 @@ final class MeetingTranscriptionService: ObservableObject { } catch { DebugLogger.shared.warning( "Diarization failed: \(error.localizedDescription)", - source: "MeetingTranscriptionService" + source: "FileTranscriptionService" ) return nil } @@ -536,7 +559,7 @@ final class MeetingTranscriptionService: ObservableObject { guard !turns.isEmpty else { DebugLogger.shared.info( "Diarization found no speaker turns", - source: "MeetingTranscriptionService" + source: "FileTranscriptionService" ) return nil } @@ -547,7 +570,7 @@ final class MeetingTranscriptionService: ObservableObject { } catch { DebugLogger.shared.warning( "Could not open audio for speaker slicing: \(error.localizedDescription)", - source: "MeetingTranscriptionService" + source: "FileTranscriptionService" ) return nil } @@ -569,7 +592,7 @@ final class MeetingTranscriptionService: ObservableObject { // at risk (a complete unlabeled transcript beats a labeled one with holes). DebugLogger.shared.warning( "Speaker labeling aborted at segment \(index + 1)/\(turns.count) (\(String(format: "%.1f", turn.startSeconds))s): \(error.localizedDescription); falling back to standard transcription", - source: "MeetingTranscriptionService" + source: "FileTranscriptionService" ) return nil } @@ -579,7 +602,7 @@ final class MeetingTranscriptionService: ObservableObject { guard let transcribed else { DebugLogger.shared.warning( "Speaker segment \(index + 1)/\(turns.count) produced no text; falling back to standard transcription", - source: "MeetingTranscriptionService" + source: "FileTranscriptionService" ) return nil } @@ -596,7 +619,7 @@ final class MeetingTranscriptionService: ObservableObject { guard !segments.isEmpty else { DebugLogger.shared.warning( "No speaker segments produced text", - source: "MeetingTranscriptionService" + source: "FileTranscriptionService" ) return nil } diff --git a/Sources/Fluid/Services/Meeting/MeetingAudioChunkWriter.swift b/Sources/Fluid/Services/Meeting/MeetingAudioChunkWriter.swift new file mode 100644 index 00000000..f45c2733 --- /dev/null +++ b/Sources/Fluid/Services/Meeting/MeetingAudioChunkWriter.swift @@ -0,0 +1,536 @@ +import AudioToolbox +@preconcurrency import AVFoundation +import CoreMedia +import CryptoKit +import Foundation + +final nonisolated class MeetingAudioChunkWriter: @unchecked Sendable { + typealias EventHandler = @Sendable (MeetingCaptureEvent) -> Void + + private final class SampleBox: @unchecked Sendable { + let sampleBuffer: CMSampleBuffer + + init(_ sampleBuffer: CMSampleBuffer) { + self.sampleBuffer = sampleBuffer + } + } + + private enum Lifecycle: Equatable { + case accepting + case stopping + case stopped + } + + private struct ActiveChunk: @unchecked Sendable { + var sequence: Int + var partialURL: URL + var finalURL: URL + var relativeFinalPath: String + var writer: AVAssetWriter + var input: AVAssetWriterInput + var start: CMTime + var end: CMTime + var discontinuities: [MeetingAudioDiscontinuity] + } + + private enum FinalizationResult: @unchecked Sendable { + case finalized(MeetingAudioChunk) + case failed(MeetingAudioChunk, String) + } + + private let queue: DispatchQueue + private let finalizationQueue: DispatchQueue + private let finalizationGroup = DispatchGroup() + private let finalizationSlots = DispatchSemaphore(value: 2) + private let pendingSlots = DispatchSemaphore(value: 24) + private let stateLock = NSLock() + private let sessionDirectory: URL + private let trackDirectory: URL + private let manifestURL: URL + private let chunkDuration: TimeInterval + private let eventHandler: EventHandler + private let encoder: JSONEncoder + + private var track: MeetingAudioTrack + private var activeChunk: ActiveChunk? + private var lifecycle: Lifecycle = .accepting + private var stopWaiters: [CheckedContinuation] = [] + private var stoppedTrack: MeetingAudioTrack? + private var nextChunkSequence = 0 + private var pendingDroppedSampleCount = 0 + private var dropReportScheduled = false + private var lastHealthEmission = Date.distantPast + private var lastLevelMeasurement = Date.distantPast + + init( + track: MeetingAudioTrack, + sessionDirectory: URL, + chunkDuration: TimeInterval, + eventHandler: @escaping EventHandler + ) throws { + self.track = track + self.sessionDirectory = sessionDirectory + self.trackDirectory = sessionDirectory + .appendingPathComponent("tracks", isDirectory: true) + .appendingPathComponent(track.kind.rawValue, isDirectory: true) + self.manifestURL = self.trackDirectory.appendingPathComponent("track.json") + self.chunkDuration = max(60, chunkDuration) + self.eventHandler = eventHandler + self.queue = DispatchQueue( + label: "com.fluidvoice.meeting.writer.\(track.kind.rawValue)", + qos: .userInitiated + ) + self.finalizationQueue = DispatchQueue( + label: "com.fluidvoice.meeting.writer.\(track.kind.rawValue).finalization", + qos: .utility + ) + + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .iso8601 + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + self.encoder = encoder + + try FileManager.default.createDirectory( + at: self.trackDirectory, + withIntermediateDirectories: true, + attributes: [.posixPermissions: NSNumber(value: Int16(0o700))] + ) + try self.persistTrackManifest() + } + + func enqueue(_ sampleBuffer: CMSampleBuffer) { + self.stateLock.lock() + guard self.lifecycle == .accepting else { + self.stateLock.unlock() + return + } + + guard self.pendingSlots.wait(timeout: .now()) == .success else { + self.pendingDroppedSampleCount += 1 + let shouldScheduleReport = !self.dropReportScheduled + self.dropReportScheduled = true + self.stateLock.unlock() + if shouldScheduleReport { + self.queue.async { [weak self] in + self?.drainDroppedSampleReport() + } + } + return + } + + let sampleBox = SampleBox(sampleBuffer) + self.queue.async { [weak self, sampleBox] in + defer { self?.pendingSlots.signal() } + self?.consume(sampleBox.sampleBuffer) + } + self.stateLock.unlock() + } + + func stop() async -> MeetingAudioTrack { + await withCheckedContinuation { continuation in + self.stateLock.lock() + switch self.lifecycle { + case .accepting: + self.lifecycle = .stopping + self.stopWaiters.append(continuation) + self.queue.async { [self] in + self.scheduleActiveChunkFinalization() + self.finalizationGroup.notify(queue: self.queue) { [self] in + self.completeStop() + } + } + self.stateLock.unlock() + case .stopping: + self.stopWaiters.append(continuation) + self.stateLock.unlock() + case .stopped: + let stoppedTrack = self.stoppedTrack ?? self.track + self.stateLock.unlock() + continuation.resume(returning: stoppedTrack) + } + } + } + + func snapshot() async -> MeetingAudioTrack { + await withCheckedContinuation { continuation in + self.queue.async { [self] in + continuation.resume(returning: self.track) + } + } + } + + private func consume(_ sampleBuffer: CMSampleBuffer) { + guard CMSampleBufferIsValid(sampleBuffer), CMSampleBufferDataIsReady(sampleBuffer) else { + self.recordDroppedSample(detail: "Capture delivered an invalid audio sample.") + return + } + let presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + guard presentationTime.isValid, presentationTime.isNumeric else { + self.recordDroppedSample(detail: "Capture delivered audio without a valid presentation timestamp.") + return + } + + if let activeChunk = self.activeChunk { + let elapsed = CMTimeGetSeconds(presentationTime - activeChunk.start) + let backwards = presentationTime < activeChunk.end + let gap = CMTimeGetSeconds(presentationTime - activeChunk.end) + if elapsed >= self.chunkDuration || backwards || gap > 0.5 { + if backwards || gap > 0.5 { + let kind: MeetingInterruptionKind = backwards ? .clockDiscontinuity : .sourceLost + self.activeChunk?.discontinuities.append(MeetingAudioDiscontinuity( + kind: kind, + presentationTime: Self.mediaTime(presentationTime), + gapSeconds: backwards ? nil : gap, + detail: backwards ? "Presentation timestamp moved backwards." : "Unexpected audio gap." + )) + } + self.scheduleActiveChunkFinalization() + } + } + + do { + if self.activeChunk == nil { + try self.beginChunk(with: sampleBuffer, at: presentationTime) + } + guard var activeChunk = self.activeChunk else { return } + guard activeChunk.input.isReadyForMoreMediaData else { + self.recordDroppedSample(detail: "Audio encoder backpressure caused a dropped sample.") + return + } + guard activeChunk.input.append(sampleBuffer) else { + throw MeetingCaptureError.writerFailed( + activeChunk.writer.error?.localizedDescription ?? "The audio encoder rejected a sample." + ) + } + + let duration = Self.sampleDuration(sampleBuffer) + activeChunk.end = presentationTime + duration + self.activeChunk = activeChunk + self.track.health.status = .healthy + self.track.health.lastPresentationTime = Self.mediaTime(presentationTime) + let now = Date() + self.track.health.lastSampleAt = now + if now.timeIntervalSince(self.lastLevelMeasurement) >= 0.1 { + self.lastLevelMeasurement = now + self.track.health.level = Self.normalizedLevel(sampleBuffer) + } + self.track.health.detail = nil + self.emitHealthIfNeeded() + } catch { + self.track.health.status = .degraded + self.track.health.detail = error.localizedDescription + self.eventHandler(.interrupted( + kind: .writerFailure, + trackID: self.track.id, + detail: error.localizedDescription + )) + } + } + + private func beginChunk(with sampleBuffer: CMSampleBuffer, at start: CMTime) throws { + guard let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer), + let asbd = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription)?.pointee, + asbd.mSampleRate > 0, + asbd.mChannelsPerFrame > 0 + else { + throw MeetingCaptureError.unsupportedAudioFormat + } + + let channelCount = min(2, max(1, Int(asbd.mChannelsPerFrame))) + let bitRate = channelCount == 1 ? 64_000 : 96_000 + let sequence = self.nextChunkSequence + self.nextChunkSequence += 1 + let stem = String(format: "%06d", sequence) + let partialURL = self.trackDirectory.appendingPathComponent("\(stem).partial.m4a") + let finalURL = self.trackDirectory.appendingPathComponent("\(stem).m4a") + try? FileManager.default.removeItem(at: partialURL) + + let writer = try AVAssetWriter(outputURL: partialURL, fileType: .m4a) + let outputSettings: [String: Any] = [ + AVFormatIDKey: kAudioFormatMPEG4AAC, + AVSampleRateKey: asbd.mSampleRate, + AVNumberOfChannelsKey: channelCount, + AVEncoderBitRateKey: bitRate, + ] + let input = AVAssetWriterInput( + mediaType: .audio, + outputSettings: outputSettings, + sourceFormatHint: formatDescription + ) + input.expectsMediaDataInRealTime = true + guard writer.canAdd(input) else { throw MeetingCaptureError.unsupportedAudioFormat } + writer.add(input) + guard writer.startWriting() else { + throw MeetingCaptureError.writerFailed( + writer.error?.localizedDescription ?? "The audio writer could not start." + ) + } + writer.startSession(atSourceTime: start) + + self.track.format = MeetingAudioFormat( + codec: "aac-lc", + sampleRate: asbd.mSampleRate, + channelCount: channelCount, + bitRate: bitRate + ) + let relativePath = finalURL.path.replacingOccurrences( + of: self.sessionDirectory.path + "/", + with: "" + ) + self.activeChunk = ActiveChunk( + sequence: sequence, + partialURL: partialURL, + finalURL: finalURL, + relativeFinalPath: relativePath, + writer: writer, + input: input, + start: start, + end: start, + discontinuities: [] + ) + } + + private func scheduleActiveChunkFinalization() { + guard let activeChunk = self.activeChunk else { return } + self.activeChunk = nil + activeChunk.input.markAsFinished() + + guard self.finalizationSlots.wait(timeout: .now()) == .success else { + activeChunk.writer.cancelWriting() + self.applyFinalization(.failed( + Self.failedChunk(from: activeChunk), + "Audio chunk finalization was saturated; the chunk was closed without blocking capture." + )) + return + } + + self.finalizationGroup.enter() + self.finalizationQueue.async { [self, activeChunk] in + let result = Self.finalize(activeChunk) + self.queue.async { [self] in + self.applyFinalization(result) + self.finalizationSlots.signal() + self.finalizationGroup.leave() + } + } + } + + private func applyFinalization(_ result: FinalizationResult) { + switch result { + case let .finalized(chunk): + self.track.chunks.append(chunk) + try? self.persistTrackManifest() + if let format = self.track.format { + self.eventHandler(.chunkFinalized(trackID: self.track.id, chunk: chunk, format: format)) + } + case let .failed(chunk, detail): + self.track.chunks.append(chunk) + self.track.health.status = .degraded + self.track.health.detail = detail + try? self.persistTrackManifest() + self.eventHandler(.interrupted(kind: .writerFailure, trackID: self.track.id, detail: detail)) + } + } + + private func completeStop() { + self.track.chunks.sort { $0.sequence < $1.sequence } + self.track.health.status = self.track.chunks.contains(where: { $0.finalizationState == .finalized }) + ? .stopped + : .unavailable + if self.track.health.status == .stopped { + self.track.health.detail = nil + } + try? self.persistTrackManifest() + + self.stateLock.lock() + self.lifecycle = .stopped + self.stoppedTrack = self.track + let waiters = self.stopWaiters + self.stopWaiters = [] + self.stateLock.unlock() + waiters.forEach { $0.resume(returning: self.track) } + } + + private nonisolated static func finalize(_ activeChunk: ActiveChunk) -> FinalizationResult { + let completion = DispatchSemaphore(value: 0) + activeChunk.writer.finishWriting { completion.signal() } + guard completion.wait(timeout: .now() + 4) == .success else { + activeChunk.writer.cancelWriting() + return .failed( + self.failedChunk(from: activeChunk), + "Timed out while finalizing the audio chunk." + ) + } + guard activeChunk.writer.status == .completed else { + return .failed( + self.failedChunk(from: activeChunk), + activeChunk.writer.error?.localizedDescription ?? "The audio chunk could not be finalized." + ) + } + + do { + try? FileManager.default.removeItem(at: activeChunk.finalURL) + try FileManager.default.moveItem(at: activeChunk.partialURL, to: activeChunk.finalURL) + try FileManager.default.setAttributes( + [.posixPermissions: NSNumber(value: Int16(0o600))], + ofItemAtPath: activeChunk.finalURL.path + ) + let values = try activeChunk.finalURL.resourceValues(forKeys: [.fileSizeKey]) + return try .finalized(MeetingAudioChunk( + id: UUID(), + sequence: activeChunk.sequence, + relativeFilePath: activeChunk.relativeFinalPath, + presentationStart: self.mediaTime(activeChunk.start), + presentationEnd: self.mediaTime(activeChunk.end), + discontinuities: activeChunk.discontinuities, + sha256: self.sha256(of: activeChunk.finalURL), + byteCount: Int64(values.fileSize ?? 0), + finalizationState: .finalized + )) + } catch { + return .failed(self.failedChunk(from: activeChunk), error.localizedDescription) + } + } + + private nonisolated static func failedChunk(from activeChunk: ActiveChunk) -> MeetingAudioChunk { + MeetingAudioChunk( + id: UUID(), + sequence: activeChunk.sequence, + relativeFilePath: activeChunk.relativeFinalPath, + presentationStart: self.mediaTime(activeChunk.start), + presentationEnd: self.mediaTime(activeChunk.end), + discontinuities: activeChunk.discontinuities, + sha256: "", + byteCount: 0, + finalizationState: .failed + ) + } + + private func recordDroppedSample(detail: String) { + self.track.health.status = .degraded + self.track.health.droppedSampleCount += 1 + self.track.health.detail = detail + self.eventHandler(.interrupted(kind: .writerBackpressure, trackID: self.track.id, detail: detail)) + self.emitHealthIfNeeded(force: true) + } + + private func drainDroppedSampleReport() { + self.stateLock.lock() + let droppedCount = self.pendingDroppedSampleCount + self.pendingDroppedSampleCount = 0 + self.dropReportScheduled = false + self.stateLock.unlock() + guard droppedCount > 0 else { return } + self.track.health.status = .degraded + self.track.health.droppedSampleCount += droppedCount + let detail = "Audio writer queue dropped \(droppedCount) samples while saturated." + self.track.health.detail = detail + self.eventHandler(.interrupted(kind: .writerBackpressure, trackID: self.track.id, detail: detail)) + self.emitHealthIfNeeded(force: true) + } + + private func emitHealthIfNeeded(force: Bool = false) { + let now = Date() + guard force || now.timeIntervalSince(self.lastHealthEmission) >= 0.25 else { return } + self.lastHealthEmission = now + self.eventHandler(.trackHealth(trackID: self.track.id, health: self.track.health)) + } + + private func persistTrackManifest() throws { + let data = try self.encoder.encode(self.track) + try data.write(to: self.manifestURL, options: .atomic) + try FileManager.default.setAttributes( + [.posixPermissions: NSNumber(value: Int16(0o600))], + ofItemAtPath: self.manifestURL.path + ) + } + + private static func mediaTime(_ time: CMTime) -> MeetingMediaTime { + MeetingMediaTime(value: time.value, timescale: time.timescale) + } + + private static func sampleDuration(_ sampleBuffer: CMSampleBuffer) -> CMTime { + let duration = CMSampleBufferGetDuration(sampleBuffer) + if duration.isValid, duration.isNumeric, duration > .zero { + return duration + } + guard let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer), + let asbd = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription)?.pointee, + asbd.mSampleRate > 0 + else { return CMTime(value: 1, timescale: 48_000) } + return CMTime( + value: CMTimeValue(CMSampleBufferGetNumSamples(sampleBuffer)), + timescale: CMTimeScale(asbd.mSampleRate.rounded()) + ) + } + + private static func sha256(of url: URL) throws -> String { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + var hasher = SHA256() + while true { + let data = try handle.read(upToCount: 1_048_576) ?? Data() + if data.isEmpty { break } + hasher.update(data: data) + } + return hasher.finalize().map { String(format: "%02x", $0) }.joined() + } + + private static func normalizedLevel(_ sampleBuffer: CMSampleBuffer) -> Float { + guard let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer), + let asbd = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription)?.pointee, + asbd.mFormatID == kAudioFormatLinearPCM, + asbd.mBitsPerChannel == 32, + asbd.mFormatFlags & kAudioFormatFlagIsFloat != 0 + else { return 0 } + + var requiredSize = 0 + let sizeStatus = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer( + sampleBuffer, + bufferListSizeNeededOut: &requiredSize, + bufferListOut: nil, + bufferListSize: 0, + blockBufferAllocator: nil, + blockBufferMemoryAllocator: nil, + flags: 0, + blockBufferOut: nil + ) + guard sizeStatus == noErr, requiredSize > 0 else { return 0 } + + let rawBuffer = UnsafeMutableRawPointer.allocate( + byteCount: requiredSize, + alignment: MemoryLayout.alignment + ) + defer { rawBuffer.deallocate() } + let audioBufferList = rawBuffer.bindMemory(to: AudioBufferList.self, capacity: 1) + var retainedBlockBuffer: CMBlockBuffer? + let status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer( + sampleBuffer, + bufferListSizeNeededOut: nil, + bufferListOut: audioBufferList, + bufferListSize: requiredSize, + blockBufferAllocator: nil, + blockBufferMemoryAllocator: nil, + flags: UInt32(kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment), + blockBufferOut: &retainedBlockBuffer + ) + guard status == noErr else { return 0 } + + var peak: Float = 0 + for buffer in UnsafeMutableAudioBufferListPointer(audioBufferList) { + guard let data = buffer.mData else { continue } + let sampleCount = Int(buffer.mDataByteSize) / MemoryLayout.size + guard sampleCount > 0 else { continue } + let samples = data.assumingMemoryBound(to: Float.self) + let stride = max(1, sampleCount / 512) + var index = 0 + while index < sampleCount { + peak = max(peak, abs(samples[index])) + index += stride + } + } + guard peak > 0 else { return 0 } + let decibels = 20 * log10f(min(1, peak)) + return max(0, min(1, (decibels + 60) / 60)) + } +} diff --git a/Sources/Fluid/Services/Meeting/MeetingCaptureEngine.swift b/Sources/Fluid/Services/Meeting/MeetingCaptureEngine.swift new file mode 100644 index 00000000..d6acf263 --- /dev/null +++ b/Sources/Fluid/Services/Meeting/MeetingCaptureEngine.swift @@ -0,0 +1,655 @@ +import AppKit +@preconcurrency import AVFoundation +import CoreMedia +import Foundation +import ScreenCaptureKit + +nonisolated protocol MeetingCaptureControlling: Sendable { + func start( + session: MeetingSession, + configuration: MeetingCaptureConfiguration, + sessionDirectory: URL, + eventHandler: @escaping @Sendable (MeetingCaptureEvent) -> Void + ) async throws -> MeetingCaptureStartResult + + func stop(sessionID: MeetingSessionID) async throws -> MeetingCaptureStopResult + func shutdownForTermination() async +} + +actor MeetingCaptureEngine: MeetingCaptureControlling { + private struct ActiveCapture: @unchecked Sendable { + var sessionID: MeetingSessionID + var runtime: any MeetingCaptureRuntime + var writers: [MeetingAudioChunkWriter] + } + + private var activeCapture: ActiveCapture? + private var startingSessionID: MeetingSessionID? + private var stopTask: Task? + + func start( + session: MeetingSession, + configuration: MeetingCaptureConfiguration, + sessionDirectory: URL, + eventHandler: @escaping @Sendable (MeetingCaptureEvent) -> Void + ) async throws -> MeetingCaptureStartResult { + try configuration.validate() + guard self.activeCapture == nil, self.startingSessionID == nil, self.stopTask == nil else { + throw MeetingCaptureError.captureAlreadyActive + } + self.startingSessionID = session.id + defer { self.startingSessionID = nil } + try Self.preflightStorage(at: sessionDirectory) + try await Self.ensureMicrophonePermission() + + let tracks = Self.makeTracks(session: session, configuration: configuration) + let writers = try tracks.map { track in + try MeetingAudioChunkWriter( + track: track, + sessionDirectory: sessionDirectory, + chunkDuration: configuration.chunkDuration, + eventHandler: eventHandler + ) + } + let writersByKind = Dictionary(uniqueKeysWithValues: zip(tracks.map(\.kind), writers)) + + let runtime: any MeetingCaptureRuntime + switch configuration.mode { + case .onlineCall: + guard let application = configuration.application, + let applicationWriter = writersByKind[.applicationAudio], + let microphoneWriter = writersByKind[.microphone] + else { throw MeetingCaptureError.applicationNotSelected } + runtime = try await ScreenCaptureMeetingRuntime.make( + application: application, + microphone: configuration.microphone, + applicationWriter: applicationWriter, + microphoneWriter: microphoneWriter, + eventHandler: eventHandler + ) + case .inRoom: + guard let microphoneWriter = writersByKind[.microphone] else { + throw MeetingCaptureError.microphoneUnavailable + } + runtime = try InRoomMicrophoneCaptureRuntime( + microphone: configuration.microphone, + writer: microphoneWriter, + eventHandler: eventHandler + ) + } + + do { + try await runtime.start() + } catch { + try? await runtime.stop() + _ = await Self.stopWriters(writers) + throw error + } + + self.activeCapture = ActiveCapture( + sessionID: session.id, + runtime: runtime, + writers: writers + ) + return MeetingCaptureStartResult(tracks: tracks, firstPresentationTime: nil) + } + + func stop(sessionID: MeetingSessionID) async throws -> MeetingCaptureStopResult { + if let stopTask = self.stopTask { + return try await stopTask.value + } + guard let activeCapture = self.activeCapture else { + throw MeetingCaptureError.noActiveCapture + } + guard activeCapture.sessionID == sessionID else { + throw MeetingCaptureError.wrongActiveSession + } + + let task = Task { + let runtimeFailure: Error? + do { + try await activeCapture.runtime.stop() + runtimeFailure = nil + } catch { + runtimeFailure = error + } + let tracks = await Self.stopWriters(activeCapture.writers) + let result = MeetingCaptureStopResult(tracks: tracks, stoppedAt: Date()) + if let runtimeFailure { + throw MeetingCaptureError.captureStopFailed( + runtimeFailure.localizedDescription, + partialResult: result + ) + } + return result + } + self.stopTask = task + do { + let result = try await task.value + self.activeCapture = nil + self.stopTask = nil + return result + } catch { + self.activeCapture = nil + self.stopTask = nil + throw error + } + } + + func shutdownForTermination() async { + guard let sessionID = self.activeCapture?.sessionID else { return } + _ = try? await self.stop(sessionID: sessionID) + } + + private nonisolated static func stopWriters(_ writers: [MeetingAudioChunkWriter]) async -> [MeetingAudioTrack] { + await withTaskGroup(of: MeetingAudioTrack.self, returning: [MeetingAudioTrack].self) { group in + for writer in writers { + group.addTask { await writer.stop() } + } + var tracks: [MeetingAudioTrack] = [] + for await track in group { + tracks.append(track) + } + return tracks.sorted { $0.kind.rawValue < $1.kind.rawValue } + } + } + + private nonisolated static func makeTracks( + session: MeetingSession, + configuration: MeetingCaptureConfiguration + ) -> [MeetingAudioTrack] { + var tracks: [MeetingAudioTrack] = [] + if let application = configuration.application { + tracks.append(MeetingAudioTrack( + id: UUID(), + kind: .applicationAudio, + sourceIdentifier: application.bundleIdentifier, + sourceDisplayName: application.displayName, + format: nil, + timebase: session.timebase, + health: .waiting, + chunks: [] + )) + } + tracks.append(MeetingAudioTrack( + id: UUID(), + kind: .microphone, + sourceIdentifier: configuration.microphone.captureDeviceID, + sourceDisplayName: configuration.microphone.displayName, + format: nil, + timebase: session.timebase, + health: .waiting, + chunks: [] + )) + return tracks + } + + private nonisolated static func ensureMicrophonePermission() async throws { + switch AVCaptureDevice.authorizationStatus(for: .audio) { + case .authorized: + return + case .notDetermined: + let granted = await withCheckedContinuation { continuation in + AVCaptureDevice.requestAccess(for: .audio) { continuation.resume(returning: $0) } + } + guard granted else { throw MeetingCaptureError.microphonePermissionDenied } + case .denied, .restricted: + throw MeetingCaptureError.microphonePermissionDenied + @unknown default: + throw MeetingCaptureError.microphonePermissionDenied + } + } + + private nonisolated static func preflightStorage(at directory: URL) throws { + let values = try directory.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey]) + if let capacity = values.volumeAvailableCapacityForImportantUsage, capacity < 512 * 1024 * 1024 { + throw MeetingCaptureError.insufficientDiskSpace + } + } +} + +private nonisolated protocol MeetingCaptureRuntime: AnyObject, Sendable { + func start() async throws + func stop() async throws +} + +private final nonisolated class ScreenCaptureMeetingRuntime: NSObject, MeetingCaptureRuntime, SCStreamOutput, SCStreamDelegate, + @unchecked Sendable +{ + private let stream: SCStream + private let applicationWriter: MeetingAudioChunkWriter + private let microphoneWriter: MeetingAudioChunkWriter + private let eventHandler: @Sendable (MeetingCaptureEvent) -> Void + private let stateLock = NSLock() + private var delegateProxy: ScreenCaptureRuntimeDelegateProxy? + private var stopping = false + private var unexpectedStopReported = false + + private init( + stream: SCStream, + applicationWriter: MeetingAudioChunkWriter, + microphoneWriter: MeetingAudioChunkWriter, + eventHandler: @escaping @Sendable (MeetingCaptureEvent) -> Void + ) { + self.stream = stream + self.applicationWriter = applicationWriter + self.microphoneWriter = microphoneWriter + self.eventHandler = eventHandler + super.init() + } + + static func make( + application: MeetingApplicationIdentity, + microphone: MeetingMicrophoneIdentity, + applicationWriter: MeetingAudioChunkWriter, + microphoneWriter: MeetingAudioChunkWriter, + eventHandler: @escaping @Sendable (MeetingCaptureEvent) -> Void + ) async throws -> ScreenCaptureMeetingRuntime { + let content: SCShareableContent + do { + content = try await SCShareableContent.current + } catch { + throw MeetingCaptureError.screenCapturePermissionDenied(error.localizedDescription) + } + guard let runningApplication = content.applications.first(where: { candidate in + if let processID = application.processID { + return candidate.processID == processID + } + return candidate.bundleIdentifier == application.bundleIdentifier + }) else { + throw MeetingCaptureError.applicationUnavailable(application.displayName) + } + let display = application.displayID.flatMap { displayID in + content.displays.first(where: { $0.displayID == displayID }) + } ?? content.displays.first + guard let display else { throw MeetingCaptureError.noCaptureDisplay } + + let filter = SCContentFilter( + display: display, + including: [runningApplication], + exceptingWindows: [] + ) + let configuration = SCStreamConfiguration() + configuration.width = 2 + configuration.height = 2 + configuration.minimumFrameInterval = CMTime(value: 1, timescale: 1) + configuration.queueDepth = 3 + configuration.showsCursor = false + configuration.capturesAudio = true + configuration.sampleRate = 48_000 + configuration.channelCount = 2 + configuration.excludesCurrentProcessAudio = true + configuration.captureMicrophone = true + configuration.microphoneCaptureDeviceID = microphone.captureDeviceID + + let placeholder = ScreenCaptureRuntimeDelegateProxy() + let stream = SCStream(filter: filter, configuration: configuration, delegate: placeholder) + let runtime = ScreenCaptureMeetingRuntime( + stream: stream, + applicationWriter: applicationWriter, + microphoneWriter: microphoneWriter, + eventHandler: eventHandler + ) + placeholder.owner = runtime + runtime.delegateProxy = placeholder + try stream.addStreamOutput( + runtime, + type: .audio, + sampleHandlerQueue: DispatchQueue( + label: "com.fluidvoice.meeting.screencapture.application", + qos: .userInteractive + ) + ) + try stream.addStreamOutput( + runtime, + type: .microphone, + sampleHandlerQueue: DispatchQueue( + label: "com.fluidvoice.meeting.screencapture.microphone", + qos: .userInteractive + ) + ) + return runtime + } + + func start() async throws { + do { + try await self.stream.startCapture() + } catch { + throw MeetingCaptureError.captureStartFailed(error.localizedDescription) + } + } + + func stop() async throws { + self.markStopping() + let result = await withCheckedContinuation { continuation in + let completion = MeetingOneShotCompletion(continuation) + self.stream.stopCapture { error in + if let error { + completion.resume(.failure(error.localizedDescription)) + } else { + completion.resume(.success) + } + } + DispatchQueue.global(qos: .userInitiated).asyncAfter(deadline: .now() + 3) { + completion.resume(.failure("Timed out while stopping system-audio capture.")) + } + } + if case let .failure(detail) = result { + throw MeetingCaptureRuntimeError.stopFailed(detail) + } + } + + private func markStopping() { + self.stateLock.lock() + self.stopping = true + self.stateLock.unlock() + } + + func stream( + _ stream: SCStream, + didOutputSampleBuffer sampleBuffer: CMSampleBuffer, + of outputType: SCStreamOutputType + ) { + self.stateLock.lock() + let shouldAccept = !self.stopping + self.stateLock.unlock() + guard shouldAccept else { return } + switch outputType { + case .audio: + self.applicationWriter.enqueue(sampleBuffer) + case .microphone: + self.microphoneWriter.enqueue(sampleBuffer) + case .screen: + break + @unknown default: + break + } + } + + func handleStreamStop(error: Error) { + self.stateLock.lock() + let wasStopping = self.stopping + let shouldReport = !wasStopping && !self.unexpectedStopReported + if shouldReport { + self.unexpectedStopReported = true + self.stopping = true + } + self.stateLock.unlock() + guard shouldReport else { return } + self.eventHandler(.interrupted( + kind: .captureStoppedUnexpectedly, + trackID: nil, + detail: error.localizedDescription + )) + } +} + +private final nonisolated class ScreenCaptureRuntimeDelegateProxy: NSObject, SCStreamDelegate, @unchecked Sendable { + weak var owner: ScreenCaptureMeetingRuntime? + + func stream(_ stream: SCStream, didStopWithError error: Error) { + self.owner?.handleStreamStop(error: error) + } +} + +private final nonisolated class InRoomMicrophoneCaptureRuntime: NSObject, MeetingCaptureRuntime, + AVCaptureAudioDataOutputSampleBufferDelegate, @unchecked Sendable +{ + private let session = AVCaptureSession() + private let output = AVCaptureAudioDataOutput() + private let writer: MeetingAudioChunkWriter + private let controlQueue = DispatchQueue(label: "com.fluidvoice.meeting.inroom.control") + private let stateLock = NSLock() + private let eventHandler: @Sendable (MeetingCaptureEvent) -> Void + private var stopping = false + private var emittedUnexpectedStop = false + private var notificationObservers: [NSObjectProtocol] = [] + + init( + microphone: MeetingMicrophoneIdentity, + writer: MeetingAudioChunkWriter, + eventHandler: @escaping @Sendable (MeetingCaptureEvent) -> Void + ) throws { + self.writer = writer + self.eventHandler = eventHandler + super.init() + guard let device = AVCaptureDevice(uniqueID: microphone.captureDeviceID) else { + throw MeetingCaptureError.microphoneUnavailable + } + let input = try AVCaptureDeviceInput(device: device) + self.session.beginConfiguration() + defer { self.session.commitConfiguration() } + guard self.session.canAddInput(input), self.session.canAddOutput(self.output) else { + throw MeetingCaptureError.microphoneUnavailable + } + self.session.addInput(input) + self.session.addOutput(self.output) + self.output.setSampleBufferDelegate( + self, + queue: DispatchQueue(label: "com.fluidvoice.meeting.inroom.samples", qos: .userInteractive) + ) + self.observeSessionLifecycle() + } + + deinit { + for observer in self.notificationObservers { + NotificationCenter.default.removeObserver(observer) + } + } + + func start() async throws { + let started = await withCheckedContinuation { continuation in + self.controlQueue.async { [session] in + session.startRunning() + continuation.resume(returning: session.isRunning) + } + } + guard started else { throw MeetingCaptureError.captureStartFailed("The microphone did not start.") } + } + + func stop() async throws { + self.stateLock.withLock { self.stopping = true } + let result = await withCheckedContinuation { continuation in + let completion = MeetingOneShotCompletion(continuation) + self.controlQueue.async { [session] in + if session.isRunning { session.stopRunning() } + completion.resume(.success) + } + DispatchQueue.global(qos: .userInitiated).asyncAfter(deadline: .now() + 3) { + completion.resume(.failure("Timed out while stopping microphone capture.")) + } + } + if case let .failure(detail) = result { + throw MeetingCaptureRuntimeError.stopFailed(detail) + } + } + + func captureOutput( + _ output: AVCaptureOutput, + didOutput sampleBuffer: CMSampleBuffer, + from connection: AVCaptureConnection + ) { + self.stateLock.lock() + let shouldAccept = !self.stopping + self.stateLock.unlock() + guard shouldAccept else { return } + self.writer.enqueue(sampleBuffer) + } + + private func observeSessionLifecycle() { + let center = NotificationCenter.default + let names: [Notification.Name] = [ + AVCaptureSession.runtimeErrorNotification, + AVCaptureSession.wasInterruptedNotification, + AVCaptureSession.didStopRunningNotification, + ] + self.notificationObservers = names.map { name in + center.addObserver(forName: name, object: self.session, queue: nil) { [weak self] notification in + self?.handleUnexpectedSessionEvent(notification) + } + } + } + + private func handleUnexpectedSessionEvent(_ notification: Notification) { + let shouldEmit = self.stateLock.withLock { () -> Bool in + guard !self.stopping, !self.emittedUnexpectedStop else { return false } + self.emittedUnexpectedStop = true + return true + } + guard shouldEmit else { return } + let detail = (notification.userInfo?[AVCaptureSessionErrorKey] as? Error)?.localizedDescription + ?? "Microphone capture stopped unexpectedly." + self.eventHandler(.interrupted( + kind: .captureStoppedUnexpectedly, + trackID: nil, + detail: detail + )) + } +} + +private nonisolated enum MeetingOneShotResult: Sendable { + case success + case failure(String) +} + +private final nonisolated class MeetingOneShotCompletion: @unchecked Sendable { + private let lock = NSLock() + private var continuation: CheckedContinuation? + + init(_ continuation: CheckedContinuation) { + self.continuation = continuation + } + + func resume(_ result: MeetingOneShotResult) { + let continuation = self.lock.withLock { () -> CheckedContinuation? in + defer { self.continuation = nil } + return self.continuation + } + continuation?.resume(returning: result) + } +} + +private nonisolated enum MeetingCaptureRuntimeError: LocalizedError { + case stopFailed(String) + + var errorDescription: String? { + switch self { + case let .stopFailed(detail): + return detail + } + } +} + +nonisolated enum MeetingCaptureError: LocalizedError { + case captureAlreadyActive + case noActiveCapture + case wrongActiveSession + case applicationNotSelected + case applicationUnavailable(String) + case noCaptureDisplay + case microphoneUnavailable + case microphonePermissionDenied + case screenCapturePermissionDenied(String) + case insufficientDiskSpace + case captureStartFailed(String) + case captureStopFailed(String, partialResult: MeetingCaptureStopResult) + case unsupportedAudioFormat + case writerFailed(String) + + var errorDescription: String? { + switch self { + case .captureAlreadyActive: + return "Another meeting recording is already active." + case .noActiveCapture: + return "There is no active meeting recording." + case .wrongActiveSession: + return "A different meeting recording is active." + case .applicationNotSelected: + return "Choose the application that contains the online meeting." + case let .applicationUnavailable(name): + return "\(name) is no longer available to record." + case .noCaptureDisplay: + return "No display is available for meeting-audio capture." + case .microphoneUnavailable: + return "The selected microphone is unavailable." + case .microphonePermissionDenied: + return "Microphone permission is required to record this meeting." + case let .screenCapturePermissionDenied(detail): + return "Screen & System Audio permission is required. \(detail)" + case .insufficientDiskSpace: + return "At least 512 MB of free space is required to start recording." + case let .captureStartFailed(detail): + return "Meeting recording could not start. \(detail)" + case let .captureStopFailed(detail, _): + return "Meeting recording could not stop cleanly. \(detail)" + case .unsupportedAudioFormat: + return "The selected audio source uses an unsupported format." + case let .writerFailed(detail): + return "Meeting audio could not be saved. \(detail)" + } + } +} + +enum MeetingCaptureSourceCatalog { + static func availableApplications() async throws -> [MeetingApplicationIdentity] { + let content = try await SCShareableContent.current + let ownBundleIdentifier = Bundle.main.bundleIdentifier + return content.applications + .filter { $0.bundleIdentifier != ownBundleIdentifier } + .map { application in + let applicationWindow = content.windows.first { + $0.owningApplication?.processID == application.processID + } + let displayID = applicationWindow.flatMap { window in + content.displays + .filter { $0.frame.intersects(window.frame) } + .max { lhs, rhs in + lhs.frame.intersection(window.frame).area < rhs.frame.intersection(window.frame).area + }?.displayID + } + return MeetingApplicationIdentity( + bundleIdentifier: application.bundleIdentifier, + processID: application.processID, + displayName: application.applicationName, + displayID: displayID + ) + } + .sorted { $0.displayName.localizedCaseInsensitiveCompare($1.displayName) == .orderedAscending } + } + + static func availableMicrophones() -> [MeetingMicrophoneIdentity] { + let coreAudioDevices = AudioDevice.listInputDevices() + return AVCaptureDevice.DiscoverySession( + deviceTypes: [.microphone, .external], + mediaType: .audio, + position: .unspecified + ).devices.map { device in + let sameName = coreAudioDevices.filter { $0.name == device.localizedName } + return MeetingMicrophoneIdentity( + captureDeviceID: device.uniqueID, + coreAudioUID: sameName.count == 1 ? sameName[0].uid : nil, + displayName: device.localizedName + ) + } + } + + static func defaultMicrophone(preferredCoreAudioUID: String? = nil) throws -> MeetingMicrophoneIdentity { + let microphones = self.availableMicrophones() + if let preferredCoreAudioUID, + let preferred = microphones.first(where: { $0.coreAudioUID == preferredCoreAudioUID }) + { + return preferred + } + guard let defaultDevice = AVCaptureDevice.default(for: .audio), + let microphone = microphones.first(where: { $0.captureDeviceID == defaultDevice.uniqueID }) + else { throw MeetingCaptureError.microphoneUnavailable } + return microphone + } +} + +private extension CGRect { + var area: CGFloat { + guard !self.isNull, !self.isInfinite else { return 0 } + return max(0, self.width) * max(0, self.height) + } +} diff --git a/Sources/Fluid/Services/Meeting/MeetingModels.swift b/Sources/Fluid/Services/Meeting/MeetingModels.swift new file mode 100644 index 00000000..789a36af --- /dev/null +++ b/Sources/Fluid/Services/Meeting/MeetingModels.swift @@ -0,0 +1,662 @@ +import Foundation + +typealias MeetingSessionID = UUID +typealias MeetingAudioTrackID = UUID +typealias MeetingAudioChunkID = UUID +typealias SessionSpeakerID = UUID +typealias MeetingTranscriptSegmentID = UUID + +nonisolated enum MeetingCaptureMode: String, Codable, CaseIterable, Sendable { + case onlineCall + case inRoom +} + +nonisolated enum MeetingSessionState: String, Codable, Sendable { + case preparing + case recording + case recordingDegraded + case stopping + case processing + case completed + case interrupted + case failed +} + +nonisolated enum MeetingAudioTrackKind: String, Codable, CaseIterable, Sendable { + case applicationAudio + case microphone +} + +nonisolated enum MeetingTrackHealthStatus: String, Codable, Sendable { + case waiting + case healthy + case degraded + case unavailable + case stopped +} + +nonisolated enum MeetingInterruptionKind: String, Codable, Sendable { + case sourceLost + case sourceRecovered + case applicationExited + case microphoneDisconnected + case microphoneChanged + case permissionRevoked + case sleep + case wake + case clockDiscontinuity + case lowDiskSpace + case diskExhausted + case appTermination + case captureStoppedUnexpectedly + case writerBackpressure + case writerFailure +} + +nonisolated enum MeetingFailureDomain: String, Codable, Sendable { + case capture + case processing + case persistence +} + +nonisolated enum MeetingProcessingStage: String, Codable, Sendable { + case pending + case saving + case identifyingSpeakers + case transcribing + case finalizing + case completed +} + +nonisolated enum MeetingTranscriptStatus: String, Codable, Sendable { + case provisional + case final +} + +nonisolated enum MeetingTranscriptCompleteness: String, Codable, Sendable { + case complete + case incompleteTrack + case noSpeech + case processingFailed +} + +nonisolated enum MeetingTranscriptOverlap: String, Codable, Sendable { + case none + case overlapsOtherTrack + case ambiguous +} + +nonisolated enum MeetingMicrophoneRole: String, Codable, CaseIterable, Sendable { + case personal + case shared + case unknown +} + +nonisolated struct MeetingRecordingDefaults: Codable, Equatable, Sendable { + var isConfigured: Bool + var mode: MeetingCaptureMode + var applicationBundleIdentifier: String? + var microphoneCaptureDeviceID: String? + var microphoneCoreAudioUID: String? + var microphoneRole: MeetingMicrophoneRole + + static let unconfigured = Self( + isConfigured: false, + mode: .onlineCall, + applicationBundleIdentifier: nil, + microphoneCaptureDeviceID: nil, + microphoneCoreAudioUID: nil, + microphoneRole: .unknown + ) + + func savedApplication(in identities: [MeetingApplicationIdentity]) -> MeetingApplicationIdentity? { + guard self.isConfigured, let applicationBundleIdentifier else { return nil } + return identities.first(where: { $0.bundleIdentifier == applicationBundleIdentifier }) + } + + func savedMicrophone(in identities: [MeetingMicrophoneIdentity]) -> MeetingMicrophoneIdentity? { + guard self.isConfigured else { return nil } + return self.microphoneCaptureDeviceID.flatMap { captureDeviceID in + identities.first(where: { $0.captureDeviceID == captureDeviceID }) + } ?? self.microphoneCoreAudioUID.flatMap { coreAudioUID in + identities.first(where: { $0.coreAudioUID == coreAudioUID }) + } + } +} + +nonisolated struct MeetingApplicationIdentity: Codable, Equatable, Sendable { + var bundleIdentifier: String + var processID: Int32? + var displayName: String + var displayID: UInt32? + + init( + bundleIdentifier: String, + processID: Int32? = nil, + displayName: String, + displayID: UInt32? = nil + ) { + self.bundleIdentifier = bundleIdentifier + self.processID = processID + self.displayName = displayName + self.displayID = displayID + } +} + +nonisolated struct MeetingMicrophoneIdentity: Codable, Equatable, Sendable { + /// `AVCaptureDevice.uniqueID`, which is the identifier ScreenCaptureKit accepts. + var captureDeviceID: String + /// Kept only for reconciling the existing FluidVoice Core Audio preference. + var coreAudioUID: String? + var displayName: String + var role: MeetingMicrophoneRole + + init( + captureDeviceID: String, + coreAudioUID: String? = nil, + displayName: String, + role: MeetingMicrophoneRole = .unknown + ) { + self.captureDeviceID = captureDeviceID + self.coreAudioUID = coreAudioUID + self.displayName = displayName + self.role = role + } +} + +nonisolated struct MeetingPlatformProfile: Codable, Equatable, Sendable { + var identifier: String + var displayName: String + var version: Int + + init(identifier: String, displayName: String, version: Int = 1) { + self.identifier = identifier + self.displayName = displayName + self.version = version + } +} + +nonisolated struct MeetingCaptureConfiguration: Codable, Equatable, Sendable { + static let defaultChunkDuration: TimeInterval = 60 + + var mode: MeetingCaptureMode + var title: String + var languageCode: String + var platform: MeetingPlatformProfile? + var application: MeetingApplicationIdentity? + var microphone: MeetingMicrophoneIdentity + var chunkDuration: TimeInterval + + init( + mode: MeetingCaptureMode, + title: String, + languageCode: String = "en", + platform: MeetingPlatformProfile? = nil, + application: MeetingApplicationIdentity? = nil, + microphone: MeetingMicrophoneIdentity, + chunkDuration: TimeInterval = Self.defaultChunkDuration + ) { + self.mode = mode + self.title = title + self.languageCode = languageCode + self.platform = platform + self.application = application + self.microphone = microphone + self.chunkDuration = max(60, chunkDuration) + } + + func validate() throws { + guard self.languageCode == "en" else { + throw MeetingModelValidationError.unsupportedLanguage + } + guard !self.microphone.captureDeviceID.isEmpty else { + throw MeetingModelValidationError.missingMicrophone + } + switch (self.mode, self.application) { + case (.onlineCall, .none): + throw MeetingModelValidationError.missingOnlineApplication + case (.inRoom, .some): + throw MeetingModelValidationError.unexpectedInRoomApplication + case (.onlineCall, .some), (.inRoom, .none): + break + } + } +} + +nonisolated struct MeetingMediaTime: Codable, Equatable, Comparable, Sendable { + var value: Int64 + var timescale: Int32 + + var seconds: TimeInterval { + guard self.timescale > 0 else { return 0 } + return TimeInterval(self.value) / TimeInterval(self.timescale) + } + + static func < (lhs: Self, rhs: Self) -> Bool { + lhs.seconds < rhs.seconds + } +} + +nonisolated struct MeetingTimebaseMetadata: Codable, Equatable, Sendable { + var startedHostTime: UInt64 + var machTimebaseNumerator: UInt32 + var machTimebaseDenominator: UInt32 + var firstPresentationTime: MeetingMediaTime? +} + +nonisolated struct MeetingAudioFormat: Codable, Equatable, Sendable { + var codec: String + var sampleRate: Double + var channelCount: Int + var bitRate: Int? +} + +nonisolated struct MeetingAudioDiscontinuity: Codable, Equatable, Sendable { + var kind: MeetingInterruptionKind + var presentationTime: MeetingMediaTime? + var gapSeconds: TimeInterval? + var detail: String? +} + +nonisolated enum MeetingAudioChunkFinalizationState: String, Codable, Sendable { + case writing + case finalized + case failed +} + +nonisolated struct MeetingAudioChunk: Codable, Identifiable, Equatable, Sendable { + var id: MeetingAudioChunkID + var sequence: Int + /// Relative to the session directory so sessions can be moved or exported safely. + var relativeFilePath: String + var presentationStart: MeetingMediaTime + var presentationEnd: MeetingMediaTime + var discontinuities: [MeetingAudioDiscontinuity] + var sha256: String + var byteCount: Int64 + var finalizationState: MeetingAudioChunkFinalizationState + + func fileURL(relativeTo sessionDirectory: URL) -> URL { + sessionDirectory.appendingPathComponent(self.relativeFilePath, isDirectory: false) + } +} + +nonisolated struct MeetingTrackHealth: Codable, Equatable, Sendable { + var status: MeetingTrackHealthStatus + var lastPresentationTime: MeetingMediaTime? + var lastSampleAt: Date? + var level: Float + var droppedSampleCount: Int + var detail: String? + + static let waiting = Self( + status: .waiting, + lastPresentationTime: nil, + lastSampleAt: nil, + level: 0, + droppedSampleCount: 0, + detail: nil + ) +} + +nonisolated struct MeetingAudioTrack: Codable, Identifiable, Equatable, Sendable { + var id: MeetingAudioTrackID + var kind: MeetingAudioTrackKind + var sourceIdentifier: String + var sourceDisplayName: String + var format: MeetingAudioFormat? + var timebase: MeetingTimebaseMetadata + var health: MeetingTrackHealth + var chunks: [MeetingAudioChunk] +} + +nonisolated struct MeetingSessionEvent: Codable, Identifiable, Equatable, Sendable { + var id: UUID + var occurredAt: Date + var kind: MeetingInterruptionKind + var trackID: MeetingAudioTrackID? + var detail: String? +} + +nonisolated struct MeetingSessionFailure: Codable, Identifiable, Equatable, Sendable { + var id: UUID + var occurredAt: Date + var domain: MeetingFailureDomain + var code: String + var message: String + var recoverable: Bool +} + +nonisolated struct MeetingIdentityCandidate: Codable, Identifiable, Equatable, Sendable { + var id: UUID + var displayName: String + var source: String + var rejected: Bool + var confirmedAt: Date? +} + +nonisolated struct MeetingSessionSpeaker: Codable, Identifiable, Equatable, Sendable { + var id: SessionSpeakerID + var displayName: String + var diarizationClusterID: String? + // Optional keeps older session manifests decodable after this additive field. + // swiftlint:disable:next discouraged_optional_collection + var diarizationEmbedding: [Float]? = nil + var diarizationEmbeddingObservationCount: Int? = nil + var diarizationQuality: Float? = nil + var trackKind: MeetingAudioTrackKind + var isLocalUser: Bool + var identityCandidates: [MeetingIdentityCandidate] +} + +nonisolated struct MeetingTranscriptSegment: Codable, Identifiable, Equatable, Sendable { + var id: MeetingTranscriptSegmentID + var start: MeetingMediaTime + var end: MeetingMediaTime + var sourceTrackID: MeetingAudioTrackID + var speakerID: SessionSpeakerID? + var text: String + var revision: Int + var status: MeetingTranscriptStatus + var overlap: MeetingTranscriptOverlap + var completeness: MeetingTranscriptCompleteness +} + +nonisolated struct MeetingProcessingAttempt: Codable, Identifiable, Equatable, Sendable { + var id: UUID + var startedAt: Date + var completedAt: Date? + var stage: MeetingProcessingStage + var pipelineVersion: Int + var asrProvider: String? + var asrModel: String? + var languageCode: String? = nil + var diarizationModel: String? + var lastCompletedTrackID: MeetingAudioTrackID? + var errorCode: String? +} + +nonisolated struct MeetingRetentionState: Codable, Equatable, Sendable { + var audioDeletedAt: Date? + var meetingDeletedAt: Date? + var retainAudioUntil: Date? +} + +nonisolated struct MeetingSession: Codable, Identifiable, Equatable, Sendable { + static let currentSchemaVersion = 1 + + var schemaVersion: Int + var id: MeetingSessionID + var title: String + var languageCode: String + var mode: MeetingCaptureMode + var platform: MeetingPlatformProfile? + var capturedApplication: MeetingApplicationIdentity? + var selectedMicrophone: MeetingMicrophoneIdentity + var startedAt: Date + var endedAt: Date? + var timebase: MeetingTimebaseMetadata + var state: MeetingSessionState + var events: [MeetingSessionEvent] + var failures: [MeetingSessionFailure] + var audioTracks: [MeetingAudioTrack] + var speakers: [MeetingSessionSpeaker] + var transcriptSegments: [MeetingTranscriptSegment] + var retention: MeetingRetentionState + var processingAttempts: [MeetingProcessingAttempt] + var updatedAt: Date + + init( + id: MeetingSessionID = UUID(), + configuration: MeetingCaptureConfiguration, + startedAt: Date = Date(), + timebase: MeetingTimebaseMetadata + ) { + self.schemaVersion = Self.currentSchemaVersion + self.id = id + self.title = configuration.title + self.languageCode = configuration.languageCode + self.mode = configuration.mode + self.platform = configuration.platform + self.capturedApplication = configuration.application + self.selectedMicrophone = configuration.microphone + self.startedAt = startedAt + self.timebase = timebase + self.state = .preparing + self.events = [] + self.failures = [] + self.audioTracks = [] + self.speakers = [] + self.transcriptSegments = [] + self.retention = MeetingRetentionState( + audioDeletedAt: nil, + meetingDeletedAt: nil, + retainAudioUntil: Calendar.current.date(byAdding: .day, value: 7, to: startedAt) + ) + self.processingAttempts = [] + self.updatedAt = startedAt + } + + var duration: TimeInterval { + (self.endedAt ?? Date()).timeIntervalSince(self.startedAt) + } + + mutating func renameSpeaker(id: SessionSpeakerID, to displayName: String) throws { + let trimmed = displayName.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { throw MeetingDomainError.emptySpeakerName } + guard let index = self.speakers.firstIndex(where: { $0.id == id }) else { + throw MeetingDomainError.speakerNotFound + } + self.speakers[index].displayName = trimmed + self.updatedAt = Date() + } + + func validateForPersistence() throws { + guard self.schemaVersion > 0, + self.schemaVersion <= Self.currentSchemaVersion + else { + throw MeetingModelValidationError.unsupportedSchema(self.schemaVersion) + } + guard self.languageCode == "en" else { + throw MeetingModelValidationError.unsupportedLanguage + } + guard !self.selectedMicrophone.captureDeviceID.isEmpty else { + throw MeetingModelValidationError.missingMicrophone + } + if let endedAt = self.endedAt, endedAt < self.startedAt { + throw MeetingModelValidationError.invalidTimeRange + } + guard self.timebase.machTimebaseDenominator > 0 else { + throw MeetingModelValidationError.invalidTimebase + } + switch (self.mode, self.capturedApplication) { + case (.onlineCall, .none): + throw MeetingModelValidationError.missingOnlineApplication + case (.inRoom, .some): + throw MeetingModelValidationError.unexpectedInRoomApplication + case (.onlineCall, .some), (.inRoom, .none): + break + } + + let trackIDs = Set(self.audioTracks.map(\.id)) + guard trackIDs.count == self.audioTracks.count else { + throw MeetingModelValidationError.duplicateTrackID + } + let trackKinds = self.audioTracks.map(\.kind) + guard Set(trackKinds).count == trackKinds.count else { + throw MeetingModelValidationError.duplicateTrackKind + } + if [.recording, .recordingDegraded, .stopping, .processing, .completed, .interrupted] + .contains(self.state) + { + let expectedKinds: Set = self.mode == .onlineCall + ? [.applicationAudio, .microphone] + : [.microphone] + guard Set(trackKinds) == expectedKinds else { + throw MeetingModelValidationError.invalidTrackSet + } + } + + var chunkIDs = Set() + for track in self.audioTracks { + let sequences = track.chunks.map(\.sequence) + guard sequences == Array(0..= chunk.presentationStart else { + throw MeetingModelValidationError.invalidTimeRange + } + let pathComponents = URL(fileURLWithPath: chunk.relativeFilePath).pathComponents + guard !chunk.relativeFilePath.hasPrefix("/"), !pathComponents.contains("..") else { + throw MeetingModelValidationError.invalidRelativePath + } + for discontinuity in chunk.discontinuities { + if let presentationTime = discontinuity.presentationTime { + try Self.validateMediaTime(presentationTime) + } + } + if chunk.finalizationState == .finalized { + guard !chunk.sha256.isEmpty, chunk.byteCount > 0 else { + throw MeetingModelValidationError.incompleteFinalizedChunk + } + } + } + } + if let firstPresentationTime = self.timebase.firstPresentationTime { + try Self.validateMediaTime(firstPresentationTime) + } + + let speakerIDs = Set(self.speakers.map(\.id)) + guard speakerIDs.count == self.speakers.count else { + throw MeetingModelValidationError.duplicateSpeakerID + } + let segmentIDs = Set(self.transcriptSegments.map(\.id)) + guard segmentIDs.count == self.transcriptSegments.count else { + throw MeetingModelValidationError.duplicateSegmentID + } + for segment in self.transcriptSegments { + try Self.validateMediaTime(segment.start) + try Self.validateMediaTime(segment.end) + guard segment.end >= segment.start else { + throw MeetingModelValidationError.invalidTimeRange + } + guard trackIDs.contains(segment.sourceTrackID) else { + throw MeetingModelValidationError.missingSegmentTrack + } + if let speakerID = segment.speakerID, !speakerIDs.contains(speakerID) { + throw MeetingModelValidationError.missingSegmentSpeaker + } + } + for attempt in self.processingAttempts { + if let languageCode = attempt.languageCode, + languageCode != self.languageCode + { + throw MeetingModelValidationError.unsupportedLanguage + } + } + } + + private static func validateMediaTime(_ time: MeetingMediaTime) throws { + guard time.timescale > 0 else { + throw MeetingModelValidationError.invalidMediaTimescale + } + } +} + +nonisolated enum MeetingDomainError: LocalizedError, Equatable { + case emptySpeakerName + case speakerNotFound + + var errorDescription: String? { + switch self { + case .emptySpeakerName: + return "Speaker name cannot be empty." + case .speakerNotFound: + return "The speaker is no longer part of this meeting." + } + } +} + +nonisolated enum MeetingModelValidationError: LocalizedError, Equatable { + case unsupportedSchema(Int) + case unsupportedLanguage + case missingMicrophone + case missingOnlineApplication + case unexpectedInRoomApplication + case duplicateTrackID + case duplicateTrackKind + case invalidTrackSet + case invalidChunkSequence + case duplicateChunkID + case invalidTimeRange + case invalidMediaTimescale + case invalidTimebase + case invalidAudioFormat + case invalidRelativePath + case incompleteFinalizedChunk + case duplicateSpeakerID + case duplicateSegmentID + case missingSegmentTrack + case missingSegmentSpeaker + + var errorDescription: String? { + switch self { + case let .unsupportedSchema(version): + return "Unsupported meeting schema version \(version)." + case .unsupportedLanguage: + return "Meeting transcription currently supports English only." + case .missingMicrophone: + return "A microphone must be selected." + case .missingOnlineApplication: + return "An online meeting application must be selected." + case .unexpectedInRoomApplication: + return "In-room meetings cannot include an application-audio source." + case .duplicateTrackID, .duplicateTrackKind, .invalidTrackSet: + return "The meeting has an invalid audio track manifest." + case .invalidChunkSequence, .duplicateChunkID, .invalidTimeRange, .invalidMediaTimescale, + .invalidTimebase, .invalidAudioFormat, .invalidRelativePath, .incompleteFinalizedChunk: + return "The meeting has an invalid audio chunk manifest." + case .duplicateSpeakerID, .duplicateSegmentID, .missingSegmentTrack, .missingSegmentSpeaker: + return "The meeting transcript contains invalid references." + } + } +} + +nonisolated enum MeetingCaptureEvent: Sendable { + case trackHealth(trackID: MeetingAudioTrackID, health: MeetingTrackHealth) + case chunkFinalized(trackID: MeetingAudioTrackID, chunk: MeetingAudioChunk, format: MeetingAudioFormat) + case interrupted(kind: MeetingInterruptionKind, trackID: MeetingAudioTrackID?, detail: String?) +} + +nonisolated struct MeetingCaptureStartResult: Sendable { + var tracks: [MeetingAudioTrack] + var firstPresentationTime: MeetingMediaTime? +} + +nonisolated struct MeetingCaptureStopResult: Sendable { + var tracks: [MeetingAudioTrack] + var stoppedAt: Date +} + +nonisolated struct MeetingProcessingResult: Sendable { + var speakers: [MeetingSessionSpeaker] + var segments: [MeetingTranscriptSegment] + var attempt: MeetingProcessingAttempt +} diff --git a/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swift b/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swift new file mode 100644 index 00000000..e4d96877 --- /dev/null +++ b/Sources/Fluid/Services/Meeting/MeetingProcessingPipeline.swift @@ -0,0 +1,947 @@ +@preconcurrency import AVFoundation +import CoreMedia +import CryptoKit +import Foundation + +@MainActor +protocol MeetingProcessingControlling: AnyObject { + func process( + session: MeetingSession, + sessionDirectory: URL, + progress: @escaping @MainActor (MeetingProcessingStage) -> Void + ) async throws -> MeetingProcessingResult +} + +actor MeetingProcessingSerializationGate { + static let shared = MeetingProcessingSerializationGate() + + private var isLeased = false + private var waiters: [CheckedContinuation] = [] + + func acquire() async { + if !self.isLeased { + self.isLeased = true + return + } + await withCheckedContinuation { continuation in + self.waiters.append(continuation) + } + } + + func release() { + guard !self.waiters.isEmpty else { + self.isLeased = false + return + } + self.waiters.removeFirst().resume() + } +} + +/// Keeps only one normalized centroid per session speaker. Matching is +/// conservative and one-to-one within each writer chunk to avoid false merges. +nonisolated struct MeetingSpeakerEmbeddingIndex: Sendable { + nonisolated struct Prototype: Equatable, Sendable { + var speakerID: SessionSpeakerID + var embedding: [Float] + var observationCount: Int + var averageQuality: Float + } + + let maximumCosineDistance: Float + let ambiguityMargin: Float + private(set) var prototypes: [SessionSpeakerID: Prototype] = [:] + + init(maximumCosineDistance: Float = 0.35, ambiguityMargin: Float = 0.08) { + self.maximumCosineDistance = maximumCosineDistance + self.ambiguityMargin = ambiguityMargin + } + + func bestMatch( + for embedding: [Float], + excluding excludedSpeakerIDs: Set = [] + ) -> SessionSpeakerID? { + guard let normalized = Self.normalized(embedding) else { return nil } + let candidates = self.prototypes.values.compactMap { prototype -> (SessionSpeakerID, Float)? in + guard !excludedSpeakerIDs.contains(prototype.speakerID), + prototype.embedding.count == normalized.count, + let distance = Self.cosineDistance(normalized, prototype.embedding) + else { return nil } + return (prototype.speakerID, distance) + }.sorted { + if $0.1 == $1.1 { return $0.0.uuidString < $1.0.uuidString } + return $0.1 < $1.1 + } + guard let closest = candidates.first, + closest.1 <= self.maximumCosineDistance + else { return nil } + if candidates.count > 1, candidates[1].1 - closest.1 < self.ambiguityMargin { + return nil + } + return closest.0 + } + + mutating func observe( + speakerID: SessionSpeakerID, + embedding: [Float], + quality: Float + ) { + guard let normalized = Self.normalized(embedding) else { return } + guard let existing = self.prototypes[speakerID], + existing.embedding.count == normalized.count + else { + self.prototypes[speakerID] = Prototype( + speakerID: speakerID, + embedding: normalized, + observationCount: 1, + averageQuality: max(0, min(1, quality)) + ) + return + } + + let nextCount = existing.observationCount + 1 + let retainedCount = min(existing.observationCount, 31) + let divisor = Float(retainedCount + 1) + let combined = zip(existing.embedding, normalized).map { + ($0 * Float(retainedCount) + $1) / divisor + } + guard let updatedEmbedding = Self.normalized(combined) else { return } + let boundedQuality = max(0, min(1, quality)) + let averageQuality = ( + existing.averageQuality * Float(existing.observationCount) + boundedQuality + ) / Float(nextCount) + self.prototypes[speakerID] = Prototype( + speakerID: speakerID, + embedding: updatedEmbedding, + observationCount: nextCount, + averageQuality: averageQuality + ) + } + + func prototype(for speakerID: SessionSpeakerID) -> Prototype? { + self.prototypes[speakerID] + } + + var speakerIDs: Set { + Set(self.prototypes.keys) + } + + static func cosineDistance(_ lhs: [Float], _ rhs: [Float]) -> Float? { + guard lhs.count == rhs.count, !lhs.isEmpty else { return nil } + var dot: Float = 0 + var lhsNorm: Float = 0 + var rhsNorm: Float = 0 + for (left, right) in zip(lhs, rhs) { + guard left.isFinite, right.isFinite else { return nil } + dot += left * right + lhsNorm += left * left + rhsNorm += right * right + } + guard lhsNorm > 0, rhsNorm > 0 else { return nil } + return 1 - max(-1, min(1, dot / sqrt(lhsNorm * rhsNorm))) + } + + // swiftlint:disable:next discouraged_optional_collection + private static func normalized(_ embedding: [Float]) -> [Float]? { + guard let distance = cosineDistance(embedding, embedding), distance.isFinite else { return nil } + let magnitude = sqrt(embedding.reduce(Float.zero) { $0 + $1 * $1 }) + guard magnitude > 0, magnitude.isFinite else { return nil } + return embedding.map { $0 / magnitude } + } +} + +nonisolated enum MeetingLocalSpeakerEvidenceSelector { + static func candidate( + cleanDurationByCluster: [SessionSpeakerID: TimeInterval], + prototypeSpeakerIDs: Set + ) -> SessionSpeakerID? { + let ranked = cleanDurationByCluster + .filter { $0.value >= 1 && prototypeSpeakerIDs.contains($0.key) } + .sorted { + if $0.value == $1.value { return $0.key.uuidString < $1.key.uuidString } + return $0.value > $1.value + } + guard let strongest = ranked.first else { return nil } + guard ranked.count > 1 else { return strongest.key } + let runnerUpDuration = ranked[1].value + guard strongest.value >= max(runnerUpDuration * 1.75, runnerUpDuration + 2) else { + return nil + } + return strongest.key + } +} + +@MainActor +private final class MeetingProviderLanguagePin { + private enum Desired { + case none + case apple(String) + case cohere(SettingsStore.CohereLanguage) + case nemotron(SettingsStore.NemotronLanguage) + } + + private enum Restoration { + case none + case apple(String) + case cohere(SettingsStore.CohereLanguage) + case nemotron(SettingsStore.NemotronLanguage) + } + + let languageCode: String + private let settings: SettingsStore + private let desired: Desired + private let restoration: Restoration + + init?(languageCode: String, model: SettingsStore.SpeechModel, settings: SettingsStore) { + guard let route = VoiceEngineLanguageCatalog.routes( + forLanguageID: languageCode, + availableModels: [model] + ).first else { return nil } + self.languageCode = route.language.id + self.settings = settings + switch route.binding { + case .automatic, .whisper: + self.desired = .none + self.restoration = .none + case let .appleSpeech(localeIdentifier): + self.desired = .apple(localeIdentifier) + self.restoration = .apple(settings.selectedAppleSpeechLocaleIdentifier) + case let .cohere(language): + self.desired = .cohere(language) + self.restoration = .cohere(settings.selectedCohereLanguage) + case let .nemotron(language): + self.desired = .nemotron(language) + self.restoration = .nemotron(settings.selectedNemotronLanguage) + } + self.apply() + } + + func apply() { + switch self.desired { + case .none: break + case let .apple(identifier): self.settings.selectedAppleSpeechLocaleIdentifier = identifier + case let .cohere(language): self.settings.selectedCohereLanguage = language + case let .nemotron(language): self.settings.selectedNemotronLanguage = language + } + } + + func restore() { + switch self.restoration { + case .none: + break + case let .apple(identifier): + self.settings.selectedAppleSpeechLocaleIdentifier = identifier + case let .cohere(language): + self.settings.selectedCohereLanguage = language + case let .nemotron(language): + self.settings.selectedNemotronLanguage = language + } + } +} + +@MainActor +final class MeetingProcessingPipeline: MeetingProcessingControlling { + static let pipelineVersion = 1 + + private enum LabeledPassError: Error { + case emptyTurn + } + + private struct TimelineInterval { + var start: TimeInterval + var end: TimeInterval + } + + private struct StagedMicrophoneTurn { + var chunkID: MeetingAudioChunkID + var index: Int + var clusterID: SessionSpeakerID + var clusterLabel: String + var start: TimeInterval + var end: TimeInterval + var text: String + var overlapsRemote: Bool + } + + private struct Accumulator { + var speakers: [MeetingSessionSpeaker] = [] + var segments: [MeetingTranscriptSegment] = [] + var speakerIDByKey: [String: SessionSpeakerID] = [:] + var embeddingIndexByTrack: [MeetingAudioTrackKind: MeetingSpeakerEmbeddingIndex] = [:] + var nextRemoteSpeaker = 1 + var nextMicrophoneSpeaker = 1 + + mutating func speaker( + key: String, + displayName: @autoclosure () -> String, + trackKind: MeetingAudioTrackKind, + isLocalUser: Bool, + clusterID: String? + ) -> SessionSpeakerID { + if let existing = self.speakerIDByKey[key] { return existing } + let id = MeetingProcessingPipeline.stableUUID("speaker:\(key)") + self.speakerIDByKey[key] = id + self.speakers.append(MeetingSessionSpeaker( + id: id, + displayName: displayName(), + diarizationClusterID: clusterID, + trackKind: trackKind, + isLocalUser: isLocalUser, + identityCandidates: [] + )) + return id + } + + mutating func resolveCluster( + trackKind: MeetingAudioTrackKind, + newClusterKey: String, + profile: SpeakerDiarizationService.SpeakerProfile?, + excluding: Set + ) -> (speakerID: SessionSpeakerID, matchedExisting: Bool) { + var index = self.embeddingIndexByTrack[trackKind] ?? MeetingSpeakerEmbeddingIndex() + let existing = profile.flatMap { + index.bestMatch(for: $0.embedding, excluding: excluding) + } + let speakerID = existing ?? MeetingProcessingPipeline.stableUUID("speaker:\(newClusterKey)") + if let profile { + index.observe( + speakerID: speakerID, + embedding: profile.embedding, + quality: profile.averageQuality + ) + } + self.embeddingIndexByTrack[trackKind] = index + return (speakerID, existing != nil) + } + + mutating func addResolvedSpeaker( + id: SessionSpeakerID, + key: String, + displayName: String, + clusterID: String?, + trackKind: MeetingAudioTrackKind, + isLocalUser: Bool + ) { + let prototype = self.embeddingIndexByTrack[trackKind]?.prototype(for: id) + if let existingIndex = self.speakers.firstIndex(where: { $0.id == id }) { + self.speakers[existingIndex].diarizationEmbedding = prototype?.embedding + self.speakers[existingIndex].diarizationEmbeddingObservationCount = prototype?.observationCount + self.speakers[existingIndex].diarizationQuality = prototype?.averageQuality + return + } + self.speakerIDByKey[key] = id + self.speakers.append(MeetingSessionSpeaker( + id: id, + displayName: displayName, + diarizationClusterID: clusterID, + diarizationEmbedding: prototype?.embedding, + diarizationEmbeddingObservationCount: prototype?.observationCount, + diarizationQuality: prototype?.averageQuality, + trackKind: trackKind, + isLocalUser: isLocalUser, + identityCandidates: [] + )) + } + } + + private struct ProcessingContext { + let origin: TimeInterval + let sessionDirectory: URL + let provider: any TranscriptionProvider + let languageCode: String + let asrService: ASRService + let languagePin: MeetingProviderLanguagePin + } + + private let asrServiceProvider: @MainActor () -> ASRService + private let serializationGate: MeetingProcessingSerializationGate + + init( + asrServiceProvider: @escaping @MainActor () -> ASRService, + serializationGate: MeetingProcessingSerializationGate = .shared + ) { + self.asrServiceProvider = asrServiceProvider + self.serializationGate = serializationGate + } + + func process( + session: MeetingSession, + sessionDirectory: URL, + progress: @escaping @MainActor (MeetingProcessingStage) -> Void + ) async throws -> MeetingProcessingResult { + guard session.languageCode == "en" else { + throw MeetingProcessingError.unsupportedLanguage + } + guard session.audioTracks.contains(where: { !$0.chunks.isEmpty }) else { + throw MeetingProcessingError.noRecoverableAudio + } + + await self.serializationGate.acquire() + do { + let result = try await self.processWithLease( + session: session, + sessionDirectory: sessionDirectory, + progress: progress + ) + await self.serializationGate.release() + return result + } catch { + await self.serializationGate.release() + throw error + } + } + + private func processWithLease( + session: MeetingSession, + sessionDirectory: URL, + progress: @escaping @MainActor (MeetingProcessingStage) -> Void + ) async throws -> MeetingProcessingResult { + let asrService = self.asrServiceProvider() + guard !asrService.isRunning else { throw MeetingProcessingError.dictationActive } + + let selectedModel = SettingsStore.shared.selectedSpeechModel + guard let languagePin = MeetingProviderLanguagePin( + languageCode: session.languageCode, + model: selectedModel, + settings: .shared + ) else { + throw MeetingProcessingError.providerDoesNotSupportLanguage(session.languageCode) + } + defer { languagePin.restore() } + + progress(.saving) + try await asrService.ensureAsrReady() + let provider = asrService.fileTranscriptionProvider + guard provider.isReady else { throw MeetingProcessingError.modelUnavailable } + + var attempt = MeetingProcessingAttempt( + id: session.processingAttempts.last(where: { $0.completedAt == nil })?.id ?? UUID(), + startedAt: Date(), + completedAt: nil, + stage: .identifyingSpeakers, + pipelineVersion: Self.pipelineVersion, + asrProvider: provider.name, + asrModel: selectedModel.rawValue, + languageCode: languagePin.languageCode, + diarizationModel: SpeakerDiarizationService.isSupported ? "FluidAudio-offline-v1" : nil, + lastCompletedTrackID: nil, + errorCode: nil + ) + progress(.identifyingSpeakers) + + let origin = session.audioTracks + .flatMap(\.chunks) + .map(\.presentationStart.seconds) + .min() ?? 0 + var accumulator = Accumulator() + var remoteSpeech: [TimelineInterval] = [] + + let context = ProcessingContext( + origin: origin, + sessionDirectory: sessionDirectory, + provider: provider, + languageCode: languagePin.languageCode, + asrService: asrService, + languagePin: languagePin + ) + if let applicationTrack = session.audioTracks.first(where: { $0.kind == .applicationAudio }) { + try await self.processApplicationTrack( + applicationTrack, + context: context, + accumulator: &accumulator, + remoteSpeech: &remoteSpeech + ) + attempt.lastCompletedTrackID = applicationTrack.id + } + + progress(.transcribing) + if let microphoneTrack = session.audioTracks.first(where: { $0.kind == .microphone }) { + try await self.processMicrophoneTrack( + microphoneTrack, + session: session, + context: context, + remoteSpeech: remoteSpeech, + accumulator: &accumulator + ) + attempt.lastCompletedTrackID = microphoneTrack.id + } + + progress(.finalizing) + accumulator.segments.sort { + if $0.start == $1.start { return $0.sourceTrackID.uuidString < $1.sourceTrackID.uuidString } + return $0.start < $1.start + } + guard !accumulator.segments.isEmpty else { throw MeetingProcessingError.noSpeech } + attempt.stage = .completed + attempt.completedAt = Date() + progress(.completed) + return MeetingProcessingResult( + speakers: accumulator.speakers, + segments: accumulator.segments, + attempt: attempt + ) + } + + private func processApplicationTrack( + _ track: MeetingAudioTrack, + context: ProcessingContext, + accumulator: inout Accumulator, + remoteSpeech: inout [TimelineInterval] + ) async throws { + let diarizer = SpeakerDiarizationService() + for chunk in track.chunks where chunk.finalizationState == .finalized { + let url = chunk.fileURL(relativeTo: context.sessionDirectory) + let chunkOffset = chunk.presentationStart.seconds - context.origin + if SpeakerDiarizationService.isSupported { + do { + let diarization = try await diarizer.diarizeWithProfiles(fileURL: url) + let turns = diarization.turns + if !turns.isEmpty { + remoteSpeech.append(contentsOf: turns.map { turn in + TimelineInterval( + start: chunkOffset + turn.startSeconds, + end: chunkOffset + turn.endSeconds + ) + }) + var transcribedTurns: [(Int, SpeakerDiarizationService.SpeakerTurn, String)] = [] + for (index, turn) in turns.enumerated() { + let text = try await self.transcribeTurn( + turn, + fileURL: url, + provider: context.provider, + languageCode: context.languageCode, + asrService: context.asrService, + languagePin: context.languagePin + ) + guard !text.isEmpty else { throw LabeledPassError.emptyTurn } + transcribedTurns.append((index, turn, text)) + } + var speakerIDByLabel: [String: SessionSpeakerID] = [:] + var usedSpeakerIDs = Set() + for (_, turn, _) in transcribedTurns where speakerIDByLabel[turn.speakerLabel] == nil { + let resolved = accumulator.resolveCluster( + trackKind: .applicationAudio, + newClusterKey: "remote:\(chunk.id.uuidString):\(turn.speakerLabel)", + profile: diarization.profilesByLabel[turn.speakerLabel], + excluding: usedSpeakerIDs + ) + let displayName: String + if resolved.matchedExisting { + displayName = "Speaker" + } else { + displayName = "Speaker \(accumulator.nextRemoteSpeaker)" + accumulator.nextRemoteSpeaker += 1 + } + accumulator.addResolvedSpeaker( + id: resolved.speakerID, + key: "remote-cluster:\(resolved.speakerID.uuidString)", + displayName: displayName, + clusterID: turn.speakerLabel, + trackKind: .applicationAudio, + isLocalUser: false + ) + speakerIDByLabel[turn.speakerLabel] = resolved.speakerID + usedSpeakerIDs.insert(resolved.speakerID) + } + for (index, turn, text) in transcribedTurns { + let absoluteStart = chunkOffset + turn.startSeconds + let absoluteEnd = chunkOffset + turn.endSeconds + guard let speakerID = speakerIDByLabel[turn.speakerLabel] else { continue } + accumulator.segments.append(Self.segment( + key: "remote:\(chunk.id.uuidString):\(index)", + trackID: track.id, + speakerID: speakerID, + start: absoluteStart, + end: absoluteEnd, + text: text, + overlap: .none + )) + } + continue + } + } catch { + DebugLogger.shared.warning( + "Meeting remote diarization failed; preserving an unlabeled track transcript", + source: "MeetingProcessingPipeline" + ) + } + } + if let fallback = try await self.transcribeWholeChunk( + url, + provider: context.provider, + languageCode: context.languageCode, + asrService: context.asrService, + languagePin: context.languagePin + ) { + let duration = try await Task.detached(priority: .userInitiated) { + try Self.audioDuration(url) + }.value + let key = "meeting-audio" + let speakerID = accumulator.speaker( + key: key, + displayName: "Meeting audio", + trackKind: .applicationAudio, + isLocalUser: false, + clusterID: nil + ) + remoteSpeech.append(TimelineInterval(start: chunkOffset, end: chunkOffset + duration)) + accumulator.segments.append(Self.segment( + key: "remote-fallback:\(chunk.id.uuidString)", + trackID: track.id, + speakerID: speakerID, + start: chunkOffset, + end: chunkOffset + duration, + text: fallback, + overlap: .none + )) + } + } + } + + private func processMicrophoneTrack( + _ track: MeetingAudioTrack, + session: MeetingSession, + context: ProcessingContext, + remoteSpeech: [TimelineInterval], + accumulator: inout Accumulator + ) async throws { + let diarizer = SpeakerDiarizationService() + var stagedTurns: [StagedMicrophoneTurn] = [] + var cleanDurationByCluster: [SessionSpeakerID: TimeInterval] = [:] + for chunk in track.chunks where chunk.finalizationState == .finalized { + let url = chunk.fileURL(relativeTo: context.sessionDirectory) + let chunkOffset = chunk.presentationStart.seconds - context.origin + if SpeakerDiarizationService.isSupported { + do { + let diarization = try await diarizer.diarizeWithProfiles(fileURL: url) + let turns = diarization.turns + if !turns.isEmpty { + var transcribedTurns: [(Int, SpeakerDiarizationService.SpeakerTurn, String)] = [] + for (index, turn) in turns.enumerated() { + let text = try await self.transcribeTurn( + turn, + fileURL: url, + provider: context.provider, + languageCode: context.languageCode, + asrService: context.asrService, + languagePin: context.languagePin + ) + guard !text.isEmpty else { throw LabeledPassError.emptyTurn } + transcribedTurns.append((index, turn, text)) + } + var clusterIDByLabel: [String: SessionSpeakerID] = [:] + var usedClusterIDs = Set() + for (_, turn, _) in transcribedTurns where clusterIDByLabel[turn.speakerLabel] == nil { + let resolved = accumulator.resolveCluster( + trackKind: .microphone, + newClusterKey: "microphone:\(chunk.id.uuidString):\(turn.speakerLabel)", + profile: diarization.profilesByLabel[turn.speakerLabel], + excluding: usedClusterIDs + ) + clusterIDByLabel[turn.speakerLabel] = resolved.speakerID + usedClusterIDs.insert(resolved.speakerID) + } + for (index, turn, text) in transcribedTurns { + let start = chunkOffset + turn.startSeconds + let end = chunkOffset + turn.endSeconds + let overlapsRemote = remoteSpeech.contains { + min(end, $0.end) - max(start, $0.start) > 0.15 + } + guard let clusterID = clusterIDByLabel[turn.speakerLabel] else { continue } + stagedTurns.append(StagedMicrophoneTurn( + chunkID: chunk.id, + index: index, + clusterID: clusterID, + clusterLabel: turn.speakerLabel, + start: start, + end: end, + text: text, + overlapsRemote: overlapsRemote + )) + if !overlapsRemote { + cleanDurationByCluster[clusterID, default: 0] += max(0, end - start) + } + } + continue + } + } catch { + DebugLogger.shared.warning( + "Meeting microphone speech detection failed; preserving an unlabeled track transcript", + source: "MeetingProcessingPipeline" + ) + } + } + if let fallback = try await self.transcribeWholeChunk( + url, + provider: context.provider, + languageCode: context.languageCode, + asrService: context.asrService, + languagePin: context.languagePin + ) { + let duration = try await Task.detached(priority: .userInitiated) { + try Self.audioDuration(url) + }.value + let speakerID = accumulator.speaker( + key: "microphone-unknown", + displayName: "Microphone / Unknown", + trackKind: .microphone, + isLocalUser: false, + clusterID: nil + ) + accumulator.segments.append(Self.segment( + key: "microphone-fallback:\(chunk.id.uuidString)", + trackID: track.id, + speakerID: speakerID, + start: chunkOffset, + end: chunkOffset + duration, + text: fallback, + overlap: .ambiguous + )) + } + } + + if session.mode == .inRoom { + for turn in stagedTurns { + let isNewSpeaker = !accumulator.speakers.contains { $0.id == turn.clusterID } + if isNewSpeaker { + let displayName = "Room Speaker \(accumulator.nextMicrophoneSpeaker)" + accumulator.nextMicrophoneSpeaker += 1 + accumulator.addResolvedSpeaker( + id: turn.clusterID, + key: "room-cluster:\(turn.clusterID.uuidString)", + displayName: displayName, + clusterID: turn.clusterLabel, + trackKind: .microphone, + isLocalUser: false + ) + } else { + accumulator.addResolvedSpeaker( + id: turn.clusterID, + key: "room-cluster:\(turn.clusterID.uuidString)", + displayName: "Room Speaker", + clusterID: turn.clusterLabel, + trackKind: .microphone, + isLocalUser: false + ) + } + accumulator.segments.append(Self.segment( + key: "microphone:\(turn.chunkID.uuidString):\(turn.index)", + trackID: track.id, + speakerID: turn.clusterID, + start: turn.start, + end: turn.end, + text: turn.text, + overlap: .none + )) + } + return + } + + let localClusterID = session.selectedMicrophone.role == .personal + ? MeetingLocalSpeakerEvidenceSelector.candidate( + cleanDurationByCluster: cleanDurationByCluster, + prototypeSpeakerIDs: accumulator.embeddingIndexByTrack[.microphone]?.speakerIDs ?? [] + ) + : nil + if let localClusterID, + let evidenceTurn = stagedTurns.first(where: { $0.clusterID == localClusterID }) + { + accumulator.addResolvedSpeaker( + id: localClusterID, + key: "local-user", + displayName: "You", + clusterID: evidenceTurn.clusterLabel, + trackKind: .microphone, + isLocalUser: true + ) + } + let unknownSpeakerID = accumulator.speaker( + key: "microphone-unknown", + displayName: "Microphone / Unknown", + trackKind: .microphone, + isLocalUser: false, + clusterID: nil + ) + for turn in stagedTurns { + let isCleanLocalTurn = turn.clusterID == localClusterID && !turn.overlapsRemote + accumulator.segments.append(Self.segment( + key: "microphone:\(turn.chunkID.uuidString):\(turn.index)", + trackID: track.id, + speakerID: isCleanLocalTurn ? localClusterID : unknownSpeakerID, + start: turn.start, + end: turn.end, + text: turn.text, + overlap: turn.overlapsRemote ? .overlapsOtherTrack : .none + )) + } + } + + private func transcribeTurn( + _ turn: SpeakerDiarizationService.SpeakerTurn, + fileURL: URL, + provider: any TranscriptionProvider, + languageCode: String, + asrService: ASRService, + languagePin: MeetingProviderLanguagePin + ) async throws -> String { + var samples = try await Task.detached(priority: .userInitiated) { + try Self.readSamples( + fileURL: fileURL, + startSeconds: turn.startSeconds, + endSeconds: turn.endSeconds + ) + }.value + let minimumSamples = 17_600 + if samples.count < minimumSamples { + samples.append(contentsOf: repeatElement(0, count: minimumSamples - samples.count)) + } + let result = try await self.transcribeSamples( + samples, + provider: provider, + languageCode: languageCode, + asrService: asrService, + languagePin: languagePin + ) + return result.text.trimmingCharacters(in: .whitespacesAndNewlines) + } + + private func transcribeWholeChunk( + _ fileURL: URL, + provider: any TranscriptionProvider, + languageCode: String, + asrService: ASRService, + languagePin: MeetingProviderLanguagePin + ) async throws -> String? { + let result: ASRTranscriptionResult + if provider.prefersNativeFileTranscription { + languagePin.apply() + result = try await asrService.transcribeMeetingFile(at: fileURL, provider: provider) + } else { + let samples = try await Task.detached(priority: .userInitiated) { + try Self.readSamples(fileURL: fileURL, startSeconds: 0, endSeconds: nil) + }.value + guard samples.count >= 16_000 else { return nil } + result = try await self.transcribeSamples( + samples, + provider: provider, + languageCode: languageCode, + asrService: asrService, + languagePin: languagePin + ) + } + let text = result.text.trimmingCharacters(in: .whitespacesAndNewlines) + return text.isEmpty ? nil : text + } + + private func transcribeSamples( + _ samples: [Float], + provider: any TranscriptionProvider, + languageCode: String, + asrService: ASRService, + languagePin: MeetingProviderLanguagePin + ) async throws -> ASRTranscriptionResult { + languagePin.apply() + return try await asrService.transcribeMeetingSamples( + samples, + provider: provider, + languageCode: languageCode + ) + } + + private nonisolated static func readSamples( + fileURL: URL, + startSeconds: TimeInterval, + endSeconds: TimeInterval? + ) throws -> [Float] { + let audioFile = try AVAudioFile(forReading: fileURL) + let sampleRate = audioFile.processingFormat.sampleRate + guard sampleRate > 0 else { throw MeetingProcessingError.audioUnreadable } + let fileDuration = Double(audioFile.length) / sampleRate + let start = max(0, min(startSeconds, fileDuration)) + let end = max(start, min(endSeconds ?? fileDuration, fileDuration)) + let startFrame = AVAudioFramePosition((start * sampleRate).rounded(.down)) + let frameCount = AVAudioFrameCount(((end - start) * sampleRate).rounded(.up)) + guard frameCount > 0, + let buffer = AVAudioPCMBuffer( + pcmFormat: audioFile.processingFormat, + frameCapacity: frameCount + ) + else { return [] } + audioFile.framePosition = startFrame + try audioFile.read(into: buffer, frameCount: frameCount) + return try AudioBufferConverter.monoSamples(from: buffer, targetSampleRate: 16_000) + } + + private nonisolated static func audioDuration(_ url: URL) throws -> TimeInterval { + let file = try AVAudioFile(forReading: url) + guard file.processingFormat.sampleRate > 0 else { throw MeetingProcessingError.audioUnreadable } + return Double(file.length) / file.processingFormat.sampleRate + } + + private static func segment( + key: String, + trackID: MeetingAudioTrackID, + speakerID: SessionSpeakerID?, + start: TimeInterval, + end: TimeInterval, + text: String, + overlap: MeetingTranscriptOverlap + ) -> MeetingTranscriptSegment { + MeetingTranscriptSegment( + id: self.stableUUID("segment:\(key)"), + start: self.mediaTime(start), + end: self.mediaTime(max(start, end)), + sourceTrackID: trackID, + speakerID: speakerID, + text: text, + revision: 0, + status: .final, + overlap: overlap, + completeness: .complete + ) + } + + private static func mediaTime(_ seconds: TimeInterval) -> MeetingMediaTime { + MeetingMediaTime(value: Int64((max(0, seconds) * 1000).rounded()), timescale: 1000) + } + + private nonisolated static func stableUUID(_ key: String) -> UUID { + var bytes = Array(SHA256.hash(data: Data(key.utf8)).prefix(16)) + bytes[6] = (bytes[6] & 0x0f) | 0x50 + bytes[8] = (bytes[8] & 0x3f) | 0x80 + return UUID(uuid: ( + bytes[0], bytes[1], bytes[2], bytes[3], + bytes[4], bytes[5], bytes[6], bytes[7], + bytes[8], bytes[9], bytes[10], bytes[11], + bytes[12], bytes[13], bytes[14], bytes[15] + )) + } +} + +nonisolated enum MeetingProcessingError: LocalizedError { + case unsupportedLanguage + case noRecoverableAudio + case dictationActive + case modelUnavailable + case providerDoesNotSupportLanguage(String) + case audioUnreadable + case noSpeech + + var errorDescription: String? { + switch self { + case .unsupportedLanguage: + return "Meeting transcription currently supports English only." + case .noRecoverableAudio: + return "No finalized meeting audio is available to transcribe." + case .dictationActive: + return "Finish the active dictation before transcribing this meeting." + case .modelUnavailable: + return "The selected speech model is not ready." + case let .providerDoesNotSupportLanguage(languageCode): + return "The selected speech model cannot transcribe meeting language \(languageCode)." + case .audioUnreadable: + return "A finalized meeting audio chunk could not be read." + case .noSpeech: + return "No speech was detected in the saved meeting audio." + } + } +} diff --git a/Sources/Fluid/Services/Meeting/MeetingSessionCoordinator.swift b/Sources/Fluid/Services/Meeting/MeetingSessionCoordinator.swift new file mode 100644 index 00000000..1637d959 --- /dev/null +++ b/Sources/Fluid/Services/Meeting/MeetingSessionCoordinator.swift @@ -0,0 +1,806 @@ +import Combine +import Darwin +import Foundation + +enum MeetingCoordinatorState: Equatable { + case idle + case preparing(MeetingSessionID) + case recording(MeetingSessionID) + case recordingDegraded(MeetingSessionID) + case stopping(MeetingSessionID) + case processing(MeetingSessionID, MeetingProcessingStage) + case completed(MeetingSessionID) + case interrupted(MeetingSessionID) + case failed(MeetingSessionID?, MeetingSessionFailure) +} + +struct MeetingAudioActivityLease: Equatable, Sendable { + var id: UUID +} + +@MainActor +protocol MeetingAudioActivityArbitrating: AnyObject { + func acquireMeetingCapture() throws -> MeetingAudioActivityLease + func release(_ lease: MeetingAudioActivityLease) +} + +@MainActor +final class AudioActivityArbiter: MeetingAudioActivityArbitrating { + private let asrServiceProvider: @MainActor () throws -> ASRService + private var meetingLease: MeetingAudioActivityLease? + private var sharedActivityLease: ASRActivityLease? + private weak var activeASRService: ASRService? + + init(asrServiceProvider: @escaping @MainActor () throws -> ASRService) { + self.asrServiceProvider = asrServiceProvider + } + + func acquireMeetingCapture() throws -> MeetingAudioActivityLease { + guard self.meetingLease == nil else { throw MeetingCoordinatorError.recordingAlreadyActive } + let asrService = try self.asrServiceProvider() + let sharedLease = try asrService.acquireExclusiveActivity(.meeting) + let lease = MeetingAudioActivityLease(id: UUID()) + self.activeASRService = asrService + self.sharedActivityLease = sharedLease + self.meetingLease = lease + return lease + } + + func release(_ lease: MeetingAudioActivityLease) { + guard self.meetingLease == lease else { return } + if let sharedActivityLease = self.sharedActivityLease { + self.activeASRService?.releaseExclusiveActivity(sharedActivityLease) + } + self.activeASRService = nil + self.sharedActivityLease = nil + self.meetingLease = nil + } +} + +@MainActor +final class MeetingSessionCoordinator: ObservableObject { + @Published private(set) var state: MeetingCoordinatorState = .idle + @Published private(set) var activeSession: MeetingSession? + @Published private(set) var latestCompletedSession: MeetingSession? + @Published private(set) var trackHealth: [MeetingAudioTrackKind: MeetingTrackHealth] = [:] + + private let store: any MeetingSessionStoring + private let capture: any MeetingCaptureControlling + private let processing: any MeetingProcessingControlling + private let audioArbiter: any MeetingAudioActivityArbitrating + private let preferredMicrophoneUID: @MainActor () -> String? + + private var activityLease: MeetingAudioActivityLease? + private var captureGeneration: UUID? + private var operationGeneration: UUID? + private var stopTask: Task? + private var interruptionTask: Task? + private var terminationTask: Task? + private var persistenceTail: Task? + + init( + store: any MeetingSessionStoring, + capture: any MeetingCaptureControlling, + processing: any MeetingProcessingControlling, + audioArbiter: any MeetingAudioActivityArbitrating, + preferredMicrophoneUID: @escaping @MainActor () -> String? = { nil } + ) { + self.store = store + self.capture = capture + self.processing = processing + self.audioArbiter = audioArbiter + self.preferredMicrophoneUID = preferredMicrophoneUID + } + + var currentSession: MeetingSession? { + self.activeSession ?? self.latestCompletedSession + } + + var isRecording: Bool { + switch self.state { + case .recording, .recordingDegraded: + return true + default: + return false + } + } + + var isProcessing: Bool { + if case .processing = self.state { return true } + return false + } + + var elapsedTime: TimeInterval { + guard let session = self.activeSession else { return 0 } + return max(0, (session.endedAt ?? Date()).timeIntervalSince(session.startedAt)) + } + + var sourceDisplayName: String { + self.currentSession?.capturedApplication?.displayName + ?? (self.currentSession?.mode == .inRoom ? "In-room meeting" : "Meeting audio") + } + + var microphoneDisplayName: String { + self.currentSession?.selectedMicrophone.displayName ?? "Microphone" + } + + func defaultConfiguration( + mode: MeetingCaptureMode, + title: String + ) async throws -> MeetingCaptureConfiguration { + let microphone = try MeetingCaptureSourceCatalog.defaultMicrophone( + preferredCoreAudioUID: self.preferredMicrophoneUID() + ) + let application: MeetingApplicationIdentity? + if mode == .onlineCall { + let applications = try await MeetingCaptureSourceCatalog.availableApplications() + let preferredBundleIdentifiers = [ + "us.zoom.xos", + "com.google.Chrome", + "com.microsoft.teams2", + ] + application = preferredBundleIdentifiers.lazy.compactMap { bundleIdentifier in + applications.first(where: { $0.bundleIdentifier == bundleIdentifier }) + }.first ?? applications.first + guard application != nil else { throw MeetingCaptureError.applicationNotSelected } + } else { + application = nil + } + return MeetingCaptureConfiguration( + mode: mode, + title: title, + application: application, + microphone: microphone + ) + } + + @discardableResult + func startRecording(configuration: MeetingCaptureConfiguration) async throws -> MeetingSession { + try configuration.validate() + guard self.activeSession == nil, + self.stopTask == nil, + self.interruptionTask == nil, + self.terminationTask == nil + else { + throw MeetingCoordinatorError.recordingAlreadyActive + } + + let lease = try self.audioArbiter.acquireMeetingCapture() + self.activityLease = lease + let generation = UUID() + self.captureGeneration = generation + self.operationGeneration = generation + let timebase = Self.makeTimebase() + var session = MeetingSession(configuration: configuration, timebase: timebase) + self.activeSession = session + self.state = .preparing(session.id) + var captureStarted = false + var captureStartAttempted = false + + do { + try await self.store.create(session) + let sessionDirectory = try await self.store.sessionDirectory(for: session.id) + captureStartAttempted = true + let startResult = try await self.capture.start( + session: session, + configuration: configuration, + sessionDirectory: sessionDirectory + ) { [weak self] event in + Task { @MainActor [weak self] in + self?.handleCaptureEvent(event, generation: generation) + } + } + captureStarted = true + guard self.operationGeneration == generation, !Task.isCancelled else { + throw CancellationError() + } + session.audioTracks = startResult.tracks + if let firstPresentationTime = startResult.firstPresentationTime { + session.timebase.firstPresentationTime = firstPresentationTime + } + session.state = .recording + session.updatedAt = Date() + self.activeSession = session + self.trackHealth = Dictionary(uniqueKeysWithValues: session.audioTracks.map { ($0.kind, $0.health) }) + self.state = .recording(session.id) + try await self.store.save(session) + return session + } catch { + if self.operationGeneration != generation { + if captureStarted { + _ = try? await self.capture.stop(sessionID: session.id) + } + throw error + } + if captureStarted { + do { + let stopResult = try await self.capture.stop(sessionID: session.id) + Self.apply(stopResult, to: &session) + } catch let stopError { + if let partialResult = Self.partialStopResult(from: stopError) { + Self.apply(partialResult, to: &session) + } else { + await self.capture.shutdownForTermination() + session.endedAt = session.endedAt ?? Date() + } + } + } + let hasRecoverableAudio = Self.hasRecoverableAudio(session) + let failureDomain: MeetingFailureDomain = captureStarted + ? .persistence + : (captureStartAttempted ? .capture : .persistence) + let failure = Self.failure( + from: error, + domain: failureDomain, + recoverable: hasRecoverableAudio + ) + session.state = .failed + session.failures.append(failure) + session.updatedAt = Date() + try? await self.store.save(session) + self.activeSession = hasRecoverableAudio ? session : nil + self.trackHealth = hasRecoverableAudio + ? Dictionary(uniqueKeysWithValues: session.audioTracks.map { ($0.kind, $0.health) }) + : [:] + self.captureGeneration = nil + self.operationGeneration = nil + self.state = .failed(session.id, failure) + self.releaseActivityLease() + throw error + } + } + + @discardableResult + func stopAndTranscribe() async throws -> MeetingSession { + if let stopTask = self.stopTask { return try await stopTask.value } + guard let session = self.activeSession else { throw MeetingCoordinatorError.noActiveMeeting } + guard self.interruptionTask == nil, self.terminationTask == nil else { + throw MeetingCoordinatorError.activityInProgress + } + guard [.preparing, .recording, .recordingDegraded, .stopping].contains(session.state), + let generation = self.operationGeneration + else { throw MeetingCoordinatorError.activityInProgress } + + let task = Task { @MainActor [weak self] () throws -> MeetingSession in + guard let self else { throw MeetingCoordinatorError.noActiveMeeting } + return try await self.performStopAndTranscribe(generation: generation) + } + self.stopTask = task + do { + let session = try await task.value + self.stopTask = nil + return session + } catch { + self.stopTask = nil + throw error + } + } + + @discardableResult + func retryProcessing() async throws -> MeetingSession { + guard var session = self.activeSession, + self.interruptionTask == nil, + self.terminationTask == nil, + session.endedAt != nil, + session.audioTracks.contains(where: { !$0.chunks.isEmpty }) + else { throw MeetingCoordinatorError.noRecoverableAudio } + if self.activityLease == nil { + self.activityLease = try self.audioArbiter.acquireMeetingCapture() + } + let generation = UUID() + self.operationGeneration = generation + session.state = .processing + session.processingAttempts.append(Self.pendingProcessingAttempt()) + session.updatedAt = Date() + self.activeSession = session + try? await self.store.save(session) + return try await self.process(session, generation: generation) + } + + func renameSpeaker(id: SessionSpeakerID, to displayName: String) async throws { + guard var session = self.currentSession else { throw MeetingCoordinatorError.noActiveMeeting } + try session.renameSpeaker(id: id, to: displayName) + try await self.store.save(session) + if self.activeSession?.id == session.id { + self.activeSession = session + } else { + self.latestCompletedSession = session + } + } + + func resetForNewMeeting() throws { + guard !self.isRecording, + !self.isProcessing, + self.stopTask == nil, + self.interruptionTask == nil, + self.terminationTask == nil + else { + throw MeetingCoordinatorError.activityInProgress + } + self.activeSession = nil + self.trackHealth = [:] + self.state = .idle + } + + func restoreRecoverableSessionIfNeeded() async { + guard self.activeSession == nil else { return } + guard var session = try? await self.store.loadRecoverable().first else { return } + let expectedKinds: Set = session.mode == .onlineCall + ? [.applicationAudio, .microphone] + : [.microphone] + guard Set(session.audioTracks.map(\.kind)) == expectedKinds else { + let failure = Self.failure( + from: MeetingCoordinatorError.noRecoverableAudio, + domain: .persistence, + recoverable: false + ) + session.state = .failed + session.failures.append(failure) + session.updatedAt = Date() + try? await self.store.save(session) + self.activeSession = session + self.state = .failed(session.id, failure) + return + } + + if session.state == .failed, + Self.hasRecoverableAudio(session), + let failure = session.failures.last, + failure.recoverable + { + self.activeSession = session + self.trackHealth = Dictionary(uniqueKeysWithValues: session.audioTracks.map { ($0.kind, $0.health) }) + self.state = .failed(session.id, failure) + return + } + + let previousState = session.state + session.state = .interrupted + if session.endedAt == nil { + session.endedAt = session.audioTracks + .compactMap(\.health.lastSampleAt) + .max() ?? session.updatedAt + } + session.events.append(MeetingSessionEvent( + id: UUID(), + occurredAt: Date(), + kind: .captureStoppedUnexpectedly, + trackID: nil, + detail: previousState == .processing + ? "FluidVoice restarted before transcription completed." + : "FluidVoice restarted before capture finalized." + )) + if session.audioTracks.contains(where: { track in + track.chunks.contains { $0.finalizationState == .finalized } + }), !session.processingAttempts.contains(where: { $0.completedAt == nil }) { + session.processingAttempts.append(Self.pendingProcessingAttempt()) + } + session.updatedAt = Date() + try? await self.store.save(session) + self.activeSession = session + self.trackHealth = Dictionary(uniqueKeysWithValues: session.audioTracks.map { ($0.kind, $0.health) }) + self.state = .interrupted(session.id) + } + + func shutdownForTermination() async { + if let terminationTask = self.terminationTask { + await terminationTask.value + return + } + let task = Task { @MainActor [weak self] in + guard let self else { return } + await self.performShutdownForTermination() + } + self.terminationTask = task + await task.value + } + + private func performShutdownForTermination() async { + self.captureGeneration = nil + self.operationGeneration = nil + self.stopTask?.cancel() + self.interruptionTask?.cancel() + + guard var session = self.activeSession else { + await self.capture.shutdownForTermination() + self.releaseActivityLease() + return + } + + let wasCapturing = [.preparing, .recording, .recordingDegraded, .stopping].contains(session.state) + if wasCapturing { + self.state = .stopping(session.id) + do { + let stopResult = try await self.capture.stop(sessionID: session.id) + Self.apply(stopResult, to: &session) + } catch { + if let partialResult = Self.partialStopResult(from: error) { + Self.apply(partialResult, to: &session) + } else { + await self.capture.shutdownForTermination() + session.endedAt = session.endedAt ?? Date() + } + session.failures.append(Self.failure(from: error, domain: .capture, recoverable: true)) + } + } else { + await self.capture.shutdownForTermination() + } + + session.state = .interrupted + if !session.events.contains(where: { $0.kind == .appTermination }) { + session.events.append(MeetingSessionEvent( + id: UUID(), + occurredAt: Date(), + kind: .appTermination, + trackID: nil, + detail: wasCapturing + ? "Capture stopped at app termination; transcription is pending." + : "Transcription paused at app termination and will resume next launch." + )) + } + if session.audioTracks.contains(where: { track in + track.chunks.contains { $0.finalizationState == .finalized } + }), !session.processingAttempts.contains(where: { $0.completedAt == nil }) { + session.processingAttempts.append(Self.pendingProcessingAttempt()) + } + session.updatedAt = Date() + await self.flushQueuedPersistence() + try? await self.store.save(session) + self.activeSession = session + self.state = .interrupted(session.id) + self.releaseActivityLease() + } + + private func performStopAndTranscribe(generation: UUID) async throws -> MeetingSession { + guard var session = self.activeSession else { throw MeetingCoordinatorError.noActiveMeeting } + self.state = .stopping(session.id) + session.state = .stopping + session.updatedAt = Date() + self.activeSession = session + await self.flushQueuedPersistence() + // Capture finalization takes priority over persistence availability. + try? await self.store.save(session) + + let stopResult: MeetingCaptureStopResult + do { + stopResult = try await self.capture.stop(sessionID: session.id) + } catch { + await self.handleStopFailure(error, session: session, generation: generation) + throw error + } + guard self.operationGeneration == generation, !Task.isCancelled else { + throw CancellationError() + } + self.captureGeneration = nil + Self.apply(stopResult, to: &session) + session.state = .processing + session.processingAttempts.append(Self.pendingProcessingAttempt()) + session.updatedAt = Date() + self.activeSession = session + self.trackHealth = Dictionary(uniqueKeysWithValues: session.audioTracks.map { ($0.kind, $0.health) }) + try? await self.store.save(session) + return try await self.process(session, generation: generation) + } + + private func process(_ inputSession: MeetingSession, generation: UUID) async throws -> MeetingSession { + var session = inputSession + let sessionDirectory = try await self.store.sessionDirectory(for: session.id) + do { + let result = try await self.processing.process( + session: session, + sessionDirectory: sessionDirectory + ) { [weak self] stage in + self?.updateProcessingStage(stage, generation: generation) + } + guard self.operationGeneration == generation, !Task.isCancelled else { + throw CancellationError() + } + session.speakers = result.speakers + session.transcriptSegments = result.segments + session.processingAttempts.removeAll { + $0.completedAt == nil && $0.id != result.attempt.id + } + if let index = session.processingAttempts.lastIndex(where: { $0.id == result.attempt.id }) { + session.processingAttempts[index] = result.attempt + } else { + session.processingAttempts.append(result.attempt) + } + session.state = .completed + session.updatedAt = Date() + await self.flushQueuedPersistence() + try await self.store.save(session) + self.activeSession = nil + self.latestCompletedSession = session + self.state = .completed(session.id) + self.operationGeneration = nil + self.releaseActivityLease() + return session + } catch { + guard self.operationGeneration == generation else { + throw error + } + if error is CancellationError { + session.state = .interrupted + session.updatedAt = Date() + await self.flushQueuedPersistence() + try? await self.store.save(session) + self.activeSession = session + self.state = .interrupted(session.id) + self.operationGeneration = nil + self.releaseActivityLease() + throw error + } + let failure = Self.failure(from: error, domain: .processing, recoverable: true) + if let lastIndex = session.processingAttempts.indices.last { + session.processingAttempts[lastIndex].errorCode = failure.code + session.processingAttempts[lastIndex].completedAt = Date() + } + session.failures.append(failure) + session.state = .failed + session.updatedAt = Date() + await self.flushQueuedPersistence() + try? await self.store.save(session) + self.activeSession = session + self.state = .failed(session.id, failure) + self.operationGeneration = nil + self.releaseActivityLease() + throw error + } + } + + private func updateProcessingStage(_ stage: MeetingProcessingStage, generation: UUID) { + guard self.operationGeneration == generation, var session = self.activeSession else { return } + session.state = .processing + if let lastIndex = session.processingAttempts.indices.last { + session.processingAttempts[lastIndex].stage = stage + } + session.updatedAt = Date() + self.activeSession = session + self.state = .processing(session.id, stage) + self.enqueuePersistence(session) + } + + private func handleCaptureEvent(_ event: MeetingCaptureEvent, generation: UUID) { + guard self.captureGeneration == generation, var session = self.activeSession else { return } + switch event { + case let .trackHealth(trackID, health): + guard let index = session.audioTracks.firstIndex(where: { $0.id == trackID }) else { return } + session.audioTracks[index].health = health + self.trackHealth[session.audioTracks[index].kind] = health + if health.status == .degraded { + session.state = .recordingDegraded + self.state = .recordingDegraded(session.id) + } + case let .chunkFinalized(trackID, chunk, format): + guard let index = session.audioTracks.firstIndex(where: { $0.id == trackID }) else { return } + if !session.audioTracks[index].chunks.contains(where: { $0.id == chunk.id }) { + session.audioTracks[index].chunks.append(chunk) + } + session.audioTracks[index].format = format + if let currentFirstPresentationTime = session.timebase.firstPresentationTime { + session.timebase.firstPresentationTime = min(currentFirstPresentationTime, chunk.presentationStart) + } else { + session.timebase.firstPresentationTime = chunk.presentationStart + } + case let .interrupted(kind, trackID, detail): + session.events.append(MeetingSessionEvent( + id: UUID(), + occurredAt: Date(), + kind: kind, + trackID: trackID, + detail: detail + )) + if kind == .captureStoppedUnexpectedly { + session.state = .interrupted + self.state = .interrupted(session.id) + session.updatedAt = Date() + self.activeSession = session + self.captureGeneration = nil + self.operationGeneration = nil + self.enqueuePersistence(session) + self.beginUnexpectedStop(sessionID: session.id) + return + } else if session.state == .recording { + session.state = .recordingDegraded + self.state = .recordingDegraded(session.id) + } + } + session.updatedAt = Date() + self.activeSession = session + self.enqueuePersistence(session) + } + + private func beginUnexpectedStop(sessionID: MeetingSessionID) { + guard self.interruptionTask == nil, self.terminationTask == nil else { return } + self.interruptionTask = Task { @MainActor [weak self] in + await self?.finishUnexpectedStop(sessionID: sessionID) + } + } + + private func finishUnexpectedStop(sessionID: MeetingSessionID) async { + guard var session = self.activeSession, session.id == sessionID else { + self.interruptionTask = nil + return + } + do { + let stopResult = try await self.capture.stop(sessionID: sessionID) + Self.apply(stopResult, to: &session) + } catch { + if let partialResult = Self.partialStopResult(from: error) { + Self.apply(partialResult, to: &session) + } else { + await self.capture.shutdownForTermination() + session.endedAt = session.endedAt ?? Date() + } + session.failures.append(Self.failure(from: error, domain: .capture, recoverable: true)) + } + guard self.terminationTask == nil, !Task.isCancelled else { return } + + session.state = .interrupted + if session.audioTracks.contains(where: { track in + track.chunks.contains { $0.finalizationState == .finalized } + }), !session.processingAttempts.contains(where: { $0.completedAt == nil }) { + session.processingAttempts.append(Self.pendingProcessingAttempt()) + } + session.updatedAt = Date() + await self.flushQueuedPersistence() + try? await self.store.save(session) + self.activeSession = session + self.trackHealth = Dictionary(uniqueKeysWithValues: session.audioTracks.map { ($0.kind, $0.health) }) + self.state = .interrupted(session.id) + self.releaseActivityLease() + self.interruptionTask = nil + } + + private func handleStopFailure( + _ error: Error, + session inputSession: MeetingSession, + generation: UUID + ) async { + guard self.operationGeneration == generation else { return } + var session = inputSession + self.captureGeneration = nil + self.operationGeneration = nil + if let partialResult = Self.partialStopResult(from: error) { + Self.apply(partialResult, to: &session) + } else { + await self.capture.shutdownForTermination() + session.endedAt = session.endedAt ?? Date() + } + let failure = Self.failure(from: error, domain: .capture, recoverable: true) + session.failures.append(failure) + session.events.append(MeetingSessionEvent( + id: UUID(), + occurredAt: Date(), + kind: .captureStoppedUnexpectedly, + trackID: nil, + detail: error.localizedDescription + )) + session.state = .interrupted + if session.audioTracks.contains(where: { track in + track.chunks.contains { $0.finalizationState == .finalized } + }), !session.processingAttempts.contains(where: { $0.completedAt == nil }) { + session.processingAttempts.append(Self.pendingProcessingAttempt()) + } + session.updatedAt = Date() + await self.flushQueuedPersistence() + try? await self.store.save(session) + self.activeSession = session + self.trackHealth = Dictionary(uniqueKeysWithValues: session.audioTracks.map { ($0.kind, $0.health) }) + self.state = .interrupted(session.id) + self.releaseActivityLease() + } + + private func enqueuePersistence(_ session: MeetingSession) { + let previous = self.persistenceTail + let store = self.store + self.persistenceTail = Task { + _ = await previous?.value + try? await store.save(session) + } + } + + private func flushQueuedPersistence() async { + _ = await self.persistenceTail?.value + self.persistenceTail = nil + } + + private func releaseActivityLease() { + guard let lease = self.activityLease else { return } + self.audioArbiter.release(lease) + self.activityLease = nil + } + + private static func makeTimebase() -> MeetingTimebaseMetadata { + var info = mach_timebase_info_data_t() + mach_timebase_info(&info) + return MeetingTimebaseMetadata( + startedHostTime: mach_absolute_time(), + machTimebaseNumerator: info.numer, + machTimebaseDenominator: info.denom, + firstPresentationTime: nil + ) + } + + private static func pendingProcessingAttempt() -> MeetingProcessingAttempt { + MeetingProcessingAttempt( + id: UUID(), + startedAt: Date(), + completedAt: nil, + stage: .pending, + pipelineVersion: MeetingProcessingPipeline.pipelineVersion, + asrProvider: nil, + asrModel: nil, + diarizationModel: nil, + lastCompletedTrackID: nil, + errorCode: nil + ) + } + + private static func apply(_ stopResult: MeetingCaptureStopResult, to session: inout MeetingSession) { + session.audioTracks = stopResult.tracks + session.endedAt = stopResult.stoppedAt + session.timebase.firstPresentationTime = self.firstPresentationTime(in: stopResult.tracks) + } + + private static func partialStopResult(from error: Error) -> MeetingCaptureStopResult? { + guard let captureError = error as? MeetingCaptureError else { return nil } + switch captureError { + case let .captureStopFailed(_, partialResult): + return partialResult + default: + return nil + } + } + + private static func hasRecoverableAudio(_ session: MeetingSession) -> Bool { + session.endedAt != nil && session.audioTracks.contains { track in + track.chunks.contains { + $0.finalizationState == .finalized && $0.byteCount > 0 + } + } + } + + private static func firstPresentationTime(in tracks: [MeetingAudioTrack]) -> MeetingMediaTime? { + tracks.flatMap(\.chunks).map(\.presentationStart).min() + } + + private static func failure( + from error: Error, + domain: MeetingFailureDomain, + recoverable: Bool + ) -> MeetingSessionFailure { + let nsError = error as NSError + return MeetingSessionFailure( + id: UUID(), + occurredAt: Date(), + domain: domain, + code: "\(nsError.domain).\(nsError.code)", + message: error.localizedDescription, + recoverable: recoverable + ) + } +} + +enum MeetingCoordinatorError: LocalizedError { + case recordingAlreadyActive + case noActiveMeeting + case noRecoverableAudio + case activityInProgress + case dictationActive + + var errorDescription: String? { + switch self { + case .recordingAlreadyActive: + return "Another meeting recording is already active." + case .noActiveMeeting: + return "There is no active meeting." + case .noRecoverableAudio: + return "This meeting has no finalized audio to retry." + case .activityInProgress: + return "Stop the active meeting activity first." + case .dictationActive: + return "Stop dictation before starting a meeting recording." + } + } +} diff --git a/Sources/Fluid/Services/Meeting/MeetingSessionStore.swift b/Sources/Fluid/Services/Meeting/MeetingSessionStore.swift new file mode 100644 index 00000000..daeea5bf --- /dev/null +++ b/Sources/Fluid/Services/Meeting/MeetingSessionStore.swift @@ -0,0 +1,311 @@ +import Foundation + +nonisolated protocol MeetingSessionStoring: Sendable { + func create(_ session: MeetingSession) async throws + func save(_ session: MeetingSession) async throws + func load(id: MeetingSessionID) async throws -> MeetingSession? + func loadAll() async throws -> [MeetingSession] + func loadRecoverable() async throws -> [MeetingSession] + func sessionDirectory(for id: MeetingSessionID) async throws -> URL +} + +private final nonisolated class MeetingSessionFileSystem: @unchecked Sendable { + let manager: FileManager + + init(manager: FileManager) { + self.manager = manager + } +} + +actor MeetingSessionStore: MeetingSessionStoring { + static let shared = MeetingSessionStore() + + private struct SessionIndex: Codable { + static let currentSchemaVersion = 1 + + var schemaVersion: Int + var sessionIDs: [MeetingSessionID] + var updatedAt: Date + } + + private let fileSystem: MeetingSessionFileSystem + private let rootDirectory: URL + private let encoder: JSONEncoder + private let decoder: JSONDecoder + + init(rootDirectory: URL? = nil, fileManager: FileManager = .default) { + self.fileSystem = MeetingSessionFileSystem(manager: fileManager) + if let rootDirectory { + self.rootDirectory = rootDirectory + } else { + let applicationSupport = fileManager.urls( + for: .applicationSupportDirectory, + in: .userDomainMask + ).first ?? fileManager.homeDirectoryForCurrentUser + .appendingPathComponent("Library/Application Support", isDirectory: true) + self.rootDirectory = applicationSupport + .appendingPathComponent("FluidVoice", isDirectory: true) + .appendingPathComponent("Meetings", isDirectory: true) + } + + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .iso8601 + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + self.encoder = encoder + + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + self.decoder = decoder + } + + func create(_ session: MeetingSession) throws { + try session.validateForPersistence() + try self.prepareRootDirectory() + let directory = self.sessionDirectoryURL(for: session.id) + guard !self.fileSystem.manager.fileExists(atPath: directory.path) else { + throw MeetingSessionStoreError.sessionAlreadyExists + } + try self.createPrivateDirectory(directory) + try self.createPrivateDirectory(directory.appendingPathComponent("tracks", isDirectory: true)) + try self.writeSession(session) + try self.addToIndex(session.id) + } + + func save(_ session: MeetingSession) throws { + try session.validateForPersistence() + try self.prepareRootDirectory() + let directory = self.sessionDirectoryURL(for: session.id) + if !self.fileSystem.manager.fileExists(atPath: directory.path) { + try self.createPrivateDirectory(directory) + try self.createPrivateDirectory(directory.appendingPathComponent("tracks", isDirectory: true)) + } + try self.writeSession(session) + try self.addToIndex(session.id) + } + + func load(id: MeetingSessionID) throws -> MeetingSession? { + let url = self.sessionManifestURL(for: id) + guard self.fileSystem.manager.fileExists(atPath: url.path) else { return nil } + let session = try self.decodeSession(at: url) + let reconciled = try self.reconcileTrackManifests(in: session) + if reconciled != session { + try self.writeSession(reconciled) + } + return reconciled + } + + func loadAll() throws -> [MeetingSession] { + try self.prepareRootDirectory() + let indexedIDs = (try? self.readIndex().sessionIDs) ?? [] + let discoveredIDs = try self.discoverSessionIDs() + let allIDs = Array(Set(indexedIDs).union(discoveredIDs)) + + var sessions: [MeetingSession] = [] + sessions.reserveCapacity(allIDs.count) + for id in allIDs { + guard let session = try? self.load(id: id) else { continue } + sessions.append(session) + } + return sessions.sorted { $0.startedAt > $1.startedAt } + } + + func loadRecoverable() throws -> [MeetingSession] { + try self.loadAll().filter { + switch $0.state { + case .preparing, .recording, .recordingDegraded, .stopping, .processing, .interrupted: + return true + case .failed: + return $0.failures.last?.recoverable == true && + $0.endedAt != nil && + $0.audioTracks.contains { track in + track.chunks.contains { + $0.finalizationState == .finalized && $0.byteCount > 0 + } + } + case .completed: + return false + } + } + } + + func sessionDirectory(for id: MeetingSessionID) throws -> URL { + try self.prepareRootDirectory() + let directory = self.sessionDirectoryURL(for: id) + if !self.fileSystem.manager.fileExists(atPath: directory.path) { + try self.createPrivateDirectory(directory) + try self.createPrivateDirectory(directory.appendingPathComponent("tracks", isDirectory: true)) + } + return directory + } + + private func writeSession(_ session: MeetingSession) throws { + let data = try self.encoder.encode(session) + try self.atomicPrivateWrite(data, to: self.sessionManifestURL(for: session.id)) + } + + private func decodeSession(at url: URL) throws -> MeetingSession { + let data = try Data(contentsOf: url, options: .mappedIfSafe) + let session = try self.decoder.decode(MeetingSession.self, from: data) + try session.validateForPersistence() + return session + } + + private func reconcileTrackManifests(in inputSession: MeetingSession) throws -> MeetingSession { + var session = inputSession + let expectedKinds: Set = session.mode == .onlineCall + ? [.applicationAudio, .microphone] + : [.microphone] + let sessionDirectory = self.sessionDirectoryURL(for: session.id) + + for kind in expectedKinds { + let manifestURL = sessionDirectory + .appendingPathComponent("tracks", isDirectory: true) + .appendingPathComponent(kind.rawValue, isDirectory: true) + .appendingPathComponent("track.json", isDirectory: false) + guard self.fileSystem.manager.fileExists(atPath: manifestURL.path), + let data = try? Data(contentsOf: manifestURL, options: .mappedIfSafe), + var track = try? self.decoder.decode(MeetingAudioTrack.self, from: data), + track.kind == kind + else { continue } + + track.chunks = track.chunks.map { chunk in + self.reconcileChunkFile(chunk, sessionDirectory: sessionDirectory) + }.sorted { $0.sequence < $1.sequence } + + if let index = session.audioTracks.firstIndex(where: { $0.kind == kind }) { + guard session.audioTracks[index].id == track.id else { continue } + session.audioTracks[index] = track + } else { + session.audioTracks.append(track) + } + } + + session.audioTracks.sort { $0.kind.rawValue < $1.kind.rawValue } + let firstPresentationTime = session.audioTracks + .flatMap(\.chunks) + .map(\.presentationStart) + .min() + if let firstPresentationTime { + session.timebase.firstPresentationTime = firstPresentationTime + } + try session.validateForPersistence() + return session + } + + private func reconcileChunkFile( + _ inputChunk: MeetingAudioChunk, + sessionDirectory: URL + ) -> MeetingAudioChunk { + guard inputChunk.finalizationState == .finalized else { return inputChunk } + let fileURL = inputChunk.fileURL(relativeTo: sessionDirectory).standardizedFileURL + let rootPath = sessionDirectory.standardizedFileURL.path + "/" + guard fileURL.path.hasPrefix(rootPath), + self.fileSystem.manager.fileExists(atPath: fileURL.path), + let values = try? fileURL.resourceValues(forKeys: [.fileSizeKey]), + let fileSize = values.fileSize, + fileSize > 0 + else { + var failedChunk = inputChunk + failedChunk.sha256 = "" + failedChunk.byteCount = 0 + failedChunk.finalizationState = .failed + return failedChunk + } + var reconciled = inputChunk + reconciled.byteCount = Int64(fileSize) + return reconciled + } + + private func prepareRootDirectory() throws { + if !self.fileSystem.manager.fileExists(atPath: self.rootDirectory.path) { + try self.createPrivateDirectory(self.rootDirectory) + } + } + + private func createPrivateDirectory(_ url: URL) throws { + try self.fileSystem.manager.createDirectory( + at: url, + withIntermediateDirectories: true, + attributes: [.posixPermissions: NSNumber(value: Int16(0o700))] + ) + try self.fileSystem.manager.setAttributes( + [.posixPermissions: NSNumber(value: Int16(0o700))], + ofItemAtPath: url.path + ) + } + + private func atomicPrivateWrite(_ data: Data, to destination: URL) throws { + try data.write(to: destination, options: .atomic) + try self.fileSystem.manager.setAttributes( + [.posixPermissions: NSNumber(value: Int16(0o600))], + ofItemAtPath: destination.path + ) + } + + private func readIndex() throws -> SessionIndex { + let url = self.rootDirectory.appendingPathComponent("index.json", isDirectory: false) + guard self.fileSystem.manager.fileExists(atPath: url.path) else { + return SessionIndex( + schemaVersion: SessionIndex.currentSchemaVersion, + sessionIDs: [], + updatedAt: Date() + ) + } + let data = try Data(contentsOf: url, options: .mappedIfSafe) + return try self.decoder.decode(SessionIndex.self, from: data) + } + + private func addToIndex(_ id: MeetingSessionID) throws { + var index = (try? self.readIndex()) ?? SessionIndex( + schemaVersion: SessionIndex.currentSchemaVersion, + sessionIDs: [], + updatedAt: Date() + ) + if !index.sessionIDs.contains(id) { + index.sessionIDs.append(id) + } + index.updatedAt = Date() + let data = try self.encoder.encode(index) + try self.atomicPrivateWrite( + data, + to: self.rootDirectory.appendingPathComponent("index.json", isDirectory: false) + ) + } + + private func discoverSessionIDs() throws -> [MeetingSessionID] { + let urls = try self.fileSystem.manager.contentsOfDirectory( + at: self.rootDirectory, + includingPropertiesForKeys: [.isDirectoryKey], + options: [.skipsHiddenFiles] + ) + return urls.compactMap { url in + guard (try? url.resourceValues(forKeys: [.isDirectoryKey]).isDirectory) == true else { + return nil + } + return UUID(uuidString: url.lastPathComponent) + } + } + + private func sessionDirectoryURL(for id: MeetingSessionID) -> URL { + self.rootDirectory.appendingPathComponent(id.uuidString, isDirectory: true) + } + + private func sessionManifestURL(for id: MeetingSessionID) -> URL { + self.sessionDirectoryURL(for: id) + .appendingPathComponent("session.json", isDirectory: false) + } +} + +enum MeetingSessionStoreError: LocalizedError { + case sessionAlreadyExists + case unsupportedSchema(Int) + + var errorDescription: String? { + switch self { + case .sessionAlreadyExists: + return "A meeting session with this identifier already exists." + case let .unsupportedSchema(version): + return "This meeting was created with unsupported schema version \(version)." + } + } +} diff --git a/Sources/Fluid/Services/MenuBarManager.swift b/Sources/Fluid/Services/MenuBarManager.swift index 0e513a8a..9f96e74f 100644 --- a/Sources/Fluid/Services/MenuBarManager.swift +++ b/Sources/Fluid/Services/MenuBarManager.swift @@ -5,11 +5,30 @@ import SwiftUI enum MenuBarNavigationDestination: String { case customDictionary + case meetingTranscription case preferences } @MainActor final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate { + private enum MeetingMenuActivity: Equatable { + case inactive + case preparing + case recording + case stopping + case processing + case interrupted + case failed + case completed + } + + private struct MeetingMenuPresentation { + var activity: MeetingMenuActivity = .inactive + var sourceName: String? + var startedAt: Date? + var attentionStatus: String? + } + private var statusItem: NSStatusItem? private var menu: NSMenu? private var isSetup: Bool = false @@ -21,10 +40,19 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate { private var rollbackMenuItem: NSMenuItem? private var microphoneMenuItem: NSMenuItem? private var microphoneSubmenu: NSMenu? + private var stopMeetingRecordingMenuItem: NSMenuItem? + private var meetingStatusMenuItem: NSMenuItem? + private var openMeetingTranscriptionMenuItem: NSMenuItem? + private var meetingMenuSeparator: NSMenuItem? + private var meetingMenuPresentation = MeetingMenuPresentation() + private var meetingStopRequested = false + private var stopMeetingRecordingHandler: (() -> Void)? // References to app state private weak var asrService: ASRService? + private weak var meetingCoordinator: MeetingSessionCoordinator? private var cancellables = Set() + private var meetingCancellables = Set() /// Overlay management (persistent, independent of window lifecycle) private var overlayVisible: Bool = false @@ -110,6 +138,167 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate { .store(in: &self.cancellables) } + func configure(meetingCoordinator: MeetingSessionCoordinator) { + guard self.meetingCoordinator !== meetingCoordinator else { + self.applyMeetingPresentation( + state: meetingCoordinator.state, + activeSession: meetingCoordinator.activeSession, + latestCompletedSession: meetingCoordinator.latestCompletedSession + ) + return + } + + self.meetingCancellables.removeAll() + self.meetingCoordinator = meetingCoordinator + self.stopMeetingRecordingHandler = { [weak meetingCoordinator] in + guard let meetingCoordinator else { return } + Task { @MainActor in + do { + _ = try await meetingCoordinator.stopAndTranscribe() + } catch { + DebugLogger.shared.error( + "Menu action: Stop meeting recording failed: \(error.localizedDescription)", + source: "MenuBarManager" + ) + } + } + } + + Publishers.CombineLatest3( + meetingCoordinator.$state, + meetingCoordinator.$activeSession, + meetingCoordinator.$latestCompletedSession + ) + .receive(on: DispatchQueue.main) + .sink { [weak self] state, activeSession, latestCompletedSession in + self?.applyMeetingPresentation( + state: state, + activeSession: activeSession, + latestCompletedSession: latestCompletedSession + ) + } + .store(in: &self.meetingCancellables) + } + + private func applyMeetingPresentation( + state: MeetingCoordinatorState, + activeSession: MeetingSession?, + latestCompletedSession: MeetingSession? + ) { + let previousActivity = self.meetingMenuPresentation.activity + switch state { + case .recording, .recordingDegraded: + break + default: + self.meetingStopRequested = false + } + + switch state { + case .idle: + self.meetingMenuPresentation = MeetingMenuPresentation() + case .preparing: + self.meetingMenuPresentation = MeetingMenuPresentation( + activity: .preparing, + sourceName: activeSession?.capturedApplication?.displayName, + startedAt: activeSession?.startedAt + ) + case .recording, .recordingDegraded: + self.meetingMenuPresentation = MeetingMenuPresentation( + activity: .recording, + sourceName: activeSession?.capturedApplication?.displayName ?? "In-room meeting", + startedAt: activeSession?.startedAt + ) + case .stopping: + self.meetingMenuPresentation = MeetingMenuPresentation( + activity: .stopping, + sourceName: activeSession?.capturedApplication?.displayName ?? "In-room meeting", + startedAt: activeSession?.startedAt + ) + case .processing: + self.meetingMenuPresentation = MeetingMenuPresentation( + activity: .processing, + sourceName: activeSession?.capturedApplication?.displayName, + startedAt: nil + ) + case .completed: + self.meetingMenuPresentation = MeetingMenuPresentation( + activity: .completed, + sourceName: latestCompletedSession?.capturedApplication?.displayName, + startedAt: nil + ) + case .interrupted: + self.meetingMenuPresentation = MeetingMenuPresentation( + activity: .interrupted, + sourceName: activeSession?.capturedApplication?.displayName, + startedAt: activeSession?.startedAt, + attentionStatus: Self.interruptionStatus(for: activeSession) + ) + case let .failed(_, failure): + self.meetingMenuPresentation = MeetingMenuPresentation( + activity: .failed, + sourceName: activeSession?.capturedApplication?.displayName, + startedAt: activeSession?.startedAt, + attentionStatus: Self.failureStatus(for: activeSession, failure: failure) + ) + } + + if self.meetingMenuPresentation.activity != previousActivity { + self.announceMeetingActivity(self.meetingMenuPresentation) + } + + self.updateMenuBarIcon() + self.updateMenu() + } + + private func announceMeetingActivity(_ presentation: MeetingMenuPresentation) { + let announcement: String? + switch presentation.activity { + case .inactive: + announcement = nil + case .preparing: + announcement = "Starting meeting recording" + case .recording: + announcement = "Meeting recording started" + case .stopping: + announcement = "Stopping meeting recording" + case .processing: + announcement = "Meeting transcription started" + case .interrupted: + announcement = presentation.attentionStatus ?? "Meeting recording interrupted" + case .failed: + announcement = presentation.attentionStatus ?? "Meeting setup failed" + case .completed: + announcement = "Meeting transcription complete" + } + guard let announcement else { return } + AccessibilityNotification.Announcement(announcement).post() + } + + private static func interruptionStatus(for session: MeetingSession?) -> String { + guard let session else { return "Meeting setup interrupted" } + if session.endedAt != nil, !session.processingAttempts.isEmpty { + return "Meeting transcription interrupted" + } + return "Meeting recording interrupted" + } + + private static func failureStatus( + for session: MeetingSession?, + failure: MeetingSessionFailure + ) -> String { + if failure.domain == .processing { + return "Meeting transcription failed" + } + let hasRecoverableAudio = session.map { session in + session.endedAt != nil && session.audioTracks.contains { track in + track.chunks.contains { + $0.finalizationState == .finalized && $0.byteCount > 0 + } + } + } ?? false + return hasRecoverableAudio ? "Meeting recording failed" : "Meeting setup failed" + } + private func handleOverlayState(isRunning: Bool, asrService: ASRService) { self.overlayBench("handle_state isRunning=\(isRunning) overlayVisible=\(self.overlayVisible) processing=\(self.isProcessingActive) mode=\(self.currentOverlayMode.rawValue)") @@ -462,9 +651,80 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate { // Use MenuBarIcon asset - vectorized from logo if let image = NSImage(named: "MenuBarIcon") { - image.isTemplate = true // Adapts to light/dark mode and tints red when recording - statusItem.button?.image = image + switch self.meetingMenuPresentation.activity { + case .recording, .stopping: + statusItem.button?.image = self.menuBarActivityImage(baseImage: image, activity: .recording) + statusItem.button?.setAccessibilityLabel( + self.meetingMenuPresentation.activity == .stopping + ? "FluidVoice, stopping meeting recording" + : "FluidVoice, meeting recording" + ) + case .preparing, .processing: + statusItem.button?.image = self.menuBarActivityImage(baseImage: image, activity: .processing) + statusItem.button?.setAccessibilityLabel( + self.meetingMenuPresentation.activity == .preparing + ? "FluidVoice, starting meeting recording" + : "FluidVoice, transcribing meeting" + ) + case .interrupted, .failed: + statusItem.button?.image = self.menuBarActivityImage(baseImage: image, activity: .interrupted) + let status = self.meetingMenuPresentation.attentionStatus ?? "Meeting needs attention" + statusItem.button?.setAccessibilityLabel("FluidVoice, \(status.lowercased())") + case .inactive: + image.isTemplate = true + statusItem.button?.image = image + statusItem.button?.setAccessibilityLabel("FluidVoice") + case .completed: + image.isTemplate = true + statusItem.button?.image = image + statusItem.button?.setAccessibilityLabel("FluidVoice, meeting transcription complete") + } + } + } + + private func menuBarActivityImage(baseImage: NSImage, activity: MeetingMenuActivity) -> NSImage { + let image = NSImage(size: NSSize(width: 18, height: 18), flipped: false) { _ in + baseImage.draw( + in: NSRect(x: 0, y: 1, width: 15, height: 15), + from: .zero, + operation: .sourceOver, + fraction: 1 + ) + + let badgeRect = NSRect(x: 10, y: 0, width: 8, height: 8) + let badgePath = NSBezierPath() + switch activity { + case .recording, .stopping: + badgePath.appendOval(in: badgeRect) + badgePath.appendRoundedRect( + NSRect(x: 12.25, y: 2.25, width: 3.5, height: 3.5), + xRadius: 0.7, + yRadius: 0.7 + ) + case .preparing, .processing: + badgePath.move(to: NSPoint(x: badgeRect.midX, y: badgeRect.maxY)) + badgePath.line(to: NSPoint(x: badgeRect.maxX, y: badgeRect.midY)) + badgePath.line(to: NSPoint(x: badgeRect.midX, y: badgeRect.minY)) + badgePath.line(to: NSPoint(x: badgeRect.minX, y: badgeRect.midY)) + badgePath.close() + badgePath.appendOval(in: NSRect(x: 12.5, y: 2.5, width: 3, height: 3)) + case .interrupted, .failed: + badgePath.move(to: NSPoint(x: badgeRect.midX, y: badgeRect.maxY)) + badgePath.line(to: NSPoint(x: badgeRect.maxX, y: badgeRect.minY)) + badgePath.line(to: NSPoint(x: badgeRect.minX, y: badgeRect.minY)) + badgePath.close() + badgePath.appendOval(in: NSRect(x: 13, y: 2.2, width: 2, height: 2)) + case .inactive, .completed: + break + } + badgePath.windingRule = .evenOdd + NSColor.black.setFill() + badgePath.fill() + return true } + image.isTemplate = true + image.accessibilityDescription = "FluidVoice meeting status" + return image } private func buildMenuStructure() { @@ -472,6 +732,33 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate { menu.removeAllItems() + let stopMeetingItem = NSMenuItem( + title: "Stop Meeting Recording", + action: #selector(stopMeetingRecording), + keyEquivalent: "" + ) + stopMeetingItem.target = self + menu.addItem(stopMeetingItem) + self.stopMeetingRecordingMenuItem = stopMeetingItem + + let meetingStatusItem = NSMenuItem(title: "", action: nil, keyEquivalent: "") + meetingStatusItem.isEnabled = false + menu.addItem(meetingStatusItem) + self.meetingStatusMenuItem = meetingStatusItem + + let openMeetingItem = NSMenuItem( + title: "Open Meeting Transcription", + action: #selector(openMeetingTranscription), + keyEquivalent: "" + ) + openMeetingItem.target = self + menu.addItem(openMeetingItem) + self.openMeetingTranscriptionMenuItem = openMeetingItem + + let meetingSeparator = NSMenuItem.separator() + menu.addItem(meetingSeparator) + self.meetingMenuSeparator = meetingSeparator + // Status indicator with hotkey info self.statusMenuItem = NSMenuItem(title: "", action: nil, keyEquivalent: "") self.statusMenuItem?.isEnabled = false @@ -563,10 +850,25 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate { } private func updateMenuItemsText() { + self.updateMeetingMenuItemsText() + // Update status text with hotkey info let hotkeyDisplay = SettingsStore.shared.primaryDictationShortcutDisplayString let hotkeyInfo = hotkeyDisplay.isEmpty ? "" : " (\(hotkeyDisplay))" - let statusTitle = self.isRecording ? "Recording...\(hotkeyInfo)" : "Ready to Record\(hotkeyInfo)" + let statusTitle: String = switch self.meetingMenuPresentation.activity { + case .preparing: + "Starting Meeting Recording…" + case .recording: + "Meeting Recording in Progress" + case .stopping: + "Stopping Meeting Recording…" + case .processing: + "Transcribing Meeting…" + case .interrupted, .failed: + self.meetingMenuPresentation.attentionStatus ?? "Meeting Needs Attention" + case .inactive, .completed: + self.isRecording ? "Recording...\(hotkeyInfo)" : "Ready to Record\(hotkeyInfo)" + } self.statusMenuItem?.title = statusTitle self.copyLastTranscriptMenuItem?.isEnabled = self.canCopyLastTranscript self.microphoneMenuItem?.isEnabled = true @@ -575,6 +877,62 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate { self.rollbackMenuItem?.isEnabled = SimpleUpdater.shared.hasRollbackBackup() } + private func updateMeetingMenuItemsText(now: Date = Date()) { + let activity = self.meetingMenuPresentation.activity + let hasMeetingStatus = activity != .inactive + let isRecording = activity == .recording + let isStopping = activity == .stopping || self.meetingStopRequested + + self.stopMeetingRecordingMenuItem?.isHidden = !isRecording && !isStopping + self.stopMeetingRecordingMenuItem?.isEnabled = isRecording && + !self.meetingStopRequested && + self.stopMeetingRecordingHandler != nil + self.stopMeetingRecordingMenuItem?.title = isStopping + ? "Stopping Meeting Recording…" + : "Stop Meeting Recording" + self.meetingStatusMenuItem?.isHidden = !hasMeetingStatus || activity == .completed + self.openMeetingTranscriptionMenuItem?.isHidden = !hasMeetingStatus + self.meetingMenuSeparator?.isHidden = !hasMeetingStatus + + switch activity { + case .inactive: + break + case .preparing: + self.meetingStatusMenuItem?.title = "Starting meeting recording…" + self.openMeetingTranscriptionMenuItem?.title = "Open Meeting Transcription" + case .recording: + let source = self.meetingMenuPresentation.sourceName ?? "Meeting" + let elapsed = self.meetingMenuPresentation.startedAt.map { Self.elapsedText(now.timeIntervalSince($0)) } ?? "0:00" + self.meetingStatusMenuItem?.title = "Recording \(source) · \(elapsed)" + self.openMeetingTranscriptionMenuItem?.title = "Open Meeting Transcription" + case .stopping: + self.meetingStatusMenuItem?.title = "Finalizing meeting audio…" + self.openMeetingTranscriptionMenuItem?.title = "Open Meeting Transcription" + case .processing: + self.meetingStatusMenuItem?.title = "Transcribing meeting…" + self.openMeetingTranscriptionMenuItem?.title = "Open Meeting Transcription" + case .interrupted: + self.meetingStatusMenuItem?.title = self.meetingMenuPresentation.attentionStatus ?? "Recording interrupted" + self.openMeetingTranscriptionMenuItem?.title = "Open Meeting Transcription" + case .failed: + self.meetingStatusMenuItem?.title = self.meetingMenuPresentation.attentionStatus ?? "Meeting setup failed" + self.openMeetingTranscriptionMenuItem?.title = "Open Meeting Transcription" + case .completed: + self.openMeetingTranscriptionMenuItem?.title = "Open Latest Meeting Transcript" + } + } + + private static func elapsedText(_ duration: TimeInterval) -> String { + let total = max(0, Int(duration.rounded(.down))) + let hours = total / 3600 + let minutes = (total % 3600) / 60 + let seconds = total % 60 + if hours > 0 { + return String(format: "%d:%02d:%02d", hours, minutes, seconds) + } + return String(format: "%d:%02d", minutes, seconds) + } + func menuWillOpen(_ menu: NSMenu) { if menu === self.menu { self.updateMenuItemsText() @@ -867,6 +1225,21 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate { self.openNavigationDestination(.customDictionary) } + @objc private func openMeetingTranscription() { + self.openNavigationDestination(.meetingTranscription) + } + + @objc private func stopMeetingRecording() { + guard self.meetingMenuPresentation.activity == .recording, + !self.meetingStopRequested + else { + return + } + self.meetingStopRequested = true + self.updateMeetingMenuItemsText() + self.stopMeetingRecordingHandler?() + } + private func openNavigationDestination(_ destination: MenuBarNavigationDestination) { // Ensure a fresh one-shot request every time the menu item is clicked. self.requestedNavigationDestination = nil diff --git a/Sources/Fluid/Services/SpeakerDiarizationService.swift b/Sources/Fluid/Services/SpeakerDiarizationService.swift index e21c6de1..cb76d2f1 100644 --- a/Sources/Fluid/Services/SpeakerDiarizationService.swift +++ b/Sources/Fluid/Services/SpeakerDiarizationService.swift @@ -9,7 +9,7 @@ actor SpeakerDiarizationService { static let isSupported = true /// A contiguous stretch of speech attributed to one speaker. - struct SpeakerTurn: Sendable, Equatable { + nonisolated struct SpeakerTurn: Sendable, Equatable { let speakerLabel: String let startSeconds: Double let endSeconds: Double @@ -19,6 +19,16 @@ actor SpeakerDiarizationService { } } + nonisolated struct SpeakerProfile: Sendable, Equatable { + let embedding: [Float] + let averageQuality: Float + } + + nonisolated struct DiarizationOutput: Sendable, Equatable { + let turns: [SpeakerTurn] + let profilesByLabel: [String: SpeakerProfile] + } + private let manager: OfflineDiarizerManager private var modelsReady = false @@ -36,11 +46,19 @@ actor SpeakerDiarizationService { /// Speaker labels are assigned in order of first appearance ("Speaker 1" spoke first). /// Returns an empty array when no speech is detected. func diarize(fileURL: URL) async throws -> [SpeakerTurn] { + try await self.diarizeWithProfiles(fileURL: fileURL).turns + } + + /// Includes one bounded centroid per local cluster so callers can stitch the + /// same voice across independently processed files without retaining audio. + func diarizeWithProfiles(fileURL: URL) async throws -> DiarizationOutput { if !self.modelsReady { - DebugLogger.shared.info( - "SpeakerDiarizationService: preparing diarizer models", - source: "SpeakerDiarizationService" - ) + await MainActor.run { + DebugLogger.shared.info( + "SpeakerDiarizationService: preparing diarizer models", + source: "SpeakerDiarizationService" + ) + } try await self.manager.prepareModels() self.modelsReady = true } @@ -50,7 +68,7 @@ actor SpeakerDiarizationService { result = try await self.manager.process(fileURL) } catch let error as OfflineDiarizationError { if case .noSpeechDetected = error { - return [] + return DiarizationOutput(turns: [], profilesByLabel: [:]) } throw error } @@ -77,7 +95,24 @@ actor SpeakerDiarizationService { ) } - return Self.mergeAdjacentTurns(turns) + var profilesByLabel: [String: SpeakerProfile] = [:] + for (clusterID, label) in labelByClusterId { + let clusterSegments = chronological.filter { $0.speakerId == clusterID } + let embedding = result.speakerDatabase?[clusterID] ?? clusterSegments.first?.embedding ?? [] + guard !embedding.isEmpty else { continue } + let quality = clusterSegments.isEmpty + ? 0 + : clusterSegments.reduce(Float.zero) { $0 + $1.qualityScore } / Float(clusterSegments.count) + profilesByLabel[label] = SpeakerProfile( + embedding: embedding, + averageQuality: quality + ) + } + + return DiarizationOutput( + turns: Self.mergeAdjacentTurns(turns), + profilesByLabel: profilesByLabel + ) } /// Merge consecutive turns from the same speaker separated by short gaps. @@ -120,7 +155,7 @@ actor SpeakerDiarizationService { actor SpeakerDiarizationService { static let isSupported = false - struct SpeakerTurn: Sendable, Equatable { + nonisolated struct SpeakerTurn: Sendable, Equatable { let speakerLabel: String let startSeconds: Double let endSeconds: Double @@ -130,9 +165,23 @@ actor SpeakerDiarizationService { } } + nonisolated struct SpeakerProfile: Sendable, Equatable { + let embedding: [Float] + let averageQuality: Float + } + + nonisolated struct DiarizationOutput: Sendable, Equatable { + let turns: [SpeakerTurn] + let profilesByLabel: [String: SpeakerProfile] + } + init(expectedSpeakers: Int? = nil) {} func diarize(fileURL: URL) async throws -> [SpeakerTurn] { + try await self.diarizeWithProfiles(fileURL: fileURL).turns + } + + func diarizeWithProfiles(fileURL: URL) async throws -> DiarizationOutput { throw NSError( domain: "SpeakerDiarizationService", code: -1, diff --git a/Sources/Fluid/Services/WhisperProvider.swift b/Sources/Fluid/Services/WhisperProvider.swift index c2e47a1a..66bd1099 100644 --- a/Sources/Fluid/Services/WhisperProvider.swift +++ b/Sources/Fluid/Services/WhisperProvider.swift @@ -292,6 +292,10 @@ final class WhisperProvider: TranscriptionProvider { } func transcribe(_ samples: [Float]) async throws -> ASRTranscriptionResult { + try await self.transcribe(samples, languageCode: nil) + } + + func transcribe(_ samples: [Float], languageCode: String?) async throws -> ASRTranscriptionResult { let minSamples = 16_000 guard samples.count >= minSamples else { throw NSError( @@ -311,7 +315,7 @@ final class WhisperProvider: TranscriptionProvider { let transcript = try await session.run( samples, - options: RunOptions(timestamps: .segment) + options: RunOptions(timestamps: .segment, language: languageCode) ) let fullText = transcript.text.trimmingCharacters(in: .whitespacesAndNewlines) return ASRTranscriptionResult(text: fullText, confidence: 1.0) diff --git a/Sources/Fluid/UI/FileTranscriptionView.swift b/Sources/Fluid/UI/FileTranscriptionView.swift new file mode 100644 index 00000000..70dd395a --- /dev/null +++ b/Sources/Fluid/UI/FileTranscriptionView.swift @@ -0,0 +1,807 @@ +import SwiftUI +import UniformTypeIdentifiers + +struct FileTranscriptionView: View { + @ObservedObject var asrService: ASRService + @ObservedObject private var transcriptionService: FileTranscriptionService + @ObservedObject private var fileHistoryStore = FileTranscriptionHistoryStore.shared + @ObservedObject private var settings = SettingsStore.shared + @State private var selectedFileURL: URL? + @Environment(\.theme) private var theme + + init(asrService: ASRService, transcriptionService: FileTranscriptionService) { + self.asrService = asrService + _transcriptionService = ObservedObject(wrappedValue: transcriptionService) + } + + @State private var showingFilePicker = false + @State private var showingExportDialog = false + @State private var exportResult: TranscriptionResult? + @State private var exportFormat: ExportFormat = .text + @State private var showingCopyConfirmation = false + @State private var isDropTargeted = false + @State private var dropErrorMessage: String? + + enum ExportFormat: String, CaseIterable { + case text = "Text (.txt)" + case json = "JSON (.json)" + + var fileExtension: String { + switch self { + case .text: return "txt" + case .json: return "json" + } + } + } + + private var selectedFileIsVideo: Bool { + guard let fileExtension = (self.selectedFileURL ?? self.transcriptionService.currentFileURL)? + .pathExtension.lowercased() + else { return false } + return UTType(filenameExtension: fileExtension)?.conforms(to: .movie) ?? false + } + + private var conflictingActivity: ASRExclusiveActivity? { + guard let activity = self.asrService.activeExclusiveActivity, + activity != .fileTranscription + else { + return nil + } + return activity + } + + var body: some View { + VStack(spacing: 0) { + // Header + VStack(spacing: 8) { + Image(systemName: "waveform.circle.fill") + .font(.system(size: 48)) + .foregroundStyle(Color.fluidGreen.gradient) + + Text("File Transcription") + .font(.title2) + .fontWeight(.semibold) + + Text("Choose an audio or video file to transcribe") + .font(.subheadline) + .foregroundColor(.secondary) + } + .padding(.top, 40) + .padding(.bottom, 30) + + // Main Content Area + ScrollView { + VStack(spacing: 24) { + if let activity = self.conflictingActivity { + self.activityConflictCard(activity: activity) + } + + // File Selection Card + self.fileSelectionCard + + // Progress Card (only show when transcribing) + if self.transcriptionService.isTranscribing { + self.progressCard + } + + // Results Card (only show when we have results) + if let result = transcriptionService.result { + self.resultsCard(result: result) + } + + // Error Card (only show when we have an error) + if let error = transcriptionService.error { + self.errorCard(error: error) + } + + // Drop error (unsupported file type) + if let message = self.dropErrorMessage { + self.dropErrorCard(message: message) + } + + // Recent transcriptions (persisted history) + if !self.fileHistoryStore.entries.isEmpty { + Divider() + .padding(.vertical, 8) + self.recentTranscriptionsSection + } + } + .padding(24) + } + } + .frame(maxWidth: .infinity, maxHeight: .infinity) + .background(self.theme.palette.windowBackground) + .overlay(alignment: .topTrailing) { + if self.showingCopyConfirmation { + Text("Copied!") + .font(.caption) + .padding(.horizontal, 12) + .padding(.vertical, 6) + .background(Color.fluidGreen.opacity(0.9)) + .foregroundColor(.white) + .cornerRadius(8) + .padding() + .transition(.move(edge: .top).combined(with: .opacity)) + } + } + .fileExporter( + isPresented: self.$showingExportDialog, + document: TranscriptionDocument( + result: self.exportResult ?? TranscriptionResult(text: "", confidence: 0, duration: 0, processingTime: 0, fileName: "transcript"), + format: self.exportFormat, + service: self.transcriptionService + ), + contentType: self.exportFormat == .text ? .plainText : .json, + defaultFilename: "\((self.exportResult?.fileName).map { "\($0)_transcript" } ?? "transcript").\(self.exportFormat.fileExtension)" + ) { exportCompletion in + switch exportCompletion { + case .success: + DebugLogger.shared.info("File exported successfully", source: "FileTranscriptionView") + case let .failure(error): + DebugLogger.shared.error("Export failed: \(error)", source: "FileTranscriptionView") + } + self.exportResult = nil + } + .onChange(of: self.transcriptionService.isTranscribing) { _, isTranscribing in + guard isTranscribing else { return } + AccessibilityNotification.Announcement("File transcription started").post() + } + .onChange(of: self.transcriptionService.result?.id) { _, resultID in + guard resultID != nil else { return } + AccessibilityNotification.Announcement("File transcription complete").post() + } + .onChange(of: self.transcriptionService.error) { _, error in + guard let error, !error.isEmpty else { return } + AccessibilityNotification.Announcement("File transcription failed. \(error)").post() + } + .onAppear { + if self.selectedFileURL == nil { + self.selectedFileURL = self.transcriptionService.currentFileURL + } + } + } + + private func activityConflictCard(activity: ASRExclusiveActivity) -> some View { + Label( + "File transcription is paused while \(activity.displayName) is active.", + systemImage: activity == .meeting ? "person.2.wave.2.fill" : "waveform.badge.mic" + ) + .font(.subheadline) + .foregroundStyle(self.theme.palette.secondaryText) + .frame(maxWidth: .infinity, alignment: .leading) + .padding() + .background( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill(self.theme.palette.cardBackground) + .overlay( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .stroke(self.theme.palette.cardBorder.opacity(0.45), lineWidth: 1) + ) + ) + .accessibilityLabel( + "File transcription unavailable. Wait for the active \(activity.displayName) to finish." + ) + } + + // MARK: - File Selection Card + + private var fileSelectionCard: some View { + VStack(spacing: 16) { + if let fileURL = selectedFileURL ?? self.transcriptionService.currentFileURL { + // Show selected file + HStack { + Image(systemName: "doc.fill") + .font(.title2) + .foregroundColor(Color.fluidGreen) + + VStack(alignment: .leading, spacing: 4) { + Text(fileURL.lastPathComponent) + .font(.headline) + + Text(self.formatFileSize(fileURL: fileURL)) + .font(.caption) + .foregroundColor(.secondary) + } + + Spacer() + + Button(action: { + self.selectedFileURL = nil + self.transcriptionService.reset() + }) { + Image(systemName: "xmark.circle.fill") + .foregroundColor(.secondary) + } + .buttonStyle(.plain) + .disabled(self.transcriptionService.isTranscribing) + .help(self.transcriptionService.isTranscribing ? "Wait for transcription to finish" : "Remove file") + } + .padding() + .background( + RoundedRectangle(cornerRadius: 8, style: .continuous) + .fill(self.theme.palette.cardBackground) + .overlay( + RoundedRectangle(cornerRadius: 8, style: .continuous) + .stroke(self.theme.palette.cardBorder.opacity(0.5), lineWidth: 1) + ) + ) + + // Speaker labeling options (unavailable on Intel Macs) + if SpeakerDiarizationService.isSupported { + VStack(spacing: 10) { + Toggle(isOn: self.$settings.fileTranscriptionSpeakerLabelsEnabled) { + VStack(alignment: .leading, spacing: 2) { + Text("Label speakers") + .font(.subheadline) + + Text(self.selectedFileIsVideo + ? "Available for audio files only" + : "Identify who said what (downloads speaker models on first use)") + .font(.caption) + .foregroundColor(.secondary) + } + } + .toggleStyle(.switch) + .disabled(self.selectedFileIsVideo || self.transcriptionService.isTranscribing) + + if self.settings.fileTranscriptionSpeakerLabelsEnabled, !self.selectedFileIsVideo { + HStack { + Text("Number of speakers") + .font(.subheadline) + + Spacer() + + Picker("", selection: self.$settings.fileTranscriptionExpectedSpeakerCount) { + Text("Auto").tag(0) + ForEach(2...8, id: \.self) { count in + Text("\(count)").tag(count) + } + } + .pickerStyle(.menu) + .labelsHidden() + .frame(width: 90) + .disabled(self.transcriptionService.isTranscribing) + } + } + } + .padding() + .background( + RoundedRectangle(cornerRadius: 8, style: .continuous) + .fill(self.theme.palette.cardBackground) + .overlay( + RoundedRectangle(cornerRadius: 8, style: .continuous) + .stroke(self.theme.palette.cardBorder.opacity(0.5), lineWidth: 1) + ) + ) + } + + // Transcribe Button + Button(action: { + Task { + await self.transcribeFile() + } + }) { + HStack { + Image(systemName: "waveform") + Text("Transcribe") + } + .frame(maxWidth: .infinity) + .padding(.vertical, 12) + } + .buttonStyle(.borderedProminent) + .disabled( + self.transcriptionService.isTranscribing || + self.conflictingActivity != nil + ) + .help( + self.conflictingActivity != nil + ? "Wait for the active transcription to finish" + : "Transcribe the selected file" + ) + + } else { + // File picker button – whole area is tappable; supports drag-and-drop + Button(action: { + self.showingFilePicker = true + }) { + VStack(spacing: 12) { + Image(systemName: "arrow.up.doc.fill") + .font(.system(size: 32)) + + Text("Drag and drop a file here, or click to open") + .font(.headline) + + Text(FileTranscriptionService.supportedFormatsDescription) + .font(.caption) + .foregroundColor(.secondary) + } + .frame(maxWidth: .infinity) + .padding(32) + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + .background( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill(self.theme.palette.cardBackground) + .overlay( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .stroke(self.theme.palette.cardBorder.opacity(0.45), lineWidth: 1) + ) + ) + .overlay( + RoundedRectangle(cornerRadius: 12) + .strokeBorder(style: StrokeStyle(lineWidth: 2, dash: [8])) + .foregroundColor(Color.fluidGreen.opacity(self.isDropTargeted ? 0.7 : 0.3)) + ) + .onDrop(of: [.fileURL], isTargeted: self.$isDropTargeted) { providers in + self.handleDrop(providers: providers) + } + } + } + .fileImporter( + isPresented: self.$showingFilePicker, + allowedContentTypes: FileTranscriptionService.allowedContentTypes, + allowsMultipleSelection: false + ) { result in + switch result { + case let .success(urls): + if let url = urls.first { + self.selectedFileURL = url + self.transcriptionService.reset() + } + case let .failure(error): + DebugLogger.shared.error("File picker error: \(error)", source: "FileTranscriptionView") + } + } + } + + // MARK: - Progress Card + + private var progressCard: some View { + VStack(spacing: 16) { + ProgressView(value: self.transcriptionService.progress) + .progressViewStyle(.linear) + + HStack { + ProgressView() + .controlSize(.small) + .fixedSize() + + Text(self.transcriptionService.currentStatus) + .font(.subheadline) + .foregroundColor(.secondary) + } + } + .padding() + .background( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill(self.theme.palette.cardBackground) + .overlay( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .stroke(self.theme.palette.cardBorder.opacity(0.45), lineWidth: 1) + ) + ) + } + + // MARK: - Results Card + + private func resultsCard(result: TranscriptionResult) -> some View { + VStack(alignment: .leading, spacing: 16) { + // Header with stats + HStack { + VStack(alignment: .leading, spacing: 4) { + Text("Transcription Complete") + .font(.headline) + + HStack(spacing: 16) { + Label("\(String(format: "%.1f", result.duration))s", systemImage: "clock") + Label("\(String(format: "%.0f%%", result.confidence * 100))", systemImage: "checkmark.circle") + Label( + "\(String(format: "%.1f", result.duration / result.processingTime))x", + systemImage: "speedometer" + ) + } + .font(.caption) + .foregroundColor(.secondary) + } + + Spacer() + + // Action buttons + HStack(spacing: 8) { + Button(action: { + self.copyToClipboard(result.text) + }) { + Image(systemName: "doc.on.doc") + } + .help("Copy to clipboard") + + Button(action: { + self.exportResult = result + self.showingExportDialog = true + }) { + Image(systemName: "square.and.arrow.up") + } + .help("Export transcription") + } + .buttonStyle(.borderless) + } + + if let notice = transcriptionService.fallbackNotice { + Label(notice, systemImage: "exclamationmark.triangle.fill") + .font(.caption) + .foregroundColor(.secondary) + } + + Divider() + + // Transcription text + ScrollView { + if !result.speakerSegments.isEmpty { + VStack(alignment: .leading, spacing: 12) { + ForEach(result.speakerSegments) { segment in + VStack(alignment: .leading, spacing: 2) { + HStack(spacing: 6) { + Text(segment.speaker) + .font(.caption) + .fontWeight(.semibold) + .foregroundColor(Color.fluidGreen) + + Text(segment.timestampText) + .font(.caption2) + .foregroundColor(.secondary) + } + + Text(segment.text) + .font(.body) + .textSelection(.enabled) + } + } + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding() + } else { + Text(result.text) + .font(.body) + .textSelection(.enabled) + .frame(maxWidth: .infinity, alignment: .leading) + .padding() + } + } + .frame(maxHeight: 300) + .background( + RoundedRectangle(cornerRadius: 8, style: .continuous) + .fill(self.theme.palette.contentBackground) + .overlay( + RoundedRectangle(cornerRadius: 8, style: .continuous) + .stroke(self.theme.palette.cardBorder.opacity(0.45), lineWidth: 1) + ) + ) + } + .padding() + .background( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill(self.theme.palette.cardBackground) + .overlay( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .stroke(self.theme.palette.cardBorder.opacity(0.45), lineWidth: 1) + ) + ) + } + + // MARK: - Recent Transcriptions Section + + private var recentTranscriptionsSection: some View { + VStack(alignment: .leading, spacing: 16) { + HStack { + Text("Recent transcriptions") + .font(.headline) + Spacer() + if !self.fileHistoryStore.entries.isEmpty { + Button("Clear all") { + self.fileHistoryStore.clearAll() + } + .buttonStyle(.plain) + .foregroundColor(.secondary) + .font(.caption) + } + } + + VStack(spacing: 8) { + ForEach(self.fileHistoryStore.entries) { entry in + self.recentEntryRow(entry: entry) + } + } + + if let entry = self.fileHistoryStore.selectedEntry { + self.historyDetailCard(entry: entry) + } + } + } + + private func recentEntryRow(entry: FileTranscriptionEntry) -> some View { + let isSelected = self.fileHistoryStore.selectedEntryID == entry.id + return Button(action: { + self.fileHistoryStore.selectedEntryID = entry.id + }) { + HStack { + Image(systemName: "doc.text.fill") + .font(.body) + .foregroundColor(Color.fluidGreen) + .frame(width: 24) + + VStack(alignment: .leading, spacing: 2) { + Text(entry.fileName) + .font(.system(size: 14, weight: .medium)) + .lineLimit(1) + Text(entry.relativeTimeString) + .font(.caption) + .foregroundColor(.secondary) + Text(entry.previewText) + .font(.caption) + .foregroundColor(.secondary) + .lineLimit(1) + } + + Spacer() + + if isSelected { + Image(systemName: "chevron.right.circle.fill") + .foregroundColor(Color.fluidGreen) + } + } + .padding(12) + .background( + RoundedRectangle(cornerRadius: 8, style: .continuous) + .fill(self.theme.palette.cardBackground) + .overlay( + RoundedRectangle(cornerRadius: 8, style: .continuous) + .stroke(isSelected ? Color.fluidGreen.opacity(0.5) : self.theme.palette.cardBorder.opacity(0.3), lineWidth: isSelected ? 2 : 1) + ) + ) + } + .buttonStyle(.plain) + } + + private func historyDetailCard(entry: FileTranscriptionEntry) -> some View { + let result = entry.toTranscriptionResult() + return VStack(alignment: .leading, spacing: 16) { + HStack { + VStack(alignment: .leading, spacing: 4) { + Text("From history") + .font(.headline) + HStack(spacing: 16) { + Label("\(String(format: "%.1f", entry.duration))s", systemImage: "clock") + Label("\(String(format: "%.0f%%", entry.confidence * 100))", systemImage: "checkmark.circle") + Label(entry.fullDateString, systemImage: "calendar") + } + .font(.caption) + .foregroundColor(.secondary) + } + Spacer() + HStack(spacing: 8) { + Button(action: { self.copyToClipboard(entry.text) }) { + Image(systemName: "doc.on.doc") + } + .help("Copy to clipboard") + Button(action: { + self.exportResult = result + self.showingExportDialog = true + }) { + Image(systemName: "square.and.arrow.up") + } + .help("Export transcription") + Button(action: { + self.fileHistoryStore.deleteEntry(id: entry.id) + }) { + Image(systemName: "trash") + } + .help("Remove from history") + } + .buttonStyle(.borderless) + } + Divider() + ScrollView { + Text(entry.text) + .font(.body) + .textSelection(.enabled) + .frame(maxWidth: .infinity, alignment: .leading) + .padding() + } + .frame(maxHeight: 300) + .background( + RoundedRectangle(cornerRadius: 8, style: .continuous) + .fill(self.theme.palette.contentBackground) + .overlay( + RoundedRectangle(cornerRadius: 8, style: .continuous) + .stroke(self.theme.palette.cardBorder.opacity(0.45), lineWidth: 1) + ) + ) + } + .padding() + .background( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill(self.theme.palette.cardBackground) + .overlay( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .stroke(self.theme.palette.cardBorder.opacity(0.45), lineWidth: 1) + ) + ) + } + + // MARK: - Error Card + + private func errorCard(error: String) -> some View { + HStack { + Image(systemName: "exclamationmark.triangle.fill") + .foregroundColor(.red) + + Text(error) + .font(.subheadline) + + Spacer() + + Button("Dismiss") { + self.transcriptionService.reset() + } + .buttonStyle(.borderless) + } + .padding() + .background( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill(Color.red.opacity(0.12)) + .overlay( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .stroke(Color.red.opacity(0.4), lineWidth: 1) + ) + ) + } + + // MARK: - Drop Error Card + + private func dropErrorCard(message: String) -> some View { + HStack { + Image(systemName: "exclamationmark.triangle.fill") + .foregroundColor(.red) + + Text(message) + .font(.subheadline) + + Spacer() + + Button("Dismiss") { + self.dropErrorMessage = nil + } + .buttonStyle(.borderless) + } + .padding() + .background( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill(Color.red.opacity(0.12)) + .overlay( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .stroke(Color.red.opacity(0.4), lineWidth: 1) + ) + ) + } + + // MARK: - Helper Functions + + private static let supportedFileExtensions = FileTranscriptionService.supportedFileExtensions + + private static let dropErrorCopy = FileTranscriptionService.dropErrorCopy + + private func handleDrop(providers: [NSItemProvider]) -> Bool { + guard let provider = providers.first else { return false } + provider.loadItem(forTypeIdentifier: UTType.fileURL.identifier, options: nil) { item, _ in + let url: URL? = (item as? URL) ?? (item as? Data).flatMap { URL(dataRepresentation: $0, relativeTo: nil) } + guard let url = url else { return } + let ext = url.pathExtension.lowercased() + guard Self.supportedFileExtensions.contains(ext) else { + DispatchQueue.main.async { + self.dropErrorMessage = Self.dropErrorCopy + DispatchQueue.main.asyncAfter(deadline: .now() + 3) { + self.dropErrorMessage = nil + } + } + return + } + DispatchQueue.main.async { + self.selectedFileURL = url + self.transcriptionService.reset() + self.dropErrorMessage = nil + } + } + return true + } + + private func transcribeFile() async { + guard let fileURL = selectedFileURL else { return } + + do { + _ = try await self.transcriptionService.transcribeFile(fileURL) + } catch { + DebugLogger.shared.error("Transcription error: \(error)", source: "FileTranscriptionView") + } + } + + private func formatFileSize(fileURL: URL) -> String { + guard let attributes = try? FileManager.default.attributesOfItem(atPath: fileURL.path), + let fileSize = attributes[.size] as? Int64 + else { + return "Unknown size" + } + + let formatter = ByteCountFormatter() + formatter.countStyle = .file + return formatter.string(fromByteCount: fileSize) + } + + private func copyToClipboard(_ text: String) { + NSPasteboard.general.clearContents() + NSPasteboard.general.setString(text, forType: .string) + + withAnimation { + self.showingCopyConfirmation = true + } + + DispatchQueue.main.asyncAfter(deadline: .now() + 2) { + withAnimation { + self.showingCopyConfirmation = false + } + } + } +} + +// MARK: - Document for Export + +struct TranscriptionDocument: FileDocument { + static var readableContentTypes: [UTType] { + [.plainText, .json] + } + + let result: TranscriptionResult + let format: FileTranscriptionView.ExportFormat + let service: FileTranscriptionService + + init( + result: TranscriptionResult, + format: FileTranscriptionView.ExportFormat, + service: FileTranscriptionService + ) { + self.result = result + self.format = format + self.service = service + } + + init(configuration: ReadConfiguration) throws { + throw CocoaError(.fileReadUnknown) + } + + func fileWrapper(configuration: WriteConfiguration) throws -> FileWrapper { + let tempURL = FileManager.default.temporaryDirectory.appendingPathComponent("temp.\(self.format.fileExtension)") + + switch self.format { + case .text: + try self.service.exportToText(self.result, to: tempURL) + case .json: + try self.service.exportToJSON(self.result, to: tempURL) + } + + let data = try Data(contentsOf: tempURL) + try? FileManager.default.removeItem(at: tempURL) + + return FileWrapper(regularFileWithContents: data) + } +} + +// MARK: - Preview + +#Preview { + let asrService = ASRService() + FileTranscriptionView( + asrService: asrService, + transcriptionService: FileTranscriptionService(asrService: asrService) + ) + .frame(width: 700, height: 800) +} diff --git a/Sources/Fluid/UI/MeetingTranscriptionView.swift b/Sources/Fluid/UI/MeetingTranscriptionView.swift index 129889b1..0896335c 100644 --- a/Sources/Fluid/UI/MeetingTranscriptionView.swift +++ b/Sources/Fluid/UI/MeetingTranscriptionView.swift @@ -1,738 +1,2036 @@ +import AppKit +import AVFoundation +import CoreGraphics import SwiftUI -import UniformTypeIdentifiers + +nonisolated struct MeetingApplicationOption: Identifiable, Equatable, Sendable { + let id: String + let identity: MeetingApplicationIdentity + + init(identity: MeetingApplicationIdentity) { + self.identity = identity + self.id = "\(identity.bundleIdentifier)-\(identity.processID ?? 0)" + } +} + +struct MeetingMicrophoneOption: Identifiable, Equatable { + let id: String + let identity: MeetingMicrophoneIdentity + + init(identity: MeetingMicrophoneIdentity) { + self.identity = identity + self.id = identity.captureDeviceID + } +} + +struct MeetingTranscriptionSetupDraft: Equatable { + var mode: MeetingCaptureMode = .onlineCall + var title: String = Self.defaultTitle() + var selectedApplicationID: String? + var selectedMicrophoneID: String? + var microphoneRole: MeetingMicrophoneRole = .unknown + + init(settings: SettingsStore = .shared) { + let defaults = settings.meetingRecordingDefaults + self.mode = defaults.mode + self.title = Self.defaultTitle() + self.selectedApplicationID = nil + self.selectedMicrophoneID = defaults.microphoneCaptureDeviceID + self.microphoneRole = defaults.microphoneRole + } + + static func defaultTitle(now: Date = Date()) -> String { + "Meeting · \(now.formatted(date: .abbreviated, time: .shortened))" + } +} + +struct MeetingSetupReadiness: Equatable { + var isCheckingSources: Bool + var meetingAudioStatus: String + var meetingAudioReady: Bool + var microphoneStatus: String + var microphoneReady: Bool + var modelStatus: String + var modelReady: Bool + var storageStatus: String + var storageReady: Bool + var activityStatus: String + var activityReady: Bool + var showMicrophoneSettingsAction: Bool + var showScreenRecordingSettingsAction: Bool + var blockingMessage: String? + + static let checking = Self( + isCheckingSources: true, + meetingAudioStatus: "Checking…", + meetingAudioReady: false, + microphoneStatus: "Checking…", + microphoneReady: false, + modelStatus: "Available after recording", + modelReady: false, + storageStatus: "Checking…", + storageReady: false, + activityStatus: "Checking…", + activityReady: false, + showMicrophoneSettingsAction: false, + showScreenRecordingSettingsAction: false, + blockingMessage: "Checking recording access and sources." + ) +} struct MeetingTranscriptionView: View { - let asrService: ASRService - @StateObject private var transcriptionService: MeetingTranscriptionService - @ObservedObject private var fileHistoryStore = FileTranscriptionHistoryStore.shared - @ObservedObject private var settings = SettingsStore.shared - @State private var selectedFileURL: URL? + @ObservedObject var coordinator: MeetingSessionCoordinator + @ObservedObject var asrService: ASRService + let onOpenVoiceEngine: () -> Void + @Environment(\.theme) private var theme + @Environment(\.accessibilityReduceMotion) private var accessibilityReduceMotion + + @State private var setupDraft: MeetingTranscriptionSetupDraft + @State private var setupDraftBeforeEditing: MeetingTranscriptionSetupDraft + @State private var isShowingMeetingSettings: Bool + @State private var applications: [MeetingApplicationOption] = [] + @State private var microphones: [MeetingMicrophoneOption] = [] + @State private var isRefreshingSources = false + @State private var isStarting = false + @State private var isStopping = false + @State private var isRetrying = false + @State private var actionErrorMessage: String? + @State private var cachedMicrophoneStatus: AVAuthorizationStatus = .notDetermined + @State private var cachedScreenCaptureAccess = false + @State private var cachedModelReady = false + @State private var cachedStorageStatus = "Checking…" + @State private var cachedStorageReady = false + @State private var meetingHistory: [MeetingSession] = [] + @State private var selectedHistorySessionID: MeetingSessionID? + @State private var meetingHistoryError: String? + @AppStorage("MeetingHistoryInspectorVisible") private var isMeetingHistoryVisible = true - init(asrService: ASRService) { + init( + coordinator: MeetingSessionCoordinator, + asrService: ASRService, + onOpenVoiceEngine: @escaping () -> Void + ) { + self.coordinator = coordinator self.asrService = asrService - _transcriptionService = StateObject(wrappedValue: MeetingTranscriptionService(asrService: asrService)) + self.onOpenVoiceEngine = onOpenVoiceEngine + + let initialDraft = MeetingTranscriptionSetupDraft(settings: .shared) + self._setupDraft = State(initialValue: initialDraft) + self._setupDraftBeforeEditing = State(initialValue: initialDraft) + self._isShowingMeetingSettings = State(initialValue: !SettingsStore.shared.meetingRecordingDefaults.isConfigured) + } + + var body: some View { + VStack(spacing: 0) { + MeetingTranscriptionHeader( + state: self.canvasState, + isMeetingHistoryVisible: self.isMeetingHistoryVisible, + onNewMeeting: self.startNewMeeting, + onOpenMeetingSettings: self.openMeetingSettings, + onToggleMeetingHistory: { + let willShowHistory = !self.isMeetingHistoryVisible + withAnimation(self.accessibilityReduceMotion ? nil : .easeInOut(duration: 0.2)) { + self.isMeetingHistoryVisible.toggle() + } + if willShowHistory, self.meetingHistory.isEmpty { + Task { await self.loadMeetingHistory() } + } + } + ) + + Divider() + + HStack(spacing: 0) { + MeetingTranscriptionCanvas( + setupDraft: self.$setupDraft, + state: self.canvasState, + applications: self.applications, + microphones: self.microphones, + readiness: self.readiness, + errorMessage: self.actionErrorMessage, + onRefreshSources: self.refreshSourcesFromUserAction, + onStart: self.startRecording, + onStop: self.stopAndTranscribe, + onRetry: self.retryProcessing, + onRevealAudio: self.revealCapturedAudio, + onCopyTranscript: self.copyTranscript, + onOpenMeetingSettings: self.openMeetingSettings, + isRetrying: self.isRetrying + ) + + if self.isMeetingHistoryVisible { + Divider() + MeetingHistoryInspector( + sessions: self.meetingHistory, + selectedSessionID: self.$selectedHistorySessionID, + errorMessage: self.meetingHistoryError, + onRefresh: { Task { await self.loadMeetingHistory() } } + ) + .frame(width: 290) + .transition(.move(edge: .trailing).combined(with: .opacity)) + } + } + } + .background(self.theme.palette.windowBackground) + .clipped() + .task { + async let sources: Void = self.refreshSources(requestPermissions: false) + if self.isMeetingHistoryVisible { + await self.loadMeetingHistory() + } + await sources + } + .onChange(of: self.setupDraft.mode) { _, _ in + Task { await self.refreshSources(requestPermissions: false) } + } + .onChange(of: self.asrService.isAsrReady) { _, isReady in + self.cachedModelReady = isReady || self.asrService.modelsExistOnDisk + } + .onChange(of: self.asrService.modelsExistOnDisk) { _, existsOnDisk in + self.cachedModelReady = self.asrService.isAsrReady || existsOnDisk + } + .onChange(of: self.coordinator.latestCompletedSession?.id) { _, _ in + Task { await self.loadMeetingHistory() } + } + .onReceive(NotificationCenter.default.publisher(for: NSApplication.didBecomeActiveNotification)) { _ in + Task { + async let sources: Void = self.refreshSources(requestPermissions: false) + if self.isMeetingHistoryVisible { + await self.loadMeetingHistory() + } + await sources + } + } + .sheet(isPresented: self.$isShowingMeetingSettings) { + MeetingRecordingSettingsSheet( + draft: self.$setupDraft, + applications: self.applications, + microphones: self.microphones, + readiness: self.readiness, + isFirstSetup: !SettingsStore.shared.meetingRecordingDefaults.isConfigured, + onRefreshSources: self.refreshSourcesFromUserAction, + onOpenMicrophoneSettings: { Self.openMicrophoneSettings() }, + onOpenScreenRecordingSettings: { Self.openScreenRecordingSettings() }, + onOpenVoiceEngine: self.onOpenVoiceEngine, + onCancel: self.cancelMeetingSettings, + onSave: self.saveMeetingSettings + ) + .interactiveDismissDisabled(!SettingsStore.shared.meetingRecordingDefaults.isConfigured) + } + } + + private var canvasState: MeetingTranscriptionCanvasState { + if let selectedHistorySession, self.canBrowseMeetingHistory { + return .result(selectedHistorySession) + } + + switch self.coordinator.state { + case .idle: + return .setup(isStarting: self.isStarting, recentSession: self.coordinator.latestCompletedSession) + case .preparing: + return .setup(isStarting: true, recentSession: self.coordinator.latestCompletedSession) + case .recording, .recordingDegraded: + guard let session = self.coordinator.activeSession else { + return .failed(session: nil, message: "The active meeting could not be loaded.") + } + if self.isStopping { + return .stopping(session: session, trackHealth: self.coordinator.trackHealth) + } + return .recording(session: session, trackHealth: self.coordinator.trackHealth) + case .stopping: + guard let session = self.coordinator.activeSession else { + return .failed(session: nil, message: "The meeting is stopping, but its session could not be loaded.") + } + return .stopping(session: session, trackHealth: self.coordinator.trackHealth) + case let .processing(_, stage): + guard let session = self.coordinator.activeSession else { + return .failed(session: nil, message: "The meeting is processing, but its session could not be loaded.") + } + return .processing(session: session, stage: stage) + case .completed: + guard let session = self.coordinator.latestCompletedSession else { + return .failed(session: nil, message: "The completed transcript could not be loaded.") + } + return .result(session) + case .interrupted: + let session = self.coordinator.activeSession + let message = session?.endedAt != nil && session?.processingAttempts.isEmpty == false + ? "Transcription was interrupted. Captured audio is ready to retry." + : "Recording was interrupted. Captured audio has been preserved on this Mac." + return .failed( + session: session, + message: message + ) + case let .failed(_, failure): + return .failed(session: self.coordinator.activeSession, message: failure.message) + } + } + + private var selectedHistorySession: MeetingSession? { + guard let selectedHistorySessionID else { return nil } + return self.meetingHistory.first(where: { $0.id == selectedHistorySessionID }) + } + + private var canBrowseMeetingHistory: Bool { + switch self.coordinator.state { + case .idle, .completed, .interrupted, .failed: + return true + case .preparing, .recording, .recordingDegraded, .stopping, .processing: + return false + } + } + + private var readiness: MeetingSetupReadiness { + let microphoneStatus = self.cachedMicrophoneStatus + let microphoneReady = microphoneStatus == .authorized + let meetingAudioReady = self.setupDraft.mode == .inRoom || self.cachedScreenCaptureAccess + let modelReady = self.cachedModelReady + let conflictingActivity = self.asrService.activeExclusiveActivity + let activityReady = conflictingActivity == nil + + let microphoneStatusText: String + switch microphoneStatus { + case .authorized: + microphoneStatusText = self.microphones.isEmpty ? "No microphone found" : "Ready" + case .notDetermined: + microphoneStatusText = "Access required" + case .denied: + microphoneStatusText = "Access denied" + case .restricted: + microphoneStatusText = "Access restricted" + @unknown default: + microphoneStatusText = "Access unavailable" + } + + let blockingMessage: String? + if self.isRefreshingSources { + blockingMessage = "Checking recording access and sources." + } else if let conflictingActivity { + blockingMessage = "Wait for the active \(conflictingActivity.displayName) to finish." + } else if microphoneStatus == .restricted { + blockingMessage = "Microphone access is restricted by system policy." + } else if !microphoneReady { + blockingMessage = "Allow microphone access, then refresh sources." + } else if self.microphones.isEmpty { + blockingMessage = "Connect a microphone, then refresh sources." + } else if self.setupDraft.mode == .onlineCall, !meetingAudioReady { + blockingMessage = "Allow Screen & System Audio access, then refresh sources." + } else if !self.cachedStorageReady { + blockingMessage = "Free at least 512 MB of storage before recording." + } else { + blockingMessage = nil + } + + return MeetingSetupReadiness( + isCheckingSources: self.isRefreshingSources, + meetingAudioStatus: meetingAudioReady ? "Ready" : "Access required", + meetingAudioReady: meetingAudioReady, + microphoneStatus: microphoneStatusText, + microphoneReady: microphoneReady && !self.microphones.isEmpty, + modelStatus: modelReady ? "Installed · runs after Stop" : "Will prepare after Stop", + modelReady: modelReady, + storageStatus: self.cachedStorageStatus, + storageReady: self.cachedStorageReady, + activityStatus: conflictingActivity.map { "Wait for \($0.displayName)" } ?? "Ready", + activityReady: activityReady, + showMicrophoneSettingsAction: microphoneStatus == .denied, + showScreenRecordingSettingsAction: self.setupDraft.mode == .onlineCall && !meetingAudioReady, + blockingMessage: blockingMessage + ) + } + + private func refreshSourcesFromUserAction() { + Task { await self.refreshSources(requestPermissions: true) } + } + + @MainActor + private func refreshSources(requestPermissions: Bool) async { + guard !self.isRefreshingSources else { return } + self.isRefreshingSources = true + self.refreshCachedReadiness() + defer { + self.refreshCachedReadiness() + self.isRefreshingSources = false + } + + self.actionErrorMessage = nil + + if requestPermissions, AVCaptureDevice.authorizationStatus(for: .audio) == .notDetermined { + _ = await Self.requestMicrophoneAccess() + } + + if AVCaptureDevice.authorizationStatus(for: .audio) == .authorized { + let identities = MeetingCaptureSourceCatalog.availableMicrophones() + self.microphones = identities.map(MeetingMicrophoneOption.init) + self.selectPreferredMicrophone(from: identities) + } else { + self.microphones = [] + self.setupDraft.selectedMicrophoneID = nil + } + + guard self.setupDraft.mode == .onlineCall else { return } + + if requestPermissions, !CGPreflightScreenCaptureAccess() { + _ = CGRequestScreenCaptureAccess() + } + + guard CGPreflightScreenCaptureAccess() else { + self.applications = [] + self.setupDraft.selectedApplicationID = nil + return + } + + do { + let identities = try await MeetingCaptureSourceCatalog.availableApplications() + self.applications = identities.map(MeetingApplicationOption.init) + self.selectPreferredApplication(from: identities) + } catch { + self.applications = [] + self.setupDraft.selectedApplicationID = nil + self.actionErrorMessage = error.localizedDescription + } + } + + private func selectPreferredMicrophone(from identities: [MeetingMicrophoneIdentity]) { + if let selectedID = setupDraft.selectedMicrophoneID, + identities.contains(where: { $0.captureDeviceID == selectedID }) + { + return + } + + let settings = SettingsStore.shared + let defaults = settings.meetingRecordingDefaults + let preferredCoreAudioUID = defaults.microphoneCoreAudioUID ?? settings.preferredInputDeviceUID + let savedMicrophone = defaults.savedMicrophone(in: identities) + if defaults.isConfigured { + self.setupDraft.selectedMicrophoneID = savedMicrophone?.captureDeviceID + return + } + + let preferred = savedMicrophone ?? (try? MeetingCaptureSourceCatalog.defaultMicrophone( + preferredCoreAudioUID: preferredCoreAudioUID + )) + self.setupDraft.selectedMicrophoneID = preferred?.captureDeviceID ?? identities.first?.captureDeviceID + } + + private func selectPreferredApplication(from identities: [MeetingApplicationIdentity]) { + let options = identities.map(MeetingApplicationOption.init) + if let selectedID = setupDraft.selectedApplicationID, + options.contains(where: { $0.id == selectedID }) + { + return + } + + let defaults = SettingsStore.shared.meetingRecordingDefaults + if defaults.isConfigured { + let savedIdentity = defaults.savedApplication(in: identities) + self.setupDraft.selectedApplicationID = savedIdentity.map { MeetingApplicationOption(identity: $0).id } + return + } + + let preferredBundleIdentifiers = [ + "us.zoom.xos", + "com.google.Chrome", + "com.microsoft.teams2", + ] + let preferredIdentity = preferredBundleIdentifiers.lazy.compactMap { bundleIdentifier in + identities.first(where: { $0.bundleIdentifier == bundleIdentifier }) + }.first ?? identities.first + self.setupDraft.selectedApplicationID = preferredIdentity.map { MeetingApplicationOption(identity: $0).id } + } + + private func startRecording() { + guard !self.isStarting else { return } + let readiness = self.readiness + guard readiness.activityReady, + readiness.storageReady, + readiness.microphoneReady, + self.setupDraft.mode == .inRoom || readiness.meetingAudioReady, + let configuration = self.captureConfiguration + else { + self.actionErrorMessage = readiness.blockingMessage ?? "Finish meeting setup before recording." + return + } + self.isStarting = true + self.actionErrorMessage = nil + + Task { + defer { self.isStarting = false } + do { + _ = try await self.coordinator.startRecording(configuration: configuration) + } catch { + self.actionErrorMessage = error.localizedDescription + } + } + } + + private var captureConfiguration: MeetingCaptureConfiguration? { + let title = self.setupDraft.title.trimmingCharacters(in: .whitespacesAndNewlines) + guard !title.isEmpty, + let microphoneOption = self.microphones.first(where: { $0.id == self.setupDraft.selectedMicrophoneID }) + else { + return nil + } + + var microphone = microphoneOption.identity + microphone.role = self.setupDraft.mode == .onlineCall ? self.setupDraft.microphoneRole : .unknown + + let applicationOption = self.applications.first(where: { $0.id == self.setupDraft.selectedApplicationID }) + if self.setupDraft.mode == .onlineCall, applicationOption == nil { return nil } + let application = self.setupDraft.mode == .onlineCall ? applicationOption?.identity : nil + + return MeetingCaptureConfiguration( + mode: self.setupDraft.mode, + title: title, + platform: application.map { + MeetingPlatformProfile(identifier: $0.bundleIdentifier, displayName: $0.displayName) + }, + application: application, + microphone: microphone + ) + } + + private func stopAndTranscribe() { + guard !self.isStopping else { return } + self.isStopping = true + self.actionErrorMessage = nil + Task { + defer { self.isStopping = false } + do { + _ = try await self.coordinator.stopAndTranscribe() + } catch { + self.actionErrorMessage = error.localizedDescription + } + } + } + + private func retryProcessing() { + guard !self.isRetrying else { return } + self.isRetrying = true + self.actionErrorMessage = nil + Task { + defer { self.isRetrying = false } + do { + _ = try await self.coordinator.retryProcessing() + } catch { + self.actionErrorMessage = error.localizedDescription + } + } + } + + private func revealCapturedAudio(_ session: MeetingSession) { + Task { + do { + let directory = try await MeetingSessionStore.shared.sessionDirectory(for: session.id) + let firstAudioURL = session.audioTracks + .flatMap(\.chunks) + .first(where: { $0.finalizationState == .finalized })? + .fileURL(relativeTo: directory) + NSWorkspace.shared.activateFileViewerSelecting([firstAudioURL ?? directory]) + } catch { + self.actionErrorMessage = "Captured audio could not be revealed: \(error.localizedDescription)" + } + } + } + + private func startNewMeeting() { + do { + try self.coordinator.resetForNewMeeting() + self.selectedHistorySessionID = nil + self.setupDraft.title = MeetingTranscriptionSetupDraft.defaultTitle() + self.actionErrorMessage = nil + } catch { + self.actionErrorMessage = error.localizedDescription + } + } + + @MainActor + private func loadMeetingHistory() async { + do { + self.meetingHistory = try await MeetingSessionStore.shared.loadAll() + self.meetingHistoryError = nil + if let selectedHistorySessionID, + !self.meetingHistory.contains(where: { $0.id == selectedHistorySessionID }) + { + self.selectedHistorySessionID = nil + } + } catch { + self.meetingHistoryError = "Meeting history could not be loaded." + } + } + + private func openMeetingSettings() { + self.setupDraftBeforeEditing = self.setupDraft + self.isShowingMeetingSettings = true + } + + private func cancelMeetingSettings() { + self.setupDraft = self.setupDraftBeforeEditing + self.isShowingMeetingSettings = false + Task { await self.refreshSources(requestPermissions: false) } + } + + private func saveMeetingSettings() { + guard let microphone = self.microphones.first(where: { $0.id == self.setupDraft.selectedMicrophoneID }) else { + self.actionErrorMessage = "Choose an available microphone before saving." + return + } + + let application = self.applications.first(where: { $0.id == self.setupDraft.selectedApplicationID }) + if self.setupDraft.mode == .onlineCall, application == nil { + self.actionErrorMessage = "Choose an available meeting application before saving." + return + } + + let settings = SettingsStore.shared + settings.meetingRecordingDefaults = MeetingRecordingDefaults( + isConfigured: true, + mode: self.setupDraft.mode, + applicationBundleIdentifier: application?.identity.bundleIdentifier, + microphoneCaptureDeviceID: microphone.identity.captureDeviceID, + microphoneCoreAudioUID: microphone.identity.coreAudioUID, + microphoneRole: self.setupDraft.microphoneRole + ) + + self.setupDraftBeforeEditing = self.setupDraft + self.actionErrorMessage = nil + self.isShowingMeetingSettings = false + } + + private func copyTranscript(_ session: MeetingSession) { + let speakerNames = Dictionary(uniqueKeysWithValues: session.speakers.map { ($0.id, $0.displayName) }) + let text = session.transcriptSegments.map { segment in + let speaker = segment.speakerID.flatMap { speakerNames[$0] } ?? "Unknown speaker" + return "[\(Self.timestampText(segment.start.seconds))] \(speaker): \(segment.text)" + }.joined(separator: "\n\n") + guard !text.isEmpty else { return } + NSPasteboard.general.clearContents() + NSPasteboard.general.setString(text, forType: .string) + } + + private static func requestMicrophoneAccess() async -> Bool { + await withCheckedContinuation { continuation in + AVCaptureDevice.requestAccess(for: .audio) { granted in + continuation.resume(returning: granted) + } + } + } + + private static func openMicrophoneSettings() { + guard let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone") + else { return } + NSWorkspace.shared.open(url) + } + + private static func openScreenRecordingSettings() { + guard let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture") + else { return } + NSWorkspace.shared.open(url) + } + + private func refreshCachedReadiness() { + self.cachedMicrophoneStatus = AVCaptureDevice.authorizationStatus(for: .audio) + self.cachedScreenCaptureAccess = CGPreflightScreenCaptureAccess() + self.cachedModelReady = self.asrService.isAsrReady || + self.asrService.modelsExistOnDisk || + SettingsStore.shared.selectedSpeechModel.isInstalled + let storage = Self.storageReadiness() + self.cachedStorageStatus = storage.status + self.cachedStorageReady = storage.ready + } + + private static func storageReadiness() -> (status: String, ready: Bool) { + guard let applicationSupport = FileManager.default.urls( + for: .applicationSupportDirectory, + in: .userDomainMask + ).first, + let capacity = try? applicationSupport.resourceValues( + forKeys: [.volumeAvailableCapacityForImportantUsageKey] + ).volumeAvailableCapacityForImportantUsage + else { + return ("Storage availability unavailable", false) + } + let requiredBytes: Int64 = 512 * 1024 * 1024 + let formatter = ByteCountFormatter() + formatter.countStyle = .file + return ("\(formatter.string(fromByteCount: capacity)) available", capacity >= requiredBytes) + } + + private static func timestampText(_ seconds: TimeInterval) -> String { + let total = max(0, Int(seconds.rounded(.down))) + let hours = total / 3600 + let minutes = (total % 3600) / 60 + let seconds = total % 60 + if hours > 0 { + return String(format: "%d:%02d:%02d", hours, minutes, seconds) + } + return String(format: "%02d:%02d", minutes, seconds) + } +} + +enum MeetingTranscriptionCanvasState { + case setup(isStarting: Bool, recentSession: MeetingSession?) + case recording(session: MeetingSession, trackHealth: [MeetingAudioTrackKind: MeetingTrackHealth]) + case stopping(session: MeetingSession, trackHealth: [MeetingAudioTrackKind: MeetingTrackHealth]) + case processing(session: MeetingSession, stage: MeetingProcessingStage) + case result(MeetingSession) + case failed(session: MeetingSession?, message: String) +} + +struct MeetingTranscriptionCanvas: View { + @Binding var setupDraft: MeetingTranscriptionSetupDraft + + let state: MeetingTranscriptionCanvasState + let applications: [MeetingApplicationOption] + let microphones: [MeetingMicrophoneOption] + let readiness: MeetingSetupReadiness + let errorMessage: String? + let onRefreshSources: () -> Void + let onStart: () -> Void + let onStop: () -> Void + let onRetry: () -> Void + let onRevealAudio: (MeetingSession) -> Void + let onCopyTranscript: (MeetingSession) -> Void + let onOpenMeetingSettings: () -> Void + let isRetrying: Bool + + @Environment(\.theme) private var theme + + var body: some View { + ScrollView { + Group { + switch self.state { + case let .setup(isStarting, recentSession): + MeetingSetupCanvas( + draft: self.$setupDraft, + applications: self.applications, + microphones: self.microphones, + readiness: self.readiness, + isStarting: isStarting, + errorMessage: self.errorMessage, + recentSession: recentSession, + onRefreshSources: self.onRefreshSources, + onStart: self.onStart, + onOpenMeetingSettings: self.onOpenMeetingSettings + ) + case let .recording(session, trackHealth): + MeetingRecordingCanvas( + session: session, + trackHealth: trackHealth, + isStopping: false, + onStop: self.onStop + ) + case let .stopping(session, trackHealth): + MeetingRecordingCanvas( + session: session, + trackHealth: trackHealth, + isStopping: true, + onStop: self.onStop + ) + case let .processing(session, stage): + MeetingProcessingCanvas(session: session, stage: stage) + case let .result(session): + MeetingResultCanvas( + session: session, + onCopyTranscript: self.onCopyTranscript + ) + case let .failed(session, message): + MeetingFailureCanvas( + session: session, + message: self.errorMessage ?? message, + isRetrying: self.isRetrying, + onRetry: self.onRetry, + onRevealAudio: self.onRevealAudio + ) + } + } + .frame(maxWidth: 820) + .padding(self.theme.metrics.spacing.xxl) + .frame(maxWidth: .infinity) + } + .frame(maxWidth: .infinity, maxHeight: .infinity) + .background(self.theme.palette.windowBackground) + } +} + +private struct MeetingTranscriptionHeader: View { + let state: MeetingTranscriptionCanvasState + let isMeetingHistoryVisible: Bool + let onNewMeeting: () -> Void + let onOpenMeetingSettings: () -> Void + let onToggleMeetingHistory: () -> Void + + @Environment(\.theme) private var theme + + var body: some View { + HStack(spacing: self.theme.metrics.spacing.md) { + Image(systemName: "person.2.wave.2.fill") + .font(self.theme.typography.titleIcon) + .foregroundStyle(self.theme.palette.accent) + + Text("Meeting Transcription") + .font(self.theme.typography.title) + .foregroundStyle(self.theme.palette.primaryText) + + Spacer() + + HStack(spacing: self.theme.metrics.spacing.xs) { + if self.canStartNewMeeting { + MeetingHeaderIconButton( + systemImage: "plus", + label: "New Meeting", + action: self.onNewMeeting + ) + .keyboardShortcut("n", modifiers: .command) + .accessibilityHint("Clear the current meeting and return to recording setup") + } + + MeetingHeaderIconButton( + systemImage: "gearshape", + label: "Meeting Settings", + action: self.onOpenMeetingSettings + ) + .accessibilityHint("Change the saved recording application, microphone, and meeting defaults") + + MeetingHeaderIconButton( + systemImage: self.isMeetingHistoryVisible + ? "rectangle.righthalf.inset.filled" + : "sidebar.right", + label: self.isMeetingHistoryVisible ? "Hide meeting history" : "Show meeting history", + isSelected: self.isMeetingHistoryVisible, + action: self.onToggleMeetingHistory + ) + } + } + .padding(.horizontal, self.theme.metrics.spacing.xxl) + .padding(.vertical, self.theme.metrics.spacing.lg) + } + + private var canStartNewMeeting: Bool { + switch self.state { + case .result, .failed: + return true + case .setup, .recording, .stopping, .processing: + return false + } + } +} + +private struct MeetingHeaderIconButton: View { + let systemImage: String + let label: String + var isSelected = false + let action: () -> Void + + @Environment(\.theme) private var theme + @State private var isHovered = false + + var body: some View { + Button(action: self.action) { + Image(systemName: self.systemImage) + .font(.system(size: 14, weight: .semibold)) + .foregroundStyle(self.isSelected ? self.theme.palette.accent : self.theme.palette.primaryText) + .frame(width: 32, height: 30) + .background(self.backgroundColor, in: RoundedRectangle( + cornerRadius: self.theme.metrics.corners.md, + style: .continuous + )) + .overlay { + RoundedRectangle(cornerRadius: self.theme.metrics.corners.md, style: .continuous) + .stroke(self.theme.palette.separator.opacity(0.55), lineWidth: 1) + } + } + .buttonStyle(.plain) + .contentShape(RoundedRectangle(cornerRadius: self.theme.metrics.corners.md, style: .continuous)) + .onHover { self.isHovered = $0 } + .help(self.label) + .accessibilityLabel(self.label) + } + + private var backgroundColor: Color { + if self.isSelected { + return self.theme.palette.accent.opacity(0.14) + } + if self.isHovered { + return self.theme.palette.contentBackground.opacity(0.9) + } + return self.theme.palette.contentBackground.opacity(0.55) + } +} + +private struct MeetingHistoryInspector: View { + let sessions: [MeetingSession] + @Binding var selectedSessionID: MeetingSessionID? + let errorMessage: String? + let onRefresh: () -> Void + + @Environment(\.theme) private var theme + @State private var searchText = "" + + private var filteredSessions: [MeetingSession] { + let query = self.searchText.trimmingCharacters(in: .whitespacesAndNewlines) + guard !query.isEmpty else { return self.sessions } + return self.sessions.filter { session in + session.title.localizedCaseInsensitiveContains(query) || + session.capturedApplication?.displayName.localizedCaseInsensitiveContains(query) == true || + session.speakers.contains(where: { $0.displayName.localizedCaseInsensitiveContains(query) }) + } + } + + var body: some View { + VStack(spacing: 0) { + HStack(spacing: self.theme.metrics.spacing.sm) { + Text("Meetings") + .font(self.theme.typography.sectionTitle) + Text("\(self.sessions.count)") + .font(self.theme.typography.badge) + .foregroundStyle(self.theme.palette.secondaryText) + Spacer() + Button(action: self.onRefresh) { + Image(systemName: "arrow.clockwise") + } + .buttonStyle(.borderless) + .help("Refresh meeting history") + .accessibilityLabel("Refresh meeting history") + } + .padding(.horizontal, self.theme.metrics.spacing.lg) + .padding(.top, self.theme.metrics.spacing.lg) + .padding(.bottom, self.theme.metrics.spacing.md) + + TextField("Search meetings", text: self.$searchText) + .textFieldStyle(.roundedBorder) + .padding(.horizontal, self.theme.metrics.spacing.lg) + .padding(.bottom, self.theme.metrics.spacing.md) + + Divider() + + if let errorMessage { + ContentUnavailableView( + "History unavailable", + systemImage: "exclamationmark.triangle", + description: Text(errorMessage) + ) + } else if self.filteredSessions.isEmpty { + ContentUnavailableView( + self.searchText.isEmpty ? "No meetings yet" : "No matching meetings", + systemImage: self.searchText.isEmpty ? "person.2.wave.2" : "magnifyingglass", + description: Text(self.searchText.isEmpty + ? "Completed meetings will appear here." + : "Try a different title, app, or speaker.") + ) + } else { + List(selection: self.$selectedSessionID) { + ForEach(self.filteredSessions) { session in + MeetingHistoryRow(session: session) + .tag(session.id) + } + } + .listStyle(.sidebar) + .scrollContentBackground(.hidden) + } + } + .background(self.theme.palette.sidebarBackground) + } +} + +private struct MeetingHistoryRow: View { + let session: MeetingSession + + @Environment(\.theme) private var theme + + var body: some View { + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.xs) { + HStack(spacing: self.theme.metrics.spacing.sm) { + Image(systemName: self.statusIcon) + .foregroundStyle(self.statusColor) + .accessibilityHidden(true) + Text(self.session.title) + .font(self.theme.typography.bodySmallStrong) + .lineLimit(1) + Spacer(minLength: 0) + } + + Text(self.session.startedAt.formatted(date: .abbreviated, time: .shortened)) + .font(self.theme.typography.caption) + .foregroundStyle(self.theme.palette.secondaryText) + .lineLimit(1) + + HStack(spacing: self.theme.metrics.spacing.sm) { + Text(Self.durationText(self.session.duration)) + Text("·") + Text(self.sourceName) + .lineLimit(1) + if !self.session.speakers.isEmpty { + Text("·") + Text("\(self.session.speakers.count) speakers") + } + } + .font(self.theme.typography.caption) + .foregroundStyle(self.theme.palette.tertiaryText) + } + .padding(.vertical, self.theme.metrics.spacing.xs) + .accessibilityElement(children: .combine) + .accessibilityLabel("\(self.session.title), \(self.statusText), \(self.sourceName)") + } + + private var sourceName: String { + self.session.capturedApplication?.displayName ?? "In-room" + } + + private var statusText: String { + switch self.session.state { + case .completed: "Completed" + case .failed: "Failed" + case .interrupted: "Interrupted" + case .processing: "Processing" + case .recording, .recordingDegraded, .preparing, .stopping: "Recording" + } + } + + private var statusIcon: String { + switch self.session.state { + case .completed: "checkmark.circle.fill" + case .failed: "xmark.circle.fill" + case .interrupted: "exclamationmark.circle.fill" + case .processing: "ellipsis.circle.fill" + case .recording, .recordingDegraded, .preparing, .stopping: "record.circle.fill" + } + } + + private var statusColor: Color { + switch self.session.state { + case .completed: self.theme.palette.success + case .failed: Color(nsColor: .systemRed) + case .interrupted: self.theme.palette.warning + case .processing: self.theme.palette.accent + case .recording, .recordingDegraded, .preparing, .stopping: self.theme.palette.warning + } + } + + private static func durationText(_ duration: TimeInterval) -> String { + let totalMinutes = max(0, Int(duration / 60)) + if totalMinutes >= 60 { + return "\(totalMinutes / 60)h \(totalMinutes % 60)m" + } + return "\(max(1, totalMinutes))m" + } +} + +private struct MeetingRecordingSettingsSheet: View { + @Binding var draft: MeetingTranscriptionSetupDraft + + let applications: [MeetingApplicationOption] + let microphones: [MeetingMicrophoneOption] + let readiness: MeetingSetupReadiness + let isFirstSetup: Bool + let onRefreshSources: () -> Void + let onOpenMicrophoneSettings: @MainActor @Sendable () -> Void + let onOpenScreenRecordingSettings: @MainActor @Sendable () -> Void + let onOpenVoiceEngine: () -> Void + let onCancel: () -> Void + let onSave: () -> Void + + @Environment(\.theme) private var theme + + private var canSave: Bool { + guard self.draft.selectedMicrophoneID != nil else { return false } + return self.draft.mode == .inRoom || self.draft.selectedApplicationID != nil + } + + private var separationDescription: String { + if !CPUArchitecture.isAppleSilicon { + return "Plain transcript on Intel" + } + return self.readiness.modelReady ? "Automatic after Stop" : "Prepares after Stop" } - @State private var showingFilePicker = false - @State private var showingExportDialog = false - @State private var exportResult: TranscriptionResult? - @State private var exportFormat: ExportFormat = .text - @State private var showingCopyConfirmation = false - @State private var isDropTargeted = false - @State private var dropErrorMessage: String? + private var saveHelp: String? { + guard !self.canSave else { return nil } + if let blockingMessage = readiness.blockingMessage { + return blockingMessage + } + if self.draft.selectedMicrophoneID == nil { + return "Choose an available microphone to save this setup." + } + return "Choose an available meeting application to save this setup." + } + + var body: some View { + VStack(spacing: 0) { + HStack(spacing: self.theme.metrics.spacing.md) { + Image(systemName: "gearshape.fill") + .font(self.theme.typography.titleIcon) + .foregroundStyle(self.theme.palette.accent) + + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.xs) { + Text(self.isFirstSetup ? "Set up meeting recording" : "Meeting Settings") + .font(self.theme.typography.title) + Text("Saved for future meetings until you change it.") + .font(self.theme.typography.bodySmall) + .foregroundStyle(self.theme.palette.secondaryText) + } + Spacer() + } + .padding(.horizontal, self.theme.metrics.spacing.xl) + .padding(.vertical, self.theme.metrics.spacing.lg) + + Divider() + + ScrollView { + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.lg) { + ThemedCard(style: .subtle, padding: 0) { + VStack(spacing: 0) { + MeetingAdaptiveSetupRow( + title: "Meeting type", + detail: "Choose the setup you use most often." + ) { + Picker("Meeting type", selection: self.$draft.mode) { + Text("Online call").tag(MeetingCaptureMode.onlineCall) + Text("In-room").tag(MeetingCaptureMode.inRoom) + } + .pickerStyle(.segmented) + .labelsHidden() + .frame(width: 320, alignment: .trailing) + .accessibilityLabel("Default meeting type") + } + + if self.draft.mode == .onlineCall { + Divider() + MeetingAdaptiveSetupRow( + title: "Meeting audio", + detail: "FluidVoice records audio from this application." + ) { + Picker("Meeting audio", selection: self.$draft.selectedApplicationID) { + Text("Choose application…").tag(String?.none) + ForEach(self.applications) { option in + Text(option.identity.displayName).tag(Optional(option.id)) + } + } + .labelsHidden() + .frame(width: 320, alignment: .trailing) + .accessibilityLabel("Default meeting application") + } + } + + Divider() + MeetingAdaptiveSetupRow( + title: "Microphone", + detail: "Used for your voice and in-room meetings." + ) { + Picker("Microphone", selection: self.$draft.selectedMicrophoneID) { + Text("Choose microphone…").tag(String?.none) + ForEach(self.microphones) { option in + Text(option.identity.displayName).tag(Optional(option.id)) + } + } + .labelsHidden() + .frame(width: 320, alignment: .trailing) + .accessibilityLabel("Default meeting microphone") + } + + if self.draft.mode == .onlineCall { + Divider() + MeetingAdaptiveSetupRow( + title: "Microphone use", + detail: "A personal mic can identify clean speech as You." + ) { + Picker("Microphone use", selection: self.$draft.microphoneRole) { + Text("Only me").tag(MeetingMicrophoneRole.personal) + Text("Shared").tag(MeetingMicrophoneRole.shared) + Text("Not sure").tag(MeetingMicrophoneRole.unknown) + } + .labelsHidden() + .frame(width: 320, alignment: .trailing) + .accessibilityLabel("Default microphone use") + } + } + + Divider() + MeetingAdaptiveSetupRow(title: "Language") { + Text("English") + .font(self.theme.typography.bodyStrong) + .frame(maxWidth: .infinity, alignment: .leading) + } + + Divider() + MeetingAdaptiveSetupRow(title: "Speaker separation") { + Text(self.separationDescription) + .font(self.theme.typography.bodyStrong) + .frame(maxWidth: .infinity, alignment: .leading) + } + } + } + + ViewThatFits(in: .horizontal) { + HStack(spacing: self.theme.metrics.spacing.sm) { + self.repairActions + } + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.sm) { + self.repairActions + } + } - enum ExportFormat: String, CaseIterable { - case text = "Text (.txt)" - case json = "JSON (.json)" + Label("Recording and transcription stay on this Mac.", systemImage: "lock.fill") + .font(self.theme.typography.caption) + .foregroundStyle(self.theme.palette.secondaryText) - var fileExtension: String { - switch self { - case .text: return "txt" - case .json: return "json" + if let saveHelp { + Label(saveHelp, systemImage: "exclamationmark.circle") + .font(self.theme.typography.caption) + .foregroundStyle(self.theme.palette.warning) + } + } + .padding(self.theme.metrics.spacing.xl) } + + Divider() + + HStack(spacing: self.theme.metrics.spacing.md) { + Button(self.isFirstSetup ? "Not Now" : "Cancel", action: self.onCancel) + .fluidButton(.compact, size: .medium) + .keyboardShortcut(.cancelAction) + Spacer() + Button(self.readiness.isCheckingSources ? "Refreshing…" : "Refresh Sources", systemImage: "arrow.clockwise") { + self.onRefreshSources() + } + .fluidButton(.compact, size: .medium) + .disabled(self.readiness.isCheckingSources) + + Button(self.isFirstSetup ? "Save Setup" : "Save Changes", action: self.onSave) + .fluidButton(.accent, size: .medium) + .disabled(!self.canSave) + .keyboardShortcut(.defaultAction) + } + .padding(self.theme.metrics.spacing.lg) + } + .frame(minWidth: 560, idealWidth: 720, maxWidth: 720) + .frame(minHeight: 540, idealHeight: 620) + .background(self.theme.palette.windowBackground) + } + + @ViewBuilder + private var repairActions: some View { + if self.readiness.showMicrophoneSettingsAction { + Button("Microphone Settings", systemImage: "mic.fill", action: self.onOpenMicrophoneSettings) + .fluidButton(.compact, size: .small) + } + if self.readiness.showScreenRecordingSettingsAction { + Button( + "Screen Recording Settings", + systemImage: "rectangle.inset.filled.and.person.filled", + action: self.onOpenScreenRecordingSettings + ) + .fluidButton(.compact, size: .small) + } + if !self.readiness.modelReady { + Button("Voice Engine", systemImage: "waveform", action: self.onOpenVoiceEngine) + .fluidButton(.compact, size: .small) + } + } +} + +private struct MeetingSetupCanvas: View { + @Binding var draft: MeetingTranscriptionSetupDraft + + let applications: [MeetingApplicationOption] + let microphones: [MeetingMicrophoneOption] + let readiness: MeetingSetupReadiness + let isStarting: Bool + let errorMessage: String? + let recentSession: MeetingSession? + let onRefreshSources: () -> Void + let onStart: () -> Void + let onOpenMeetingSettings: () -> Void + + @Environment(\.theme) private var theme + + private var requiredSystemsReady: Bool { + !self.readiness.isCheckingSources && + self.readiness.microphoneReady && + self.readiness.storageReady && + self.readiness.activityReady && + (self.draft.mode == .inRoom || self.readiness.meetingAudioReady) + } + + private var canStart: Bool { + guard self.requiredSystemsReady, + self.draft.selectedMicrophoneID != nil, + !self.draft.title.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + else { + return false + } + return self.draft.mode == .inRoom || self.draft.selectedApplicationID != nil + } + + private var startHelp: String? { + if let blockingMessage = readiness.blockingMessage, !self.requiredSystemsReady { + return blockingMessage + } + if self.draft.selectedMicrophoneID == nil { + return "Choose a microphone to continue." + } + if self.draft.mode == .onlineCall, self.draft.selectedApplicationID == nil { + return "Choose the application playing meeting audio." + } + if self.draft.title.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + return "Add a title for this meeting." + } + return nil + } + + private var modeName: String { + self.draft.mode == .onlineCall ? "Online call" : "In-room meeting" + } + + private var applicationName: String? { + guard self.draft.mode == .onlineCall else { return nil } + return self.applications.first(where: { $0.id == self.draft.selectedApplicationID })?.identity.displayName + } + + private var microphoneName: String { + self.microphones.first(where: { $0.id == self.draft.selectedMicrophoneID })?.identity.displayName + ?? "Choose microphone" + } + + private var microphoneRoleName: String? { + guard self.draft.mode == .onlineCall else { return nil } + switch self.draft.microphoneRole { + case .personal: return "Personal mic" + case .shared: return "Shared mic" + case .unknown: return "Mic use not set" } } - private var selectedFileIsVideo: Bool { - guard let fileExtension = selectedFileURL?.pathExtension.lowercased() else { return false } - return UTType(filenameExtension: fileExtension)?.conforms(to: .movie) ?? false + private var setupSummary: String { + [self.applicationName, self.microphoneName, self.microphoneRoleName] + .compactMap { $0 } + .joined(separator: " · ") } var body: some View { - VStack(spacing: 0) { - // Header - VStack(spacing: 8) { - Image(systemName: "waveform.circle.fill") - .font(.system(size: 48)) - .foregroundStyle(Color.fluidGreen.gradient) - - Text("Meeting Transcription") - .font(.title2) - .fontWeight(.semibold) - - Text("Choose an audio or video file to transcribe") - .font(.subheadline) - .foregroundColor(.secondary) - } - .padding(.top, 40) - .padding(.bottom, 30) - - // Main Content Area - ScrollView { - VStack(spacing: 24) { - // File Selection Card - self.fileSelectionCard + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.xl) { + ThemedCard(style: .subtle, padding: 0) { + VStack(spacing: 0) { + ViewThatFits(in: .horizontal) { + HStack(alignment: .center, spacing: self.theme.metrics.spacing.lg) { + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.xs) { + Text(self.modeName) + .font(self.theme.typography.sectionTitle) + .foregroundStyle(self.theme.palette.primaryText) + Text(self.setupSummary) + .font(self.theme.typography.bodySmall) + .foregroundStyle(self.theme.palette.secondaryText) + .lineLimit(1) + .truncationMode(.middle) + } + Spacer(minLength: self.theme.metrics.spacing.lg) + Button("Edit setup…", systemImage: "slider.horizontal.3", action: self.onOpenMeetingSettings) + .fluidButton(.compact, size: .small) + } - // Progress Card (only show when transcribing) - if self.transcriptionService.isTranscribing { - self.progressCard + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.md) { + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.xs) { + Text(self.modeName) + .font(self.theme.typography.sectionTitle) + Text(self.setupSummary) + .font(self.theme.typography.bodySmall) + .foregroundStyle(self.theme.palette.secondaryText) + } + Button("Edit setup…", systemImage: "slider.horizontal.3", action: self.onOpenMeetingSettings) + .fluidButton(.compact, size: .small) + } } + .padding(.horizontal, self.theme.metrics.spacing.lg) + .padding(.vertical, self.theme.metrics.spacing.md) + .accessibilityElement(children: .contain) - // Results Card (only show when we have results) - if let result = transcriptionService.result { - self.resultsCard(result: result) - } + Divider() - // Error Card (only show when we have an error) - if let error = transcriptionService.error { - self.errorCard(error: error) + MeetingAdaptiveSetupRow(title: "Meeting title") { + TextField("Meeting title", text: self.$draft.title) + .textFieldStyle(.roundedBorder) + .accessibilityLabel("Meeting title") } - // Drop error (unsupported file type) - if let message = self.dropErrorMessage { - self.dropErrorCard(message: message) - } + Divider() - // Recent transcriptions (persisted history) - if !self.fileHistoryStore.entries.isEmpty { - Divider() - .padding(.vertical, 8) - self.recentTranscriptionsSection + ViewThatFits(in: .horizontal) { + HStack(spacing: self.theme.metrics.spacing.md) { + MeetingReadyStatus( + isReady: self.canStart, + title: self.canStart ? "Ready to record" : "Setup needs attention", + detail: self.canStart + ? "English · \(self.readiness.storageStatus)" + : (self.startHelp ?? "Check the recording setup.") + ) + Spacer(minLength: self.theme.metrics.spacing.md) + Button("Refresh", systemImage: "arrow.clockwise", action: self.onRefreshSources) + .fluidButton(.compact, size: .small) + .disabled(self.readiness.isCheckingSources || self.isStarting) + } + + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.md) { + MeetingReadyStatus( + isReady: self.canStart, + title: self.canStart ? "Ready to record" : "Setup needs attention", + detail: self.canStart + ? "English · \(self.readiness.storageStatus)" + : (self.startHelp ?? "Check the recording setup.") + ) + Button("Refresh", systemImage: "arrow.clockwise", action: self.onRefreshSources) + .fluidButton(.compact, size: .small) + .disabled(self.readiness.isCheckingSources || self.isStarting) + } } + .padding(.horizontal, self.theme.metrics.spacing.lg) + .padding(.vertical, self.theme.metrics.spacing.md) } - .padding(24) } - } - .frame(maxWidth: .infinity, maxHeight: .infinity) - .background(self.theme.palette.windowBackground) - .overlay(alignment: .topTrailing) { - if self.showingCopyConfirmation { - Text("Copied!") - .font(.caption) - .padding(.horizontal, 12) - .padding(.vertical, 6) - .background(Color.fluidGreen.opacity(0.9)) - .foregroundColor(.white) - .cornerRadius(8) - .padding() - .transition(.move(edge: .top).combined(with: .opacity)) - } - } - .fileExporter( - isPresented: self.$showingExportDialog, - document: TranscriptionDocument( - result: self.exportResult ?? TranscriptionResult(text: "", confidence: 0, duration: 0, processingTime: 0, fileName: "transcript"), - format: self.exportFormat, - service: self.transcriptionService - ), - contentType: self.exportFormat == .text ? .plainText : .json, - defaultFilename: "\((self.exportResult?.fileName).map { "\($0)_transcript" } ?? "transcript").\(self.exportFormat.fileExtension)" - ) { exportCompletion in - switch exportCompletion { - case .success: - DebugLogger.shared.info("File exported successfully", source: "MeetingTranscriptionView") - case let .failure(error): - DebugLogger.shared.error("Export failed: \(error)", source: "MeetingTranscriptionView") - } - self.exportResult = nil - } - } - - // MARK: - File Selection Card - - private var fileSelectionCard: some View { - VStack(spacing: 16) { - if let fileURL = selectedFileURL { - // Show selected file - HStack { - Image(systemName: "doc.fill") - .font(.title2) - .foregroundColor(Color.fluidGreen) - VStack(alignment: .leading, spacing: 4) { - Text(fileURL.lastPathComponent) - .font(.headline) + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.sm) { + Label("Recording and transcription stay on this Mac.", systemImage: "lock.fill") + Label( + self.draft.mode == .onlineCall + ? "Headphones give the cleanest speaker separation." + : "Place the Mac where every speaker can be heard clearly.", + systemImage: self.draft.mode == .onlineCall ? "headphones" : "mic.fill" + ) + Label("Make sure everyone knows the meeting is being recorded.", systemImage: "person.2") + } + .font(self.theme.typography.caption) + .foregroundStyle(self.theme.palette.secondaryText) + + if let errorMessage, !errorMessage.isEmpty { + Label(errorMessage, systemImage: "exclamationmark.triangle.fill") + .font(self.theme.typography.bodySmall) + .foregroundStyle(self.theme.palette.warning) + .accessibilityLabel("Could not start recording. \(errorMessage)") + } - Text(self.formatFileSize(fileURL: fileURL)) - .font(.caption) - .foregroundColor(.secondary) + HStack { + Spacer() + Button(action: self.onStart) { + if self.isStarting { + HStack(spacing: self.theme.metrics.spacing.sm) { + ProgressView() + .controlSize(.small) + Text("Starting…") + } + } else { + Label("Start recording", systemImage: "record.circle") } + } + .fluidButton(.accent, size: .medium) + .disabled(!self.canStart || self.isStarting) + .keyboardShortcut(.defaultAction) + } - Spacer() + if let recentSession { + Divider() + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.sm) { + Text("Recent meeting") + .font(self.theme.typography.sectionTitle) - Button(action: { - self.selectedFileURL = nil - self.transcriptionService.reset() - }) { - Image(systemName: "xmark.circle.fill") - .foregroundColor(.secondary) + HStack { + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.xs) { + Text(recentSession.title) + .font(self.theme.typography.bodyStrong) + Text(recentSession.startedAt.formatted(date: .abbreviated, time: .shortened)) + .font(self.theme.typography.caption) + .foregroundStyle(self.theme.palette.secondaryText) + } + Spacer() + Text(Self.durationText(recentSession.duration)) + .font(self.theme.typography.codeCaption) + .foregroundStyle(self.theme.palette.secondaryText) } - .buttonStyle(.plain) + .padding(self.theme.metrics.spacing.md) + .background(self.theme.palette.contentBackground, in: RoundedRectangle( + cornerRadius: self.theme.metrics.corners.md, + style: .continuous + )) } - .padding() - .background( - RoundedRectangle(cornerRadius: 8, style: .continuous) - .fill(self.theme.palette.cardBackground) - .overlay( - RoundedRectangle(cornerRadius: 8, style: .continuous) - .stroke(self.theme.palette.cardBorder.opacity(0.5), lineWidth: 1) - ) - ) + } + } + } - // Speaker labeling options (unavailable on Intel Macs) - if SpeakerDiarizationService.isSupported { - VStack(spacing: 10) { - Toggle(isOn: self.$settings.fileTranscriptionSpeakerLabelsEnabled) { - VStack(alignment: .leading, spacing: 2) { - Text("Label speakers") - .font(.subheadline) - - Text(self.selectedFileIsVideo - ? "Available for audio files only" - : "Identify who said what (downloads speaker models on first use)") - .font(.caption) - .foregroundColor(.secondary) - } - } - .toggleStyle(.switch) - .disabled(self.selectedFileIsVideo) + private static func durationText(_ duration: TimeInterval) -> String { + let total = max(0, Int(duration.rounded(.down))) + return String(format: "%d:%02d", total / 60, total % 60) + } +} - if self.settings.fileTranscriptionSpeakerLabelsEnabled, !self.selectedFileIsVideo { - HStack { - Text("Number of speakers") - .font(.subheadline) +private struct MeetingRecordingCanvas: View { + let session: MeetingSession + let trackHealth: [MeetingAudioTrackKind: MeetingTrackHealth] + let isStopping: Bool + let onStop: () -> Void - Spacer() + @Environment(\.theme) private var theme - Picker("", selection: self.$settings.fileTranscriptionExpectedSpeakerCount) { - Text("Auto").tag(0) - ForEach(2...8, id: \.self) { count in - Text("\(count)").tag(count) - } - } - .pickerStyle(.menu) - .labelsHidden() - .frame(width: 90) - } - } - } - .padding() - .background( - RoundedRectangle(cornerRadius: 8, style: .continuous) - .fill(self.theme.palette.cardBackground) - .overlay( - RoundedRectangle(cornerRadius: 8, style: .continuous) - .stroke(self.theme.palette.cardBorder.opacity(0.5), lineWidth: 1) - ) + var body: some View { + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.xl) { + TimelineView(.periodic(from: .now, by: 1)) { context in + HStack { + Label( + self.isStopping ? "Stopping…" : "Recording", + systemImage: self.isStopping ? "stop.circle.fill" : "record.circle.fill" ) - } + .font(self.theme.typography.sectionTitle) + .foregroundStyle(self.isStopping ? self.theme.palette.warning : Color(nsColor: .systemRed)) - // Transcribe Button - Button(action: { - Task { - await self.transcribeFile() - } - }) { - HStack { - Image(systemName: "waveform") - Text("Transcribe") - } - .frame(maxWidth: .infinity) - .padding(.vertical, 12) - } - .buttonStyle(.borderedProminent) - .disabled(self.transcriptionService.isTranscribing) + Text(Self.durationText(context.date.timeIntervalSince(self.session.startedAt))) + .font(self.theme.typography.codeCaption) + .foregroundStyle(self.theme.palette.primaryText) - } else { - // File picker button – whole area is tappable; supports drag-and-drop - Button(action: { - self.showingFilePicker = true - }) { - VStack(spacing: 12) { - Image(systemName: "arrow.up.doc.fill") - .font(.system(size: 32)) - - Text("Drag and drop a file here, or click to open") - .font(.headline) - - Text(MeetingTranscriptionService.supportedFormatsDescription) - .font(.caption) - .foregroundColor(.secondary) - } - .frame(maxWidth: .infinity) - .padding(32) - .contentShape(Rectangle()) + Spacer() + + Text(self.sourceSummary) + .font(self.theme.typography.caption) + .foregroundStyle(self.theme.palette.secondaryText) + .lineLimit(1) } - .buttonStyle(.plain) - .background( - RoundedRectangle(cornerRadius: 12, style: .continuous) - .fill(self.theme.palette.cardBackground) - .overlay( - RoundedRectangle(cornerRadius: 12, style: .continuous) - .stroke(self.theme.palette.cardBorder.opacity(0.45), lineWidth: 1) - ) + .accessibilityElement(children: .combine) + .accessibilityLabel( + "\(self.isStopping ? "Meeting recording is stopping" : "Meeting recording in progress"), " + + "\(Self.durationText(context.date.timeIntervalSince(self.session.startedAt))) elapsed, " + + self.sourceSummary ) - .overlay( - RoundedRectangle(cornerRadius: 12) - .strokeBorder(style: StrokeStyle(lineWidth: 2, dash: [8])) - .foregroundColor(Color.fluidGreen.opacity(self.isDropTargeted ? 0.7 : 0.3)) - ) - .onDrop(of: [.fileURL], isTargeted: self.$isDropTargeted) { providers in - self.handleDrop(providers: providers) - } } - } - .fileImporter( - isPresented: self.$showingFilePicker, - allowedContentTypes: MeetingTranscriptionService.allowedContentTypes, - allowsMultipleSelection: false - ) { result in - switch result { - case let .success(urls): - if let url = urls.first { - self.selectedFileURL = url - self.transcriptionService.reset() + + ThemedCard(style: .prominent) { + VStack(spacing: self.theme.metrics.spacing.lg) { + if self.session.mode == .onlineCall { + MeetingTrackHealthRow( + title: "Meeting audio", + health: self.trackHealth[.applicationAudio] ?? .waiting + ) + } + MeetingTrackHealthRow( + title: "Microphone", + health: self.trackHealth[.microphone] ?? .waiting + ) } - case let .failure(error): - DebugLogger.shared.error("File picker error: \(error)", source: "MeetingTranscriptionView") } - } - } - - // MARK: - Progress Card - private var progressCard: some View { - VStack(spacing: 16) { - ProgressView(value: self.transcriptionService.progress) - .progressViewStyle(.linear) + Text( + self.isStopping + ? "Saving captured audio before offline transcription begins…" + : "When the meeting ends, stop recording to transcribe it offline." + ) + .font(self.theme.typography.body) + .foregroundStyle(self.theme.palette.secondaryText) HStack { - ProgressView() - .controlSize(.small) - .fixedSize() + Spacer() - Text(self.transcriptionService.currentStatus) - .font(.subheadline) - .foregroundColor(.secondary) + Button(action: self.onStop) { + if self.isStopping { + HStack(spacing: self.theme.metrics.spacing.sm) { + ProgressView() + .controlSize(.small) + Text("Stopping…") + } + } else { + Label("Stop & Transcribe", systemImage: "stop.fill") + } + } + .fluidButton(.destructive, size: .medium) + .disabled(self.isStopping) + .keyboardShortcut(.defaultAction) } } - .padding() - .background( - RoundedRectangle(cornerRadius: 12, style: .continuous) - .fill(self.theme.palette.cardBackground) - .overlay( - RoundedRectangle(cornerRadius: 12, style: .continuous) - .stroke(self.theme.palette.cardBorder.opacity(0.45), lineWidth: 1) - ) - ) } - // MARK: - Results Card - - private func resultsCard(result: TranscriptionResult) -> some View { - VStack(alignment: .leading, spacing: 16) { - // Header with stats - HStack { - VStack(alignment: .leading, spacing: 4) { - Text("Transcription Complete") - .font(.headline) - - HStack(spacing: 16) { - Label("\(String(format: "%.1f", result.duration))s", systemImage: "clock") - Label("\(String(format: "%.0f%%", result.confidence * 100))", systemImage: "checkmark.circle") - Label( - "\(String(format: "%.1f", result.duration / result.processingTime))x", - systemImage: "speedometer" - ) - } - .font(.caption) - .foregroundColor(.secondary) - } + private var sourceSummary: String { + let microphoneName = self.session.selectedMicrophone.displayName + guard self.session.mode == .onlineCall else { return microphoneName } + let applicationName = self.session.capturedApplication?.displayName ?? "Meeting audio" + return "\(applicationName) · \(microphoneName)" + } - Spacer() + private static func durationText(_ duration: TimeInterval) -> String { + let total = max(0, Int(duration.rounded(.down))) + let hours = total / 3600 + let minutes = (total % 3600) / 60 + let seconds = total % 60 + if hours > 0 { + return String(format: "%d:%02d:%02d", hours, minutes, seconds) + } + return String(format: "%d:%02d", minutes, seconds) + } +} - // Action buttons - HStack(spacing: 8) { - Button(action: { - self.copyToClipboard(result.text) - }) { - Image(systemName: "doc.on.doc") - } - .help("Copy to clipboard") +private struct MeetingProcessingCanvas: View { + let session: MeetingSession + let stage: MeetingProcessingStage - Button(action: { - self.exportResult = result - self.showingExportDialog = true - }) { - Image(systemName: "square.and.arrow.up") - } - .help("Export transcription") - } - .buttonStyle(.borderless) - } + @Environment(\.theme) private var theme - if let notice = transcriptionService.fallbackNotice { - Label(notice, systemImage: "exclamationmark.triangle.fill") - .font(.caption) - .foregroundColor(.secondary) - } + private let stages: [MeetingProcessingStage] = [ + .saving, + .identifyingSpeakers, + .transcribing, + .finalizing, + ] - Divider() + var body: some View { + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.xl) { + HStack(spacing: self.theme.metrics.spacing.md) { + ProgressView() + .controlSize(.regular) + .fixedSize() - // Transcription text - ScrollView { - if !result.speakerSegments.isEmpty { - VStack(alignment: .leading, spacing: 12) { - ForEach(result.speakerSegments) { segment in - VStack(alignment: .leading, spacing: 2) { - HStack(spacing: 6) { - Text(segment.speaker) - .font(.caption) - .fontWeight(.semibold) - .foregroundColor(Color.fluidGreen) - - Text(segment.timestampText) - .font(.caption2) - .foregroundColor(.secondary) - } + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.xs) { + Text("Transcribing meeting…") + .font(self.theme.typography.sectionTitle) + .accessibilityAddTraits(.isHeader) + Text("You can close this window. Processing will continue.") + .font(self.theme.typography.bodySmall) + .foregroundStyle(self.theme.palette.secondaryText) + } + } - Text(segment.text) - .font(.body) - .textSelection(.enabled) - } + ThemedCard { + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.lg) { + ForEach(Array(self.stages.enumerated()), id: \.element) { index, stage in + HStack(spacing: self.theme.metrics.spacing.md) { + Image(systemName: self.icon(for: stage, index: index)) + .foregroundStyle(self.color(for: stage, index: index)) + .frame(width: 20) + Text(Self.title(for: stage)) + .font(self.theme.typography.body) + Spacer() } + .accessibilityElement(children: .combine) + .accessibilityLabel("\(Self.title(for: stage)), \(self.accessibilityStatus(for: stage, index: index))") } - .frame(maxWidth: .infinity, alignment: .leading) - .padding() - } else { - Text(result.text) - .font(.body) - .textSelection(.enabled) - .frame(maxWidth: .infinity, alignment: .leading) - .padding() } } - .frame(maxHeight: 300) - .background( - RoundedRectangle(cornerRadius: 8, style: .continuous) - .fill(self.theme.palette.contentBackground) - .overlay( - RoundedRectangle(cornerRadius: 8, style: .continuous) - .stroke(self.theme.palette.cardBorder.opacity(0.45), lineWidth: 1) - ) - ) + + Text(self.session.title) + .font(self.theme.typography.caption) + .foregroundStyle(self.theme.palette.secondaryText) } - .padding() - .background( - RoundedRectangle(cornerRadius: 12, style: .continuous) - .fill(self.theme.palette.cardBackground) - .overlay( - RoundedRectangle(cornerRadius: 12, style: .continuous) - .stroke(self.theme.palette.cardBorder.opacity(0.45), lineWidth: 1) - ) - ) } - // MARK: - Recent Transcriptions Section + private var activeIndex: Int { + self.stages.firstIndex(of: self.stage) ?? 0 + } - private var recentTranscriptionsSection: some View { - VStack(alignment: .leading, spacing: 16) { - HStack { - Text("Recent transcriptions") - .font(.headline) - Spacer() - if !self.fileHistoryStore.entries.isEmpty { - Button("Clear all") { - self.fileHistoryStore.clearAll() - } - .buttonStyle(.plain) - .foregroundColor(.secondary) - .font(.caption) - } - } + private func icon(for stage: MeetingProcessingStage, index: Int) -> String { + if self.stage == .completed || index < self.activeIndex { return "checkmark.circle.fill" } + if stage == self.stage { return "circle.dotted" } + return "circle" + } - VStack(spacing: 8) { - ForEach(self.fileHistoryStore.entries) { entry in - self.recentEntryRow(entry: entry) - } - } + private func color(for stage: MeetingProcessingStage, index: Int) -> Color { + if self.stage == .completed || index < self.activeIndex { return self.theme.palette.success } + if stage == self.stage { return self.theme.palette.accent } + return self.theme.palette.tertiaryText + } - if let entry = self.fileHistoryStore.selectedEntry { - self.historyDetailCard(entry: entry) - } + private func accessibilityStatus(for stage: MeetingProcessingStage, index: Int) -> String { + if self.stage == .completed || index < self.activeIndex { return "complete" } + if stage == self.stage { return "in progress" } + return "waiting" + } + + private static func title(for stage: MeetingProcessingStage) -> String { + switch stage { + case .pending: return "Waiting" + case .saving: return "Saving" + case .identifyingSpeakers: return "Identifying speakers" + case .transcribing: return "Transcribing" + case .finalizing: return "Finalizing" + case .completed: return "Complete" } } +} - private func recentEntryRow(entry: FileTranscriptionEntry) -> some View { - let isSelected = self.fileHistoryStore.selectedEntryID == entry.id - return Button(action: { - self.fileHistoryStore.selectedEntryID = entry.id - }) { - HStack { - Image(systemName: "doc.text.fill") - .font(.body) - .foregroundColor(Color.fluidGreen) - .frame(width: 24) - - VStack(alignment: .leading, spacing: 2) { - Text(entry.fileName) - .font(.system(size: 14, weight: .medium)) - .lineLimit(1) - Text(entry.relativeTimeString) - .font(.caption) - .foregroundColor(.secondary) - Text(entry.previewText) - .font(.caption) - .foregroundColor(.secondary) - .lineLimit(1) - } +private struct MeetingResultCanvas: View { + let session: MeetingSession + let onCopyTranscript: (MeetingSession) -> Void + + @Environment(\.theme) private var theme + private var speakerNames: [SessionSpeakerID: String] { + Dictionary(uniqueKeysWithValues: self.session.speakers.map { ($0.id, $0.displayName) }) + } + + var body: some View { + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.xl) { + HStack(alignment: .firstTextBaseline) { + Text(self.session.title) + .font(self.theme.typography.title) Spacer() + Label(self.statusText, systemImage: self.statusIcon) + .font(self.theme.typography.badge) + .foregroundStyle(self.statusColor) + } - if isSelected { - Image(systemName: "chevron.right.circle.fill") - .foregroundColor(Color.fluidGreen) + ViewThatFits(in: .horizontal) { + HStack(spacing: self.theme.metrics.spacing.lg) { + self.meetingMetadata + } + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.sm) { + self.meetingMetadata } } - .padding(12) - .background( - RoundedRectangle(cornerRadius: 8, style: .continuous) - .fill(self.theme.palette.cardBackground) - .overlay( - RoundedRectangle(cornerRadius: 8, style: .continuous) - .stroke(isSelected ? Color.fluidGreen.opacity(0.5) : self.theme.palette.cardBorder.opacity(0.3), lineWidth: isSelected ? 2 : 1) - ) - ) - } - .buttonStyle(.plain) - } - - private func historyDetailCard(entry: FileTranscriptionEntry) -> some View { - let result = entry.toTranscriptionResult() - return VStack(alignment: .leading, spacing: 16) { - HStack { - VStack(alignment: .leading, spacing: 4) { - Text("From history") - .font(.headline) - HStack(spacing: 16) { - Label("\(String(format: "%.1f", entry.duration))s", systemImage: "clock") - Label("\(String(format: "%.0f%%", entry.confidence * 100))", systemImage: "checkmark.circle") - Label(entry.fullDateString, systemImage: "calendar") + .font(self.theme.typography.caption) + .foregroundStyle(self.theme.palette.secondaryText) + + if !self.session.speakers.isEmpty { + ScrollView(.horizontal, showsIndicators: false) { + HStack(spacing: self.theme.metrics.spacing.sm) { + ForEach(self.session.speakers) { speaker in + Label( + self.displayName(for: speaker), + systemImage: speaker.isLocalUser ? "person.crop.circle.badge.checkmark" : "person.crop.circle" + ) + .font(self.theme.typography.badge) + .padding(.horizontal, self.theme.metrics.spacing.md) + .padding(.vertical, self.theme.metrics.spacing.sm) + .background(self.theme.palette.contentBackground, in: Capsule()) + } } - .font(.caption) - .foregroundColor(.secondary) } - Spacer() - HStack(spacing: 8) { - Button(action: { self.copyToClipboard(entry.text) }) { - Image(systemName: "doc.on.doc") - } - .help("Copy to clipboard") - Button(action: { - self.exportResult = result - self.showingExportDialog = true - }) { - Image(systemName: "square.and.arrow.up") - } - .help("Export transcription") - Button(action: { - self.fileHistoryStore.deleteEntry(id: entry.id) - }) { - Image(systemName: "trash") + } + + ThemedCard { + if self.session.transcriptSegments.isEmpty { + ContentUnavailableView( + "No transcript text", + systemImage: "waveform.slash", + description: Text("The recording is preserved if you need to retry processing.") + ) + .frame(maxWidth: .infinity, minHeight: 180) + } else { + LazyVStack(alignment: .leading, spacing: self.theme.metrics.spacing.lg) { + ForEach(self.session.transcriptSegments) { segment in + MeetingTranscriptSegmentRow( + segment: segment, + speakerName: segment.speakerID.flatMap { self.speakerNames[$0] } ?? "Unknown speaker" + ) + } } - .help("Remove from history") + .frame(maxWidth: .infinity, alignment: .leading) } - .buttonStyle(.borderless) } - Divider() - ScrollView { - Text(entry.text) - .font(.body) - .textSelection(.enabled) - .frame(maxWidth: .infinity, alignment: .leading) - .padding() - } - .frame(maxHeight: 300) - .background( - RoundedRectangle(cornerRadius: 8, style: .continuous) - .fill(self.theme.palette.contentBackground) - .overlay( - RoundedRectangle(cornerRadius: 8, style: .continuous) - .stroke(self.theme.palette.cardBorder.opacity(0.45), lineWidth: 1) - ) - ) + + Button("Copy Transcript", systemImage: "doc.on.doc") { + self.onCopyTranscript(self.session) + } + .fluidButton(.compact, size: .medium) + .disabled(self.session.transcriptSegments.isEmpty) } - .padding() - .background( - RoundedRectangle(cornerRadius: 12, style: .continuous) - .fill(self.theme.palette.cardBackground) - .overlay( - RoundedRectangle(cornerRadius: 12, style: .continuous) - .stroke(self.theme.palette.cardBorder.opacity(0.45), lineWidth: 1) - ) - ) } - // MARK: - Error Card - - private func errorCard(error: String) -> some View { - HStack { - Image(systemName: "exclamationmark.triangle.fill") - .foregroundColor(.red) - - Text(error) - .font(.subheadline) - - Spacer() - - Button("Dismiss") { - self.transcriptionService.reset() - } - .buttonStyle(.borderless) + private func displayName(for speaker: MeetingSessionSpeaker) -> String { + guard speaker.isLocalUser, + speaker.displayName.caseInsensitiveCompare("You") != .orderedSame + else { + return speaker.displayName } - .padding() - .background( - RoundedRectangle(cornerRadius: 12, style: .continuous) - .fill(Color.red.opacity(0.12)) - .overlay( - RoundedRectangle(cornerRadius: 12, style: .continuous) - .stroke(Color.red.opacity(0.4), lineWidth: 1) - ) + return "\(speaker.displayName) (You)" + } + + @ViewBuilder + private var meetingMetadata: some View { + Label( + self.session.startedAt.formatted(date: .abbreviated, time: .shortened), + systemImage: "calendar" ) + Label(Self.durationText(self.session.duration), systemImage: "clock") + Label(self.sourceName, systemImage: self.session.mode == .onlineCall ? "macwindow" : "person.3") + Label(self.session.selectedMicrophone.displayName, systemImage: "mic") } - // MARK: - Drop Error Card + private var sourceName: String { + self.session.capturedApplication?.displayName ?? "In-room meeting" + } - private func dropErrorCard(message: String) -> some View { - HStack { - Image(systemName: "exclamationmark.triangle.fill") - .foregroundColor(.red) + private var statusText: String { + switch self.session.state { + case .completed: "Completed" + case .failed: "Failed" + case .interrupted: "Interrupted" + case .processing: "Processing" + case .recording, .recordingDegraded, .preparing, .stopping: "Recording" + } + } - Text(message) - .font(.subheadline) + private var statusIcon: String { + switch self.session.state { + case .completed: "checkmark.circle.fill" + case .failed: "xmark.circle.fill" + case .interrupted: "exclamationmark.circle.fill" + case .processing: "ellipsis.circle.fill" + case .recording, .recordingDegraded, .preparing, .stopping: "record.circle.fill" + } + } - Spacer() + private var statusColor: Color { + switch self.session.state { + case .completed: self.theme.palette.success + case .failed: Color(nsColor: .systemRed) + case .interrupted: self.theme.palette.warning + case .processing: self.theme.palette.accent + case .recording, .recordingDegraded, .preparing, .stopping: self.theme.palette.warning + } + } - Button("Dismiss") { - self.dropErrorMessage = nil - } - .buttonStyle(.borderless) + private static func durationText(_ duration: TimeInterval) -> String { + let total = max(0, Int(duration.rounded(.down))) + let hours = total / 3600 + let minutes = (total % 3600) / 60 + let seconds = total % 60 + if hours > 0 { + return String(format: "%d:%02d:%02d", hours, minutes, seconds) } - .padding() - .background( - RoundedRectangle(cornerRadius: 12, style: .continuous) - .fill(Color.red.opacity(0.12)) - .overlay( - RoundedRectangle(cornerRadius: 12, style: .continuous) - .stroke(Color.red.opacity(0.4), lineWidth: 1) - ) - ) + return String(format: "%d:%02d", minutes, seconds) } +} + +private struct MeetingFailureCanvas: View { + let session: MeetingSession? + let message: String + let isRetrying: Bool + let onRetry: () -> Void + let onRevealAudio: (MeetingSession) -> Void - // MARK: - Helper Functions + @Environment(\.theme) private var theme - private static let supportedFileExtensions = MeetingTranscriptionService.supportedFileExtensions + private var hasRecoverableAudio: Bool { + guard let session, session.endedAt != nil else { return false } + return session.audioTracks + .flatMap(\.chunks) + .contains(where: { $0.finalizationState == .finalized && $0.byteCount > 0 }) + } - private static let dropErrorCopy = MeetingTranscriptionService.dropErrorCopy + private var title: String { + guard let session else { return "Meeting setup failed" } + let lastFailureDomain = session.failures.last?.domain + if lastFailureDomain == .processing || + (session.state == .interrupted && !session.processingAttempts.isEmpty) + { + return session.state == .interrupted + ? "Meeting transcription was interrupted" + : "Meeting transcription failed" + } + if self.hasRecoverableAudio { + return session.state == .interrupted + ? "Meeting recording was interrupted" + : "Meeting recording failed" + } + return "Meeting setup failed" + } - private func handleDrop(providers: [NSItemProvider]) -> Bool { - guard let provider = providers.first else { return false } - provider.loadItem(forTypeIdentifier: UTType.fileURL.identifier, options: nil) { item, _ in - let url: URL? = (item as? URL) ?? (item as? Data).flatMap { URL(dataRepresentation: $0, relativeTo: nil) } - guard let url = url else { return } - let ext = url.pathExtension.lowercased() - guard Self.supportedFileExtensions.contains(ext) else { - DispatchQueue.main.async { - self.dropErrorMessage = Self.dropErrorCopy - DispatchQueue.main.asyncAfter(deadline: .now() + 3) { - self.dropErrorMessage = nil + var body: some View { + ThemedCard(style: .prominent) { + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.lg) { + Label(self.title, systemImage: "exclamationmark.triangle.fill") + .font(self.theme.typography.sectionTitle) + .foregroundStyle(self.theme.palette.warning) + .accessibilityAddTraits(.isHeader) + Text(self.message) + .font(self.theme.typography.body) + .textSelection(.enabled) + if self.hasRecoverableAudio { + Text("The captured audio has been preserved on this Mac.") + .font(self.theme.typography.bodySmall) + .foregroundStyle(self.theme.palette.secondaryText) + } + HStack(spacing: self.theme.metrics.spacing.sm) { + if let session, self.hasRecoverableAudio { + Button("Reveal Audio", systemImage: "folder", action: { self.onRevealAudio(session) }) + .fluidButton(.compact, size: .medium) + Button(action: self.onRetry) { + if self.isRetrying { + HStack(spacing: self.theme.metrics.spacing.sm) { + ProgressView().controlSize(.small) + Text("Retrying…") + } + } else { + Label("Retry Transcription", systemImage: "arrow.clockwise") + } + } + .fluidButton(.accent, size: .medium) + .disabled(self.isRetrying) } } - return - } - DispatchQueue.main.async { - self.selectedFileURL = url - self.transcriptionService.reset() - self.dropErrorMessage = nil } } - return true } +} - private func transcribeFile() async { - guard let fileURL = selectedFileURL else { return } - - do { - _ = try await self.transcriptionService.transcribeFile(fileURL) - } catch { - DebugLogger.shared.error("Transcription error: \(error)", source: "MeetingTranscriptionView") - } - } +private struct MeetingAdaptiveSetupRow: View { + let title: String + var detail: String? + @ViewBuilder let content: Content - private func formatFileSize(fileURL: URL) -> String { - guard let attributes = try? FileManager.default.attributesOfItem(atPath: fileURL.path), - let fileSize = attributes[.size] as? Int64 - else { - return "Unknown size" - } + @Environment(\.theme) private var theme - let formatter = ByteCountFormatter() - formatter.countStyle = .file - return formatter.string(fromByteCount: fileSize) + init( + title: String, + detail: String? = nil, + @ViewBuilder content: () -> Content + ) { + self.title = title + self.detail = detail + self.content = content() } - private func copyToClipboard(_ text: String) { - NSPasteboard.general.clearContents() - NSPasteboard.general.setString(text, forType: .string) + var body: some View { + ViewThatFits(in: .horizontal) { + HStack(alignment: .center, spacing: self.theme.metrics.spacing.lg) { + self.label + .frame(width: 164, alignment: .leading) + self.content + .frame(width: 320, alignment: .leading) + Spacer(minLength: 0) + } - withAnimation { - self.showingCopyConfirmation = true + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.sm) { + self.label + self.content + .frame(maxWidth: .infinity, alignment: .leading) + } } + .padding(.horizontal, self.theme.metrics.spacing.lg) + .padding(.vertical, self.theme.metrics.spacing.md) + } - DispatchQueue.main.asyncAfter(deadline: .now() + 2) { - withAnimation { - self.showingCopyConfirmation = false + private var label: some View { + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.xs) { + Text(self.title) + .font(self.theme.typography.bodyStrong) + .foregroundStyle(self.theme.palette.primaryText) + if let detail { + Text(detail) + .font(self.theme.typography.caption) + .foregroundStyle(self.theme.palette.secondaryText) + .fixedSize(horizontal: false, vertical: true) } } } } -// MARK: - Document for Export +private struct MeetingReadyStatus: View { + let isReady: Bool + let title: String + let detail: String + + @Environment(\.theme) private var theme -struct TranscriptionDocument: FileDocument { - static var readableContentTypes: [UTType] { - [.plainText, .json] + var body: some View { + HStack(spacing: self.theme.metrics.spacing.md) { + Image(systemName: self.isReady ? "checkmark.circle.fill" : "exclamationmark.circle.fill") + .foregroundStyle(self.isReady ? self.theme.palette.success : self.theme.palette.warning) + .accessibilityHidden(true) + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.xs) { + Text(self.title) + .font(self.theme.typography.bodyStrong) + .foregroundStyle(self.theme.palette.primaryText) + Text(self.detail) + .font(self.theme.typography.caption) + .foregroundStyle(self.theme.palette.secondaryText) + .fixedSize(horizontal: false, vertical: true) + } + } + .accessibilityElement(children: .combine) + .accessibilityLabel("\(self.title). \(self.detail)") } +} - let result: TranscriptionResult - let format: MeetingTranscriptionView.ExportFormat - let service: MeetingTranscriptionService +private struct MeetingTrackHealthRow: View { + let title: String + let health: MeetingTrackHealth - init( - result: TranscriptionResult, - format: MeetingTranscriptionView.ExportFormat, - service: MeetingTranscriptionService - ) { - self.result = result - self.format = format - self.service = service + @Environment(\.theme) private var theme + + private var statusText: String { + switch self.health.status { + case .waiting: return "Waiting for audio" + case .healthy: return "Healthy" + case .degraded: return self.health.detail ?? "Audio source degraded" + case .unavailable: return self.health.detail ?? "Audio source unavailable" + case .stopped: return "Stopped" + } } - init(configuration: ReadConfiguration) throws { - throw CocoaError(.fileReadUnknown) + private var isHealthy: Bool { + self.health.status == .healthy } - func fileWrapper(configuration: WriteConfiguration) throws -> FileWrapper { - let tempURL = FileManager.default.temporaryDirectory.appendingPathComponent("temp.\(self.format.fileExtension)") + var body: some View { + HStack(spacing: self.theme.metrics.spacing.md) { + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.xs) { + Text(self.title) + .font(self.theme.typography.bodyStrong) + Text(self.statusText) + .font(self.theme.typography.caption) + .foregroundStyle(self.theme.palette.secondaryText) + .lineLimit(2) + } - switch self.format { - case .text: - try self.service.exportToText(self.result, to: tempURL) - case .json: - try self.service.exportToJSON(self.result, to: tempURL) - } + Spacer() - let data = try Data(contentsOf: tempURL) - try? FileManager.default.removeItem(at: tempURL) + ProgressView(value: min(max(Double(self.health.level), 0), 1)) + .progressViewStyle(.linear) + .frame(width: 150) + .accessibilityLabel("\(self.title) level") - return FileWrapper(regularFileWithContents: data) + Label( + self.statusText, + systemImage: self.isHealthy ? "checkmark.circle.fill" : "exclamationmark.circle.fill" + ) + .labelStyle(.iconOnly) + .foregroundStyle(self.isHealthy ? self.theme.palette.success : self.theme.palette.warning) + .accessibilityLabel(self.statusText) + } } } -// MARK: - Preview +private struct MeetingTranscriptSegmentRow: View { + let segment: MeetingTranscriptSegment + let speakerName: String + + @Environment(\.theme) private var theme + + var body: some View { + Grid(alignment: .topLeading, horizontalSpacing: self.theme.metrics.spacing.lg) { + GridRow { + Text(Self.timestampText(self.segment.start.seconds)) + .font(self.theme.typography.codeCaption) + .foregroundStyle(self.theme.palette.tertiaryText) + + VStack(alignment: .leading, spacing: self.theme.metrics.spacing.xs) { + Text(self.speakerName) + .font(self.theme.typography.captionStrong) + .foregroundStyle(self.theme.palette.accent) + Text(self.segment.text) + .font(self.theme.typography.body) + .foregroundStyle(self.theme.palette.primaryText) + .textSelection(.enabled) + } + } + } + .accessibilityElement(children: .combine) + .accessibilityLabel( + "\(Self.timestampText(self.segment.start.seconds)), \(self.speakerName), \(self.segment.text)" + ) + } -#Preview { - MeetingTranscriptionView(asrService: ASRService()) - .frame(width: 700, height: 800) + private static func timestampText(_ seconds: TimeInterval) -> String { + let total = max(0, Int(seconds.rounded(.down))) + let hours = total / 3600 + let minutes = (total % 3600) / 60 + let seconds = total % 60 + if hours > 0 { + return String(format: "%d:%02d:%02d", hours, minutes, seconds) + } + return String(format: "%02d:%02d", minutes, seconds) + } } diff --git a/Tests/FluidDictationIntegrationTests/SpeakerTurnMergingTests.swift b/Tests/FluidDictationIntegrationTests/SpeakerTurnMergingTests.swift index 77f00d22..5fbe0451 100644 --- a/Tests/FluidDictationIntegrationTests/SpeakerTurnMergingTests.swift +++ b/Tests/FluidDictationIntegrationTests/SpeakerTurnMergingTests.swift @@ -1,4 +1,5 @@ @testable import FluidVoice_Debug +import Foundation import XCTest #if arch(arm64) @@ -108,8 +109,8 @@ final class SpeakerTranscriptSegmentTests: XCTestCase { func testPlainTextIncludesHourTimestamp() { let segment = SpeakerTranscriptSegment( speaker: "Speaker 2", - startSeconds: 3_661, - endSeconds: 3_670, + startSeconds: 3661, + endSeconds: 3670, text: "Still here" ) @@ -118,3 +119,1089 @@ final class SpeakerTranscriptSegmentTests: XCTestCase { } #endif + +final class MeetingSpeakerEmbeddingIndexTests: XCTestCase { + func testSameVoiceAcrossChunksMatchesStableSpeakerID() { + let speakerID = UUID() + var index = MeetingSpeakerEmbeddingIndex() + index.observe(speakerID: speakerID, embedding: [1, 0, 0], quality: 0.9) + + XCTAssertEqual(index.bestMatch(for: [0.99, 0.01, 0]), speakerID) + } + + func testDistinctVoiceDoesNotMerge() { + let speakerID = UUID() + var index = MeetingSpeakerEmbeddingIndex() + index.observe(speakerID: speakerID, embedding: [1, 0, 0], quality: 0.9) + + XCTAssertNil(index.bestMatch(for: [0, 1, 0])) + } + + func testAmbiguousNearestVoicesDoNotMerge() { + var index = MeetingSpeakerEmbeddingIndex() + index.observe(speakerID: UUID(), embedding: [1, 0.02, 0], quality: 0.9) + index.observe(speakerID: UUID(), embedding: [1, -0.02, 0], quality: 0.9) + + XCTAssertNil(index.bestMatch(for: [1, 0, 0])) + } + + func testChunkLocalOneToOneExclusionPreventsDuplicateAssignment() { + let speakerID = UUID() + var index = MeetingSpeakerEmbeddingIndex() + index.observe(speakerID: speakerID, embedding: [1, 0, 0], quality: 0.9) + + XCTAssertNil(index.bestMatch(for: [1, 0, 0], excluding: [speakerID])) + } + + func testPersonalMicNeedsOneDominantEmbeddedClusterForYou() { + let localID = UUID() + let secondaryID = UUID() + let selected = MeetingLocalSpeakerEvidenceSelector.candidate( + cleanDurationByCluster: [localID: 12, secondaryID: 3], + prototypeSpeakerIDs: [localID, secondaryID] + ) + + XCTAssertEqual(selected, localID) + } + + func testAmbiguousPersonalMicClustersDoNotSelectYou() { + let firstID = UUID() + let secondID = UUID() + let selected = MeetingLocalSpeakerEvidenceSelector.candidate( + cleanDurationByCluster: [firstID: 8, secondID: 7], + prototypeSpeakerIDs: [firstID, secondID] + ) + + XCTAssertNil(selected) + } + + func testClusterWithoutPositiveEmbeddingEvidenceDoesNotSelectYou() { + let clusterID = UUID() + let selected = MeetingLocalSpeakerEvidenceSelector.candidate( + cleanDurationByCluster: [clusterID: 12], + prototypeSpeakerIDs: [] + ) + + XCTAssertNil(selected) + } +} + +final class MeetingSessionModelTests: XCTestCase { + func testMeetingRecordingDefaultsRoundTripStableSourceIdentifiers() throws { + let defaults = MeetingRecordingDefaults( + isConfigured: true, + mode: .onlineCall, + applicationBundleIdentifier: "com.google.Chrome", + microphoneCaptureDeviceID: "av-device-id", + microphoneCoreAudioUID: "core-audio-uid", + microphoneRole: .personal + ) + + let encoded = try JSONEncoder().encode(defaults) + let decoded = try JSONDecoder().decode(MeetingRecordingDefaults.self, from: encoded) + + XCTAssertEqual(decoded, defaults) + XCTAssertEqual(decoded.applicationBundleIdentifier, "com.google.Chrome") + XCTAssertEqual(decoded.microphoneCaptureDeviceID, "av-device-id") + XCTAssertEqual(decoded.microphoneCoreAudioUID, "core-audio-uid") + } + + func testUnconfiguredMeetingDefaultsRemainConservative() { + let defaults = MeetingRecordingDefaults.unconfigured + + XCTAssertFalse(defaults.isConfigured) + XCTAssertEqual(defaults.mode, .onlineCall) + XCTAssertEqual(defaults.microphoneRole, .unknown) + XCTAssertNil(defaults.applicationBundleIdentifier) + XCTAssertNil(defaults.microphoneCaptureDeviceID) + } + + func testConfiguredMeetingDefaultsResolveOnlySavedSources() { + let defaults = MeetingRecordingDefaults( + isConfigured: true, + mode: .onlineCall, + applicationBundleIdentifier: "com.google.Chrome", + microphoneCaptureDeviceID: "saved-device", + microphoneCoreAudioUID: "saved-core-audio", + microphoneRole: .personal + ) + let applications = [ + MeetingApplicationIdentity(bundleIdentifier: "us.zoom.xos", displayName: "Zoom"), + MeetingApplicationIdentity(bundleIdentifier: "com.google.Chrome", displayName: "Google Chrome"), + ] + let microphones = [ + MeetingMicrophoneIdentity(captureDeviceID: "other-device", displayName: "Other"), + MeetingMicrophoneIdentity(captureDeviceID: "saved-device", displayName: "Saved"), + ] + + XCTAssertEqual(defaults.savedApplication(in: applications)?.bundleIdentifier, "com.google.Chrome") + XCTAssertEqual(defaults.savedMicrophone(in: microphones)?.captureDeviceID, "saved-device") + XCTAssertNil(defaults.savedApplication(in: [applications[0]])) + XCTAssertNil(defaults.savedMicrophone(in: [microphones[0]])) + } + + func testCodableRoundTripPreservesStableMeetingIdentitiesAndTrackRelationships() throws { + let session = MeetingModelFixture.makeSession() + + let encoded = try JSONEncoder().encode(session) + let decoded = try JSONDecoder().decode(MeetingSession.self, from: encoded) + + XCTAssertEqual(decoded, session) + XCTAssertEqual(decoded.id, MeetingModelFixture.sessionID) + XCTAssertEqual(decoded.languageCode, "en") + XCTAssertEqual(decoded.audioTracks.map(\.kind.rawValue).sorted(), [ + MeetingAudioTrackKind.applicationAudio.rawValue, + MeetingAudioTrackKind.microphone.rawValue, + ]) + XCTAssertEqual(decoded.audioTracks.map(\.id), [ + MeetingModelFixture.applicationTrackID, + MeetingModelFixture.microphoneTrackID, + ]) + XCTAssertEqual(decoded.audioTracks[0].chunks.map(\.sequence), [0, 1]) + XCTAssertEqual(decoded.audioTracks[0].chunks.map(\.id), [ + MeetingModelFixture.applicationChunk0ID, + MeetingModelFixture.applicationChunk1ID, + ]) + + let segment = try XCTUnwrap(decoded.transcriptSegments.first) + XCTAssertEqual(segment.id, MeetingModelFixture.segmentID) + XCTAssertTrue(decoded.audioTracks.contains { $0.id == segment.sourceTrackID }) + XCTAssertTrue(decoded.speakers.contains { $0.id == segment.speakerID }) + XCTAssertEqual(decoded.processingAttempts.first?.languageCode, "en") + } + + func testSpeakerRenameChangesOnlyDisplayMetadata() throws { + var session = MeetingModelFixture.makeSession() + let originalSegments = session.transcriptSegments + let originalTrackIDs = session.audioTracks.map(\.id) + let originalClusterID = session.speakers.first?.diarizationClusterID + + try session.renameSpeaker(id: MeetingModelFixture.speakerID, to: " Erik ") + + let speaker = try XCTUnwrap(session.speakers.first) + XCTAssertEqual(speaker.id, MeetingModelFixture.speakerID) + XCTAssertEqual(speaker.displayName, "Erik") + XCTAssertEqual(speaker.diarizationClusterID, originalClusterID) + XCTAssertEqual(session.transcriptSegments, originalSegments) + XCTAssertEqual(session.transcriptSegments.map(\.id), [MeetingModelFixture.segmentID]) + XCTAssertEqual(session.audioTracks.map(\.id), originalTrackIDs) + } + + func testRejectedSpeakerRenamesLeaveSessionIdentityUntouched() { + var session = MeetingModelFixture.makeSession() + let originalSpeakers = session.speakers + let originalSegments = session.transcriptSegments + let originalUpdatedAt = session.updatedAt + + XCTAssertThrowsError(try session.renameSpeaker(id: MeetingModelFixture.speakerID, to: " \n ")) { + XCTAssertEqual($0 as? MeetingDomainError, .emptySpeakerName) + } + XCTAssertThrowsError(try session.renameSpeaker(id: UUID(), to: "Erik")) { + XCTAssertEqual($0 as? MeetingDomainError, .speakerNotFound) + } + + XCTAssertEqual(session.speakers, originalSpeakers) + XCTAssertEqual(session.transcriptSegments, originalSegments) + XCTAssertEqual(session.updatedAt, originalUpdatedAt) + } + + func testTranscriptRevisionKeepsStableSegmentAndDomainReferences() { + var segment = MeetingModelFixture.makeSession().transcriptSegments[0] + let originalID = segment.id + let originalTrackID = segment.sourceTrackID + let originalSpeakerID = segment.speakerID + + segment.text = "Revised final text" + segment.revision += 1 + segment.status = .final + + XCTAssertEqual(segment.id, originalID) + XCTAssertEqual(segment.sourceTrackID, originalTrackID) + XCTAssertEqual(segment.speakerID, originalSpeakerID) + XCTAssertEqual(segment.revision, 2) + XCTAssertEqual(segment.status, .final) + } + + func testCaptureConfigurationPinsEnglishAndSeparatesMicrophoneIdentifiers() { + let configuration = MeetingCaptureConfiguration( + mode: .inRoom, + title: "Room sync", + microphone: MeetingMicrophoneIdentity( + captureDeviceID: "avcapture-device-id", + coreAudioUID: "core-audio-uid", + displayName: "Studio microphone", + role: .shared + ), + chunkDuration: 5 + ) + + XCTAssertEqual(configuration.languageCode, "en") + XCTAssertEqual(configuration.chunkDuration, 60) + XCTAssertEqual(configuration.microphone.captureDeviceID, "avcapture-device-id") + XCTAssertEqual(configuration.microphone.coreAudioUID, "core-audio-uid") + XCTAssertEqual(configuration.microphone.role, .shared) + XCTAssertNil(configuration.application) + } + + func testCaptureConfigurationRejectsInvalidModeSourcesLanguageAndMicrophone() { + var configuration = MeetingModelFixture.makeOnlineConfiguration() + XCTAssertNoThrow(try configuration.validate()) + + configuration.languageCode = "fr" + self.assertConfigurationValidationError(.unsupportedLanguage, configuration: configuration) + + configuration = MeetingModelFixture.makeOnlineConfiguration() + configuration.microphone.captureDeviceID = "" + self.assertConfigurationValidationError(.missingMicrophone, configuration: configuration) + + configuration = MeetingModelFixture.makeOnlineConfiguration() + configuration.application = nil + self.assertConfigurationValidationError(.missingOnlineApplication, configuration: configuration) + + configuration = MeetingModelFixture.makeOnlineConfiguration() + configuration.mode = .inRoom + self.assertConfigurationValidationError(.unexpectedInRoomApplication, configuration: configuration) + } + + func testPersistenceValidationRequiresExpectedUniqueTrackSet() { + XCTAssertNoThrow(try MeetingModelFixture.makeSession().validateForPersistence()) + + self.assertSessionValidationError(.missingMicrophone) { + $0.selectedMicrophone.captureDeviceID = "" + } + self.assertSessionValidationError(.invalidTrackSet) { + $0.audioTracks.removeLast() + } + self.assertSessionValidationError(.duplicateTrackID) { + $0.audioTracks[1].id = $0.audioTracks[0].id + } + self.assertSessionValidationError(.duplicateTrackKind) { + $0.audioTracks[1].kind = .applicationAudio + } + } + + func testPersistenceValidationRequiresOrderedRecoverableChunks() { + self.assertSessionValidationError(.invalidChunkSequence) { + $0.audioTracks[0].chunks.reverse() + } + self.assertSessionValidationError(.duplicateChunkID) { + $0.audioTracks[1].chunks[0].id = $0.audioTracks[0].chunks[0].id + } + self.assertSessionValidationError(.invalidMediaTimescale) { + $0.audioTracks[0].chunks[0].presentationStart.timescale = 0 + } + self.assertSessionValidationError(.invalidTimeRange) { + $0.audioTracks[0].chunks[0].presentationEnd = MeetingModelFixture.mediaTime(-1) + } + self.assertSessionValidationError(.incompleteFinalizedChunk) { + $0.audioTracks[0].chunks[0].sha256 = "" + } + } + + func testPersistenceValidationRequiresUniqueResolvableTranscriptReferences() { + self.assertSessionValidationError(.duplicateSpeakerID) { + $0.speakers.append($0.speakers[0]) + } + self.assertSessionValidationError(.duplicateSegmentID) { + $0.transcriptSegments.append($0.transcriptSegments[0]) + } + self.assertSessionValidationError(.missingSegmentTrack) { + $0.transcriptSegments[0].sourceTrackID = UUID() + } + self.assertSessionValidationError(.missingSegmentSpeaker) { + $0.transcriptSegments[0].speakerID = UUID() + } + } + + func testProcessingAttemptLanguageMustMatchSessionLanguage() { + self.assertSessionValidationError(.unsupportedLanguage) { + $0.processingAttempts[0].languageCode = "fr" + } + } + + private func assertConfigurationValidationError( + _ expected: MeetingModelValidationError, + configuration: MeetingCaptureConfiguration, + file: StaticString = #filePath, + line: UInt = #line + ) { + XCTAssertThrowsError(try configuration.validate(), file: file, line: line) { + XCTAssertEqual($0 as? MeetingModelValidationError, expected, file: file, line: line) + } + } + + private func assertSessionValidationError( + _ expected: MeetingModelValidationError, + mutating mutate: (inout MeetingSession) -> Void, + file: StaticString = #filePath, + line: UInt = #line + ) { + var session = MeetingModelFixture.makeSession() + mutate(&session) + XCTAssertThrowsError(try session.validateForPersistence(), file: file, line: line) { + XCTAssertEqual($0 as? MeetingModelValidationError, expected, file: file, line: line) + } + } +} + +private enum MeetingModelFixture { + static let sessionID = Self.uuid(1) + static let applicationTrackID = Self.uuid(2) + static let microphoneTrackID = Self.uuid(3) + static let applicationChunk0ID = Self.uuid(4) + static let applicationChunk1ID = Self.uuid(5) + static let microphoneChunkID = Self.uuid(6) + static let speakerID = Self.uuid(7) + static let segmentID = Self.uuid(8) + + static let startedAt = Date(timeIntervalSince1970: 1_750_000_000) + static let endedAt = Date(timeIntervalSince1970: 1_750_000_012) + + static func makeOnlineConfiguration() -> MeetingCaptureConfiguration { + MeetingCaptureConfiguration( + mode: .onlineCall, + title: "Design sync", + platform: MeetingPlatformProfile(identifier: "zoom", displayName: "Zoom", version: 1), + application: MeetingApplicationIdentity( + bundleIdentifier: "us.zoom.xos", + processID: 321, + displayName: "Zoom", + displayID: 1 + ), + microphone: MeetingMicrophoneIdentity( + captureDeviceID: "capture-mic", + coreAudioUID: "core-audio-mic", + displayName: "MacBook Microphone", + role: .personal + ) + ) + } + + static func makeSession() -> MeetingSession { + let timebase = MeetingTimebaseMetadata( + startedHostTime: 42, + machTimebaseNumerator: 1, + machTimebaseDenominator: 1, + firstPresentationTime: self.mediaTime(0) + ) + var session = MeetingSession( + id: self.sessionID, + configuration: self.makeOnlineConfiguration(), + startedAt: self.startedAt, + timebase: timebase + ) + let format = MeetingAudioFormat(codec: "aac", sampleRate: 48_000, channelCount: 1, bitRate: 96_000) + let health = MeetingTrackHealth( + status: .stopped, + lastPresentationTime: self.mediaTime(12), + lastSampleAt: self.endedAt, + level: 0, + droppedSampleCount: 0, + detail: nil + ) + + session.state = .completed + session.endedAt = self.endedAt + session.audioTracks = [ + MeetingAudioTrack( + id: self.applicationTrackID, + kind: .applicationAudio, + sourceIdentifier: "us.zoom.xos", + sourceDisplayName: "Zoom", + format: format, + timebase: timebase, + health: health, + chunks: [ + self.chunk( + id: self.applicationChunk0ID, + sequence: 0, + path: "application/0000.m4a", + start: 0, + end: 6, + checksum: "application-0" + ), + self.chunk( + id: self.applicationChunk1ID, + sequence: 1, + path: "application/0001.m4a", + start: 6, + end: 12, + checksum: "application-1" + ), + ] + ), + MeetingAudioTrack( + id: self.microphoneTrackID, + kind: .microphone, + sourceIdentifier: "capture-mic", + sourceDisplayName: "MacBook Microphone", + format: format, + timebase: timebase, + health: health, + chunks: [ + self.chunk( + id: self.microphoneChunkID, + sequence: 0, + path: "microphone/0000.m4a", + start: 0, + end: 12, + checksum: "microphone-0" + ), + ] + ), + ] + session.speakers = [ + MeetingSessionSpeaker( + id: self.speakerID, + displayName: "Speaker 1", + diarizationClusterID: "cluster-7", + trackKind: .applicationAudio, + isLocalUser: false, + identityCandidates: [ + MeetingIdentityCandidate( + id: Self.uuid(9), + displayName: "Erik", + source: "manual", + rejected: false, + confirmedAt: nil + ), + ] + ), + ] + session.transcriptSegments = [ + MeetingTranscriptSegment( + id: self.segmentID, + start: self.mediaTime(4), + end: self.mediaTime(7), + sourceTrackID: self.applicationTrackID, + speakerID: self.speakerID, + text: "Initial provisional text", + revision: 1, + status: .provisional, + overlap: .none, + completeness: .complete + ), + ] + session.events = [ + MeetingSessionEvent( + id: Self.uuid(10), + occurredAt: self.endedAt, + kind: .sourceRecovered, + trackID: self.applicationTrackID, + detail: "Recovered on a chunk boundary" + ), + ] + session.processingAttempts = [ + MeetingProcessingAttempt( + id: Self.uuid(11), + startedAt: self.endedAt, + completedAt: self.endedAt.addingTimeInterval(3), + stage: .completed, + pipelineVersion: 1, + asrProvider: "fixture-asr", + asrModel: "fixture-model", + languageCode: "en", + diarizationModel: "fixture-diarization", + lastCompletedTrackID: self.microphoneTrackID, + errorCode: nil + ), + ] + session.retention = MeetingRetentionState( + audioDeletedAt: nil, + meetingDeletedAt: nil, + retainAudioUntil: self.startedAt.addingTimeInterval(7 * 24 * 60 * 60) + ) + session.updatedAt = self.endedAt.addingTimeInterval(3) + return session + } + + private static func chunk( + id: MeetingAudioChunkID, + sequence: Int, + path: String, + start: Int64, + end: Int64, + checksum: String + ) -> MeetingAudioChunk { + MeetingAudioChunk( + id: id, + sequence: sequence, + relativeFilePath: path, + presentationStart: self.mediaTime(start), + presentationEnd: self.mediaTime(end), + discontinuities: [], + sha256: checksum, + byteCount: 1024, + finalizationState: .finalized + ) + } + + static func mediaTime(_ seconds: Int64) -> MeetingMediaTime { + MeetingMediaTime(value: seconds * 48_000, timescale: 48_000) + } + + private static func uuid(_ finalByte: UInt8) -> UUID { + UUID(uuid: (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, finalByte)) + } +} + +@MainActor +final class MeetingSessionCoordinatorTests: XCTestCase { + func testConcurrentStartCreatesExactlyOneSessionAndCapture() async throws { + let harness = self.makeHarness(blockStart: true) + let configuration = MeetingModelFixture.makeOnlineConfiguration() + + let firstStart = Task { @MainActor in + try await harness.coordinator.startRecording(configuration: configuration) + } + await harness.capture.waitForStartCall() + + let duplicateStart = Task { @MainActor in + try await harness.coordinator.startRecording(configuration: configuration) + } + var duplicateWasRejected = false + do { + _ = try await duplicateStart.value + XCTFail("A concurrent Start must not create a second session.") + } catch let error as MeetingCoordinatorError { + if case .recordingAlreadyActive = error { + duplicateWasRejected = true + } else { + XCTFail("Expected recordingAlreadyActive, got \(error).") + } + } catch { + XCTFail("Unexpected duplicate Start error: \(error)") + } + + await harness.capture.resumeStart() + let session = try await firstStart.value + let captureSnapshot = await harness.capture.snapshot() + let storeSnapshot = await harness.store.snapshot() + + XCTAssertTrue(duplicateWasRejected) + XCTAssertEqual(captureSnapshot.startCallCount, 1) + XCTAssertEqual(storeSnapshot.createCallCount, 1) + XCTAssertEqual(harness.arbiter.acquireCallCount, 1) + XCTAssertEqual(harness.coordinator.state, .recording(session.id)) + } + + func testConcurrentStopJoinsOneFinalizationAndPersistsLegalStateOrder() async throws { + let harness = self.makeHarness(blockStop: true) + let configuration = MeetingModelFixture.makeOnlineConfiguration() + let started = try await harness.coordinator.startRecording(configuration: configuration) + + let firstStop = Task { @MainActor in + try await harness.coordinator.stopAndTranscribe() + } + await harness.capture.waitForStopCall() + let duplicateStop = Task { @MainActor in + try await harness.coordinator.stopAndTranscribe() + } + await Task.yield() + await harness.capture.resumeStop() + + let firstResult = try await firstStop.value + let duplicateResult = try await duplicateStop.value + let captureSnapshot = await harness.capture.snapshot() + let storeSnapshot = await harness.store.snapshot() + + XCTAssertEqual(firstResult.id, started.id) + XCTAssertEqual(duplicateResult.id, firstResult.id) + XCTAssertEqual(captureSnapshot.stopCallCount, 1) + XCTAssertEqual(harness.processing.processCallCount, 1) + XCTAssertEqual(harness.arbiter.releaseCallCount, 1) + XCTAssertEqual(harness.coordinator.state, .completed(started.id)) + XCTAssertEqual(storeSnapshot.sessions[started.id]?.state, .completed) + XCTAssertEqual(self.collapsingAdjacentDuplicates(storeSnapshot.persistedStates), [ + .preparing, + .recording, + .stopping, + .processing, + .completed, + ]) + } + + func testStaleCaptureEventCannotMutateNewGeneration() async throws { + let harness = self.makeHarness() + let firstSession = try await harness.coordinator.startRecording( + configuration: MeetingModelFixture.makeOnlineConfiguration() + ) + _ = try await harness.coordinator.stopAndTranscribe() + let completedBeforeEvent = try XCTUnwrap(harness.coordinator.latestCompletedSession) + let secondSession = try await harness.coordinator.startRecording( + configuration: MeetingModelFixture.makeOnlineConfiguration() + ) + + await harness.capture.emit(.trackHealth( + trackID: MeetingModelFixture.applicationTrackID, + health: MeetingTrackHealth( + status: .degraded, + lastPresentationTime: MeetingModelFixture.mediaTime(999), + lastSampleAt: Date(), + level: 1, + droppedSampleCount: 99, + detail: "stale callback" + ) + ), handlerIndex: 0) + await Task.yield() + await Task.yield() + + let storeSnapshot = await harness.store.snapshot() + XCTAssertNotEqual(firstSession.id, secondSession.id) + XCTAssertEqual(harness.coordinator.state, .recording(secondSession.id)) + XCTAssertEqual(harness.coordinator.activeSession?.id, secondSession.id) + XCTAssertEqual(harness.coordinator.latestCompletedSession, completedBeforeEvent) + XCTAssertEqual(harness.coordinator.trackHealth[.applicationAudio]?.status, .waiting) + XCTAssertEqual(storeSnapshot.sessions[firstSession.id]?.state, .completed) + XCTAssertEqual(storeSnapshot.sessions[secondSession.id]?.state, .recording) + } + + func testStoppingContinuesWhenPreStopPersistenceFails() async throws { + let harness = self.makeHarness(failingSaveState: .stopping) + let started = try await harness.coordinator.startRecording( + configuration: MeetingModelFixture.makeOnlineConfiguration() + ) + + let completed = try await harness.coordinator.stopAndTranscribe() + let captureSnapshot = await harness.capture.snapshot() + let storeSnapshot = await harness.store.snapshot() + + XCTAssertEqual(completed.id, started.id) + XCTAssertEqual(captureSnapshot.stopCallCount, 1) + XCTAssertEqual(harness.processing.processCallCount, 1) + XCTAssertEqual(harness.coordinator.state, .completed(started.id)) + XCTAssertEqual(storeSnapshot.sessions[started.id]?.state, .completed) + XCTAssertEqual(storeSnapshot.rejectedSaveStates, [.stopping]) + } + + func testPostStartPersistenceFailureStopsCaptureAndReleasesLease() async { + let harness = self.makeHarness(failingSaveState: .recording) + + do { + _ = try await harness.coordinator.startRecording( + configuration: MeetingModelFixture.makeOnlineConfiguration() + ) + XCTFail("Start must report the persistence failure.") + } catch let error as MeetingCoordinatorTestError { + XCTAssertEqual(error, .rejectedSave(.recording)) + } catch { + XCTFail("Unexpected error: \(error)") + } + + let captureSnapshot = await harness.capture.snapshot() + let storeSnapshot = await harness.store.snapshot() + XCTAssertEqual(captureSnapshot.startCallCount, 1) + XCTAssertEqual(captureSnapshot.stopCallCount, 1) + XCTAssertEqual(harness.processing.processCallCount, 0) + XCTAssertEqual(harness.arbiter.releaseCallCount, 1) + let recoveredSession = harness.coordinator.activeSession + XCTAssertNotNil(recoveredSession) + XCTAssertNotNil(recoveredSession?.endedAt) + XCTAssertEqual(recoveredSession?.failures.last?.recoverable, true) + XCTAssertEqual(recoveredSession.flatMap { storeSnapshot.sessions[$0.id] }?.state, .failed) + XCTAssertEqual(storeSnapshot.rejectedSaveStates, [.recording]) + guard case .failed = harness.coordinator.state else { + return XCTFail("Coordinator must expose the failed Start after cleanup.") + } + } + + func testRecoverableFailedSessionRestoresWithoutLosingFailureState() async throws { + var failedSession = MeetingModelFixture.makeSession() + failedSession.state = .failed + failedSession.failures.append(MeetingSessionFailure( + id: UUID(), + occurredAt: Date(), + domain: .processing, + code: "fixture.processing", + message: "Fixture processing failed", + recoverable: true + )) + + let store = FakeMeetingSessionStore(failingSaveState: nil) + await store.seed(failedSession) + let coordinator = MeetingSessionCoordinator( + store: store, + capture: FakeMeetingCaptureController(blockStart: false, blockStop: false, failStop: false), + processing: FakeMeetingProcessingController(), + audioArbiter: FakeMeetingAudioActivityArbiter() + ) + + await coordinator.restoreRecoverableSessionIfNeeded() + + XCTAssertEqual(coordinator.activeSession?.id, failedSession.id) + XCTAssertEqual(coordinator.activeSession?.state, .failed) + guard case let .failed(id, failure) = coordinator.state else { + return XCTFail("Recoverable failed session must reopen as failed.") + } + XCTAssertEqual(id, failedSession.id) + XCTAssertEqual(failure.domain, .processing) + } + + func testFailedStopPreservesRecoverableAudioAndReleasesLease() async throws { + let harness = self.makeHarness(failStop: true) + let started = try await harness.coordinator.startRecording( + configuration: MeetingModelFixture.makeOnlineConfiguration() + ) + + do { + _ = try await harness.coordinator.stopAndTranscribe() + XCTFail("Stop must surface the capture failure.") + } catch { + XCTAssertTrue(error is MeetingCaptureError) + } + + let captureSnapshot = await harness.capture.snapshot() + let storeSnapshot = await harness.store.snapshot() + XCTAssertEqual(captureSnapshot.stopCallCount, 1) + XCTAssertEqual(harness.processing.processCallCount, 0) + XCTAssertEqual(harness.arbiter.releaseCallCount, 1) + XCTAssertEqual(harness.coordinator.state, .interrupted(started.id)) + XCTAssertEqual(harness.coordinator.activeSession?.id, started.id) + XCTAssertNotNil(harness.coordinator.activeSession?.endedAt) + XCTAssertEqual(storeSnapshot.sessions[started.id]?.state, .interrupted) + } + + func testConcurrentTerminationFinalizesAndReleasesExactlyOnce() async throws { + let harness = self.makeHarness(blockStop: true) + let started = try await harness.coordinator.startRecording( + configuration: MeetingModelFixture.makeOnlineConfiguration() + ) + + let first = Task { @MainActor in await harness.coordinator.shutdownForTermination() } + await harness.capture.waitForStopCall() + let second = Task { @MainActor in await harness.coordinator.shutdownForTermination() } + await Task.yield() + await harness.capture.resumeStop() + await first.value + await second.value + + let captureSnapshot = await harness.capture.snapshot() + XCTAssertEqual(captureSnapshot.stopCallCount, 1) + XCTAssertEqual(harness.arbiter.releaseCallCount, 1) + XCTAssertEqual(harness.coordinator.state, .interrupted(started.id)) + XCTAssertEqual( + harness.coordinator.activeSession?.events.filter { $0.kind == .appTermination }.count, + 1 + ) + } + + func testUnexpectedCaptureStopFinalizesAndReleasesLease() async throws { + let harness = self.makeHarness() + let started = try await harness.coordinator.startRecording( + configuration: MeetingModelFixture.makeOnlineConfiguration() + ) + + await harness.capture.emit(.interrupted( + kind: .captureStoppedUnexpectedly, + trackID: nil, + detail: "fixture stream ended" + )) + await harness.capture.waitForStopCall() + for _ in 0..<4 { + await Task.yield() + } + + XCTAssertEqual(harness.coordinator.state, .interrupted(started.id)) + XCTAssertEqual(harness.arbiter.releaseCallCount, 1) + XCTAssertEqual(harness.processing.processCallCount, 0) + } + + private func makeHarness( + blockStart: Bool = false, + blockStop: Bool = false, + failStop: Bool = false, + failingSaveState: MeetingSessionState? = nil + ) -> MeetingCoordinatorHarness { + let store = FakeMeetingSessionStore(failingSaveState: failingSaveState) + let capture = FakeMeetingCaptureController( + blockStart: blockStart, + blockStop: blockStop, + failStop: failStop + ) + let processing = FakeMeetingProcessingController() + let arbiter = FakeMeetingAudioActivityArbiter() + return MeetingCoordinatorHarness( + coordinator: MeetingSessionCoordinator( + store: store, + capture: capture, + processing: processing, + audioArbiter: arbiter + ), + store: store, + capture: capture, + processing: processing, + arbiter: arbiter + ) + } + + private func collapsingAdjacentDuplicates(_ states: [MeetingSessionState]) -> [MeetingSessionState] { + states.reduce(into: []) { result, state in + if result.last != state { + result.append(state) + } + } + } +} + +@MainActor +private struct MeetingCoordinatorHarness { + let coordinator: MeetingSessionCoordinator + let store: FakeMeetingSessionStore + let capture: FakeMeetingCaptureController + let processing: FakeMeetingProcessingController + let arbiter: FakeMeetingAudioActivityArbiter +} + +private enum MeetingCoordinatorTestError: Error, Equatable { + case rejectedSave(MeetingSessionState) +} + +private actor FakeMeetingSessionStore: MeetingSessionStoring { + struct Snapshot: Sendable { + var sessions: [MeetingSessionID: MeetingSession] + var persistedStates: [MeetingSessionState] + var rejectedSaveStates: [MeetingSessionState] + var createCallCount: Int + } + + private var sessions: [MeetingSessionID: MeetingSession] = [:] + private var persistedStates: [MeetingSessionState] = [] + private var rejectedSaveStates: [MeetingSessionState] = [] + private var createCallCount = 0 + private var failingSaveState: MeetingSessionState? + + init(failingSaveState: MeetingSessionState?) { + self.failingSaveState = failingSaveState + } + + func create(_ session: MeetingSession) throws { + try session.validateForPersistence() + self.createCallCount += 1 + self.sessions[session.id] = session + self.persistedStates.append(session.state) + } + + func save(_ session: MeetingSession) throws { + try session.validateForPersistence() + if self.failingSaveState == session.state { + self.failingSaveState = nil + self.rejectedSaveStates.append(session.state) + throw MeetingCoordinatorTestError.rejectedSave(session.state) + } + self.sessions[session.id] = session + self.persistedStates.append(session.state) + } + + func load(id: MeetingSessionID) -> MeetingSession? { + self.sessions[id] + } + + func loadAll() -> [MeetingSession] { + Array(self.sessions.values) + } + + func loadRecoverable() -> [MeetingSession] { + self.sessions.values.filter { + switch $0.state { + case .preparing, .recording, .recordingDegraded, .stopping, .processing, .interrupted: + return true + case .failed: + return $0.failures.last?.recoverable == true && + $0.endedAt != nil && + $0.audioTracks.contains { track in + track.chunks.contains { + $0.finalizationState == .finalized && $0.byteCount > 0 + } + } + case .completed: + return false + } + } + } + + func seed(_ session: MeetingSession) { + self.sessions[session.id] = session + } + + func sessionDirectory(for id: MeetingSessionID) -> URL { + URL(fileURLWithPath: NSTemporaryDirectory(), isDirectory: true) + .appendingPathComponent("meeting-coordinator-tests", isDirectory: true) + .appendingPathComponent(id.uuidString, isDirectory: true) + } + + func snapshot() -> Snapshot { + Snapshot( + sessions: self.sessions, + persistedStates: self.persistedStates, + rejectedSaveStates: self.rejectedSaveStates, + createCallCount: self.createCallCount + ) + } +} + +private actor FakeMeetingCaptureController: MeetingCaptureControlling { + struct Snapshot: Sendable { + var startCallCount: Int + var stopCallCount: Int + var shutdownCallCount: Int + } + + private let shouldBlockStart: Bool + private let shouldBlockStop: Bool + private let shouldFailStop: Bool + private var startCallCount = 0 + private var stopCallCount = 0 + private var shutdownCallCount = 0 + private var eventHandlers: [@Sendable (MeetingCaptureEvent) -> Void] = [] + private var startObservers: [CheckedContinuation] = [] + private var stopObservers: [CheckedContinuation] = [] + private var blockedStarts: [CheckedContinuation] = [] + private var blockedStops: [CheckedContinuation] = [] + + init(blockStart: Bool, blockStop: Bool, failStop: Bool) { + self.shouldBlockStart = blockStart + self.shouldBlockStop = blockStop + self.shouldFailStop = failStop + } + + func start( + session _: MeetingSession, + configuration _: MeetingCaptureConfiguration, + sessionDirectory _: URL, + eventHandler: @escaping @Sendable (MeetingCaptureEvent) -> Void + ) async -> MeetingCaptureStartResult { + self.startCallCount += 1 + self.eventHandlers.append(eventHandler) + self.startObservers.forEach { $0.resume() } + self.startObservers = [] + if self.shouldBlockStart { + await withCheckedContinuation { self.blockedStarts.append($0) } + } + return MeetingCaptureStartResult( + tracks: MeetingCoordinatorFixture.recordingTracks(), + firstPresentationTime: nil + ) + } + + func stop(sessionID _: MeetingSessionID) async throws -> MeetingCaptureStopResult { + self.stopCallCount += 1 + self.stopObservers.forEach { $0.resume() } + self.stopObservers = [] + if self.shouldBlockStop { + await withCheckedContinuation { self.blockedStops.append($0) } + } + let result = MeetingCaptureStopResult( + tracks: MeetingModelFixture.makeSession().audioTracks, + stoppedAt: Date() + ) + if self.shouldFailStop { + throw MeetingCaptureError.captureStopFailed("fixture stop failed", partialResult: result) + } + return result + } + + func shutdownForTermination() { + self.shutdownCallCount += 1 + } + + func waitForStartCall() async { + if self.startCallCount > 0 { return } + await withCheckedContinuation { self.startObservers.append($0) } + } + + func waitForStopCall() async { + if self.stopCallCount > 0 { return } + await withCheckedContinuation { self.stopObservers.append($0) } + } + + func resumeStart() { + self.blockedStarts.forEach { $0.resume() } + self.blockedStarts = [] + } + + func resumeStop() { + self.blockedStops.forEach { $0.resume() } + self.blockedStops = [] + } + + func emit(_ event: MeetingCaptureEvent, handlerIndex: Int = 0) { + guard self.eventHandlers.indices.contains(handlerIndex) else { return } + self.eventHandlers[handlerIndex](event) + } + + func snapshot() -> Snapshot { + Snapshot( + startCallCount: self.startCallCount, + stopCallCount: self.stopCallCount, + shutdownCallCount: self.shutdownCallCount + ) + } +} + +@MainActor +private final class FakeMeetingProcessingController: MeetingProcessingControlling { + private(set) var processCallCount = 0 + + func process( + session _: MeetingSession, + sessionDirectory _: URL, + progress: @escaping @MainActor (MeetingProcessingStage) -> Void + ) async throws -> MeetingProcessingResult { + self.processCallCount += 1 + progress(.saving) + progress(.identifyingSpeakers) + progress(.transcribing) + progress(.finalizing) + progress(.completed) + + let fixture = MeetingModelFixture.makeSession() + return MeetingProcessingResult( + speakers: fixture.speakers, + segments: fixture.transcriptSegments, + attempt: MeetingProcessingAttempt( + id: UUID(), + startedAt: Date(), + completedAt: Date(), + stage: .completed, + pipelineVersion: 1, + asrProvider: "test-asr", + asrModel: "test-model", + diarizationModel: "test-diarization", + lastCompletedTrackID: MeetingModelFixture.microphoneTrackID, + errorCode: nil + ) + ) + } +} + +@MainActor +private final class FakeMeetingAudioActivityArbiter: MeetingAudioActivityArbitrating { + private(set) var acquireCallCount = 0 + private(set) var releaseCallCount = 0 + private var activeLease: MeetingAudioActivityLease? + + func acquireMeetingCapture() throws -> MeetingAudioActivityLease { + self.acquireCallCount += 1 + let lease = MeetingAudioActivityLease(id: UUID()) + self.activeLease = lease + return lease + } + + func release(_ lease: MeetingAudioActivityLease) { + guard self.activeLease == lease else { return } + self.activeLease = nil + self.releaseCallCount += 1 + } +} + +private enum MeetingCoordinatorFixture { + static func recordingTracks() -> [MeetingAudioTrack] { + MeetingModelFixture.makeSession().audioTracks.map { track in + var track = track + track.format = nil + track.health = .waiting + track.chunks = [] + return track + } + } +}