From a35ad292e8780d2daba3c1330c00f7094b37355e Mon Sep 17 00:00:00 2001 From: Dario Lencina Date: Mon, 10 Aug 2026 23:35:59 -0700 Subject: [PATCH] Multicam PR4: synced photo capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Schedule-at-timestamp synced stills across the rig, behind ENABLE_MULTICAM. Wire (additive, same evolution pattern): - ScheduledCapture (action 26): director -> camera, carrying the shutter instant in the camera's own SyncClock domain (director applied the offset), the shared director-clock anchor (the alignment key stamped into each clip), capture id / session id, and the camera's 1-based index. - CameraStateResponse gains capture_id_echo for the ack. RemoteCmd ScheduledCapture / ScheduledCaptureAck, both dispatch switches, round-trips. Camera side (SessionCoordinator, single-cam untouched): - On ScheduledCapture: ack (or nack if the fire time is > 1s past) immediately, then schedule the shutter. The delay runs OFF the actor and only enqueues FireScheduledCapture, so the capture is pulled by the message pump in order and never races a state transition (the flagged concern). The photo saves locally as today, additionally stamped with CaptureSyncMetadata: EXIF UserComment JSON + DateTimeOriginal/SubSec, and a shared RS___cam originalFilename so any editor can group and align the angles. A stamping failure never costs the user the photo. Director side (MulticamController): - capturePhoto(): picks fireAt = now + 150ms, sends per-camera ScheduledCapture with fireAt + lane.offset to every linked multicam lane; if any offset is missing, falls back to a plain TakePic fan-out under the same shot id. Aggregate state capturingPhoto(captureId, acksRemaining); best-effort policy — per-lane 3s ack timeout marks a silent camera failed; returns to monitoring when all lanes acked/nacked/timed out. Per-lane outcome (captured/failed) surfaced to the tile badge. UI: the MulticamView shutter (reusing the 1:1 ShutterButton + activity ring) fires capturePhoto(); tiles show a captured/failed badge. Tests: serialization round-trips; MulticamControllerTests (per-lane offsets => different fire instants, non-multicam lane excluded, ack aggregation, timeout completion, fallback); camera-side (past-fire nack, valid fire acks immediately then pulls the shutter). Full suite green (677 tests). Co-Authored-By: Claude Fable 5 --- RemoteCam/CameraLink.swift | 4 + RemoteCam/CaptureSyncMetadata.swift | 47 ++++++ RemoteCam/FlatBufferSchemas.fbs | 20 ++- RemoteCam/FlatBufferSchemas_generated.swift | 50 +++++- RemoteCam/MulticamController.swift | 153 +++++++++++++++++- RemoteCam/MulticamView.swift | 40 +++++ RemoteCam/MulticamViewController.swift | 8 + RemoteCam/MulticamViewModel.swift | 6 + RemoteCam/RemoteCmdFlatBuffers.swift | 44 +++++ RemoteCam/RemoteCmds.swift | 44 +++++ RemoteCam/SessionCoordinator.swift | 119 +++++++++++++- RemoteCamTests/MulticamControllerTests.swift | 112 +++++++++++++ RemoteCamTests/MulticamViewModelTests.swift | 3 +- RemoteCamTests/RemoteCamSessionTests.swift | 41 +++++ .../RemoteCmdSerializationTests.swift | 28 ++++ 15 files changed, 708 insertions(+), 11 deletions(-) diff --git a/RemoteCam/CameraLink.swift b/RemoteCam/CameraLink.swift index 2d3128b..572088a 100644 --- a/RemoteCam/CameraLink.swift +++ b/RemoteCam/CameraLink.swift @@ -51,6 +51,10 @@ final class CameraLink { /// `SessionCoordinator.monitorReceivedVP9Frame`, but per camera). var sawVP9 = false + /// How this camera answered the most recent synced capture (nil before the + /// first). Drives the tile's captured/failed badge. + var captureOutcome: CaptureOutcome? + /// This camera's own preview decoder + stall watchdog. Frames tagged with /// this peer's id are fed here; its `onImage` drives exactly this lane's /// tile, so a frame from another camera never touches it. diff --git a/RemoteCam/CaptureSyncMetadata.swift b/RemoteCam/CaptureSyncMetadata.swift index 376e34a..128456e 100644 --- a/RemoteCam/CaptureSyncMetadata.swift +++ b/RemoteCam/CaptureSyncMetadata.swift @@ -8,6 +8,8 @@ import AVFoundation import Foundation +import ImageIO +import UniformTypeIdentifiers /// Alignment metadata attached to every clip and photo captured in a multicam /// director session. Each camera saves full-res media locally; these fields @@ -73,6 +75,51 @@ struct CaptureSyncMetadata: Codable, Equatable { return try? JSONDecoder().decode(CaptureSyncMetadata.self, from: data) } + /// Re-encode `imageData` with this shot's sync fields embedded in EXIF, so + /// the alignment travels inside the photo itself (survives export/AirDrop): + /// the JSON in `UserComment`, and the anchor instant in + /// `DateTimeOriginal`/`SubSecTimeOriginal`. Format (JPEG/HEIC) is preserved. + /// Returns the original data unchanged if re-encoding isn't possible, so a + /// stamping failure never costs the user the photo. + func stamped(_ imageData: Data) -> Data { + guard let source = CGImageSourceCreateWithData(imageData as CFData, nil), + let type = CGImageSourceGetType(source) else { return imageData } + + var properties = (CGImageSourceCopyPropertiesAtIndex(source, 0, nil) + as? [CFString: Any]) ?? [:] + var exif = (properties[kCGImagePropertyExifDictionary] as? [CFString: Any]) ?? [:] + + if let json = jsonString() { + exif[kCGImagePropertyExifUserComment] = json + } + let anchorSeconds = Double(anchorMillis) / 1000.0 + let date = Date(timeIntervalSince1970: anchorSeconds) + exif[kCGImagePropertyExifDateTimeOriginal] = Self.exifDateFormatter.string(from: date) + exif[kCGImagePropertyExifSubsecTimeOriginal] = String(format: "%03d", anchorMillis % 1000) + properties[kCGImagePropertyExifDictionary] = exif + + let output = NSMutableData() + guard let dest = CGImageDestinationCreateWithData( + output, type, 1, nil) else { return imageData } + CGImageDestinationAddImageFromSource(dest, source, 0, properties as CFDictionary) + guard CGImageDestinationFinalize(dest) else { return imageData } + return output as Data + } + + /// A Photos `originalFilename` for this shot, e.g. + /// `RS___cam2.heic`. Groups a capture's files across cameras. + func photoFilename(isHEIC: Bool) -> String { + "\(filenamePrefix).\(isHEIC ? "heic" : "jpg")" + } + + /// EXIF wants `yyyy:MM:dd HH:mm:ss` in the local zone. + private static let exifDateFormatter: DateFormatter = { + let f = DateFormatter() + f.dateFormat = "yyyy:MM:dd HH:mm:ss" + f.locale = Locale(identifier: "en_US_POSIX") + return f + }() + private static func shortID(_ uuidString: String) -> String { String(uuidString.prefix(8)).lowercased() } diff --git a/RemoteCam/FlatBufferSchemas.fbs b/RemoteCam/FlatBufferSchemas.fbs index 3dfee4b..5c82674 100644 --- a/RemoteCam/FlatBufferSchemas.fbs +++ b/RemoteCam/FlatBufferSchemas.fbs @@ -40,10 +40,16 @@ enum CommandAction : byte { EndSession = 23, // either side: "I am leaving on purpose" SetCameraPreviewMode = 24, // monitor -> camera: local preview on/standby; // only sent to peers advertising supports_preview_mode - ClockSyncPing = 25 // director -> camera: clock-offset probe carrying the + ClockSyncPing = 25, // director -> camera: clock-offset probe carrying the // director's send time; the camera answers with a // ClockSyncPing response echoing it plus its own clock. // Only sent to peers advertising supports_multicam. + ScheduledCapture = 26 // director -> camera: take a photo at a wall-clock + // instant expressed in the camera's own clock domain + // (the director already applied the per-camera offset), + // so N cameras fire together. Only sent to peers + // advertising supports_multicam; the camera acks with a + // CameraStateResponse echoing the capture id. } // Whether the camera device drives its own on-screen live preview. On is the @@ -163,6 +169,17 @@ table CommandParameters { focus_point_y: float; // upright-display space; origin top-left) camera_preview_mode: CameraPreviewModeEnum; // payload for SetCameraPreviewMode clock_sync_t0_ms: uint64; // payload for ClockSyncPing: director clock at send + // ScheduledCapture payload. `fire_at_camera_clock_ms` is the shutter instant + // in the CAMERA's SyncClock domain (director already added the offset), used + // to schedule. `anchor_ms` is the same instant in the DIRECTOR's clock — the + // value is identical across every camera in the shot, so it is the alignment + // key stamped into each clip. `capture_id`/`capture_session_id` group the + // shot; `capture_camera_index` is this camera's 1-based slot for filenames. + capture_fire_at_camera_clock_ms: uint64; + capture_anchor_ms: uint64; + capture_id: string; + capture_session_id: string; + capture_camera_index: int; } // MARK: - Command Structure @@ -285,6 +302,7 @@ table CameraStateResponse { // Appended fields only below this line (FlatBuffers schema evolution). clock_sync_echo_t0_ms: uint64; // ClockSyncPing response: echoed director t0 clock_sync_camera_clock_ms: uint64; // ClockSyncPing response: camera clock at receipt + capture_id_echo: string; // ScheduledCapture ack: the accepted capture id } // MARK: - Frame Data diff --git a/RemoteCam/FlatBufferSchemas_generated.swift b/RemoteCam/FlatBufferSchemas_generated.swift index ef8adaa..f8674ee 100644 --- a/RemoteCam/FlatBufferSchemas_generated.swift +++ b/RemoteCam/FlatBufferSchemas_generated.swift @@ -34,8 +34,9 @@ public enum RemoteShutter_CommandAction: Int8, Enum, Verifiable { case endsession = 23 case setcamerapreviewmode = 24 case clocksyncping = 25 + case scheduledcapture = 26 - public static var max: RemoteShutter_CommandAction { return .clocksyncping } + public static var max: RemoteShutter_CommandAction { return .scheduledcapture } public static var min: RemoteShutter_CommandAction { return .unknown } } @@ -333,6 +334,11 @@ public struct RemoteShutter_CommandParameters: FlatBufferObject, Verifiable { case focusPointY = 38 case cameraPreviewMode = 40 case clockSyncT0Ms = 42 + case captureFireAtCameraClockMs = 44 + case captureAnchorMs = 46 + case captureId = 48 + case captureSessionId = 50 + case captureCameraIndex = 52 var v: Int32 { Int32(self.rawValue) } var p: VOffset { self.rawValue } } @@ -360,7 +366,14 @@ public struct RemoteShutter_CommandParameters: FlatBufferObject, Verifiable { public var focusPointY: Float32 { let o = _accessor.offset(VTOFFSET.focusPointY.v); return o == 0 ? 0.0 : _accessor.readBuffer(of: Float32.self, at: o) } public var cameraPreviewMode: RemoteShutter_CameraPreviewModeEnum { let o = _accessor.offset(VTOFFSET.cameraPreviewMode.v); return o == 0 ? .unknown : RemoteShutter_CameraPreviewModeEnum(rawValue: _accessor.readBuffer(of: Int8.self, at: o)) ?? .unknown } public var clockSyncT0Ms: UInt64 { let o = _accessor.offset(VTOFFSET.clockSyncT0Ms.v); return o == 0 ? 0 : _accessor.readBuffer(of: UInt64.self, at: o) } - public static func startCommandParameters(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 20) } + public var captureFireAtCameraClockMs: UInt64 { let o = _accessor.offset(VTOFFSET.captureFireAtCameraClockMs.v); return o == 0 ? 0 : _accessor.readBuffer(of: UInt64.self, at: o) } + public var captureAnchorMs: UInt64 { let o = _accessor.offset(VTOFFSET.captureAnchorMs.v); return o == 0 ? 0 : _accessor.readBuffer(of: UInt64.self, at: o) } + public var captureId: String? { let o = _accessor.offset(VTOFFSET.captureId.v); return o == 0 ? nil : _accessor.string(at: o) } + public var captureIdSegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.captureId.v) } + public var captureSessionId: String? { let o = _accessor.offset(VTOFFSET.captureSessionId.v); return o == 0 ? nil : _accessor.string(at: o) } + public var captureSessionIdSegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.captureSessionId.v) } + public var captureCameraIndex: Int32 { let o = _accessor.offset(VTOFFSET.captureCameraIndex.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int32.self, at: o) } + public static func startCommandParameters(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 25) } public static func add(sendToRemote: Bool, _ fbb: inout FlatBufferBuilder) { fbb.add(element: sendToRemote, def: false, at: VTOFFSET.sendToRemote.p) } public static func add(zoomFactor: Double, _ fbb: inout FlatBufferBuilder) { fbb.add(element: zoomFactor, def: 0.0, at: VTOFFSET.zoomFactor.p) } @@ -382,6 +395,11 @@ public struct RemoteShutter_CommandParameters: FlatBufferObject, Verifiable { public static func add(focusPointY: Float32, _ fbb: inout FlatBufferBuilder) { fbb.add(element: focusPointY, def: 0.0, at: VTOFFSET.focusPointY.p) } public static func add(cameraPreviewMode: RemoteShutter_CameraPreviewModeEnum, _ fbb: inout FlatBufferBuilder) { fbb.add(element: cameraPreviewMode.rawValue, def: 0, at: VTOFFSET.cameraPreviewMode.p) } public static func add(clockSyncT0Ms: UInt64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: clockSyncT0Ms, def: 0, at: VTOFFSET.clockSyncT0Ms.p) } + public static func add(captureFireAtCameraClockMs: UInt64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: captureFireAtCameraClockMs, def: 0, at: VTOFFSET.captureFireAtCameraClockMs.p) } + public static func add(captureAnchorMs: UInt64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: captureAnchorMs, def: 0, at: VTOFFSET.captureAnchorMs.p) } + public static func add(captureId: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: captureId, at: VTOFFSET.captureId.p) } + public static func add(captureSessionId: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: captureSessionId, at: VTOFFSET.captureSessionId.p) } + public static func add(captureCameraIndex: Int32, _ fbb: inout FlatBufferBuilder) { fbb.add(element: captureCameraIndex, def: 0, at: VTOFFSET.captureCameraIndex.p) } public static func endCommandParameters(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } public static func createCommandParameters( _ fbb: inout FlatBufferBuilder, @@ -404,7 +422,12 @@ public struct RemoteShutter_CommandParameters: FlatBufferObject, Verifiable { focusPointX: Float32 = 0.0, focusPointY: Float32 = 0.0, cameraPreviewMode: RemoteShutter_CameraPreviewModeEnum = .unknown, - clockSyncT0Ms: UInt64 = 0 + clockSyncT0Ms: UInt64 = 0, + captureFireAtCameraClockMs: UInt64 = 0, + captureAnchorMs: UInt64 = 0, + captureIdOffset captureId: Offset = Offset(), + captureSessionIdOffset captureSessionId: Offset = Offset(), + captureCameraIndex: Int32 = 0 ) -> Offset { let __start = RemoteShutter_CommandParameters.startCommandParameters(&fbb) RemoteShutter_CommandParameters.add(sendToRemote: sendToRemote, &fbb) @@ -427,6 +450,11 @@ public struct RemoteShutter_CommandParameters: FlatBufferObject, Verifiable { RemoteShutter_CommandParameters.add(focusPointY: focusPointY, &fbb) RemoteShutter_CommandParameters.add(cameraPreviewMode: cameraPreviewMode, &fbb) RemoteShutter_CommandParameters.add(clockSyncT0Ms: clockSyncT0Ms, &fbb) + RemoteShutter_CommandParameters.add(captureFireAtCameraClockMs: captureFireAtCameraClockMs, &fbb) + RemoteShutter_CommandParameters.add(captureAnchorMs: captureAnchorMs, &fbb) + RemoteShutter_CommandParameters.add(captureId: captureId, &fbb) + RemoteShutter_CommandParameters.add(captureSessionId: captureSessionId, &fbb) + RemoteShutter_CommandParameters.add(captureCameraIndex: captureCameraIndex, &fbb) return RemoteShutter_CommandParameters.endCommandParameters(&fbb, start: __start) } @@ -452,6 +480,11 @@ public struct RemoteShutter_CommandParameters: FlatBufferObject, Verifiable { try _v.visit(field: VTOFFSET.focusPointY.p, fieldName: "focusPointY", required: false, type: Float32.self) try _v.visit(field: VTOFFSET.cameraPreviewMode.p, fieldName: "cameraPreviewMode", required: false, type: RemoteShutter_CameraPreviewModeEnum.self) try _v.visit(field: VTOFFSET.clockSyncT0Ms.p, fieldName: "clockSyncT0Ms", required: false, type: UInt64.self) + try _v.visit(field: VTOFFSET.captureFireAtCameraClockMs.p, fieldName: "captureFireAtCameraClockMs", required: false, type: UInt64.self) + try _v.visit(field: VTOFFSET.captureAnchorMs.p, fieldName: "captureAnchorMs", required: false, type: UInt64.self) + try _v.visit(field: VTOFFSET.captureId.p, fieldName: "captureId", required: false, type: ForwardOffset.self) + try _v.visit(field: VTOFFSET.captureSessionId.p, fieldName: "captureSessionId", required: false, type: ForwardOffset.self) + try _v.visit(field: VTOFFSET.captureCameraIndex.p, fieldName: "captureCameraIndex", required: false, type: Int32.self) _v.finish() } } @@ -1103,6 +1136,7 @@ public struct RemoteShutter_CameraStateResponse: FlatBufferObject, Verifiable { case currentZoom = 22 case clockSyncEchoT0Ms = 24 case clockSyncCameraClockMs = 26 + case captureIdEcho = 28 var v: Int32 { Int32(self.rawValue) } var p: VOffset { self.rawValue } } @@ -1125,7 +1159,9 @@ public struct RemoteShutter_CameraStateResponse: FlatBufferObject, Verifiable { public var currentZoom: Double { let o = _accessor.offset(VTOFFSET.currentZoom.v); return o == 0 ? 0.0 : _accessor.readBuffer(of: Double.self, at: o) } public var clockSyncEchoT0Ms: UInt64 { let o = _accessor.offset(VTOFFSET.clockSyncEchoT0Ms.v); return o == 0 ? 0 : _accessor.readBuffer(of: UInt64.self, at: o) } public var clockSyncCameraClockMs: UInt64 { let o = _accessor.offset(VTOFFSET.clockSyncCameraClockMs.v); return o == 0 ? 0 : _accessor.readBuffer(of: UInt64.self, at: o) } - public static func startCameraStateResponse(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 12) } + public var captureIdEcho: String? { let o = _accessor.offset(VTOFFSET.captureIdEcho.v); return o == 0 ? nil : _accessor.string(at: o) } + public var captureIdEchoSegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.captureIdEcho.v) } + public static func startCameraStateResponse(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 13) } public static func add(action: RemoteShutter_CommandAction, _ fbb: inout FlatBufferBuilder) { fbb.add(element: action.rawValue, def: 0, at: VTOFFSET.action.p) } public static func add(success: Bool, _ fbb: inout FlatBufferBuilder) { fbb.add(element: success, def: false, at: VTOFFSET.success.p) } @@ -1139,6 +1175,7 @@ public struct RemoteShutter_CameraStateResponse: FlatBufferObject, Verifiable { public static func add(currentZoom: Double, _ fbb: inout FlatBufferBuilder) { fbb.add(element: currentZoom, def: 0.0, at: VTOFFSET.currentZoom.p) } public static func add(clockSyncEchoT0Ms: UInt64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: clockSyncEchoT0Ms, def: 0, at: VTOFFSET.clockSyncEchoT0Ms.p) } public static func add(clockSyncCameraClockMs: UInt64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: clockSyncCameraClockMs, def: 0, at: VTOFFSET.clockSyncCameraClockMs.p) } + public static func add(captureIdEcho: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: captureIdEcho, at: VTOFFSET.captureIdEcho.p) } public static func endCameraStateResponse(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } public static func createCameraStateResponse( _ fbb: inout FlatBufferBuilder, @@ -1153,7 +1190,8 @@ public struct RemoteShutter_CameraStateResponse: FlatBufferObject, Verifiable { zoomRangeOffset zoomRange: Offset = Offset(), currentZoom: Double = 0.0, clockSyncEchoT0Ms: UInt64 = 0, - clockSyncCameraClockMs: UInt64 = 0 + clockSyncCameraClockMs: UInt64 = 0, + captureIdEchoOffset captureIdEcho: Offset = Offset() ) -> Offset { let __start = RemoteShutter_CameraStateResponse.startCameraStateResponse(&fbb) RemoteShutter_CameraStateResponse.add(action: action, &fbb) @@ -1168,6 +1206,7 @@ public struct RemoteShutter_CameraStateResponse: FlatBufferObject, Verifiable { RemoteShutter_CameraStateResponse.add(currentZoom: currentZoom, &fbb) RemoteShutter_CameraStateResponse.add(clockSyncEchoT0Ms: clockSyncEchoT0Ms, &fbb) RemoteShutter_CameraStateResponse.add(clockSyncCameraClockMs: clockSyncCameraClockMs, &fbb) + RemoteShutter_CameraStateResponse.add(captureIdEcho: captureIdEcho, &fbb) return RemoteShutter_CameraStateResponse.endCameraStateResponse(&fbb, start: __start) } @@ -1185,6 +1224,7 @@ public struct RemoteShutter_CameraStateResponse: FlatBufferObject, Verifiable { try _v.visit(field: VTOFFSET.currentZoom.p, fieldName: "currentZoom", required: false, type: Double.self) try _v.visit(field: VTOFFSET.clockSyncEchoT0Ms.p, fieldName: "clockSyncEchoT0Ms", required: false, type: UInt64.self) try _v.visit(field: VTOFFSET.clockSyncCameraClockMs.p, fieldName: "clockSyncCameraClockMs", required: false, type: UInt64.self) + try _v.visit(field: VTOFFSET.captureIdEcho.p, fieldName: "captureIdEcho", required: false, type: ForwardOffset.self) _v.finish() } } diff --git a/RemoteCam/MulticamController.swift b/RemoteCam/MulticamController.swift index 0b9d82d..d717d06 100644 --- a/RemoteCam/MulticamController.swift +++ b/RemoteCam/MulticamController.swift @@ -13,10 +13,18 @@ import Stormo import UIKit /// The director's aggregate posture across all cameras. One shutter drives -/// them all, so the capture states are aggregate, not per-camera. PR3 only -/// needs `.monitoring`; the capture cases arrive with synced photo/video. +/// them all, so the capture states are aggregate, not per-camera. enum MulticamState: Equatable { case monitoring(mode: MonitorMode) + /// A synced photo is in flight: `acksRemaining` cameras have yet to accept + /// or refuse `captureId`. Returns to `.monitoring` when it reaches zero. + case capturingPhoto(captureId: String, acksRemaining: Int) +} + +/// How a camera answered the last synced capture, for the tile badge. +enum CaptureOutcome: Equatable { + case captured + case failed } /// A UI-facing snapshot of one lane. Deliberately a value type carrying only @@ -30,6 +38,8 @@ struct MulticamLaneInfo: Equatable { /// Clock-offset estimate in ms (nil until the first pong), for a future /// sync-quality indicator; unused by PR3's UI beyond diagnostics. let clockOffsetMillis: Int64? + /// How this camera answered the last synced capture, for the tile badge. + let captureOutcome: CaptureOutcome? } /// The main-actor bridge from the controller to the multicam screen — the @@ -39,6 +49,8 @@ struct MulticamLaneInfo: Equatable { /// camera B never re-renders camera A. protocol MulticamDisplay: AnyObject { func applyLanes(_ lanes: [MulticamLaneInfo]) + /// A synced photo is (or is no longer) in flight — drives the shutter ring. + func applyCaptureInFlight(_ inFlight: Bool) func receiveFrame(_ frame: RemoteCmd.OnFrame) func exitMulticam() } @@ -110,6 +122,20 @@ public actor MulticamController { private var links: [MCPeerID: CameraLink] = [:] private var focusedPeer: MCPeerID? + /// One id per director session, part of every capture's filename group. + private let sessionID = UUID().uuidString + /// Cameras that have yet to answer the in-flight synced capture. Empty + /// unless `state` is `.capturingPhoto`. + private var capturingLanes: Set = [] + private var currentCaptureID: String? + + /// Lead time before a synced shutter fires — long enough to cover the + /// worst-case one-way latency plus a retransmit and scheduling slop, so + /// every camera has the command in hand before the instant arrives. + private let captureLeadMillis: UInt64 = 150 + /// How long a camera has to answer before it is counted as failed. + private var captureAckTimeout: TimeInterval = 3 + private weak var display: MulticamDisplay? /// The interval between clock-offset refreshes per camera. @@ -234,6 +260,18 @@ public actor MulticamController { sendTo(peer, RemoteCmd.ClockSyncPing(t0Millis: SyncClock.nowMillis())) } + case let ack as RemoteCmd.ScheduledCaptureAck: + recordCaptureAck(from: peer, outcome: ack.error == nil ? .captured : .failed) + + case is RemoteCmd.TakePicAck: + // The fallback (plain TakePic) path's positive ack. + recordCaptureAck(from: peer, outcome: .captured) + + case let resp as RemoteCmd.TakePicResp: + // Fallback failure signal; a success here is a duplicate of the ack + // and is ignored (the lane is already resolved). + if resp.error != nil { recordCaptureAck(from: peer, outcome: .failed) } + default: // Per-camera command responses (zoom/lens/flash/torch acks) update // only the focused lane's controls, wired to the UI in a later PR; @@ -331,6 +369,111 @@ public actor MulticamController { sendTo(peer, msg) } + // MARK: - Synced photo capture (all cameras) + + /// Test seam. + func setCaptureAckTimeout(_ t: TimeInterval) { captureAckTimeout = t } + func captureStateForTesting() -> (id: String, remaining: Int)? { + if case .capturingPhoto(let id, let remaining) = state { return (id, remaining) } + return nil + } + func captureOutcomeForTesting(_ peer: MCPeerID) -> CaptureOutcome? { links[peer]?.captureOutcome } + + /// Test seam: put a lane in the state a real handshake would — linked, with + /// known multicam capability and (optionally) a known clock offset — so + /// capture behavior can be asserted deterministically. + func seedLaneForTesting(_ peer: MCPeerID, supportsMulticam: Bool, offsetMillis: Int64?) { + guard let link = links[peer] else { return } + link.status = .linked + link.capabilities = RemoteCmd.CameraCapabilitiesResp( + frontCamera: nil, backCamera: nil, currentCamera: .back, + currentLens: .wideAngle, currentZoom: 1.0, + supportsMulticam: supportsMulticam, error: nil) + if let offsetMillis { + // t0 == t3 == 0 → rtt 0, midpoint 0, so offset == cameraClock. + link.clockEstimator.recordExchange( + t0Millis: 0, cameraClockMillis: UInt64(bitPattern: offsetMillis), t3Millis: 0) + } + } + + /// Fire a synced photo on every ready multicam camera. When each camera's + /// clock offset is known, the shutter is scheduled at a shared instant + /// (`ScheduledCapture`) so the exposures line up sub-frame; if any offset + /// is still missing it falls back to a plain fan-out (`TakePic`) — the same + /// shot id groups both paths. No-op if nothing is ready or a capture is + /// already in flight. + func capturePhoto() { + guard case .monitoring = state else { return } + let ready = order.compactMap { links[$0] } + .filter { $0.status == .linked && $0.supportsMulticam } + guard !ready.isEmpty else { return } + + let captureID = UUID().uuidString + capturingLanes = Set(ready.map(\.peerID)) + currentCaptureID = captureID + for link in ready { link.captureOutcome = nil } + + let everyOffsetKnown = ready.allSatisfy { $0.latestOffset != nil } + if everyOffsetKnown { + let fireAt = SyncClock.nowMillis() + captureLeadMillis + for (index, link) in ready.enumerated() { + let offset = link.latestOffset?.offsetMillis ?? 0 + let fireAtCameraClock = UInt64(Int64(fireAt) + offset) + sendTo(link.peerID, RemoteCmd.ScheduledCapture( + fireAtCameraClockMillis: fireAtCameraClock, + anchorMillis: fireAt, + captureId: captureID, + sessionId: sessionID, + cameraIndex: index + 1)) + } + } else { + // Fallback: not every clock is measured yet, so we can't promise a + // sub-frame shutter — take the shot immediately on each camera. + for link in ready { + sendTo(link.peerID, RemoteCmd.TakePic(sender: nil, sendMediaToPeer: false)) + } + } + + state = .capturingPhoto(captureId: captureID, acksRemaining: capturingLanes.count) + publishLanes() + armCaptureTimeout(captureID) + } + + /// A camera answered the in-flight capture (accepted or refused). Counts it + /// once; later duplicate acks (e.g. a scheduled camera also sends TakePic + /// acks after it fires) are ignored. + private func recordCaptureAck(from peer: MCPeerID, outcome: CaptureOutcome) { + guard case .capturingPhoto(let captureID, _) = state, + capturingLanes.contains(peer) else { return } + links[peer]?.captureOutcome = outcome + capturingLanes.remove(peer) + let remaining = capturingLanes.count + state = remaining == 0 + ? .monitoring(mode: .photo) + : .capturingPhoto(captureId: captureID, acksRemaining: remaining) + publishLanes() + } + + private func armCaptureTimeout(_ captureID: String) { + let timeout = captureAckTimeout + Task { [weak self] in + try? await Task.sleep(nanoseconds: UInt64(timeout * 1_000_000_000)) + guard let self else { return } + await self.expireCaptureAcks(captureID) + } + } + + /// Any camera that never answered by the deadline is counted as failed, so + /// the aggregate always resolves back to monitoring. + private func expireCaptureAcks(_ captureID: String) { + guard case .capturingPhoto(let inFlight, _) = state, inFlight == captureID else { return } + for peer in capturingLanes { links[peer]?.captureOutcome = .failed } + capturingLanes.removeAll() + currentCaptureID = nil + state = .monitoring(mode: .photo) + publishLanes() + } + /// A lane's stall watchdog fired — re-request a frame to unstick just that /// camera's pump (the others are unaffected). func nudgeFrame(for peer: MCPeerID) { @@ -384,15 +527,19 @@ public actor MulticamController { displayName: link.displayName, status: link.status, isFocused: peer == focusedPeer, - clockOffsetMillis: link.latestOffset?.offsetMillis) + clockOffsetMillis: link.latestOffset?.offsetMillis, + captureOutcome: link.captureOutcome) } } private func publishLanes() { let snapshot = laneSnapshot() + let inFlight: Bool + if case .capturingPhoto = state { inFlight = true } else { inFlight = false } let display = display OperationQueue.main.addOperation { display?.applyLanes(snapshot) + display?.applyCaptureInFlight(inFlight) } } diff --git a/RemoteCam/MulticamView.swift b/RemoteCam/MulticamView.swift index ca3b8af..f4f914d 100644 --- a/RemoteCam/MulticamView.swift +++ b/RemoteCam/MulticamView.swift @@ -16,6 +16,8 @@ struct MulticamView: View { /// Tap a thumbnail to make that camera the focused one. let onFocusLane: (CameraLane) -> Void + /// The synced shutter — fires a photo on every ready camera at once. + let onCapture: () -> Void var body: some View { GeometryReader { geo in @@ -30,10 +32,34 @@ struct MulticamView: View { focusedViewfinder stripOverlay(dock: dock) + + shutterOverlay(dock: dock) } } } + /// The all-camera shutter, docked on the same edge the 1:1 monitor uses so + /// the two screens feel of a piece. Reuses the monitor's `ShutterButton` + /// (and its activity ring) for a consistent feel. + @ViewBuilder + private func shutterOverlay(dock: MonitorChromeDock) -> some View { + let shutter = ShutterButton( + uiState: .photoMode, + isRecording: false, + activity: viewModel.isCapturing ? .capturing : nil, + isEnabled: !viewModel.isCapturing && viewModel.focusedLane != nil, + action: onCapture) + + switch dock { + case .bottom: + VStack { Spacer(); shutter.padding(.bottom, 24) } + case .leading: + HStack { shutter.padding(.leading, 24); Spacer() } + case .trailing: + HStack { Spacer(); shutter.padding(.trailing, 24) } + } + } + private var chromeInput: MonitorChromeInput { #if targetEnvironment(macCatalyst) return .pointer @@ -119,6 +145,20 @@ struct CameraTileView: View { .foregroundColor(.white)) } + if let outcome = lane.captureOutcome { + VStack { + HStack { + Spacer() + Image(systemName: outcome == .captured + ? "checkmark.circle.fill" : "exclamationmark.triangle.fill") + .font(.body) + .foregroundColor(outcome == .captured ? .green : .yellow) + .padding(6) + } + Spacer() + } + } + VStack { Spacer() Text(lane.displayName) diff --git a/RemoteCam/MulticamViewController.swift b/RemoteCam/MulticamViewController.swift index 0dd11c1..475feae 100644 --- a/RemoteCam/MulticamViewController.swift +++ b/RemoteCam/MulticamViewController.swift @@ -44,6 +44,10 @@ public final class MulticamViewController: UIViewController { onFocusLane: { [weak self] lane in guard let self else { return } Task { await self.controller.setFocusedPeer(lane.peerID) } + }, + onCapture: { [weak self] in + guard let self else { return } + Task { await self.controller.capturePhoto() } }) hosting = embedSwiftUIView(multicamView) @@ -106,6 +110,10 @@ extension MulticamViewController: MulticamDisplay { for lane in created { wire(lane) } } + func applyCaptureInFlight(_ inFlight: Bool) { + viewModel.isCapturing = inFlight + } + func receiveFrame(_ frame: RemoteCmd.OnFrame) { // Route to exactly the source lane's decoder; a frame for camera B // never touches camera A's tile. diff --git a/RemoteCam/MulticamViewModel.swift b/RemoteCam/MulticamViewModel.swift index 799056d..b8228b8 100644 --- a/RemoteCam/MulticamViewModel.swift +++ b/RemoteCam/MulticamViewModel.swift @@ -25,6 +25,8 @@ final class CameraLane: ObservableObject, Identifiable { @Published var status: CameraLink.Status @Published var isFocused: Bool + /// How this camera answered the last synced capture — a brief tile badge. + @Published var captureOutcome: CaptureOutcome? /// This lane's own decoder + stall watchdog. The view controller wires its /// `onImage` to set `frames.cameraImage`, and its stall/keyframe callbacks @@ -36,6 +38,7 @@ final class CameraLane: ObservableObject, Identifiable { self.displayName = info.displayName self.status = info.status self.isFocused = info.isFocused + self.captureOutcome = info.captureOutcome } } @@ -44,6 +47,8 @@ final class CameraLane: ObservableObject, Identifiable { final class MulticamViewModel: ObservableObject { @Published private(set) var lanes: [CameraLane] = [] @Published var interfaceOrientation: UIInterfaceOrientation = .portrait + /// A synced photo is in flight — drives the shutter's activity ring. + @Published var isCapturing: Bool = false var focusedLane: CameraLane? { lanes.first { $0.isFocused } } var otherLanes: [CameraLane] { lanes.filter { !$0.isFocused } } @@ -63,6 +68,7 @@ final class MulticamViewModel: ObservableObject { if let lane = existing.removeValue(forKey: info.peerID) { if lane.status != info.status { lane.status = info.status } if lane.isFocused != info.isFocused { lane.isFocused = info.isFocused } + if lane.captureOutcome != info.captureOutcome { lane.captureOutcome = info.captureOutcome } return lane } let lane = CameraLane(info: info) diff --git a/RemoteCam/RemoteCmdFlatBuffers.swift b/RemoteCam/RemoteCmdFlatBuffers.swift index ed3636e..b5a7b3a 100644 --- a/RemoteCam/RemoteCmdFlatBuffers.swift +++ b/RemoteCam/RemoteCmdFlatBuffers.swift @@ -31,6 +31,8 @@ func serializeToFlatBuffer(_ msg: Message) -> Data? { case let m as RemoteCmd.RequestKeyframe: return m.toFlatBuffer() case let m as RemoteCmd.ClockSyncPing: return m.toFlatBuffer() case let m as RemoteCmd.ClockSyncPong: return m.toFlatBuffer() + case let m as RemoteCmd.ScheduledCapture: return m.toFlatBuffer() + case let m as RemoteCmd.ScheduledCaptureAck: return m.toFlatBuffer() case let m as RemoteCmd.SetZoom: return m.toFlatBuffer() case let m as RemoteCmd.SetZoomResp: return m.toFlatBuffer() case let m as RemoteCmd.FocusAtPoint: return m.toFlatBuffer() @@ -765,6 +767,37 @@ extension RemoteCmd.ClockSyncPong { } } +extension RemoteCmd.ScheduledCapture { + func toFlatBuffer() -> Data { + var fbb = FlatBufferBuilder() + let captureIdOffset = fbb.create(string: captureId) + let sessionIdOffset = fbb.create(string: sessionId) + let params = RemoteShutter_CommandParameters.createCommandParameters( + &fbb, + captureFireAtCameraClockMs: fireAtCameraClockMillis, + captureAnchorMs: anchorMillis, + captureIdOffset: captureIdOffset, + captureSessionIdOffset: sessionIdOffset, + captureCameraIndex: Int32(cameraIndex)) + return buildCommand(&fbb, action: .scheduledcapture, parameters: params) + } +} + +extension RemoteCmd.ScheduledCaptureAck { + func toFlatBuffer() -> Data { + var fbb = FlatBufferBuilder() + let echoOffset = fbb.create(string: captureId) + let errorOffset = (error as NSError?).map { fbb.create(string: $0.localizedDescription) } ?? Offset() + let resp = RemoteShutter_CameraStateResponse.createCameraStateResponse( + &fbb, + action: .scheduledcapture, + success: error == nil, + errorOffset: errorOffset, + captureIdEchoOffset: echoOffset) + return buildResponse(&fbb, action: .scheduledcapture, response: resp) + } +} + extension RemoteCmd.ToggleFlash { func toFlatBuffer() -> Data { var fbb = FlatBufferBuilder() @@ -1128,6 +1161,14 @@ extension RemoteCmd { case .clocksyncping: return ClockSyncPing(t0Millis: params?.clockSyncT0Ms ?? 0) + case .scheduledcapture: + return ScheduledCapture( + fireAtCameraClockMillis: params?.captureFireAtCameraClockMs ?? 0, + anchorMillis: params?.captureAnchorMs ?? 0, + captureId: params?.captureId ?? "", + sessionId: params?.captureSessionId ?? "", + cameraIndex: Int(params?.captureCameraIndex ?? 0)) + case .toggleflash: return ToggleFlash() @@ -1192,6 +1233,9 @@ extension RemoteCmd { return ClockSyncPong(echoT0Millis: resp.clockSyncEchoT0Ms, cameraClockMillis: resp.clockSyncCameraClockMs) + case .scheduledcapture: + return ScheduledCaptureAck(captureId: resp.captureIdEcho ?? "", error: nsError) + case .startrecording: let startTime: Date? = resp.recordingStartTime > 0 ? Date(timeIntervalSince1970: Double(resp.recordingStartTime) / 1000.0) : nil return StartRecordingVideoAck(sender: nil, recordingStartTime: startTime, error: nsError) diff --git a/RemoteCam/RemoteCmds.swift b/RemoteCam/RemoteCmds.swift index 4ff8728..d91c676 100644 --- a/RemoteCam/RemoteCmds.swift +++ b/RemoteCam/RemoteCmds.swift @@ -220,6 +220,50 @@ public class RemoteCmd: Message, @unchecked Sendable { } } + /// Director → camera: fire the shutter at `fireAtCameraClockMillis`, a + /// wall-clock instant already translated into this camera's own + /// `SyncClock` domain (the director applied the per-camera offset), so all + /// cameras expose together. `anchorMillis` is the same instant in the + /// director's clock — identical across every camera in the shot, so it is + /// the alignment key each clip is stamped with. Only sent to peers that + /// advertised `supportsMulticam`. + public class ScheduledCapture: Message, @unchecked Sendable { + public let fireAtCameraClockMillis: UInt64 + public let anchorMillis: UInt64 + public let captureId: String + public let sessionId: String + public let cameraIndex: Int + + public init(fireAtCameraClockMillis: UInt64, + anchorMillis: UInt64, + captureId: String, + sessionId: String, + cameraIndex: Int, + sender: AnyObject? = nil) { + self.fireAtCameraClockMillis = fireAtCameraClockMillis + self.anchorMillis = anchorMillis + self.captureId = captureId + self.sessionId = sessionId + self.cameraIndex = cameraIndex + super.init(sender: sender) + } + } + + /// Camera → director: the scheduled capture was accepted (or refused). Sent + /// immediately on receipt — before the shutter actually fires — so the + /// director can aggregate acks without waiting for N photos. `error` set = + /// the camera could not schedule it (wrong state, fire time long past). + public class ScheduledCaptureAck: Message, @unchecked Sendable { + public let captureId: String + public let error: Error? + + public init(captureId: String, error: Error? = nil, sender: AnyObject? = nil) { + self.captureId = captureId + self.error = error + super.init(sender: sender) + } + } + public class OnFrame: Message, @unchecked Sendable { public let data: Data public let peerId: MCPeerID diff --git a/RemoteCam/SessionCoordinator.swift b/RemoteCam/SessionCoordinator.swift index d93a1b4..ca807be 100644 --- a/RemoteCam/SessionCoordinator.swift +++ b/RemoteCam/SessionCoordinator.swift @@ -126,6 +126,16 @@ final class DeferredPopAndScan: Message, @unchecked Sendable { /// Incompatibility detected by the transport (routed through the inbox — the /// old code mutated state directly on the delegate thread). final class IncompatibilityDetected: Message, @unchecked Sendable {} +/// Camera side: a scheduled multicam capture's fire time has arrived. Enqueued +/// by the off-actor delay task so the shutter is pulled by the message pump, +/// in order with every other state transition — never racing it. +final class FireScheduledCapture: Message, @unchecked Sendable { + let metadata: CaptureSyncMetadata + init(metadata: CaptureSyncMetadata) { + self.metadata = metadata + super.init(sender: nil) + } +} /// Retry tick for the capabilities ladder. final class RetryCapabilities: Message, @unchecked Sendable { let attempt: Int @@ -222,6 +232,18 @@ public actor SessionCoordinator { /// camera peer speaks VP9, which gates sending `RemoteCmd.RequestKeyframe`. private var monitorReceivedVP9Frame = false + /// Camera side: the sync metadata for a scheduled multicam capture that is + /// about to fire. Set when `FireScheduledCapture` triggers the shutter and + /// consumed by the next `OnPicture`, so exactly that photo is stamped and + /// saved under its `RS___cam` filename. Nil for ordinary + /// single-camera captures, which save exactly as before. + private var pendingSyncMetadata: CaptureSyncMetadata? + + /// How far in the past a scheduled fire time may be before the camera + /// refuses it (clock error and one retransmit can nudge it slightly late; + /// beyond this it is stale and would fire out of sync). + private let scheduledCaptureMaxLatenessMillis: Int64 = 1000 + /// Multicam director "collecting" mode. Off by default and only ever set /// by the scanner when `ENABLE_MULTICAM` and the monitor role, so every /// non-multicam path is byte-identical. While set, a peer connecting in @@ -971,6 +993,22 @@ public actor SessionCoordinator { await showCameraAlert(NSLocalizedString("Taking picture", comment: "")) await transition(to: .cameraTakingPic(sendMediaToPeer: pic.sendMediaToPeer, generation: generation)) + case let scheduled as RemoteCmd.ScheduledCapture: + await handleScheduledCapture(scheduled) + + case let fire as FireScheduledCapture: + // The fire instant arrived (enqueued by the off-actor delay task). + // Pull the shutter through the normal photo path, saving locally + // only — the director does not collect stills in v1 — but stamp + // this photo with its sync metadata on the way to Photos. + pendingSyncMetadata = fire.metadata + ctrl.currentCameraMode = .Photo + ctrl.updateCameraStatus() + ctrl.takePicture(false) + let generation = scheduleTimeout(.cameraTakingPic) + await showCameraAlert(NSLocalizedString("Taking picture", comment: "")) + await transition(to: .cameraTakingPic(sendMediaToPeer: false, generation: generation)) + case is RemoteCmd.ToggleCamera: do { _ = try await ctrl.toggleCamera() @@ -1173,6 +1211,41 @@ public actor SessionCoordinator { sendMessage(RemoteCmd.RequestKeyframe(sender: nil), mode: .reliable) } + /// Camera side of a synced multicam shot. Acks (or nacks) immediately so + /// the director can aggregate without waiting for the photo, then schedules + /// the shutter for the fire instant. The delay runs off the actor and only + /// enqueues `FireScheduledCapture`, so the capture is pulled by the message + /// pump in order — never racing a state transition. + private func handleScheduledCapture(_ scheduled: RemoteCmd.ScheduledCapture) async { + let now = SyncClock.nowMillis() + let lateness = Int64(now) - Int64(scheduled.fireAtCameraClockMillis) + guard lateness <= scheduledCaptureMaxLatenessMillis else { + await sendOrGoToScanning(RemoteCmd.ScheduledCaptureAck( + captureId: scheduled.captureId, + error: NSError(domain: "Scheduled capture fire time already passed", + code: 0, userInfo: nil))) + return + } + await sendOrGoToScanning(RemoteCmd.ScheduledCaptureAck(captureId: scheduled.captureId)) + + let metadata = CaptureSyncMetadata( + sessionID: scheduled.sessionId, + captureID: scheduled.captureId, + cameraIndex: scheduled.cameraIndex, + anchorMillis: scheduled.anchorMillis, + // The camera does not compute its own clock offset — the director + // did, when it translated the fire time into this clock. The + // shared `anchorMillis` is the alignment key; these are diagnostics. + clockOffsetMillis: 0, + roundTripMillis: 0) + + let delayMillis = max(0, -lateness) + Task { [weak self] in + try? await Task.sleep(nanoseconds: UInt64(delayMillis) * 1_000_000) + self?.tell(FireScheduledCapture(metadata: metadata)) + } + } + private func inCameraTakingPic(_ msg: Message, sendMediaToPeer: Bool, generation: Int) async { switch msg { case is RemoteCmd.RequestKeyframe: @@ -1183,6 +1256,7 @@ public actor SessionCoordinator { case let timeout as UICmd.StateTimeout: guard timeout.stateName == .cameraTakingPic && timeout.generation == generation else { break } + pendingSyncMetadata = nil await dismissCameraAlert() let sent = sendMessage(RemoteCmd.TakePicResp( sender: nil, @@ -1195,8 +1269,16 @@ public actor SessionCoordinator { case let picture as UICmd.OnPicture: if let pic = picture.pic { - photoLibrarySaver(pic) + // A scheduled multicam capture stamps its sync metadata and + // saves under the shared RS___cam name; an + // ordinary capture saves exactly as before. + if let metadata = pendingSyncMetadata { + saveSyncedPictureToLibrary(pic, metadata: metadata) + } else { + photoLibrarySaver(pic) + } } + pendingSyncMetadata = nil await dismissCameraAlert() guard sendMessage(RemoteCmd.TakePicAck(sender: nil)) else { await popToScanning() @@ -1498,6 +1580,11 @@ public actor SessionCoordinator { case is RemoteCmd.TakePic: await sendOrGoToScanning(RemoteCmd.TakePicResp(sender: nil, error: unableToProcessError(msg))) + case let scheduled as RemoteCmd.ScheduledCapture: + // Not in a state that can take a picture — nack so the director's + // ack aggregation completes instead of timing out. + await sendOrGoToScanning(RemoteCmd.ScheduledCaptureAck( + captureId: scheduled.captureId, error: unableToProcessError(msg))) case is RemoteCmd.ToggleCamera: await sendOrGoToScanning(RemoteCmd.ToggleCameraResp(cameraCapabilities: nil, error: unableToProcessError(msg))) case is RemoteCmd.SelectCameraDevice: @@ -2437,6 +2524,36 @@ public actor SessionCoordinator { } } + /// Save a synced multicam still: the image is stamped with the shot's sync + /// fields (EXIF) and lands under the shared `RS___cam` name, + /// so any editor can group and align the angles. Full-res stays on this + /// camera — the director does not collect stills in v1. + private nonisolated func saveSyncedPictureToLibrary(_ data: Data, + metadata: CaptureSyncMetadata) { + let stamped = metadata.stamped(data) + // HEIC magic: bytes 4..12 are "ftyp" then a heic/heix/mif1 brand. + let isHEIC = stamped.count > 12 + && stamped[4] == 0x66 && stamped[5] == 0x74 + && stamped[6] == 0x79 && stamped[7] == 0x70 + PHPhotoLibrary.requestAuthorization { status in + guard status == .authorized else { + DispatchQueue.main.async { + showPhotosAccessDeniedModal(for: .photo) + } + return + } + PHPhotoLibrary.shared().performChanges({ + let options = PHAssetResourceCreationOptions() + options.originalFilename = metadata.photoFilename(isHEIC: isHEIC) + PHAssetCreationRequest.forAsset() + .addResource(with: .photo, data: stamped, options: options) + }) { (success: Bool, _: Error?) in + print(success ? "Saved synced photo \(metadata.filenamePrefix)" + : "Failed to save synced photo!") + } + } + } + /// Monitor-side picture save (with the App Store review prompt). private nonisolated func savePictureOnMonitor(_ imageData: Data) { PHPhotoLibrary.requestAuthorization { status in diff --git a/RemoteCamTests/MulticamControllerTests.swift b/RemoteCamTests/MulticamControllerTests.swift index 9a1ab65..bdd03ce 100644 --- a/RemoteCamTests/MulticamControllerTests.swift +++ b/RemoteCamTests/MulticamControllerTests.swift @@ -13,9 +13,11 @@ import XCTest private final class FakeMulticamDisplay: MulticamDisplay, @unchecked Sendable { var lastLanes: [MulticamLaneInfo] = [] var receivedFrames: [MCPeerID] = [] + var captureInFlight = false var didExit = false func applyLanes(_ lanes: [MulticamLaneInfo]) { lastLanes = lanes } + func applyCaptureInFlight(_ inFlight: Bool) { captureInFlight = inFlight } func receiveFrame(_ frame: RemoteCmd.OnFrame) { receivedFrames.append(frame.peerId) } func exitMulticam() { didExit = true } } @@ -161,6 +163,116 @@ final class MulticamControllerTests: XCTestCase { XCTAssertNil(offsetB) } + // MARK: - Synced photo capture + + func testScheduledCaptureAppliesPerLaneOffsets() async { + let (controller, transport, _) = await makeController(peers: [camA, camB]) + await controller.seedLaneForTesting(camA, supportsMulticam: true, offsetMillis: 100) + await controller.seedLaneForTesting(camB, supportsMulticam: true, offsetMillis: -50) + transport.sentMessages.removeAll() + + await controller.capturePhoto() + await controller.waitForIdle() + + let scheduled = sent(transport, RemoteCmd.ScheduledCapture.self) + XCTAssertEqual(scheduled.count, 2) + for s in scheduled { XCTAssertEqual(s.peers.count, 1) } + + func fireAt(_ peer: MCPeerID) -> UInt64? { + (scheduled.first { $0.peers == [peer] }?.msg as? RemoteCmd.ScheduledCapture)? + .fireAtCameraClockMillis + } + // Same shared anchor + each lane's own offset ⇒ the fire instants differ + // by exactly the offset difference (100 − (−50) = 150), regardless of + // the nondeterministic base. + let a = fireAt(camA), b = fireAt(camB) + XCTAssertNotNil(a); XCTAssertNotNil(b) + XCTAssertEqual(Int64(a!) - Int64(b!), 150) + + // One shared capture id across both cameras. + let ids = Set(scheduled.compactMap { ($0.msg as? RemoteCmd.ScheduledCapture)?.captureId }) + XCTAssertEqual(ids.count, 1) + } + + func testNonMulticamLaneIsExcludedFromCapture() async { + let (controller, transport, _) = await makeController(peers: [camA, camB]) + await controller.seedLaneForTesting(camA, supportsMulticam: true, offsetMillis: 10) + await controller.seedLaneForTesting(camB, supportsMulticam: false, offsetMillis: 10) + transport.sentMessages.removeAll() + + await controller.capturePhoto() + await controller.waitForIdle() + + let scheduled = sent(transport, RemoteCmd.ScheduledCapture.self) + XCTAssertEqual(scheduled.map(\.peers), [[camA]]) + let capture = await controller.captureStateForTesting() + XCTAssertEqual(capture?.remaining, 1) + } + + func testAckAggregationReturnsToMonitoring() async { + let (controller, _, _) = await makeController(peers: [camA, camB]) + await controller.seedLaneForTesting(camA, supportsMulticam: true, offsetMillis: 0) + await controller.seedLaneForTesting(camB, supportsMulticam: true, offsetMillis: 0) + + await controller.capturePhoto() + await controller.waitForIdle() + let captureID = await controller.captureStateForTesting()?.id + XCTAssertNotNil(captureID) + + controller.didReceiveMessage(RemoteCmd.ScheduledCaptureAck(captureId: captureID!), from: camA) + await controller.waitForIdle() + let afterFirst = await controller.captureStateForTesting() + XCTAssertEqual(afterFirst?.remaining, 1) + + controller.didReceiveMessage(RemoteCmd.ScheduledCaptureAck(captureId: captureID!), from: camB) + await controller.waitForIdle() + let afterSecond = await controller.captureStateForTesting() + let outA = await controller.captureOutcomeForTesting(camA) + let outB = await controller.captureOutcomeForTesting(camB) + XCTAssertNil(afterSecond, "back to monitoring") + XCTAssertEqual(outA, .captured) + XCTAssertEqual(outB, .captured) + } + + func testCaptureTimeoutMarksTheSilentLaneAndCompletes() async { + let (controller, _, _) = await makeController(peers: [camA, camB]) + await controller.setCaptureAckTimeout(0.1) + await controller.seedLaneForTesting(camA, supportsMulticam: true, offsetMillis: 0) + await controller.seedLaneForTesting(camB, supportsMulticam: true, offsetMillis: 0) + + await controller.capturePhoto() + await controller.waitForIdle() + let captureID = await controller.captureStateForTesting()?.id + controller.didReceiveMessage(RemoteCmd.ScheduledCaptureAck(captureId: captureID!), from: camA) + await controller.waitForIdle() + + // Wait out the ack timeout; camB never answered. + try? await Task.sleep(nanoseconds: 300_000_000) + let resolved = await controller.captureStateForTesting() + let outA = await controller.captureOutcomeForTesting(camA) + let outB = await controller.captureOutcomeForTesting(camB) + XCTAssertNil(resolved, "aggregate resolves") + XCTAssertEqual(outA, .captured) + XCTAssertEqual(outB, .failed) + } + + func testFallbackToPlainTakePicWhenAnOffsetIsMissing() async { + let (controller, transport, _) = await makeController(peers: [camA, camB]) + await controller.seedLaneForTesting(camA, supportsMulticam: true, offsetMillis: 20) + await controller.seedLaneForTesting(camB, supportsMulticam: true, offsetMillis: nil) + transport.sentMessages.removeAll() + + await controller.capturePhoto() + await controller.waitForIdle() + + XCTAssertTrue(sent(transport, RemoteCmd.ScheduledCapture.self).isEmpty, + "a missing offset forces the plain fan-out") + let takes = sent(transport, RemoteCmd.TakePic.self) + XCTAssertEqual(Set(takes.flatMap(\.peers)), [camA, camB]) + let fallbackState = await controller.captureStateForTesting() + XCTAssertEqual(fallbackState?.remaining, 2) + } + // MARK: - Removal func testRemoveCameraDropsTheLaneAndRefocuses() async { diff --git a/RemoteCamTests/MulticamViewModelTests.swift b/RemoteCamTests/MulticamViewModelTests.swift index a97ce3f..1edab49 100644 --- a/RemoteCamTests/MulticamViewModelTests.swift +++ b/RemoteCamTests/MulticamViewModelTests.swift @@ -17,7 +17,8 @@ final class MulticamViewModelTests: XCTestCase { private func info(_ peer: MCPeerID, status: CameraLink.Status = .linked, focused: Bool = false) -> MulticamLaneInfo { MulticamLaneInfo(peerID: peer, displayName: peer.displayName, - status: status, isFocused: focused, clockOffsetMillis: nil) + status: status, isFocused: focused, clockOffsetMillis: nil, + captureOutcome: nil) } func testApplyAddsLanesAndReportsCreated() { diff --git a/RemoteCamTests/RemoteCamSessionTests.swift b/RemoteCamTests/RemoteCamSessionTests.swift index a335997..05039e5 100644 --- a/RemoteCamTests/RemoteCamSessionTests.swift +++ b/RemoteCamTests/RemoteCamSessionTests.swift @@ -302,6 +302,47 @@ class SessionCoordinatorTests: XCTestCase { (pongs[0].msg as? RemoteCmd.ClockSyncPong)?.cameraClockMillis ?? 0, 0) } + // MARK: - Scheduled (multicam) capture, camera side + + func testScheduledCaptureInThePastNacks() async { + await enterCamera() + harness.fakeMP.sendResult = true + + await harness.deliver(RemoteCmd.ScheduledCapture( + fireAtCameraClockMillis: 1, // long past + anchorMillis: 1, captureId: "CAP-1", sessionId: "S", cameraIndex: 1)) + + let acks = sent(RemoteCmd.ScheduledCaptureAck.self) + .compactMap { $0.msg as? RemoteCmd.ScheduledCaptureAck } + XCTAssertEqual(acks.count, 1) + XCTAssertEqual(acks[0].captureId, "CAP-1") + XCTAssertNotNil(acks[0].error, "a fire time in the past is refused") + XCTAssertTrue(camera.takePictureCalls.isEmpty, "no shutter on a nack") + } + + func testValidScheduledCaptureAcksImmediatelyAndFires() async { + await enterCamera() + harness.fakeMP.sendResult = true + + // Fire ~now: acked immediately, then the shutter pulls a moment later. + await harness.deliver(RemoteCmd.ScheduledCapture( + fireAtCameraClockMillis: SyncClock.nowMillis(), + anchorMillis: SyncClock.nowMillis(), captureId: "CAP-2", + sessionId: "S", cameraIndex: 2)) + + let acks = sent(RemoteCmd.ScheduledCaptureAck.self) + .compactMap { $0.msg as? RemoteCmd.ScheduledCaptureAck } + XCTAssertEqual(acks.map(\.captureId), ["CAP-2"]) + XCTAssertNil(acks[0].error, "accepted") + + // The fire is enqueued by an off-actor delay task; wait for it. + for _ in 0..<200 where camera.takePictureCalls.isEmpty { + try? await Task.sleep(nanoseconds: 10_000_000) + } + XCTAssertEqual(camera.takePictureCalls, [false], + "the scheduled shutter fires, saving locally (not to the director)") + } + func testMonitorPhotoModeUnbecomeMonitorPopsToConnected() async { await enterMonitor(.Photo) await harness.deliver(UICmd.UnbecomeMonitor(sender: nil)) diff --git a/RemoteCamTests/RemoteCmdSerializationTests.swift b/RemoteCamTests/RemoteCmdSerializationTests.swift index b5b9585..dafc51e 100644 --- a/RemoteCamTests/RemoteCmdSerializationTests.swift +++ b/RemoteCamTests/RemoteCmdSerializationTests.swift @@ -49,6 +49,8 @@ final class RemoteCmdSerializationTests: XCTestCase { case let m as RemoteCmd.RequestKeyframe: return m.toFlatBuffer() case let m as RemoteCmd.ClockSyncPing: return m.toFlatBuffer() case let m as RemoteCmd.ClockSyncPong: return m.toFlatBuffer() + case let m as RemoteCmd.ScheduledCapture: return m.toFlatBuffer() + case let m as RemoteCmd.ScheduledCaptureAck: return m.toFlatBuffer() case let m as RemoteCmd.SetZoom: return m.toFlatBuffer() case let m as RemoteCmd.SetZoomResp: return m.toFlatBuffer() case let m as RemoteCmd.FocusAtPoint: return m.toFlatBuffer() @@ -1197,6 +1199,32 @@ extension RemoteCmdSerializationTests { XCTAssertEqual(result.cameraClockMillis, 123_456_789_345) } + func testScheduledCapture_roundTrip() { + let result = roundTrip(RemoteCmd.ScheduledCapture( + fireAtCameraClockMillis: 1_754_800_000_123, + anchorMillis: 1_754_800_000_000, + captureId: "CAP-123", + sessionId: "SESS-9", + cameraIndex: 3)) + XCTAssertEqual(result.fireAtCameraClockMillis, 1_754_800_000_123) + XCTAssertEqual(result.anchorMillis, 1_754_800_000_000) + XCTAssertEqual(result.captureId, "CAP-123") + XCTAssertEqual(result.sessionId, "SESS-9") + XCTAssertEqual(result.cameraIndex, 3) + } + + func testScheduledCaptureAck_roundTrip() { + let ok = roundTrip(RemoteCmd.ScheduledCaptureAck(captureId: "CAP-42")) + XCTAssertEqual(ok.captureId, "CAP-42") + XCTAssertNil(ok.error) + + let nack = roundTrip(RemoteCmd.ScheduledCaptureAck( + captureId: "CAP-43", + error: NSError(domain: "too late", code: 0))) + XCTAssertEqual(nack.captureId, "CAP-43") + XCTAssertNotNil(nack.error) + } + /// The multicam capability survives the wire, and a peer that predates it /// (absent field) decodes as not-multicam-capable. func testCapabilitiesCarryMulticamSupport() {