Multicam clock sync: ClockSyncPing/Pong + ClockOffsetEstimator

RemoteCam/ClockOffsetEstimator.swift

@@ -0,0 +1,74 @@
+//
+//  ClockOffsetEstimator.swift
+//  RemoteShutter
+//
+//  Created by Dario Lencina on 2026.
+//  Copyright © 2026 Security Union. All rights reserved.
+//
+
+import Foundation
+
+/// The millisecond clock both ends of a clock-sync exchange read. Monotonic
+/// while the app runs (never jumps with NTP/wall-clock changes), but it
+/// pauses in deep sleep and restarts per boot — which is why offsets are
+/// re-estimated on every foreground and never persisted.
+enum SyncClock {
+    static func nowMillis() -> UInt64 {
+        DispatchTime.now().uptimeNanoseconds / 1_000_000
+    }
+}
+
+/// One ping/pong exchange reduced to an offset estimate.
+struct ClockOffsetSample: Equatable {
+    /// camera clock − director clock, in ms. Adding this to a director
+    /// timestamp yields the same instant on the camera's clock.
+    let offsetMillis: Int64
+    /// Round trip of the exchange; smaller = tighter estimate (the error
+    /// bound is the RTT asymmetry, at most rtt/2).
+    let roundTripMillis: Int64
+}
+
+/// NTP-style offset estimation over the session link: keep a short window of
+/// exchanges and trust the minimum-RTT one — a queued or retransmitted
+/// exchange has a large RTT and an unreliable midpoint, so it should never
+/// outvote a clean one.
+///
+/// Pure value type: callers supply every timestamp, so tests need no clocks.
+struct ClockOffsetEstimator {
+    static let windowSize = 5
+
+    private(set) var samples: [ClockOffsetSample] = []
+
+    /// Record one exchange: director sent at `t0Millis`, camera stamped
+    /// `cameraClockMillis` at receipt, director received the pong at
+    /// `t3Millis` (all in each side's own `SyncClock`). Returns the sample,
+    /// or nil for a nonsensical exchange (pong before ping — a stale reply
+    /// from before a clock reset).
+    @discardableResult
+    mutating func recordExchange(t0Millis: UInt64,
+                                 cameraClockMillis: UInt64,
+                                 t3Millis: UInt64) -> ClockOffsetSample? {
+        guard t3Millis >= t0Millis else { return nil }
+        let rtt = Int64(t3Millis - t0Millis)
+        let directorMidpoint = Int64(bitPattern: t0Millis) + rtt / 2
+        let sample = ClockOffsetSample(
+            offsetMillis: Int64(bitPattern: cameraClockMillis) - directorMidpoint,
+            roundTripMillis: rtt)
+        samples.append(sample)
+        if samples.count > Self.windowSize {
+            samples.removeFirst(samples.count - Self.windowSize)
+        }
+        return sample
+    }
+
+    /// The trusted estimate: the minimum-RTT sample in the window.
+    var best: ClockOffsetSample? {
+        samples.min { $0.roundTripMillis < $1.roundTripMillis }
+    }
+
+    /// Drop everything — called when the clocks may have moved under us
+    /// (either side backgrounded, or the peer reconnected).
+    mutating func reset() {
+        samples.removeAll()
+    }
+}

RemoteCam/FlatBufferSchemas.fbs

@@ -38,8 +38,12 @@ enum CommandAction : byte {
     RequestKeyframe = 21,      // monitor -> camera: force a VP9 keyframe
     FocusAtPoint = 22,         // monitor -> camera: focus/exposure point
     EndSession = 23,           // either side: "I am leaving on purpose"
-    SetCameraPreviewMode = 24  // monitor -> camera: local preview on/standby;
+    SetCameraPreviewMode = 24, // monitor -> camera: local preview on/standby;
                                // only sent to peers advertising supports_preview_mode
+    ClockSyncPing = 25         // director -> camera: clock-offset probe carrying the
+                               // director's send time; the camera answers with a
+                               // ClockSyncPing response echoing it plus its own clock.
+                               // Only sent to peers advertising supports_multicam.
 }
 
 // Whether the camera device drives its own on-screen live preview. On is the
@@ -158,6 +162,7 @@ table CommandParameters {
     focus_point_x: float;            // payload for FocusAtPoint (normalized 0..1,
     focus_point_y: float;            // upright-display space; origin top-left)
     camera_preview_mode: CameraPreviewModeEnum;  // payload for SetCameraPreviewMode
+    clock_sync_t0_ms: uint64;        // payload for ClockSyncPing: director clock at send
 }
 
 // MARK: - Command Structure
@@ -277,6 +282,9 @@ table CameraStateResponse {
     available_lenses: [CameraLensType];
     zoom_range: ZoomRange;
     current_zoom: double;
+    // Appended fields only below this line (FlatBuffers schema evolution).
+    clock_sync_echo_t0_ms: uint64;    // ClockSyncPing response: echoed director t0
+    clock_sync_camera_clock_ms: uint64; // ClockSyncPing response: camera clock at receipt
 }
 
 // MARK: - Frame Data

RemoteCam/FlatBufferSchemas_generated.swift

@@ -33,8 +33,9 @@ public enum RemoteShutter_CommandAction: Int8, Enum, Verifiable {
   case focusatpoint = 22
   case endsession = 23
   case setcamerapreviewmode = 24
+  case clocksyncping = 25
 
-  public static var max: RemoteShutter_CommandAction { return .setcamerapreviewmode }
+  public static var max: RemoteShutter_CommandAction { return .clocksyncping }
   public static var min: RemoteShutter_CommandAction { return .unknown }
 }
 
@@ -331,6 +332,7 @@ public struct RemoteShutter_CommandParameters: FlatBufferObject, Verifiable {
     case focusPointX = 36
     case focusPointY = 38
     case cameraPreviewMode = 40
+    case clockSyncT0Ms = 42
     var v: Int32 { Int32(self.rawValue) }
     var p: VOffset { self.rawValue }
   }
@@ -357,7 +359,8 @@ public struct RemoteShutter_CommandParameters: FlatBufferObject, Verifiable {
   public var focusPointX: Float32 { let o = _accessor.offset(VTOFFSET.focusPointX.v); return o == 0 ? 0.0 : _accessor.readBuffer(of: Float32.self, at: o) }
   public var focusPointY: Float32 { let o = _accessor.offset(VTOFFSET.focusPointY.v); return o == 0 ? 0.0 : _accessor.readBuffer(of: Float32.self, at: o) }
   public var cameraPreviewMode: RemoteShutter_CameraPreviewModeEnum { let o = _accessor.offset(VTOFFSET.cameraPreviewMode.v); return o == 0 ? .unknown : RemoteShutter_CameraPreviewModeEnum(rawValue: _accessor.readBuffer(of: Int8.self, at: o)) ?? .unknown }
-  public static func startCommandParameters(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 19) }
+  public var clockSyncT0Ms: UInt64 { let o = _accessor.offset(VTOFFSET.clockSyncT0Ms.v); return o == 0 ? 0 : _accessor.readBuffer(of: UInt64.self, at: o) }
+  public static func startCommandParameters(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 20) }
   public static func add(sendToRemote: Bool, _ fbb: inout FlatBufferBuilder) { fbb.add(element: sendToRemote, def: false,
    at: VTOFFSET.sendToRemote.p) }
   public static func add(zoomFactor: Double, _ fbb: inout FlatBufferBuilder) { fbb.add(element: zoomFactor, def: 0.0, at: VTOFFSET.zoomFactor.p) }
@@ -378,6 +381,7 @@ public struct RemoteShutter_CommandParameters: FlatBufferObject, Verifiable {
   public static func add(focusPointX: Float32, _ fbb: inout FlatBufferBuilder) { fbb.add(element: focusPointX, def: 0.0, at: VTOFFSET.focusPointX.p) }
   public static func add(focusPointY: Float32, _ fbb: inout FlatBufferBuilder) { fbb.add(element: focusPointY, def: 0.0, at: VTOFFSET.focusPointY.p) }
   public static func add(cameraPreviewMode: RemoteShutter_CameraPreviewModeEnum, _ fbb: inout FlatBufferBuilder) { fbb.add(element: cameraPreviewMode.rawValue, def: 0, at: VTOFFSET.cameraPreviewMode.p) }
+  public static func add(clockSyncT0Ms: UInt64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: clockSyncT0Ms, def: 0, at: VTOFFSET.clockSyncT0Ms.p) }
   public static func endCommandParameters(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end }
   public static func createCommandParameters(
     _ fbb: inout FlatBufferBuilder,
@@ -399,7 +403,8 @@ public struct RemoteShutter_CommandParameters: FlatBufferObject, Verifiable {
     deviceUniqueIdOffset deviceUniqueId: Offset = Offset(),
     focusPointX: Float32 = 0.0,
     focusPointY: Float32 = 0.0,
-    cameraPreviewMode: RemoteShutter_CameraPreviewModeEnum = .unknown
+    cameraPreviewMode: RemoteShutter_CameraPreviewModeEnum = .unknown,
+    clockSyncT0Ms: UInt64 = 0
   ) -> Offset {
     let __start = RemoteShutter_CommandParameters.startCommandParameters(&fbb)
     RemoteShutter_CommandParameters.add(sendToRemote: sendToRemote, &fbb)
@@ -421,6 +426,7 @@ public struct RemoteShutter_CommandParameters: FlatBufferObject, Verifiable {
     RemoteShutter_CommandParameters.add(focusPointX: focusPointX, &fbb)
     RemoteShutter_CommandParameters.add(focusPointY: focusPointY, &fbb)
     RemoteShutter_CommandParameters.add(cameraPreviewMode: cameraPreviewMode, &fbb)
+    RemoteShutter_CommandParameters.add(clockSyncT0Ms: clockSyncT0Ms, &fbb)
     return RemoteShutter_CommandParameters.endCommandParameters(&fbb, start: __start)
   }
 
@@ -445,6 +451,7 @@ public struct RemoteShutter_CommandParameters: FlatBufferObject, Verifiable {
     try _v.visit(field: VTOFFSET.focusPointX.p, fieldName: "focusPointX", required: false, type: Float32.self)
     try _v.visit(field: VTOFFSET.focusPointY.p, fieldName: "focusPointY", required: false, type: Float32.self)
     try _v.visit(field: VTOFFSET.cameraPreviewMode.p, fieldName: "cameraPreviewMode", required: false, type: RemoteShutter_CameraPreviewModeEnum.self)
+    try _v.visit(field: VTOFFSET.clockSyncT0Ms.p, fieldName: "clockSyncT0Ms", required: false, type: UInt64.self)
     _v.finish()
   }
 }
@@ -1094,6 +1101,8 @@ public struct RemoteShutter_CameraStateResponse: FlatBufferObject, Verifiable {
     case availableLenses = 18
     case zoomRange = 20
     case currentZoom = 22
+    case clockSyncEchoT0Ms = 24
+    case clockSyncCameraClockMs = 26
     var v: Int32 { Int32(self.rawValue) }
     var p: VOffset { self.rawValue }
   }
@@ -1114,7 +1123,9 @@ public struct RemoteShutter_CameraStateResponse: FlatBufferObject, Verifiable {
   public func availableLenses(at index: Int32) -> RemoteShutter_CameraLensType? { let o = _accessor.offset(VTOFFSET.availableLenses.v); return o == 0 ? RemoteShutter_CameraLensType.wideangle : RemoteShutter_CameraLensType(rawValue: _accessor.directRead(of: Int8.self, offset: _accessor.vector(at: o) + index * 1)) }
   public var zoomRange: RemoteShutter_ZoomRange? { let o = _accessor.offset(VTOFFSET.zoomRange.v); return o == 0 ? nil : RemoteShutter_ZoomRange(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) }
   public var currentZoom: Double { let o = _accessor.offset(VTOFFSET.currentZoom.v); return o == 0 ? 0.0 : _accessor.readBuffer(of: Double.self, at: o) }
-  public static func startCameraStateResponse(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 10) }
+  public var clockSyncEchoT0Ms: UInt64 { let o = _accessor.offset(VTOFFSET.clockSyncEchoT0Ms.v); return o == 0 ? 0 : _accessor.readBuffer(of: UInt64.self, at: o) }
+  public var clockSyncCameraClockMs: UInt64 { let o = _accessor.offset(VTOFFSET.clockSyncCameraClockMs.v); return o == 0 ? 0 : _accessor.readBuffer(of: UInt64.self, at: o) }
+  public static func startCameraStateResponse(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 12) }
   public static func add(action: RemoteShutter_CommandAction, _ fbb: inout FlatBufferBuilder) { fbb.add(element: action.rawValue, def: 0, at: VTOFFSET.action.p) }
   public static func add(success: Bool, _ fbb: inout FlatBufferBuilder) { fbb.add(element: success, def: false,
    at: VTOFFSET.success.p) }
@@ -1126,6 +1137,8 @@ public struct RemoteShutter_CameraStateResponse: FlatBufferObject, Verifiable {
   public static func addVectorOf(availableLenses: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: availableLenses, at: VTOFFSET.availableLenses.p) }
   public static func add(zoomRange: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: zoomRange, at: VTOFFSET.zoomRange.p) }
   public static func add(currentZoom: Double, _ fbb: inout FlatBufferBuilder) { fbb.add(element: currentZoom, def: 0.0, at: VTOFFSET.currentZoom.p) }
+  public static func add(clockSyncEchoT0Ms: UInt64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: clockSyncEchoT0Ms, def: 0, at: VTOFFSET.clockSyncEchoT0Ms.p) }
+  public static func add(clockSyncCameraClockMs: UInt64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: clockSyncCameraClockMs, def: 0, at: VTOFFSET.clockSyncCameraClockMs.p) }
   public static func endCameraStateResponse(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end }
   public static func createCameraStateResponse(
     _ fbb: inout FlatBufferBuilder,
@@ -1138,7 +1151,9 @@ public struct RemoteShutter_CameraStateResponse: FlatBufferObject, Verifiable {
     recordingStartTime: UInt64 = 0,
     availableLensesVectorOffset availableLenses: Offset = Offset(),
     zoomRangeOffset zoomRange: Offset = Offset(),
-    currentZoom: Double = 0.0
+    currentZoom: Double = 0.0,
+    clockSyncEchoT0Ms: UInt64 = 0,
+    clockSyncCameraClockMs: UInt64 = 0
   ) -> Offset {
     let __start = RemoteShutter_CameraStateResponse.startCameraStateResponse(&fbb)
     RemoteShutter_CameraStateResponse.add(action: action, &fbb)
@@ -1151,6 +1166,8 @@ public struct RemoteShutter_CameraStateResponse: FlatBufferObject, Verifiable {
     RemoteShutter_CameraStateResponse.addVectorOf(availableLenses: availableLenses, &fbb)
     RemoteShutter_CameraStateResponse.add(zoomRange: zoomRange, &fbb)
     RemoteShutter_CameraStateResponse.add(currentZoom: currentZoom, &fbb)
+    RemoteShutter_CameraStateResponse.add(clockSyncEchoT0Ms: clockSyncEchoT0Ms, &fbb)
+    RemoteShutter_CameraStateResponse.add(clockSyncCameraClockMs: clockSyncCameraClockMs, &fbb)
     return RemoteShutter_CameraStateResponse.endCameraStateResponse(&fbb, start: __start)
   }
 
@@ -1166,6 +1183,8 @@ public struct RemoteShutter_CameraStateResponse: FlatBufferObject, Verifiable {
     try _v.visit(field: VTOFFSET.availableLenses.p, fieldName: "availableLenses", required: false, type: ForwardOffset<Vector<RemoteShutter_CameraLensType, RemoteShutter_CameraLensType>>.self)
     try _v.visit(field: VTOFFSET.zoomRange.p, fieldName: "zoomRange", required: false, type: ForwardOffset<RemoteShutter_ZoomRange>.self)
     try _v.visit(field: VTOFFSET.currentZoom.p, fieldName: "currentZoom", required: false, type: Double.self)
+    try _v.visit(field: VTOFFSET.clockSyncEchoT0Ms.p, fieldName: "clockSyncEchoT0Ms", required: false, type: UInt64.self)
+    try _v.visit(field: VTOFFSET.clockSyncCameraClockMs.p, fieldName: "clockSyncCameraClockMs", required: false, type: UInt64.self)
     _v.finish()
   }
 }

RemoteCam/RemoteCmdFlatBuffers.swift

@@ -29,6 +29,8 @@ func serializeToFlatBuffer(_ msg: Message) -> Data? {
     case let m as RemoteCmd.SendFrame: return m.toFlatBuffer()
     case let m as RemoteCmd.RequestFrame: return m.toFlatBuffer()
     case let m as RemoteCmd.RequestKeyframe: return m.toFlatBuffer()
+    case let m as RemoteCmd.ClockSyncPing: return m.toFlatBuffer()
+    case let m as RemoteCmd.ClockSyncPong: return m.toFlatBuffer()
     case let m as RemoteCmd.SetZoom: return m.toFlatBuffer()
     case let m as RemoteCmd.SetZoomResp: return m.toFlatBuffer()
     case let m as RemoteCmd.FocusAtPoint: return m.toFlatBuffer()
@@ -741,6 +743,28 @@ extension RemoteCmd.RequestKeyframe {
     }
 }
 
+extension RemoteCmd.ClockSyncPing {
+    func toFlatBuffer() -> Data {
+        var fbb = FlatBufferBuilder()
+        let params = RemoteShutter_CommandParameters.createCommandParameters(
+            &fbb, clockSyncT0Ms: t0Millis)
+        return buildCommand(&fbb, action: .clocksyncping, parameters: params)
+    }
+}
+
+extension RemoteCmd.ClockSyncPong {
+    func toFlatBuffer() -> Data {
+        var fbb = FlatBufferBuilder()
+        let resp = RemoteShutter_CameraStateResponse.createCameraStateResponse(
+            &fbb,
+            action: .clocksyncping,
+            success: true,
+            clockSyncEchoT0Ms: echoT0Millis,
+            clockSyncCameraClockMs: cameraClockMillis)
+        return buildResponse(&fbb, action: .clocksyncping, response: resp)
+    }
+}
+
 extension RemoteCmd.ToggleFlash {
     func toFlatBuffer() -> Data {
         var fbb = FlatBufferBuilder()
@@ -1101,6 +1125,9 @@ extension RemoteCmd {
         case .requestkeyframe:
             return RequestKeyframe(sender: nil)
 
+        case .clocksyncping:
+            return ClockSyncPing(t0Millis: params?.clockSyncT0Ms ?? 0)
+
         case .toggleflash:
             return ToggleFlash()
 
@@ -1161,6 +1188,10 @@ extension RemoteCmd {
         let nsError: Error? = errorStr.map { NSError(domain: "RemoteCmd", code: -1, userInfo: [NSLocalizedDescriptionKey: $0]) }
 
         switch resp.action {
+        case .clocksyncping:
+            return ClockSyncPong(echoT0Millis: resp.clockSyncEchoT0Ms,
+                                 cameraClockMillis: resp.clockSyncCameraClockMs)
+
         case .startrecording:
             let startTime: Date? = resp.recordingStartTime > 0 ? Date(timeIntervalSince1970: Double(resp.recordingStartTime) / 1000.0) : nil
             return StartRecordingVideoAck(sender: nil, recordingStartTime: startTime, error: nsError)

RemoteCam/RemoteCmds.swift

@@ -194,6 +194,32 @@ public class RemoteCmd: Message, @unchecked Sendable {
         }
     }
 
+    /// Director → camera: clock-offset probe. `t0Millis` is the director's
+    /// monotonic clock at send; the camera answers immediately with a
+    /// `ClockSyncPong` echoing it. Only sent to peers advertising
+    /// `supportsMulticam` (an old peer would decode it as Unknown and drop it).
+    public class ClockSyncPing: Message, @unchecked Sendable {
+        public let t0Millis: UInt64
+        init(t0Millis: UInt64, sender: AnyObject? = nil) {
+            self.t0Millis = t0Millis
+            super.init(sender: sender)
+        }
+    }
+
+    /// Camera → director: answer to `ClockSyncPing`. Echoes the director's
+    /// `t0Millis` (so the director can compute RTT against its own clock) and
+    /// carries the camera's monotonic clock at receipt — the pair the
+    /// director's `ClockOffsetEstimator` turns into an offset sample.
+    public class ClockSyncPong: Message, @unchecked Sendable {
+        public let echoT0Millis: UInt64
+        public let cameraClockMillis: UInt64
+        init(echoT0Millis: UInt64, cameraClockMillis: UInt64, sender: AnyObject? = nil) {
+            self.echoT0Millis = echoT0Millis
+            self.cameraClockMillis = cameraClockMillis
+            super.init(sender: sender)
+        }
+    }
+
     public class OnFrame: Message, @unchecked Sendable {
         public let data: Data
         public let peerId: MCPeerID

RemoteCam/SessionCoordinator.swift

@@ -2439,9 +2439,21 @@ public actor SessionCoordinator {
 extension SessionCoordinator: MultipeerServiceDelegate {
 
     public nonisolated func didReceiveMessage(_ message: Message, from peer: MCPeerID) {
-        // `peer` is deliberately unused: this coordinator links exactly one
-        // peer, so every message's source is unambiguous. The multicam
-        // director's controller is the consumer that routes by it.
+        if let ping = message as? RemoteCmd.ClockSyncPing {
+            // Answered here, off the actor inbox: the sample's quality is the
+            // camera clock *at receipt*, and queuing behind state-machine work
+            // would smear it (same pacing argument as didReceiveFrameRequest).
+            // Stateless, so no isolation is needed.
+            _ = transportShared.value?.send(
+                RemoteCmd.ClockSyncPong(echoT0Millis: ping.t0Millis,
+                                        cameraClockMillis: SyncClock.nowMillis()),
+                to: [peer], mode: .reliable)
+            tell(UICmd.PeerTrafficObserved())
+            return
+        }
+        // `peer` is otherwise deliberately unused: this coordinator links
+        // exactly one peer, so every message's source is unambiguous. The
+        // multicam director's controller is the consumer that routes by it.
         tell(UICmd.PeerTrafficObserved())
         tell(message)
     }