From 991bd57de7677f4d2126195bc3b4b643b67c349f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 15 Apr 2026 10:10:09 +0200 Subject: [PATCH 01/40] Implement crud batches in Swift instead of Kotlin --- CHANGELOG.md | 4 + .../Kotlin/KotlinPowerSyncDatabaseImpl.swift | 24 +++- .../PowerSync/Kotlin/db/KotlinCrudBatch.swift | 27 ---- .../PowerSync/Kotlin/db/KotlinCrudEntry.swift | 44 ------- .../Kotlin/db/KotlinCrudTransaction.swift | 22 ---- .../Kotlin/db/KotlinCrudTransactions.swift | 39 ------ .../Protocol/PowerSyncDatabaseProtocol.swift | 2 +- Sources/PowerSync/Protocol/db/CrudBatch.swift | 39 ++++-- Sources/PowerSync/Protocol/db/CrudEntry.swift | 118 +++++++++++++++--- .../Protocol/db/CrudTransaction.swift | 83 ++++++++++-- Sources/PowerSync/Protocol/db/JsonParam.swift | 37 ++++++ Tests/PowerSyncTests/JsonParamTests.swift | 28 +++++ 12 files changed, 297 insertions(+), 170 deletions(-) delete mode 100644 Sources/PowerSync/Kotlin/db/KotlinCrudBatch.swift delete mode 100644 Sources/PowerSync/Kotlin/db/KotlinCrudEntry.swift delete mode 100644 Sources/PowerSync/Kotlin/db/KotlinCrudTransaction.swift delete mode 100644 Sources/PowerSync/Kotlin/db/KotlinCrudTransactions.swift create mode 100644 Tests/PowerSyncTests/JsonParamTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index fbe5a80..6eb4d7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 1.14.0 (unreleased) + +* + ## 1.13.1 * Don't attempt to create WebSocket connections on watchOS. diff --git a/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift b/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift index faea484..5a729ce 100644 --- a/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift +++ b/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift @@ -73,16 +73,30 @@ final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, } func getCrudBatch(limit: Int32 = 100) async throws -> CrudBatch? { - guard let base = try await kotlinDatabase.getCrudBatch(limit: limit) else { + var entries = try await getAll( + sql: "SELECT id, tx_id, data FROM ps_crud ORDER BY id ASC LIMIT ?", + parameters: [Int64(limit + 1)], + mapper: CrudEntry.fromCursor + ) + + if entries.isEmpty { return nil } - return try KotlinCrudBatch( - batch: base + + let hasMore = entries.count > limit + if hasMore { + entries.removeLast() + } + + return CrudBatch( + hasMore: hasMore, + crud: entries, + db: self ) } - func getCrudTransactions() -> any CrudTransactions { - return KotlinCrudTransactions(db: kotlinDatabase) + func getCrudTransactions() -> CrudTransactions { + return CrudTransactions(db: self) } func getPowerSyncVersion() async throws -> String { diff --git a/Sources/PowerSync/Kotlin/db/KotlinCrudBatch.swift b/Sources/PowerSync/Kotlin/db/KotlinCrudBatch.swift deleted file mode 100644 index f94c29c..0000000 --- a/Sources/PowerSync/Kotlin/db/KotlinCrudBatch.swift +++ /dev/null @@ -1,27 +0,0 @@ -import PowerSyncKotlin - -/// Implements `CrudBatch` using the Kotlin SDK -struct KotlinCrudBatch: CrudBatch { - let batch: PowerSyncKotlin.CrudBatch - let crud: [CrudEntry] - - init( - batch: PowerSyncKotlin.CrudBatch) - throws - { - self.batch = batch - self.crud = try batch.crud.map { try KotlinCrudEntry( - entry: $0 - ) } - } - - var hasMore: Bool { - batch.hasMore - } - - func complete( - writeCheckpoint: String? - ) async throws { - _ = try await batch.complete.invoke(p1: writeCheckpoint) - } -} diff --git a/Sources/PowerSync/Kotlin/db/KotlinCrudEntry.swift b/Sources/PowerSync/Kotlin/db/KotlinCrudEntry.swift deleted file mode 100644 index 6433854..0000000 --- a/Sources/PowerSync/Kotlin/db/KotlinCrudEntry.swift +++ /dev/null @@ -1,44 +0,0 @@ -import PowerSyncKotlin - -/// Implements `CrudEntry` using the KotlinSDK -struct KotlinCrudEntry : CrudEntry { - let entry: PowerSyncKotlin.CrudEntry - let op: UpdateType - - init ( - entry: PowerSyncKotlin.CrudEntry - ) throws { - self.entry = entry - self.op = try UpdateType.fromString(entry.op.name) - } - - var id: String { - entry.id - } - - var clientId: Int64 { - Int64(entry.clientId) - } - - var table: String { - entry.table - } - - var transactionId: Int64? { - entry.transactionId?.int64Value - } - - var metadata: String? { - entry.metadata - } - - var opData: [String : String?]? { - /// Kotlin represents this as Map, but this is - /// converted to [String: Any] by SKIEE - entry.opData?.mapValues { $0 as? String } - } - - var previousValues: [String : String?]? { - entry.previousValues?.mapValues { $0 as? String } - } -} diff --git a/Sources/PowerSync/Kotlin/db/KotlinCrudTransaction.swift b/Sources/PowerSync/Kotlin/db/KotlinCrudTransaction.swift deleted file mode 100644 index fe4e2f5..0000000 --- a/Sources/PowerSync/Kotlin/db/KotlinCrudTransaction.swift +++ /dev/null @@ -1,22 +0,0 @@ -import PowerSyncKotlin - -/// Implements `CrudTransaction` using the Kotlin SDK -struct KotlinCrudTransaction: CrudTransaction { - let transaction: PowerSyncKotlin.CrudTransaction - let crud: [CrudEntry] - - init(transaction: PowerSyncKotlin.CrudTransaction) throws { - self.transaction = transaction - self.crud = try transaction.crud.map { try KotlinCrudEntry( - entry: $0 - ) } - } - - var transactionId: Int64? { - transaction.transactionId?.int64Value - } - - func complete(writeCheckpoint: String?) async throws { - _ = try await transaction.complete.invoke(p1: writeCheckpoint) - } -} diff --git a/Sources/PowerSync/Kotlin/db/KotlinCrudTransactions.swift b/Sources/PowerSync/Kotlin/db/KotlinCrudTransactions.swift deleted file mode 100644 index 866e76e..0000000 --- a/Sources/PowerSync/Kotlin/db/KotlinCrudTransactions.swift +++ /dev/null @@ -1,39 +0,0 @@ -import PowerSyncKotlin - -struct KotlinCrudTransactions: CrudTransactions { - typealias Element = KotlinCrudTransaction - - private let db: KotlinPowerSyncDatabase - - init(db: KotlinPowerSyncDatabase) { - self.db = db - } - - public func makeAsyncIterator() -> CrudTransactionIterator { - let kotlinIterator = errorHandledCrudTransactions(db: self.db).makeAsyncIterator() - return CrudTransactionIterator(inner: kotlinIterator) - } - - struct CrudTransactionIterator: CrudTransactionsIterator { - private var inner: PowerSyncKotlin.SkieSwiftFlowIterator - - internal init(inner: PowerSyncKotlin.SkieSwiftFlowIterator) { - self.inner = inner - } - - public mutating func next() async throws -> KotlinCrudTransaction? { - if let innerTx = await self.inner.next() { - if let success = innerTx as? PowerSyncResult.Success { - let tx = success.value as! PowerSyncKotlin.CrudTransaction - return try KotlinCrudTransaction(transaction: tx) - } else if let failure = innerTx as? PowerSyncResult.Failure { - try throwPowerSyncException(exception: failure.exception) - } - - fatalError("unreachable") - } else { - return nil - } - } - } -} diff --git a/Sources/PowerSync/Protocol/PowerSyncDatabaseProtocol.swift b/Sources/PowerSync/Protocol/PowerSyncDatabaseProtocol.swift index 05e2477..a88020e 100644 --- a/Sources/PowerSync/Protocol/PowerSyncDatabaseProtocol.swift +++ b/Sources/PowerSync/Protocol/PowerSyncDatabaseProtocol.swift @@ -223,7 +223,7 @@ public protocol PowerSyncDatabaseProtocol: Queries, Sendable { /// ```Swift /// /// ``` - func getCrudTransactions() -> any CrudTransactions + func getCrudTransactions() -> CrudTransactions /// Convenience method to get the current version of PowerSync. func getPowerSyncVersion() async throws -> String diff --git a/Sources/PowerSync/Protocol/db/CrudBatch.swift b/Sources/PowerSync/Protocol/db/CrudBatch.swift index 6c770f4..1376b02 100644 --- a/Sources/PowerSync/Protocol/db/CrudBatch.swift +++ b/Sources/PowerSync/Protocol/db/CrudBatch.swift @@ -1,20 +1,29 @@ import Foundation -/// A transaction of client-side changes. -public protocol CrudBatch: Sendable { +/// A collection of client-side changes. +public struct CrudBatch: Sendable { /// Indicates if there are additional Crud items in the queue which are not included in this batch - var hasMore: Bool { get } + let hasMore: Bool /// List of client-side changes. - var crud: [any CrudEntry] { get } + let crud: [CrudEntry] + + private let db: PowerSyncDatabaseProtocol + + internal init(hasMore: Bool, crud: [CrudEntry], db: PowerSyncDatabaseProtocol) { + self.hasMore = hasMore + self.crud = crud + self.db = db + } /// Call to remove the changes from the local queue, once successfully uploaded. /// /// `writeCheckpoint` is optional. - func complete(writeCheckpoint: String?) async throws -} - -public extension CrudBatch { + func complete(writeCheckpoint: String?) async throws { + let lastId = crud.last!.clientId + try await completeCrudItems(self.db, lastId) + } + /// Call to remove the changes from the local queue, once successfully uploaded. func complete() async throws { try await self.complete( @@ -22,3 +31,17 @@ public extension CrudBatch { ) } } + +internal func completeCrudItems(_ db: any PowerSyncDatabaseProtocol, _ lastItemId: Int64, writeCheckpoint: String? = nil) async throws { + return try await db.writeTransaction { tx in + try tx.execute(sql: "DELETE FROM ps_crud WHERE id <= ?", parameters: [lastItemId]) + if writeCheckpoint != nil { + let hasCrud = (try tx.getOptional(sql: "SELECT 1 FROM ps_crud", parameters: nil) { cursor in () }) != nil + if !hasCrud { + try tx.execute(sql: "UPDATE ps_buckets SET target_op = CAST(? AS INTEGER) WHERE name = '$local'", parameters: [writeCheckpoint]) + return + } + } + try tx.execute(sql: "UPDATE ps_buckets SET target_op = 9223372036854775807 WHERE name = '$local'", parameters: nil) + } +} diff --git a/Sources/PowerSync/Protocol/db/CrudEntry.swift b/Sources/PowerSync/Protocol/db/CrudEntry.swift index 85ecfeb..32330d2 100644 --- a/Sources/PowerSync/Protocol/db/CrudEntry.swift +++ b/Sources/PowerSync/Protocol/db/CrudEntry.swift @@ -1,3 +1,5 @@ +import Foundation + /// Represents the type of CRUD update operation that can be performed on a row. public enum UpdateType: String, Codable, Sendable { /// A row has been inserted or replaced @@ -28,22 +30,22 @@ public enum UpdateType: String, Codable, Sendable { } /// Represents a CRUD (Create, Read, Update, Delete) entry in the system. -public protocol CrudEntry: Sendable { +public struct CrudEntry: Sendable { /// The unique identifier of the entry. - var id: String { get } - + let id: String + /// The client ID associated with the entry. - var clientId: Int64 { get } - + let clientId: Int64 + /// The type of update operation performed on the entry. - var op: UpdateType { get } + let op: UpdateType /// The name of the table where the entry resides. - var table: String { get } - + let table: String + /// The transaction ID associated with the entry, if any. - var transactionId: Int64? { get } - + let transactionId: Int64? + /// User-defined metadata that can be attached to writes. /// /// This is the value the `_metadata` column had when the write to the database was made, @@ -51,13 +53,101 @@ public protocol CrudEntry: Sendable { /// /// Note that the `_metadata` column and this field are only available when ``Table/trackMetadata`` /// is enabled. - var metadata: String? { get } - + let metadata: String? + /// The operation data associated with the entry, represented as a dictionary of column names to their values. - var opData: [String: String?]? { get } + let opDataTyped: JsonParam? + + /// The operation data associated with the entry, represented as a dictionary of column names to their values. + /// + /// Consider using ``CrudEntry/opDataTyped`` instead, which provides values as typed JSON. + var opData: [String: String?]? { + get { + opDataTyped?.mapValues { value in + do { + return try CrudEntry.jsonValueToString(value) + } catch { + return nil + } + } + } + } /// Previous values before this change. /// /// These values can be tracked for `UPDATE` statements when ``Table/trackPreviousValues`` is enabled. - var previousValues: [String: String?]? { get } + let previousValuesTyped: JsonParam? + + /// Previous values before this change. + /// + /// These values can be tracked for `UPDATE` statements when ``Table/trackPreviousValues`` is enabled. + /// + /// Consider using ``CrudEntry/previousValuesTyped`` instead, which provides values as typed JSON. + var previousValues: [String: String?]? { + get { + previousValuesTyped?.mapValues { value in + do { + return try CrudEntry.jsonValueToString(value) + } catch { + return nil + } + } + } + } + + private let nonExhaustive: Void // Prevent initialization outside of this package + + internal static func fromCursor(cursor: borrowing SqlCursor) throws -> CrudEntry { + let id = try cursor.getInt64(index: 0) + let txId = cursor.getInt64Optional(index: 1) + let data = try cursor.getString(index: 2) + + struct CrudJsonEntry: Decodable { + let id: String + let op: UpdateType + let data: JsonParam? + let type: String + let metadata: String? + let old: JsonParam? + } + + let decoder = JSONDecoder() + var entry: CrudJsonEntry + do { + entry = try decoder.decode(CrudJsonEntry.self, from: data.data(using: .utf8)!) + } catch { + throw error + } + + return CrudEntry( + id: entry.id, + clientId: id, + op: entry.op, + table: entry.type, + transactionId: txId, + metadata: entry.metadata, + opDataTyped: entry.data, + previousValuesTyped: entry.old, + nonExhaustive: () + ) + } + + private static func jsonValueToString(_ value: JsonValue?) throws -> String? { + try value.map { value in + switch (value) { + case .string(let value): + return value + case .int(let value): + return String(value) + case .double(let value): + return String(value) + case .bool(let value): + return String(value) + case .null: + return "null" + case .array(_), .object(_): + throw PowerSyncError.operationFailed(message: "Invalid array/object in CRUD data, should be string") + } + } + } } diff --git a/Sources/PowerSync/Protocol/db/CrudTransaction.swift b/Sources/PowerSync/Protocol/db/CrudTransaction.swift index 8a7dacf..4ce630e 100644 --- a/Sources/PowerSync/Protocol/db/CrudTransaction.swift +++ b/Sources/PowerSync/Protocol/db/CrudTransaction.swift @@ -1,22 +1,32 @@ import Foundation + /// A transaction of client-side changes. -public protocol CrudTransaction: Sendable { +public struct CrudTransaction: Sendable { /// Unique transaction id. /// /// If nil, this contains a list of changes recorded without an explicit transaction associated. - var transactionId: Int64? { get } + let transactionId: Int64 /// List of client-side changes. - var crud: [any CrudEntry] { get } + let crud: [CrudEntry] + + private let db: any PowerSyncDatabaseProtocol + + internal init(transactionId: Int64, crud: [CrudEntry], db: any PowerSyncDatabaseProtocol) { + self.transactionId = transactionId + self.crud = crud + self.db = db + } /// Call to remove the changes from the local queue, once successfully uploaded. /// /// `writeCheckpoint` is optional. - func complete(writeCheckpoint: String?) async throws -} - -public extension CrudTransaction { + func complete(writeCheckpoint: String?) async throws { + let id = self.crud.last!.clientId + try await completeCrudItems(db, id, writeCheckpoint: writeCheckpoint) + } + /// Call to remove the changes from the local queue, once successfully uploaded. func complete() async throws { try await self.complete( @@ -24,11 +34,64 @@ public extension CrudTransaction { ) } } - /// A sequence of crud transactions in a PowerSync database. /// /// For details, see ``PowerSyncDatabaseProtocol/getCrudTransactions()``. -public protocol CrudTransactions: AsyncSequence where Element: CrudTransaction, AsyncIterator: CrudTransactionsIterator {} +public struct CrudTransactions: AsyncSequence { + public typealias Element = CrudTransaction + public typealias AsyncIterator = CrudTransactionsIterator + + private let db: any PowerSyncDatabaseProtocol + + internal init(db: any PowerSyncDatabaseProtocol) { + self.db = db + } + + public func makeAsyncIterator() -> CrudTransactionsIterator { + CrudTransactionsIterator(db: db) + } +} /// The iterator returned by ``CrudTransactions``. -public protocol CrudTransactionsIterator: AsyncIteratorProtocol where Element: CrudTransaction {} +public struct CrudTransactionsIterator: AsyncIteratorProtocol { + public typealias Element = CrudTransaction + + private var lastItemId: Int64 = -1 + private let db: any PowerSyncDatabaseProtocol + + internal init(db: any PowerSyncDatabaseProtocol) { + self.db = db + } + + public mutating func next() async throws -> CrudTransaction? { + // Note: We try to avoid filtering on tx_id here because there's no index on that column. + // Starting at the first entry we want and then joining by rowid is more efficient. This is + // sound because there can't be concurrent write transactions, so transaction ids are + // increasing when we iterate over rowids. + let query = """ +WITH RECURSIVE crud_entries AS ( + SELECT id, tx_id, data FROM ps_crud WHERE id = (SELECT min(id) FROM ps_crud WHERE id > ?) + UNION ALL + SELECT ps_crud.id, ps_crud.tx_id, ps_crud.data FROM ps_crud + INNER JOIN crud_entries ON crud_entries.id + 1 = rowid + WHERE crud_entries.tx_id = ps_crud.tx_id +) +SELECT * FROM crud_entries; +""" + + let items = try await db.getAll(sql: query, parameters: [lastItemId], mapper: CrudEntry.fromCursor) + if items.isEmpty { + return nil + } + + let txId = items.first!.transactionId + let lastId = items.last!.clientId + + lastItemId = lastId + return CrudTransaction( + transactionId: txId!, + crud: items, + db: db + ) + } +} diff --git a/Sources/PowerSync/Protocol/db/JsonParam.swift b/Sources/PowerSync/Protocol/db/JsonParam.swift index 85775ae..87ea5a6 100644 --- a/Sources/PowerSync/Protocol/db/JsonParam.swift +++ b/Sources/PowerSync/Protocol/db/JsonParam.swift @@ -24,6 +24,23 @@ public enum JsonValue: Codable, Sendable, Equatable { /// A JSON object containing key-value pairs where values are `JSONValue` instances. case object([String: JsonValue]) + public init(from decoder: any Decoder) throws { + let c = try decoder.singleValueContainer() + if c.decodeNil() { self = .null } + else if let b = try? c.decode(Bool.self) { self = .bool(b) } + else if let i = try? c.decode(Int.self) { self = .int(i) } + else if let d = try? c.decode(Double.self) { self = .double(d) } + else if let s = try? c.decode(String.self) { self = .string(s) } + else if let a = try? c.decode([JsonValue].self) { self = .array(a) } + else if let o = try? c.decode([String: JsonValue].self) { self = .object(o) } + else { + throw DecodingError.typeMismatch( + JsonValue.self, + .init(codingPath: decoder.codingPath, + debugDescription: "Expected any JSON value")) + } + } + /// Converts the `JSONValue` into a native Swift representation. /// /// - Returns: A corresponding Swift type (`String`, `Int`, `Double`, `Bool`, `nil`, `[Any]`, or `[String: Any]`), @@ -50,6 +67,26 @@ public enum JsonValue: Codable, Sendable, Equatable { return anyDict } } + + public func encode(to encoder: any Encoder) throws { + var c = encoder.singleValueContainer() + switch self { + case .string(let value): + try c.encode(value) + case .int(let value): + try c.encode(value) + case .double(let value): + try c.encode(value) + case .bool(let value): + try c.encode(value) + case .null: + try c.encodeNil() + case .array(let values): + try c.encode(values) + case .object(let object): + try c.encode(object) + } + } } /// A typealias representing a top-level JSON object with string keys and `JSONValue` values. diff --git a/Tests/PowerSyncTests/JsonParamTests.swift b/Tests/PowerSyncTests/JsonParamTests.swift new file mode 100644 index 0000000..ed9c975 --- /dev/null +++ b/Tests/PowerSyncTests/JsonParamTests.swift @@ -0,0 +1,28 @@ +import Foundation +import Testing +import PowerSync + +@Suite() +struct JsonValueTests { + @Test func canDecode() throws { + let decoder = JSONDecoder() + + try #require(try decoder.decode(JsonValue.self, from: "null".data(using: .utf8)!) == .null) + try #require(try decoder.decode(JsonValue.self, from: "123".data(using: .utf8)!) == .int(123)) + try #require(try decoder.decode(JsonValue.self, from: "123.45".data(using: .utf8)!) == .double(123.45)) + try #require(try decoder.decode(JsonValue.self, from: "\"123\"".data(using: .utf8)!) == .string("123")) + try #require(try decoder.decode(JsonValue.self, from: "[1,2,3]".data(using: .utf8)!) == .array([.int(1), .int(2), .int(3)])) + try #require(try decoder.decode(JsonValue.self, from: "{\"foo\": \"bar\"}".data(using: .utf8)!) == .object(["foo": .string("bar")])) + } + + @Test func canEncode() throws { + let encoder = JSONEncoder() + + try #require(String(data: try encoder.encode(JsonValue.null), encoding: .utf8) == "null") + try #require(String(data: try encoder.encode(JsonValue.int(123)), encoding: .utf8) == "123") + try #require(String(data: try encoder.encode(JsonValue.double(123.45)), encoding: .utf8) == "123.45") + try #require(String(data: try encoder.encode(JsonValue.string("123")), encoding: .utf8) == "\"123\"") + try #require(String(data: try encoder.encode(JsonValue.array([.int(1), .int(2), .int(3)])), encoding: .utf8) == "[1,2,3]") + try #require(String(data: try encoder.encode(JsonValue.object(["foo": .string("bar")])), encoding: .utf8) == "{\"foo\":\"bar\"}") + } +} From a2d0439e16108a51887e7f591c96d229c9c2cdf4 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 15 Apr 2026 10:18:33 +0200 Subject: [PATCH 02/40] Changelog, format --- CHANGELOG.md | 2 +- Sources/PowerSync/Protocol/db/CrudBatch.swift | 14 +++---- Sources/PowerSync/Protocol/db/CrudEntry.swift | 40 +++++++++---------- .../Protocol/db/CrudTransaction.swift | 23 ++++++----- 4 files changed, 40 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6eb4d7e..60f0cbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 1.14.0 (unreleased) -* +* Add `opDataTyped` and `previousValuesTyped` to `CrudEntry`, providing typed values instead of strings. ## 1.13.1 diff --git a/Sources/PowerSync/Protocol/db/CrudBatch.swift b/Sources/PowerSync/Protocol/db/CrudBatch.swift index 1376b02..ff03e01 100644 --- a/Sources/PowerSync/Protocol/db/CrudBatch.swift +++ b/Sources/PowerSync/Protocol/db/CrudBatch.swift @@ -3,13 +3,13 @@ import Foundation /// A collection of client-side changes. public struct CrudBatch: Sendable { /// Indicates if there are additional Crud items in the queue which are not included in this batch - let hasMore: Bool + public let hasMore: Bool /// List of client-side changes. - let crud: [CrudEntry] - + public let crud: [CrudEntry] + private let db: PowerSyncDatabaseProtocol - + internal init(hasMore: Bool, crud: [CrudEntry], db: PowerSyncDatabaseProtocol) { self.hasMore = hasMore self.crud = crud @@ -19,13 +19,13 @@ public struct CrudBatch: Sendable { /// Call to remove the changes from the local queue, once successfully uploaded. /// /// `writeCheckpoint` is optional. - func complete(writeCheckpoint: String?) async throws { + public func complete(writeCheckpoint: String?) async throws { let lastId = crud.last!.clientId try await completeCrudItems(self.db, lastId) } - + /// Call to remove the changes from the local queue, once successfully uploaded. - func complete() async throws { + public func complete() async throws { try await self.complete( writeCheckpoint: nil ) diff --git a/Sources/PowerSync/Protocol/db/CrudEntry.swift b/Sources/PowerSync/Protocol/db/CrudEntry.swift index 32330d2..2499fac 100644 --- a/Sources/PowerSync/Protocol/db/CrudEntry.swift +++ b/Sources/PowerSync/Protocol/db/CrudEntry.swift @@ -32,20 +32,20 @@ public enum UpdateType: String, Codable, Sendable { /// Represents a CRUD (Create, Read, Update, Delete) entry in the system. public struct CrudEntry: Sendable { /// The unique identifier of the entry. - let id: String - + public let id: String + /// The client ID associated with the entry. - let clientId: Int64 - + public let clientId: Int64 + /// The type of update operation performed on the entry. - let op: UpdateType + public let op: UpdateType /// The name of the table where the entry resides. - let table: String - + public let table: String + /// The transaction ID associated with the entry, if any. - let transactionId: Int64? - + public let transactionId: Int64? + /// User-defined metadata that can be attached to writes. /// /// This is the value the `_metadata` column had when the write to the database was made, @@ -53,15 +53,15 @@ public struct CrudEntry: Sendable { /// /// Note that the `_metadata` column and this field are only available when ``Table/trackMetadata`` /// is enabled. - let metadata: String? - + public let metadata: String? + /// The operation data associated with the entry, represented as a dictionary of column names to their values. - let opDataTyped: JsonParam? - + public let opDataTyped: JsonParam? + /// The operation data associated with the entry, represented as a dictionary of column names to their values. /// /// Consider using ``CrudEntry/opDataTyped`` instead, which provides values as typed JSON. - var opData: [String: String?]? { + public var opData: [String: String?]? { get { opDataTyped?.mapValues { value in do { @@ -76,14 +76,14 @@ public struct CrudEntry: Sendable { /// Previous values before this change. /// /// These values can be tracked for `UPDATE` statements when ``Table/trackPreviousValues`` is enabled. - let previousValuesTyped: JsonParam? - + public let previousValuesTyped: JsonParam? + /// Previous values before this change. /// /// These values can be tracked for `UPDATE` statements when ``Table/trackPreviousValues`` is enabled. /// /// Consider using ``CrudEntry/previousValuesTyped`` instead, which provides values as typed JSON. - var previousValues: [String: String?]? { + public var previousValues: [String: String?]? { get { previousValuesTyped?.mapValues { value in do { @@ -101,7 +101,7 @@ public struct CrudEntry: Sendable { let id = try cursor.getInt64(index: 0) let txId = cursor.getInt64Optional(index: 1) let data = try cursor.getString(index: 2) - + struct CrudJsonEntry: Decodable { let id: String let op: UpdateType @@ -110,7 +110,7 @@ public struct CrudEntry: Sendable { let metadata: String? let old: JsonParam? } - + let decoder = JSONDecoder() var entry: CrudJsonEntry do { @@ -131,7 +131,7 @@ public struct CrudEntry: Sendable { nonExhaustive: () ) } - + private static func jsonValueToString(_ value: JsonValue?) throws -> String? { try value.map { value in switch (value) { diff --git a/Sources/PowerSync/Protocol/db/CrudTransaction.swift b/Sources/PowerSync/Protocol/db/CrudTransaction.swift index 4ce630e..4059e02 100644 --- a/Sources/PowerSync/Protocol/db/CrudTransaction.swift +++ b/Sources/PowerSync/Protocol/db/CrudTransaction.swift @@ -6,13 +6,13 @@ public struct CrudTransaction: Sendable { /// Unique transaction id. /// /// If nil, this contains a list of changes recorded without an explicit transaction associated. - let transactionId: Int64 + public let transactionId: Int64 /// List of client-side changes. - let crud: [CrudEntry] - + public let crud: [CrudEntry] + private let db: any PowerSyncDatabaseProtocol - + internal init(transactionId: Int64, crud: [CrudEntry], db: any PowerSyncDatabaseProtocol) { self.transactionId = transactionId self.crud = crud @@ -22,25 +22,26 @@ public struct CrudTransaction: Sendable { /// Call to remove the changes from the local queue, once successfully uploaded. /// /// `writeCheckpoint` is optional. - func complete(writeCheckpoint: String?) async throws { + public func complete(writeCheckpoint: String?) async throws { let id = self.crud.last!.clientId try await completeCrudItems(db, id, writeCheckpoint: writeCheckpoint) } - + /// Call to remove the changes from the local queue, once successfully uploaded. - func complete() async throws { + public func complete() async throws { try await self.complete( writeCheckpoint: nil ) } } + /// A sequence of crud transactions in a PowerSync database. /// /// For details, see ``PowerSyncDatabaseProtocol/getCrudTransactions()``. public struct CrudTransactions: AsyncSequence { public typealias Element = CrudTransaction public typealias AsyncIterator = CrudTransactionsIterator - + private let db: any PowerSyncDatabaseProtocol internal init(db: any PowerSyncDatabaseProtocol) { @@ -78,15 +79,15 @@ WITH RECURSIVE crud_entries AS ( ) SELECT * FROM crud_entries; """ - + let items = try await db.getAll(sql: query, parameters: [lastItemId], mapper: CrudEntry.fromCursor) if items.isEmpty { return nil } - + let txId = items.first!.transactionId let lastId = items.last!.clientId - + lastItemId = lastId return CrudTransaction( transactionId: txId!, From 7a093246375bc79a5b861d490de9537862750dc8 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 15 Apr 2026 10:38:01 +0200 Subject: [PATCH 03/40] AI feedback --- Sources/PowerSync/Protocol/db/CrudEntry.swift | 7 +------ Sources/PowerSync/Protocol/db/CrudTransaction.swift | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/Sources/PowerSync/Protocol/db/CrudEntry.swift b/Sources/PowerSync/Protocol/db/CrudEntry.swift index 2499fac..51dac38 100644 --- a/Sources/PowerSync/Protocol/db/CrudEntry.swift +++ b/Sources/PowerSync/Protocol/db/CrudEntry.swift @@ -112,12 +112,7 @@ public struct CrudEntry: Sendable { } let decoder = JSONDecoder() - var entry: CrudJsonEntry - do { - entry = try decoder.decode(CrudJsonEntry.self, from: data.data(using: .utf8)!) - } catch { - throw error - } + let entry = try decoder.decode(CrudJsonEntry.self, from: data.data(using: .utf8)!) return CrudEntry( id: entry.id, diff --git a/Sources/PowerSync/Protocol/db/CrudTransaction.swift b/Sources/PowerSync/Protocol/db/CrudTransaction.swift index 4059e02..243b690 100644 --- a/Sources/PowerSync/Protocol/db/CrudTransaction.swift +++ b/Sources/PowerSync/Protocol/db/CrudTransaction.swift @@ -6,7 +6,7 @@ public struct CrudTransaction: Sendable { /// Unique transaction id. /// /// If nil, this contains a list of changes recorded without an explicit transaction associated. - public let transactionId: Int64 + public let transactionId: Int64? /// List of client-side changes. public let crud: [CrudEntry] @@ -90,7 +90,7 @@ SELECT * FROM crud_entries; lastItemId = lastId return CrudTransaction( - transactionId: txId!, + transactionId: txId, crud: items, db: db ) From d0cecacb33c35f1d5bc56450d30f48c64e57f54a Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 15 Apr 2026 10:40:16 +0200 Subject: [PATCH 04/40] typo --- Sources/PowerSync/Protocol/db/CrudTransaction.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/PowerSync/Protocol/db/CrudTransaction.swift b/Sources/PowerSync/Protocol/db/CrudTransaction.swift index 243b690..15285b4 100644 --- a/Sources/PowerSync/Protocol/db/CrudTransaction.swift +++ b/Sources/PowerSync/Protocol/db/CrudTransaction.swift @@ -13,7 +13,7 @@ public struct CrudTransaction: Sendable { private let db: any PowerSyncDatabaseProtocol - internal init(transactionId: Int64, crud: [CrudEntry], db: any PowerSyncDatabaseProtocol) { + internal init(transactionId: Int64?, crud: [CrudEntry], db: any PowerSyncDatabaseProtocol) { self.transactionId = transactionId self.crud = crud self.db = db From 18eb1967bb1b66f11db34692746f235ee4405edd Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 15 Apr 2026 20:30:00 +0200 Subject: [PATCH 05/40] Scaffold Swift sync client --- Package.resolved | 20 +- Package.swift | 11 +- .../sync/CachingCredentialsConnector.swift | 26 ++ .../Implementation/sync/CoreSyncStatus.swift | 77 ++++ .../Implementation/sync/HttpClient.swift | 36 ++ .../Implementation/sync/Instruction.swift | 91 ++++ .../sync/PowerSyncControlArguments.swift | 78 ++++ .../Implementation/sync/Status.swift | 231 +++++++++++ .../sync/StreamingSyncClient.swift | 389 ++++++++++++++++++ .../Implementation/sync/SyncCoordinator.swift | 30 ++ .../Kotlin/KotlinPowerSyncDatabaseImpl.swift | 51 +-- .../PowerSyncBackendConnectorAdapter.swift | 37 -- .../Kotlin/sync/KotlinSyncStatus.swift | 43 -- .../Kotlin/sync/KotlinSyncStatusData.swift | 144 ------- .../Kotlin/sync/KotlinSyncStreams.swift | 125 ------ Sources/PowerSync/PowerSyncCredentials.swift | 4 - Sources/PowerSync/Protocol/db/CrudBatch.swift | 2 +- .../Protocol/sync/BucketPriority.swift | 8 +- .../Protocol/sync/PriorityStatusEntry.swift | 21 +- .../Protocol/sync/SyncStatusData.swift | 17 +- .../PowerSync/Protocol/sync/SyncStream.swift | 21 + Sources/PowerSync/Utils/BroadcastStream.swift | 44 ++ Sources/SyncPlayground/main.swift | 35 ++ 23 files changed, 1148 insertions(+), 393 deletions(-) create mode 100644 Sources/PowerSync/Implementation/sync/CachingCredentialsConnector.swift create mode 100644 Sources/PowerSync/Implementation/sync/CoreSyncStatus.swift create mode 100644 Sources/PowerSync/Implementation/sync/HttpClient.swift create mode 100644 Sources/PowerSync/Implementation/sync/Instruction.swift create mode 100644 Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift create mode 100644 Sources/PowerSync/Implementation/sync/Status.swift create mode 100644 Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift create mode 100644 Sources/PowerSync/Implementation/sync/SyncCoordinator.swift delete mode 100644 Sources/PowerSync/Kotlin/PowerSyncBackendConnectorAdapter.swift delete mode 100644 Sources/PowerSync/Kotlin/sync/KotlinSyncStatus.swift delete mode 100644 Sources/PowerSync/Kotlin/sync/KotlinSyncStatusData.swift delete mode 100644 Sources/PowerSync/Kotlin/sync/KotlinSyncStreams.swift create mode 100644 Sources/PowerSync/Utils/BroadcastStream.swift create mode 100644 Sources/SyncPlayground/main.swift diff --git a/Package.resolved b/Package.resolved index efb6530..c61f4d2 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "5d24af35f5a7f9f12e6f8486efe9c263534391d1a45bc446da2b60a7c2faf9a6", + "originHash" : "3165e395f09f43046349a7f58470df039b67a3cc3e2ce3d2c3ee2635489c6fb7", "pins" : [ { "identity" : "csqlite", @@ -27,6 +27,24 @@ "revision" : "aa35758afca6cc97db836e1545ad28e4c5ddc50d", "version" : "0.4.12" } + }, + { + "identity" : "swift-async-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-async-algorithms.git", + "state" : { + "revision" : "9d349bcc328ac3c31ce40e746b5882742a0d1272", + "version" : "1.1.3" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "6675bc0ff86e61436e615df6fc5174e043e57924", + "version" : "1.4.1" + } } ], "version" : 3 diff --git a/Package.swift b/Package.swift index 43375dd..81cf654 100644 --- a/Package.swift +++ b/Package.swift @@ -55,9 +55,9 @@ if let corePath = localCoreExtension { let package = Package( name: packageName, platforms: [ - .iOS(.v15), - .macOS(.v12), - .watchOS(.v9), + .iOS(.v18), + .macOS(.v15), + .watchOS(.v11), .tvOS(.v15), ], products: [ @@ -83,7 +83,8 @@ let package = Package( ], dependencies: conditionalDependencies + [ .package(url: "https://github.com/groue/GRDB.swift.git", from: "7.9.0"), - .package(url: "https://github.com/powersync-ja/CSQLite.git", exact: "3.51.2") + .package(url: "https://github.com/powersync-ja/CSQLite.git", exact: "3.51.2"), + .package(url: "https://github.com/apple/swift-async-algorithms.git", from: "1.1.0") ], targets: [ // Targets are the basic building blocks of a package, defining a module or a test suite. @@ -94,6 +95,7 @@ let package = Package( kotlinTargetDependency, .product(name: "PowerSyncSQLiteCore", package: corePackageName), .product(name: "CSQLite", package: "CSQLite"), + .product(name: "AsyncAlgorithms", package: "swift-async-algorithms") ] ), .target( @@ -103,6 +105,7 @@ let package = Package( .product(name: "GRDB", package: "GRDB.swift") ] ), + .executableTarget(name: "SyncPlayground", dependencies: [.target(name: "PowerSync")]), .testTarget( name: "PowerSyncTests", dependencies: ["PowerSync"] diff --git a/Sources/PowerSync/Implementation/sync/CachingCredentialsConnector.swift b/Sources/PowerSync/Implementation/sync/CachingCredentialsConnector.swift new file mode 100644 index 0000000..3de8dde --- /dev/null +++ b/Sources/PowerSync/Implementation/sync/CachingCredentialsConnector.swift @@ -0,0 +1,26 @@ +actor CachingCredentialsConnector { + private let inner: PowerSyncBackendConnectorProtocol + private var cachedCredentials: PowerSyncCredentials? = nil + + init(inner: PowerSyncBackendConnectorProtocol) { + self.inner = inner + } + + func fetchCredentials() async throws -> PowerSyncCredentials? { + if let credentials = self.cachedCredentials { + return credentials + } + + let credentials = try await self.inner.fetchCredentials() + self.cachedCredentials = credentials + return credentials + } + + func invalidateCachedCredentials() { + self.cachedCredentials = nil + } + + nonisolated func uploadData(database: any PowerSyncDatabaseProtocol) async throws { + try await self.inner.uploadData(database: database) + } +} diff --git a/Sources/PowerSync/Implementation/sync/CoreSyncStatus.swift b/Sources/PowerSync/Implementation/sync/CoreSyncStatus.swift new file mode 100644 index 0000000..049f41d --- /dev/null +++ b/Sources/PowerSync/Implementation/sync/CoreSyncStatus.swift @@ -0,0 +1,77 @@ +struct CoreDownloadSyncStatus: Decodable, Sendable { + let connected: Bool + let connecting: Bool + let priorityStatus: [PriorityStatusEntry] + let downloading: CoreSyncDownloadProgress? + let streams: [SyncStreamStatus] + + enum CodingKeys: String, CodingKey { + case connected + case connecting + case priorityStatus = "priority_status" + case downloading + case streams + } + + init() { + self.connected = false + self.connecting = false + self.priorityStatus = [] + self.downloading = nil + self.streams = [] + } + + init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.connected = try container.decode(Bool.self, forKey: .connected) + self.connecting = try container.decode(Bool.self, forKey: .connecting) + self.priorityStatus = try container.decode([PriorityStatusEntry].self, forKey: .priorityStatus) + self.downloading = try container.decodeIfPresent(CoreSyncDownloadProgress.self, forKey: .downloading) + + var streamsContainer = try container.nestedUnkeyedContainer(forKey: .streams) + var streams: [SyncStreamStatus] = [] + while !streamsContainer.isAtEnd { + streams.append(try streamsContainer.decode(DecodableSyncStreamStatus.self).inner) + } + self.streams = streams + } +} + +struct BucketProgress: Decodable { + let priority: BucketPriority + let atLast: Int64 + let sinceLast: Int64 + let targetCount: Int64 + + enum CodingKeys: String, CodingKey { + case priority + case atLast = "at_last" + case sinceLast = "since_last" + case targetCount = "target_count" + } +} + +struct CoreSyncDownloadProgress: Decodable { + let buckets: [String: BucketProgress] +} + +struct ProgressCounters: Decodable, ProgressWithOperations { + let total: Int32 + let downloaded: Int32 + + var totalOperations: Int32 { + total + } + + var downloadedOperations: Int32 { + downloaded + } +} + +private struct DecodableSyncStreamStatus: Decodable { + let inner: SyncStreamStatus + + init(from decoder: any Decoder) throws { + self.inner = try SyncStreamStatus.init(from: decoder) + } +} diff --git a/Sources/PowerSync/Implementation/sync/HttpClient.swift b/Sources/PowerSync/Implementation/sync/HttpClient.swift new file mode 100644 index 0000000..80ae471 --- /dev/null +++ b/Sources/PowerSync/Implementation/sync/HttpClient.swift @@ -0,0 +1,36 @@ +import Foundation + +/// An internal protocol for HTTP clients, we use this to mock clients in tests. +protocol HttpClient: Sendable { + func receiveSyncLines(request: URLRequest) async throws -> (HTTPURLResponse, any AsyncSequence & Sendable) + func readFully(request: URLRequest) async throws -> (HTTPURLResponse, Data) +} + +enum SyncLine { + case text(contents: String) + // In the future, we might also want to support splitting BSON objects. Currently, we stream JSON only. + //case binary(contents: Data) +} + +struct PlatformHttpClient: HttpClient { + let session: URLSession + + func receiveSyncLines(request: URLRequest) async throws -> (HTTPURLResponse, any AsyncSequence & Sendable) { + let (bytes, response) = try await session.bytes(for: request) + let jsonStreamMimeType = "application/x-ndjson" + + if response.mimeType != jsonStreamMimeType { + throw PowerSyncError.operationFailed(message: "Invalid sync lines response, (expected \(jsonStreamMimeType), got \(response.mimeType, default: "")") + } + + let syncLines = bytes.lines.map { line in SyncLine.text(contents: line) } + return (response as! HTTPURLResponse, syncLines) + } + + func readFully(request: URLRequest) async throws -> (HTTPURLResponse, Data) { + let (data, response) = try await session.data(for: request) + return (response as! HTTPURLResponse, data) + } + + static let shared = PlatformHttpClient(session: .shared) +} diff --git a/Sources/PowerSync/Implementation/sync/Instruction.swift b/Sources/PowerSync/Implementation/sync/Instruction.swift new file mode 100644 index 0000000..fe42b3e --- /dev/null +++ b/Sources/PowerSync/Implementation/sync/Instruction.swift @@ -0,0 +1,91 @@ +enum CoreLogSeverity: String, Decodable { + case debug = "DEBUG" + case info = "INFO" + case warning = "WARNING" +} + +/// An instruction sent from the core extension to the Swift sync client. +enum Instruction { + case logLine(severity: CoreLogSeverity, line: String) + case updateSyncStatus(status: CoreDownloadSyncStatus) + case establishSyncStream(request: JsonParam) + case fetchCredentials(didExpire: Bool) + case closeSyncStream(hideDisconnect: Bool) + case flushFileSystem + case didCompleteSync +} + +extension Instruction: Decodable { + enum CodingKeys: String, CodingKey { + case logLine = "LogLine" + case updateSyncStatus = "UpdateSyncStatus" + case establishSyncStream = "EstablishSyncStream" + case fetchCredentials = "FetchCredentials" + case closeSyncStream = "CloseSyncStream" + case flushFileSystem = "FlushFileSystem" + case didCompleteSync = "DidCompleteSync" + } + + enum LogLineCodingKeys: CodingKey { + case severity + case line + } + + enum UpdateSyncStatusCodingKeys: CodingKey { + case status + } + + enum EstablishSyncStreamCodingKeys: CodingKey { + case request + } + + enum FetchCredentialsCodingKeys: String, CodingKey { + case didExpire = "did_expire" + } + + enum CloseSyncStreamCodingKeys: String, CodingKey { + case hideDisconnect = "hide_disconnect" + } + + enum FlushFileSystemCodingKeys: CodingKey { + } + + enum DidCompleteSyncCodingKeys: CodingKey { + } + + init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + var allKeys = ArraySlice(container.allKeys) + guard let onlyKey = allKeys.popFirst(), allKeys.isEmpty else { + throw DecodingError.typeMismatch( + Instruction.self, + DecodingError.Context.init(codingPath: container.codingPath, debugDescription: "Invalid number of keys found, expected one.", underlyingError: nil) + ) + } + + switch onlyKey { + case .logLine: + let nestedContainer = try container.nestedContainer(keyedBy: Instruction.LogLineCodingKeys.self, forKey: .logLine) + self = Instruction.logLine( + severity: try nestedContainer.decode(CoreLogSeverity.self, forKey: Instruction.LogLineCodingKeys.severity), + line: try nestedContainer.decode(String.self, forKey: Instruction.LogLineCodingKeys.line) + ) + case .updateSyncStatus: + let nestedContainer = try container.nestedContainer(keyedBy: Instruction.UpdateSyncStatusCodingKeys.self, forKey: .updateSyncStatus) + self = Instruction.updateSyncStatus(status: try nestedContainer.decode(CoreDownloadSyncStatus.self, forKey: Instruction.UpdateSyncStatusCodingKeys.status)) + case .establishSyncStream: + let nestedContainer = try container.nestedContainer(keyedBy: Instruction.EstablishSyncStreamCodingKeys.self, forKey: .establishSyncStream) + self = Instruction.establishSyncStream(request: try nestedContainer.decode(JsonParam.self, forKey: Instruction.EstablishSyncStreamCodingKeys.request)) + case .fetchCredentials: + let nestedContainer = try container.nestedContainer(keyedBy: Instruction.FetchCredentialsCodingKeys.self, forKey: .fetchCredentials) + self = Instruction.fetchCredentials(didExpire: try nestedContainer.decode(Bool.self, forKey: Instruction.FetchCredentialsCodingKeys.didExpire)) + case .closeSyncStream: + let nestedContainer = try container.nestedContainer(keyedBy: Instruction.CloseSyncStreamCodingKeys.self, forKey: .closeSyncStream) + self = Instruction.closeSyncStream(hideDisconnect: try nestedContainer.decode(Bool.self, forKey: Instruction.CloseSyncStreamCodingKeys.hideDisconnect)) + case .flushFileSystem: + self = Instruction.flushFileSystem + case .didCompleteSync: + self = Instruction.didCompleteSync + } + } +} diff --git a/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift b/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift new file mode 100644 index 0000000..79d7d90 --- /dev/null +++ b/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift @@ -0,0 +1,78 @@ +/// Arguments to the `powersync_control()` SQL function driving the sync process. +enum PowerSyncControlArguments { + case start(_ start: StartSyncIteration) + case stop + case textLine(line: String) + case binaryLine(line: ContiguousArray) + case completedUpload + case connectionEstablished + case responseStreamEnd + case updateSubscriptions(streams: [StreamKey]) + + func execute(_ context: ConnectionContext) throws -> String { + let op: String + let param: Sendable? + + switch (self) { + case .start(let start): + op = "start" + param = String(data: try StreamingSyncClient.jsonEncoder.encode(start), encoding: .utf8) + case .stop: + op = "stop" + param = nil + case .textLine(line: let line): + op = "line_text" + param = line + case .binaryLine(line: let line): + op = "line_binary" + param = line + case .completedUpload: + op = "completed_upload" + param = nil + case .connectionEstablished: + op = "connection" + param = "established" + case .responseStreamEnd: + op = "connection" + param = "end" + case .updateSubscriptions(streams: let streams): + op = "update_subscriptions" + param = String(data: try StreamingSyncClient.jsonEncoder.encode(streams), encoding: .utf8) + } + + return try context.get(sql: "SELECT powersync_control(?, ?)", parameters: [op, param]) { cursor in + try cursor.getString(index: 0) + } + } + + func isSyncLine() -> Bool { + switch self { + case .binaryLine(line: _): + return true + case .textLine(line: _): + return true + default: + return false + } + } +} + +struct StartSyncIteration: Codable { + let parameters: JsonParam + // TODO: Schema + let includeDefaults: Bool + let activeStreams: [StreamKey] + let appMetadata: [String: String] + + enum CodingKeys: String, CodingKey { + case parameters + case includeDefaults = "include_defaults" + case activeStreams = "active_streams" + case appMetadata = "app_metadata" + } +} + +struct StreamKey: Codable { + let name: String + let params: JsonParam +} diff --git a/Sources/PowerSync/Implementation/sync/Status.swift b/Sources/PowerSync/Implementation/sync/Status.swift new file mode 100644 index 0000000..bc7c66e --- /dev/null +++ b/Sources/PowerSync/Implementation/sync/Status.swift @@ -0,0 +1,231 @@ +import Foundation +import Synchronization + +struct MutableSyncStatus: ~Copyable { + var core: CoreDownloadSyncStatus = CoreDownloadSyncStatus() + var uploading: Bool = false + var internalDownloadError: (any Error & Sendable)? + var internalUploadError: (any Error & Sendable)? +} + +fileprivate struct SyncStatusDataImpl: SyncStatusData { + let core: CoreDownloadSyncStatus + let downloadProgress: (any SyncDownloadProgress)? + let uploading: Bool + + let internalDownloadError: (any Error & Sendable)? + let internalUploadError: (any Error & Sendable)? + + init(status: borrowing MutableSyncStatus) { + self.core = status.core + self.uploading = status.uploading + self.internalUploadError = status.internalUploadError + self.internalDownloadError = status.internalDownloadError + + if let downloading = core.downloading { + self.downloadProgress = IndexedCoreDownloadProgress(inner: downloading) + } else { + self.downloadProgress = nil + } + } + + var connected: Bool { + core.connected + } + + var connecting: Bool { + core.connecting + } + + var downloading: Bool { + core.downloading != nil + } + + var lastSyncedAt: Date? { + let completeSyncStatus = core.priorityStatus.first { $0.priority == .fullSyncPriority } + return completeSyncStatus?.lastSyncedAt + } + + var hasSynced: Bool? { + lastSyncedAt != nil + } + + var downloadError: Any? { + internalDownloadError + } + + var uploadError: Any? { + internalUploadError + } + + var anyError: Any? { + downloadError ?? uploadError + } + + var priorityStatusEntries: [PriorityStatusEntry] { + core.priorityStatus + } + + var syncStreams: [SyncStreamStatus]? { + if downloadProgress != nil { + return core.streams + } else { + // core.streams includes progress information, we need to hide that since we're not currently + // downloading anything. + return core.streams.map { stream in SyncStreamStatus.init(subscription: stream.subscription) } + } + } + + func statusForPriority(_ priority: BucketPriority) -> PriorityStatusEntry { + for known in priorityStatusEntries { + // Lower-priority buckets are synced after higher-priority buckets, and since priorityStatusEntries + // is sortedwe look for the first entry that doesn't have a higher priority. + if known.priority <= priority { + return known + } + } + + // Fallback, report status for complete sync (which necessarily includes all priorities) + return PriorityStatusEntry(priority: priority, lastSyncedAt: lastSyncedAt, hasSynced: hasSynced) + } + + func forStream(stream: any SyncStreamDescription) -> SyncStreamStatus? { + for found in syncStreams! { + if found.subscription.name == stream.name && found.subscription.parameters == stream.parameters { + return found + } + } + + return nil + } +} + +fileprivate struct SyncStatusContainer: ~Copyable { + var inner: MutableSyncStatus + var snapshot: SyncStatusDataImpl + + init(inner: consuming MutableSyncStatus) { + self.snapshot = SyncStatusDataImpl(status: inner) + self.inner = inner + } +} + +final class SwiftSyncStatus: SyncStatus { + private let current: Mutex + private let listeners: BroadcastStream = BroadcastStream() + + init() { + self.current = Mutex(SyncStatusContainer(inner: MutableSyncStatus())) + } + + private func readStatus(status: (borrowing SyncStatusDataImpl) -> T) -> T { + return self.current.withLock { status($0.snapshot) } + } + + internal func mutateStatus(update: (_ status: inout MutableSyncStatus) -> Void) { + self.current.withLock { + update(&$0.inner) + $0.snapshot = SyncStatusDataImpl(status: $0.inner) + } + + self.listeners.dispatch(event: self) + } + + func asFlow() -> AsyncStream { + self.listeners.subscribe() + } + + func waitFor(_ predicate: (borrowing SwiftSyncStatus) -> Bool) async { + if predicate(self) { + return + } + + for await _ in self.asFlow() { + if predicate(self) { + return + } + } + } + + var connected: Bool { + self.readStatus { current in current.connected } + } + + var connecting: Bool { + self.readStatus { current in current.connecting } + } + + var downloading: Bool { + self.readStatus { current in current.downloading } + } + + var downloadProgress: (any SyncDownloadProgress)? { + self.readStatus { current in current.downloadProgress } + } + + var uploading: Bool { + self.readStatus { current in current.uploading } + } + + var lastSyncedAt: Date? { + self.readStatus { current in current.lastSyncedAt } + } + + var hasSynced: Bool? { + self.readStatus { current in current.hasSynced } + } + + var downloadError: Any? { + self.readStatus { current in current.downloadError } + } + + var uploadError: Any? { + self.readStatus { current in current.uploadError } + } + + var anyError: Any? { + self.readStatus { current in current.anyError } + } + + var priorityStatusEntries: [PriorityStatusEntry] { + self.readStatus { current in current.priorityStatusEntries } + } + + var syncStreams: [SyncStreamStatus]? { + self.readStatus { current in current.syncStreams } + } + + func statusForPriority(_ priority: BucketPriority) -> PriorityStatusEntry { + self.readStatus { current in current.statusForPriority(priority) } + } + + func forStream(stream: any SyncStreamDescription) -> SyncStreamStatus? { + self.readStatus { current in current.forStream(stream: stream) } + } +} + +struct IndexedCoreDownloadProgress: SyncDownloadProgress { + private let inner: CoreSyncDownloadProgress + + let totalOperations: Int32 + let downloadedOperations: Int32 + + init(inner: CoreSyncDownloadProgress) { + self.inner = inner + let (total, downloaded) = inner.buckets.values.reduce((0, 0), Self.addProgress) + self.totalOperations = total + self.downloadedOperations = downloaded + } + + func untilPriority(priority: BucketPriority) -> any ProgressWithOperations { + let (total, downloaded) = inner.buckets.values.filter{ bkt in bkt.priority >= priority }.reduce((0, 0), Self.addProgress) + return ProgressCounters(total: total, downloaded: downloaded) + } + + private static func addProgress(prev: (Int32, Int32), entry: BucketProgress) -> (Int32, Int32) { + let downloaded = Int32(entry.sinceLast) + let total = Int32(entry.targetCount - entry.atLast) + return (prev.0 + total, prev.1 + downloaded) + } +} + diff --git a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift new file mode 100644 index 0000000..00dc489 --- /dev/null +++ b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift @@ -0,0 +1,389 @@ +import AsyncAlgorithms +import Foundation +import Synchronization + +fileprivate let tag = "StreamingSyncClient" + +final class StreamingSyncClient: Sendable { + let db: KotlinPowerSyncDatabaseImpl + let options: ConnectOptions + let events = BroadcastStream() + let connector: CachingCredentialsConnector + let httpClient: any HttpClient + + init( + db: KotlinPowerSyncDatabaseImpl, + connector: PowerSyncBackendConnectorProtocol, + httpClient: any HttpClient = PlatformHttpClient.shared, + options: ConnectOptions = ConnectOptions() + ) { + self.db = db + self.connector = CachingCredentialsConnector(inner: connector) + self.httpClient = httpClient + self.options = ConnectOptions() + } + + /// Starts a task driving uploads and downloads by repeatedly connecting to the PowerSync service, + /// managing tokens and CRUD uploads. + /// + /// There should at most be one such task per database, but this internal method performs no concurrency + /// control for that. + func run() -> Task { + Task(name: "StreamingSyncClient.run") { + let signals = SyncSignals() + async let download = try downloadLoop(signals: signals) + + try await download + } + } + + private func watchPsCrudChanges(signals: SyncSignals) async throws { + + } + + private func uploadLoop(signals: SyncSignals) async throws { + // TODO: Replace with better watch mechanism + let watch = try db.watch(sql: "SELECT 1 FROM ps_crud LIMIT 1", parameters: [], mapper: { _ in () }) + .dropFirst() // Skip initial result, we just want to watch changes + .map { _ in () } + let allTriggers = AsyncAlgorithms.merge(watch, signals.signalCrudUpload.subscribe()) + + for try await item in allTriggers { + try await uploadAllCrud() + } + } + + private func uploadAllCrud() async throws { + var lastUploadItem: Int64? = nil + + while (true) { + defer { db.syncStatus.mutateStatus { $0.uploading = false } } + + do { + let nextItem = try await db.getOptional("SELECT id FROM ps_crud ORDER BY id LIMIT 1", mapper: { cursor in try cursor.getInt64(index: 1) }) + if let nextItem { + if nextItem == lastUploadItem { + db.logger.warning(""" +Potentially previously uploaded CRUD entries are still present in the upload queue. +Make sure to handle uploads and complete CRUD transactions or batches by calling and awaiting their [.complete()] method. +The next upload iteration will be delayed. +""", tag: tag) + throw PowerSyncError.operationFailed(message: "Delaying due to previously encountered CRUD item.") + } + + lastUploadItem = nextItem + db.syncStatus.mutateStatus { $0.uploading = true } + try await connector.uploadData(database: db) + } else { + // Uploading is completed + try await self.uploadLocalTarget() + break + } + } catch { + db.syncStatus.mutateStatus { + $0.uploading = false + $0.internalUploadError = error + } + + if error is CancellationError { + return + } + + db.logger.error("Error uploading crud: \(error)", tag: tag) + do { + try await Task.sleep(for: .seconds(self.options.retryDelay)) + } catch { + // Cancelled, abort + return + } + } + } + } + + private func uploadLocalTarget() async throws { + guard let _ = try await db.getOptional( + sql: "SELECT 1 FROM ps_bucket WHERE name = '$local' AND target_op = ?", + parameters: [KotlinPowerSyncDatabaseImpl.maxOpId], + mapper: { cursor in () } + ) else { + return // Nothing to update + } + + guard let seqBefore = try await db.getOptional("SELECT seq FROM main.sqlite_sequence WHERE name = 'ps_crud'", mapper: { try $0.getInt64(index: 0) }) else { + return // Nothing to update + } + + let opId = try await getWriteCheckpoint() + + try await db.writeTransaction { tx in + let anyData = try tx.getOptional(sql: "SELECT 1 FROM ps_crud LIMIT 1", parameters: nil) { cursor in 1 } + if anyData != nil { + // Additional write after we've obtained the write checkpoint + return + } + + let seqAfter = try tx.getOptional(sql: "SELECT seq FROM main.sqlite_sequence WHERE name = 'ps_crud'", parameters: nil, mapper: { try $0.getInt64(index: 0) }) + if seqBefore != seqAfter { + // New crud data may have been uploaded since we got the checkpoint, abort. + return + } + + try tx.execute(sql: "UPDATE ps_buckets SET target_op = CAST(? AS INTEGER) WHERE name = '$local'", parameters: [opId]) + } + } + + private func getWriteCheckpoint() async throws -> String { + let clientId = try await db.get("SELECT powersync_client_id()") { try $0.getString(index: 0) } + var (_, request) = try await authenticatedRequest { endpoint in + endpoint + .appending(path: "write-checkpoint2.json") + .appending(queryItems: [.init(name: "client_id", value: clientId)]) + } + let (response, data) = try await httpClient.readFully(request: request) + + if response.statusCode == 401 { + await self.invalidateCredentials() + } + if response.statusCode != 200 { + throw PowerSyncError.operationFailed(message: "Error getting write checkpoint: \(response.statusCode)") + } + + struct WriteCheckpointData: Decodable { + let write_checkpoint: String + } + + struct WriteCheckpointResponse: Decodable { + let data: WriteCheckpointData + } + + let body = try StreamingSyncClient.jsonDecoder.decode(WriteCheckpointResponse.self, from: data) + return body.data.write_checkpoint + } + + private func downloadLoop(signals: SyncSignals) async throws { + var result = SyncIterationResult() + + while (!Task.isCancelled) { + do { + result = try await ActiveSyncIteration(syncClient: self, signals: signals).run() + } catch { + result = SyncIterationResult() + + db.logger.error("Error in streamingSync: \(error)", tag: tag) + db.syncStatus.mutateStatus { $0.internalDownloadError = error } + } + + if !result.hideDisconnect { + do { + try await Task.sleep(for: .seconds(options.retryDelay)) + } catch { + // Cancelled + break + } + } + } + } + + fileprivate func invalidateCredentials() async { + await self.connector.invalidateCachedCredentials() + } + + private func authenticatedRequest(buildUrl: (URL) -> URL) async throws -> (URL, URLRequest) { + guard let credentials = try await connector.fetchCredentials() else { + throw PowerSyncError.operationFailed(message: "fetchCredentials() returned nil") + } + + guard let base = URL(string: credentials.endpoint) else { + throw PowerSyncError.operationFailed(message: "Invalid backend connector URL: \(credentials.endpoint)") + } + let url = buildUrl(base) + var request = URLRequest(url: url) + request.setValue("Token \(credentials.token)", forHTTPHeaderField: "Authorization") + return (url, request) + } + + fileprivate func fetchSyncLines(request: JsonParam) async throws -> ControlInvocationsFromStream { + var (url, httpRequest) = try await authenticatedRequest { endpoint in endpoint.appending(path: "sync/stream") } + httpRequest.httpMethod = "POST" + httpRequest.setValue("application/json", forHTTPHeaderField: "Content-Type") + httpRequest.setValue("application/x-ndjson", forHTTPHeaderField: "Accept") + httpRequest.httpBody = try StreamingSyncClient.jsonEncoder.encode(request) + + let (response, stream) = try await httpClient.receiveSyncLines(request: httpRequest) + if response.statusCode == 401 { + await invalidateCredentials() + } + if response.statusCode != 200 { + throw PowerSyncError.operationFailed(message: "POST \(url) failed with status code \(response.statusCode)") + } + + return ControlInvocationsFromStream(sequence: stream) + } + + static let jsonEncoder = JSONEncoder() + static let jsonDecoder = JSONDecoder() +} + +private struct ActiveSyncIteration: Sendable { + private let syncClient: StreamingSyncClient + private let localEvents: AsyncStream + private let signals: SyncSignals + + init(syncClient: StreamingSyncClient, signals: SyncSignals) { + self.syncClient = syncClient + self.localEvents = syncClient.events.subscribe() + self.signals = signals + } + + func run() async throws -> SyncIterationResult { + let initialInstructions = try await powersyncControl(.start(StartSyncIteration( + parameters: [:], + includeDefaults: true, + activeStreams: [], + appMetadata: [:] + ))) + + var controlArgs: (any AsyncSequence)? + + for instruction in initialInstructions { + if case .establishSyncStream(request: let request) = instruction { + let serviceEvents = try await syncClient.fetchSyncLines(request: request) + controlArgs = AsyncAlgorithms.merge(serviceEvents, localEvents) + } else { + try await self.execute(instr: instruction) + } + } + + guard let controlArgs else { + // Rust client didn't ask for a connection?? Ok then, end the iteration and retry + return SyncIterationResult() + } + + var hadSyncLine = false + for try await arg in controlArgs { + let control = try await powersyncControl(arg) + for instr in control { + if case let .closeSyncStream(hideDisconnect) = instr { + return SyncIterationResult(hideDisconnect: hideDisconnect) + } + + try await execute(instr: instr) + } + + if !hadSyncLine && arg.isSyncLine() { + // Trigger a crud upload when receiving the first sync line: We could have + // pending local writes made while disconnected, so in addition to listening on + // updates to `ps_crud`, we also need to trigger a CRUD upload in some other cases. + // We do this on the first sync line because the client is likely to be online in + // that case. + hadSyncLine = true + signals.triggerAsyncCrudUpload() + } + } + + return SyncIterationResult() + } + + private func powersyncControl(_ args: PowerSyncControlArguments) async throws -> [Instruction] { + let rawInstructions = try await syncClient.db.writeTransaction { tx in try args.execute(tx) } + return try StreamingSyncClient.jsonDecoder.decode([Instruction].self, from: rawInstructions.data(using: .utf8)!) + } + + private func execute(instr: consuming Instruction) async throws { + switch (instr) { + case .logLine(severity: let severity, line: let line): + let logger = syncClient.db.logger + switch severity { + case .debug: + logger.debug(line, tag: tag) + case .info: + logger.info(line, tag: tag) + case .warning: + logger.warning(line, tag: tag) + } + break; + case .updateSyncStatus(status: let status): + syncClient.db.syncStatus.mutateStatus { + $0.core = status + } + case .establishSyncStream(request: _): + fatalError("There can only be one establishSyncStream instruction per sync iteration") + case .closeSyncStream(hideDisconnect: _): + fatalError("Must be handled in run() loop") + case .fetchCredentials(didExpire: let didExpire): + if didExpire { + await syncClient.invalidateCredentials() + } else { + signals.triggerAsyncFetchCredentials() + } + fatalError("todo: fetchCredentials") + case .flushFileSystem: + // Noop on native platforms. + break; + case .didCompleteSync: + syncClient.db.syncStatus.mutateStatus { + $0.internalDownloadError = nil + } + } + } +} + +/// Wraps an HTTP response by mapping it to control invocations for lines. This also adds an "connection established" / "response ended" prefix and suffix. +fileprivate struct ControlInvocationsFromStream: AsyncSequence, Sendable { + typealias AsyncIterator = ControlInvocationsFromStreamIterator + typealias Element = PowerSyncControlArguments + + let sequence: any AsyncSequence & Sendable + + func makeAsyncIterator() -> ControlInvocationsFromStreamIterator { + .beforeStart(self.sequence) + } +} + +fileprivate enum ControlInvocationsFromStreamIterator: AsyncIteratorProtocol { + typealias Element = PowerSyncControlArguments + + case beforeStart(any AsyncSequence) + case isReceiving(any AsyncIteratorProtocol) + case eof + + mutating func next() async throws -> PowerSyncControlArguments? { + switch self { + case .beforeStart(let sequence): + self = .isReceiving(sequence.makeAsyncIterator()) + return .connectionEstablished + case .isReceiving(var iterator): + let next = try await iterator.next() + switch next { + case .none: + self = .eof + return .responseStreamEnd + case .some(.text(contents: let contents)): + return .textLine(line: contents) + } + case .eof: + return nil + } + } +} + +private struct SyncIterationResult { + let hideDisconnect: Bool + + init(hideDisconnect: Bool = false) { + self.hideDisconnect = hideDisconnect + } +} + +struct SyncSignals { + let signalCrudUpload = BroadcastStream() + let prefetchCredentials = BroadcastStream() + + func triggerAsyncCrudUpload() { + self.signalCrudUpload.dispatch(event: ()) + } + + func triggerAsyncFetchCredentials() { + self.prefetchCredentials.dispatch(event: ()) + } +} diff --git a/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift b/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift new file mode 100644 index 0000000..c892f65 --- /dev/null +++ b/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift @@ -0,0 +1,30 @@ +actor SyncCoordinator { + private var activeSync: Task? + + func connect(db: KotlinPowerSyncDatabaseImpl, connector: PowerSyncBackendConnectorProtocol, options: ConnectOptions) async { + if let task = activeSync { + await self.finishSyncTask(task: task) + } + + let sync = StreamingSyncClient(db: db, connector: connector) + activeSync = sync.run() + } + + func disconnect() async { + guard let task = activeSync else { + return // Not connecteed + } + + await self.finishSyncTask(task: task) + } + + private func finishSyncTask(task: Task) async { + self.activeSync = nil + task.cancel() + do { + try await task.value + } catch { + // Ignore here, the sync task itself handles errors by retrying. + } + } +} diff --git a/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift b/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift index 5a729ce..9ff03d2 100644 --- a/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift +++ b/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift @@ -9,7 +9,8 @@ final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, let logger: any LoggerProtocol private let kotlinDatabase: PowerSyncKotlin.PowerSyncDatabase private let encoder = JSONEncoder() - let currentStatus: SyncStatus + private let syncCoordinator = SyncCoordinator() + internal let syncStatus = SwiftSyncStatus() private let dbFilename: String init( @@ -23,13 +24,14 @@ final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, /// The kotlin PowerSyncDatabase.identifier currently prepends `null` to the dbFilename (for the directory). /// FIXME. Update this once we support database directory configuration. self.dbFilename = dbFilename - currentStatus = KotlinSyncStatus( - baseStatus: kotlinDatabase.currentStatus - ) + } + + var currentStatus: any SyncStatus { + syncStatus } - func waitForFirstSync() async throws { - try await kotlinDatabase.waitForFirstSync() + func waitForFirstSync() async { + await syncStatus.waitFor { $0.hasSynced == true } } func updateSchema(schema: any SchemaProtocol) async throws { @@ -38,38 +40,21 @@ final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, ) } - func waitForFirstSync(priority: Int32) async throws { - try await kotlinDatabase.waitForFirstSync( - priority: priority - ) + func waitForFirstSync(priority: Int32) async { + let priority = BucketPriority(priority) + await syncStatus.waitFor { $0.statusForPriority(priority).hasSynced == true } } func syncStream(name: String, params: JsonParam?) -> any SyncStream { let rawStream = kotlinDatabase.syncStream(name: name, parameters: params?.mapValues { $0.toKotlinMap() }) - return KotlinSyncStream(kotlinStream: rawStream) + fatalError("todo") } func connect( connector: PowerSyncBackendConnectorProtocol, options: ConnectOptions? - ) async throws { - let connectorAdapter = swiftBackendConnectorToPowerSyncConnector(connector: SwiftBackendConnectorBridge( - swiftBackendConnector: connector, db: self - )) - - let resolvedOptions = options ?? ConnectOptions() - try await kotlinDatabase.connect( - connector: connectorAdapter, - crudThrottleMs: Int64(resolvedOptions.crudThrottle * 1000), - retryDelayMs: Int64(resolvedOptions.retryDelay * 1000), - params: resolvedOptions.params.mapValues { $0.toKotlinMap() }, - options: createSyncOptions( - newClient: resolvedOptions.newClientImplementation, - userAgent: userAgent(), - loggingConfig: resolvedOptions.clientConfiguration?.requestLogger?.toKotlinConfig() - ), - appMetadata: resolvedOptions.appMetadata - ) + ) async { + await syncCoordinator.connect(db: self, connector: connector, options: options ?? ConnectOptions()) } func getCrudBatch(limit: Int32 = 100) async throws -> CrudBatch? { @@ -103,11 +88,13 @@ final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, try await kotlinDatabase.getPowerSyncVersion() } - func disconnect() async throws { - try await kotlinDatabase.disconnect() + func disconnect() async { + await syncCoordinator.disconnect() } func disconnectAndClear(clearLocal: Bool, soft: Bool) async throws { + await disconnect() + try await kotlinDatabase.disconnectAndClear( clearLocal: clearLocal, soft: soft @@ -432,6 +419,8 @@ final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, ) } } + + static let maxOpId = Int64.max } func openKotlinDBDefault( diff --git a/Sources/PowerSync/Kotlin/PowerSyncBackendConnectorAdapter.swift b/Sources/PowerSync/Kotlin/PowerSyncBackendConnectorAdapter.swift deleted file mode 100644 index 02b894b..0000000 --- a/Sources/PowerSync/Kotlin/PowerSyncBackendConnectorAdapter.swift +++ /dev/null @@ -1,37 +0,0 @@ -import OSLog -import PowerSyncKotlin - -final class SwiftBackendConnectorBridge: KotlinSwiftBackendConnector, Sendable { - let swiftBackendConnector: PowerSyncBackendConnectorProtocol - let db: any PowerSyncDatabaseProtocol - let logTag = "PowerSyncBackendConnector" - - init( - swiftBackendConnector: PowerSyncBackendConnectorProtocol, - db: any PowerSyncDatabaseProtocol - ) { - self.swiftBackendConnector = swiftBackendConnector - self.db = db - } - - func __fetchCredentials() async throws -> PowerSyncResult { - do { - let result = try await swiftBackendConnector.fetchCredentials() - return PowerSyncResult.Success(value: result?.kotlinCredentials) - } catch { - db.logger.error("Error while fetching credentials", tag: logTag) - return PowerSyncResult.Failure(exception: error.toPowerSyncError()) - } - } - - func __uploadData() async throws -> PowerSyncResult { - do { - // Pass the Swift DB protocal to the connector - try await swiftBackendConnector.uploadData(database: self.db) - return PowerSyncResult.Success(value: nil) - } catch { - db.logger.error("Error while uploading data: \(error)", tag: logTag) - return PowerSyncResult.Failure(exception: error.toPowerSyncError()) - } - } -} diff --git a/Sources/PowerSync/Kotlin/sync/KotlinSyncStatus.swift b/Sources/PowerSync/Kotlin/sync/KotlinSyncStatus.swift deleted file mode 100644 index 95a187f..0000000 --- a/Sources/PowerSync/Kotlin/sync/KotlinSyncStatus.swift +++ /dev/null @@ -1,43 +0,0 @@ -import Combine -import Foundation -import PowerSyncKotlin - -final class KotlinSyncStatus: KotlinSyncStatusDataProtocol, SyncStatus { - private let baseStatus: PowerSyncKotlin.SyncStatus - - var base: PowerSyncKotlin.SyncStatusData { - baseStatus - } - - init(baseStatus: PowerSyncKotlin.SyncStatus) { - self.baseStatus = baseStatus - } - - func asFlow() -> AsyncStream { - AsyncStream(bufferingPolicy: .bufferingNewest(1)) { continuation in - // Create an outer task to monitor cancellation - let task = Task { - do { - // Watching for changes in the database - for try await value in baseStatus.asFlow() { - // Check if the outer task is cancelled - try Task.checkCancellation() // This checks if the calling task was cancelled - - continuation.yield( - KotlinSyncStatusData(base: value) - ) - } - - continuation.finish() - } catch { - continuation.finish() - } - } - - // Propagate cancellation from the outer task to the inner task - continuation.onTermination = { @Sendable _ in - task.cancel() // This cancels the inner task when the stream is terminated - } - } - } -} diff --git a/Sources/PowerSync/Kotlin/sync/KotlinSyncStatusData.swift b/Sources/PowerSync/Kotlin/sync/KotlinSyncStatusData.swift deleted file mode 100644 index 9740694..0000000 --- a/Sources/PowerSync/Kotlin/sync/KotlinSyncStatusData.swift +++ /dev/null @@ -1,144 +0,0 @@ -import Foundation -import PowerSyncKotlin - -/// A protocol extension which allows sharing common implementation using a base sync status -protocol KotlinSyncStatusDataProtocol: SyncStatusData { - var base: PowerSyncKotlin.SyncStatusData { get } -} - -struct KotlinSyncStatusData: KotlinSyncStatusDataProtocol, - // We can't override the PowerSyncKotlin.SyncStatusData's Sendable status - @unchecked Sendable -{ - let base: PowerSyncKotlin.SyncStatusData -} - -/// Extension of `KotlinSyncStatusDataProtocol` which uses the shared `base` to implement `SyncStatusData` -extension KotlinSyncStatusDataProtocol { - var connected: Bool { - base.connected - } - - var connecting: Bool { - base.connecting - } - - var downloading: Bool { - base.downloading - } - - var uploading: Bool { - base.uploading - } - - var lastSyncedAt: Date? { - guard let lastSyncedAt = base.lastSyncedAt else { return nil } - return Date( - timeIntervalSince1970: Double( - lastSyncedAt.epochSeconds - ) - ) - } - - var downloadProgress: (any SyncDownloadProgress)? { - guard let kotlinProgress = base.downloadProgress else { return nil } - return KotlinSyncDownloadProgress(progress: kotlinProgress) - } - - var hasSynced: Bool? { - base.hasSynced?.boolValue - } - - var uploadError: Any? { - base.uploadError - } - - var downloadError: Any? { - base.downloadError - } - - var anyError: Any? { - base.anyError - } - - public var priorityStatusEntries: [PriorityStatusEntry] { - base.priorityStatusEntries.map { mapPriorityStatus($0) } - } - - public func statusForPriority(_ priority: BucketPriority) -> PriorityStatusEntry { - mapPriorityStatus( - base.statusForPriority( - priority: Int32(priority.priorityCode) - ) - ) - } - - var syncStreams: [SyncStreamStatus]? { - return base.syncStreams?.map(mapSyncStreamStatus) - } - - func forStream(stream: SyncStreamDescription) -> SyncStreamStatus? { - var rawStatus: Optional - if let kotlinStream = stream as? any HasKotlinStreamDescription { - // Fast path: Reuse Kotlin stream object for lookup. - rawStatus = base.forStream(stream: kotlinStream.kotlinDescription) - } else { - // Custom stream description, we have to convert parameters to a Kotlin map. - let parameters = stream.parameters?.mapValues { $0.toValue() } - rawStatus = syncStatusForStream(status: base, name: stream.name, parameters: parameters) - } - - return rawStatus.map(mapSyncStreamStatus) - } - - private func mapPriorityStatus(_ status: PowerSyncKotlin.PriorityStatusEntry) -> PriorityStatusEntry { - var lastSyncedAt: Date? - if let syncedAt = status.lastSyncedAt { - lastSyncedAt = Date( - timeIntervalSince1970: Double(syncedAt.epochSeconds) - ) - } - - return PriorityStatusEntry( - priority: BucketPriority(status.priority), - lastSyncedAt: lastSyncedAt, - hasSynced: status.hasSynced?.boolValue - ) - } -} - -protocol KotlinProgressWithOperationsProtocol: ProgressWithOperations { - var base: any PowerSyncKotlin.ProgressWithOperations { get } -} - -extension KotlinProgressWithOperationsProtocol { - var totalOperations: Int32 { - return base.totalOperations - } - - var downloadedOperations: Int32 { - return base.downloadedOperations - } -} - -struct KotlinProgressWithOperations: KotlinProgressWithOperationsProtocol, - // We can't mark PowerSyncKotlin.ProgressWithOperations as Sendable - @unchecked Sendable -{ - let base: PowerSyncKotlin.ProgressWithOperations -} - -struct KotlinSyncDownloadProgress: KotlinProgressWithOperationsProtocol, SyncDownloadProgress, - // We can't mark PowerSyncKotlin.SyncDownloadProgress as Sendable - @unchecked Sendable -{ - let progress: PowerSyncKotlin.SyncDownloadProgress - - var base: any PowerSyncKotlin.ProgressWithOperations { - progress - } - - func untilPriority(priority: BucketPriority) -> any ProgressWithOperations { - return KotlinProgressWithOperations(base: progress.untilPriority(priority: priority.priorityCode)) - } -} diff --git a/Sources/PowerSync/Kotlin/sync/KotlinSyncStreams.swift b/Sources/PowerSync/Kotlin/sync/KotlinSyncStreams.swift deleted file mode 100644 index cb1acc0..0000000 --- a/Sources/PowerSync/Kotlin/sync/KotlinSyncStreams.swift +++ /dev/null @@ -1,125 +0,0 @@ -import Foundation -import PowerSyncKotlin - -class KotlinStreamDescription { - let inner: T - let name: String - let parameters: JsonParam? - let kotlinParameters: [String: Any?]? - - init(inner: T) { - self.inner = inner - self.name = inner.name - self.kotlinParameters = inner.parameters - self.parameters = inner.parameters?.mapValues { JsonValue.kotlinValueToJsonParam(raw: $0) } - } -} - -protocol HasKotlinStreamDescription { - associatedtype Description: PowerSyncKotlin.SyncStreamDescription - - var stream: KotlinStreamDescription { get } -} - -extension HasKotlinStreamDescription { - var kotlinDescription: any PowerSyncKotlin.SyncStreamDescription { - self.stream.inner - } -} - -class KotlinSyncStream: SyncStream, HasKotlinStreamDescription, -// `PowerSyncKotlin.SyncStream` cannot be marked as Sendable, but is thread-safe. -@unchecked Sendable -{ - let stream: KotlinStreamDescription - - init(kotlinStream: PowerSyncKotlin.SyncStream) { - self.stream = KotlinStreamDescription(inner: kotlinStream); - } - - var name: String { - stream.name - } - - var parameters: JsonParam? { - stream.parameters - } - - func subscribe(ttl: TimeInterval?, priority: BucketPriority?) async throws -> any SyncStreamSubscription { - let kotlinTtl: Optional = if let ttl { - KotlinDouble(value: ttl) - } else { - nil - } - let kotlinPriority: Optional = if let priority { - KotlinInt(value: priority.priorityCode) - } else { - nil - } - - let kotlinSubscription = try await syncStreamSubscribeSwift( - stream: stream.inner, - ttl: kotlinTtl, - priority: kotlinPriority, - ); - return KotlinSyncStreamSubscription(kotlinStream: kotlinSubscription) - } - - func unsubscribeAll() async throws { - try await stream.inner.unsubscribeAll() - } -} - -class KotlinSyncStreamSubscription: SyncStreamSubscription, HasKotlinStreamDescription, -// `PowerSyncKotlin.SyncStreamSubscription` cannot be marked as Sendable, but is thread-safe. -@unchecked Sendable -{ - let stream: KotlinStreamDescription - - init(kotlinStream: PowerSyncKotlin.SyncStreamSubscription) { - self.stream = KotlinStreamDescription(inner: kotlinStream) - } - - var name: String { - stream.name - } - var parameters: JsonParam? { - stream.parameters - } - - func waitForFirstSync() async throws { - try await stream.inner.waitForFirstSync() - } - - func unsubscribe() async throws { - try await stream.inner.unsubscribe() - } -} - -func mapSyncStreamStatus(_ status: PowerSyncKotlin.SyncStreamStatus) -> SyncStreamStatus { - let progress = status.progress.map { ProgressNumbers(source: $0) } - let subscription = status.subscription - - return SyncStreamStatus( - progress: progress, - subscription: SyncSubscriptionDescription( - name: subscription.name, - parameters: subscription.parameters?.mapValues { JsonValue.kotlinValueToJsonParam(raw: $0) }, - active: subscription.active, - isDefault: subscription.isDefault, - hasExplicitSubscription: subscription.hasExplicitSubscription, - expiresAt: subscription.expiresAt.map { Double($0.epochSeconds) }, - lastSyncedAt: subscription.lastSyncedAt.map { Double($0.epochSeconds) } - ) - ) -} - -struct ProgressNumbers: ProgressWithOperations { - let totalOperations: Int32 - let downloadedOperations: Int32 - - init(source: PowerSyncKotlin.ProgressWithOperations) { - self.totalOperations = source.totalOperations - self.downloadedOperations = source.downloadedOperations - } -} diff --git a/Sources/PowerSync/PowerSyncCredentials.swift b/Sources/PowerSync/PowerSyncCredentials.swift index 1de8bae..e1497e1 100644 --- a/Sources/PowerSync/PowerSyncCredentials.swift +++ b/Sources/PowerSync/PowerSyncCredentials.swift @@ -39,10 +39,6 @@ public struct PowerSyncCredentials: Codable, Sendable { token = kotlin.token } - var kotlinCredentials: KotlinPowerSyncCredentials { - return KotlinPowerSyncCredentials(endpoint: endpoint, token: token, userId: nil) - } - public func endpointUri(path: String) -> String { return "\(endpoint)/\(path)" } diff --git a/Sources/PowerSync/Protocol/db/CrudBatch.swift b/Sources/PowerSync/Protocol/db/CrudBatch.swift index ff03e01..b1c30dc 100644 --- a/Sources/PowerSync/Protocol/db/CrudBatch.swift +++ b/Sources/PowerSync/Protocol/db/CrudBatch.swift @@ -42,6 +42,6 @@ internal func completeCrudItems(_ db: any PowerSyncDatabaseProtocol, _ lastItemI return } } - try tx.execute(sql: "UPDATE ps_buckets SET target_op = 9223372036854775807 WHERE name = '$local'", parameters: nil) + try tx.execute(sql: "UPDATE ps_buckets SET target_op = ? WHERE name = '$local'", parameters: [KotlinPowerSyncDatabaseImpl.maxOpId]) } } diff --git a/Sources/PowerSync/Protocol/sync/BucketPriority.swift b/Sources/PowerSync/Protocol/sync/BucketPriority.swift index 6b0f677..65627b1 100644 --- a/Sources/PowerSync/Protocol/sync/BucketPriority.swift +++ b/Sources/PowerSync/Protocol/sync/BucketPriority.swift @@ -1,7 +1,7 @@ import Foundation /// Represents the priority of a bucket, used for sorting and managing operations based on priority levels. -public struct BucketPriority: Comparable, Sendable { +public struct BucketPriority: Comparable, Sendable, Decodable { /// The priority code associated with the bucket. Higher values indicate lower priority. public let priorityCode: Int32 @@ -13,6 +13,12 @@ public struct BucketPriority: Comparable, Sendable { self.priorityCode = priorityCode } + public init(from decoder: any Decoder) throws { + let container = try decoder.singleValueContainer() + let code = try container.decode(Int32.self) + self.init(code) + } + /// Compares two `BucketPriority` instances to determine their order. /// - Parameters: /// - lhs: The left-hand side `BucketPriority` instance. diff --git a/Sources/PowerSync/Protocol/sync/PriorityStatusEntry.swift b/Sources/PowerSync/Protocol/sync/PriorityStatusEntry.swift index 10e4586..6a0814a 100644 --- a/Sources/PowerSync/Protocol/sync/PriorityStatusEntry.swift +++ b/Sources/PowerSync/Protocol/sync/PriorityStatusEntry.swift @@ -1,7 +1,7 @@ import Foundation /// Represents the status of a bucket priority, including synchronization details. -public struct PriorityStatusEntry: Sendable { +public struct PriorityStatusEntry: Sendable, Decodable { /// The priority of the bucket. public let priority: BucketPriority @@ -12,4 +12,23 @@ public struct PriorityStatusEntry: Sendable { /// Indicates whether the bucket has been successfully synchronized. /// - Note: This value is optional and may be `nil` if the synchronization status is unknown. public let hasSynced: Bool? + + enum CodingKeys: String, CodingKey { + case priority + case lastSyncedAt = "last_synced_at" + case hasSynced = "has_synced" + } + + init(priority: BucketPriority, lastSyncedAt: Date?, hasSynced: Bool?) { + self.priority = priority + self.lastSyncedAt = lastSyncedAt + self.hasSynced = hasSynced + } + + public init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.priority = try container.decode(BucketPriority.self, forKey: .priority) + self.lastSyncedAt = try container.decodeIfPresent(Int64.self, forKey: .lastSyncedAt).map { Date(timeIntervalSince1970: Double($0)) } + self.hasSynced = try container.decodeIfPresent(Bool.self, forKey: .hasSynced) + } } diff --git a/Sources/PowerSync/Protocol/sync/SyncStatusData.swift b/Sources/PowerSync/Protocol/sync/SyncStatusData.swift index 4badd51..f0d9b85 100644 --- a/Sources/PowerSync/Protocol/sync/SyncStatusData.swift +++ b/Sources/PowerSync/Protocol/sync/SyncStatusData.swift @@ -66,9 +66,24 @@ public protocol SyncStatus: SyncStatusData, Sendable { } /// Current information about a ``SyncStreamSubscription``. -public struct SyncStreamStatus { +public struct SyncStreamStatus: Sendable { /// If the sync status is currently downloading, information about download progress related to this stream. public let progress: ProgressWithOperations? /// The ``SyncSubscriptionDescription`` providing information about the subscription. public let subscription: SyncSubscriptionDescription + + enum CodingKeys: CodingKey { case progress } + + init(subscription: SyncSubscriptionDescription, progress: ProgressWithOperations? = nil) { + self.subscription = subscription + self.progress = progress + } + + init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.progress = try container.decode(ProgressCounters.self, forKey: .progress) + + // Parse from same decoder (it's [flatten]ed in Rust) + self.subscription = try SyncSubscriptionDescription(from: decoder) + } } diff --git a/Sources/PowerSync/Protocol/sync/SyncStream.swift b/Sources/PowerSync/Protocol/sync/SyncStream.swift index 34294de..8de074e 100644 --- a/Sources/PowerSync/Protocol/sync/SyncStream.swift +++ b/Sources/PowerSync/Protocol/sync/SyncStream.swift @@ -81,4 +81,25 @@ public struct SyncSubscriptionDescription: SyncStreamDescription { return self.lastSyncedAt != nil } } + + private enum CodingKeys: String, CodingKey { + case name + case parameters + case active + case isDefault = "is_default" + case hasExplicitSubscription = "has_explicit_subscription" + case expiresAt = "expires_at" + case lastSyncedAt = "last_synced_at" + } + + internal init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.name = try container.decode(String.self, forKey: .name) + self.parameters = try container.decodeIfPresent(JsonParam.self, forKey: .parameters) + self.active = try container.decode(Bool.self, forKey: .active) + self.isDefault = try container.decode(Bool.self, forKey: .isDefault) + self.hasExplicitSubscription = try container.decode(Bool.self, forKey: .hasExplicitSubscription) + self.expiresAt = try container.decodeIfPresent(Int64.self, forKey: .expiresAt).map { t in TimeInterval(t) } + self.lastSyncedAt = try container.decodeIfPresent(Int64.self, forKey: .lastSyncedAt).map { t in TimeInterval(t) } + } } diff --git a/Sources/PowerSync/Utils/BroadcastStream.swift b/Sources/PowerSync/Utils/BroadcastStream.swift new file mode 100644 index 0000000..cbc4690 --- /dev/null +++ b/Sources/PowerSync/Utils/BroadcastStream.swift @@ -0,0 +1,44 @@ +import Synchronization + +final class BroadcastStream: Sendable { + private let listeners: Mutex>> = Mutex([]) + + private func register(continuation: AsyncStream.Continuation) { + let listener = BroadcastStreamListener(continuation: continuation) + let _ = listeners.withLock { $0.insert(listener) } + + continuation.onTermination = { @Sendable [weak self] _ in + let _ = self?.listeners.withLock { + $0.remove(listener) + } + } + } + + func dispatch(event: T) { + let listeners = self.listeners.withLock { Array($0) } + for listener in listeners { + listener.continuation.yield(event) + } + } + + func subscribe(bufferingPolicy: AsyncStream.Continuation.BufferingPolicy = .unbounded) -> AsyncStream { + return AsyncStream(bufferingPolicy: bufferingPolicy) { continuation in + self.register(continuation: continuation) + } + } +} + +final private class BroadcastStreamListener: Sendable, Hashable { + let continuation: AsyncStream.Continuation + init(continuation: AsyncStream.Continuation) { + self.continuation = continuation + } + + func hash(into hasher: inout Hasher) { + hasher.combine(ObjectIdentifier(self)) + } + + static func == (lhs: BroadcastStreamListener, rhs: BroadcastStreamListener) -> Bool { + lhs === rhs + } +} diff --git a/Sources/SyncPlayground/main.swift b/Sources/SyncPlayground/main.swift new file mode 100644 index 0000000..b7ef35b --- /dev/null +++ b/Sources/SyncPlayground/main.swift @@ -0,0 +1,35 @@ +import Foundation +import PowerSync + +func start() async throws { + let schema = Schema(tables: []) + let db = PowerSyncDatabase(schema: schema) + + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + print("Is connected!") + + try await db.waitForFirstSync() +} + +final class TestConnector: PowerSyncBackendConnectorProtocol { + func fetchCredentials() async throws -> PowerSync.PowerSyncCredentials? { + let url = URL(string: "http://localhost:6060/api/auth/token")! + let (data, _) = try await URLSession.shared.data(from: url) + + struct Response: Decodable { + let token: String + } + + let response = try JSONDecoder().decode(Response.self, from: data) + return PowerSyncCredentials( + endpoint: "http://localhost:8080", + token: response.token + ) + } + + func uploadData(database: any PowerSync.PowerSyncDatabaseProtocol) async throws { + throw PowerSyncError.operationFailed(message: "todo: uploadData") + } +} + +let _ = try await start() From 7b7e5fc19902216fcace0cef25c13d1cee7bb20e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 16 Apr 2026 18:03:26 +0200 Subject: [PATCH 06/40] Drive sync process from Swift --- .../sync/CachingCredentialsConnector.swift | 4 +- .../sync/PowerSyncControlArguments.swift | 4 + .../sync/StreamingSyncClient.swift | 78 ++-- .../Implementation/sync/SyncCoordinator.swift | 14 +- .../Kotlin/KotlinPowerSyncDatabaseImpl.swift | 31 +- .../Protocol/sync/BucketPriority.swift | 7 +- Tests/PowerSyncTests/SyncTests.swift | 355 ++++++++++++++++++ .../test-utils/MockHttpClient.swift | 37 ++ .../test-utils/SyncProtocol.swift | 140 +++++++ 9 files changed, 626 insertions(+), 44 deletions(-) create mode 100644 Tests/PowerSyncTests/SyncTests.swift create mode 100644 Tests/PowerSyncTests/test-utils/MockHttpClient.swift create mode 100644 Tests/PowerSyncTests/test-utils/SyncProtocol.swift diff --git a/Sources/PowerSync/Implementation/sync/CachingCredentialsConnector.swift b/Sources/PowerSync/Implementation/sync/CachingCredentialsConnector.swift index 3de8dde..110418a 100644 --- a/Sources/PowerSync/Implementation/sync/CachingCredentialsConnector.swift +++ b/Sources/PowerSync/Implementation/sync/CachingCredentialsConnector.swift @@ -6,8 +6,8 @@ actor CachingCredentialsConnector { self.inner = inner } - func fetchCredentials() async throws -> PowerSyncCredentials? { - if let credentials = self.cachedCredentials { + func fetchCredentials(allowCached: Bool = true) async throws -> PowerSyncCredentials? { + if let credentials = self.cachedCredentials, allowCached { return credentials } diff --git a/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift b/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift index 79d7d90..97e25f9 100644 --- a/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift +++ b/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift @@ -5,6 +5,7 @@ enum PowerSyncControlArguments { case textLine(line: String) case binaryLine(line: ContiguousArray) case completedUpload + case didRefreshToken case connectionEstablished case responseStreamEnd case updateSubscriptions(streams: [StreamKey]) @@ -29,6 +30,9 @@ enum PowerSyncControlArguments { case .completedUpload: op = "completed_upload" param = nil + case .didRefreshToken: + op = "refreshed_token" + param = nil case .connectionEstablished: op = "connection" param = "established" diff --git a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift index 00dc489..3b5ca9b 100644 --- a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift +++ b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift @@ -7,20 +7,19 @@ fileprivate let tag = "StreamingSyncClient" final class StreamingSyncClient: Sendable { let db: KotlinPowerSyncDatabaseImpl let options: ConnectOptions - let events = BroadcastStream() let connector: CachingCredentialsConnector let httpClient: any HttpClient init( db: KotlinPowerSyncDatabaseImpl, connector: PowerSyncBackendConnectorProtocol, - httpClient: any HttpClient = PlatformHttpClient.shared, - options: ConnectOptions = ConnectOptions() + httpClient: any HttpClient, + options: ConnectOptions, ) { self.db = db self.connector = CachingCredentialsConnector(inner: connector) self.httpClient = httpClient - self.options = ConnectOptions() + self.options = options } /// Starts a task driving uploads and downloads by repeatedly connecting to the PowerSync service, @@ -31,9 +30,9 @@ final class StreamingSyncClient: Sendable { func run() -> Task { Task(name: "StreamingSyncClient.run") { let signals = SyncSignals() - async let download = try downloadLoop(signals: signals) - - try await download + async let download = downloadLoop(signals: signals) + + let _ = try await download } } @@ -48,7 +47,7 @@ final class StreamingSyncClient: Sendable { .map { _ in () } let allTriggers = AsyncAlgorithms.merge(watch, signals.signalCrudUpload.subscribe()) - for try await item in allTriggers { + for try await _ in allTriggers { try await uploadAllCrud() } } @@ -134,7 +133,7 @@ The next upload iteration will be delayed. private func getWriteCheckpoint() async throws -> String { let clientId = try await db.get("SELECT powersync_client_id()") { try $0.getString(index: 0) } - var (_, request) = try await authenticatedRequest { endpoint in + let (_, request) = try await authenticatedRequest { endpoint in endpoint .appending(path: "write-checkpoint2.json") .appending(queryItems: [.init(name: "client_id", value: clientId)]) @@ -148,14 +147,6 @@ The next upload iteration will be delayed. throw PowerSyncError.operationFailed(message: "Error getting write checkpoint: \(response.statusCode)") } - struct WriteCheckpointData: Decodable { - let write_checkpoint: String - } - - struct WriteCheckpointResponse: Decodable { - let data: WriteCheckpointData - } - let body = try StreamingSyncClient.jsonDecoder.decode(WriteCheckpointResponse.self, from: data) return body.data.write_checkpoint } @@ -165,7 +156,11 @@ The next upload iteration will be delayed. while (!Task.isCancelled) { do { - result = try await ActiveSyncIteration(syncClient: self, signals: signals).run() + // This async let ensures each iteration is a task scoped to this block. This allows us to spawn + // aditional tasks in run() that would get cancelled when the main iteration is complete. + async let iteration = ActiveSyncIteration(syncClient: self, signals: signals).run() + + result = try await iteration } catch { result = SyncIterationResult() @@ -226,21 +221,20 @@ The next upload iteration will be delayed. private struct ActiveSyncIteration: Sendable { private let syncClient: StreamingSyncClient - private let localEvents: AsyncStream + private let localEvents = BroadcastStream() private let signals: SyncSignals init(syncClient: StreamingSyncClient, signals: SyncSignals) { self.syncClient = syncClient - self.localEvents = syncClient.events.subscribe() self.signals = signals } func run() async throws -> SyncIterationResult { let initialInstructions = try await powersyncControl(.start(StartSyncIteration( - parameters: [:], + parameters: syncClient.options.params, includeDefaults: true, activeStreams: [], - appMetadata: [:] + appMetadata: syncClient.options.appMetadata, ))) var controlArgs: (any AsyncSequence)? @@ -248,7 +242,7 @@ private struct ActiveSyncIteration: Sendable { for instruction in initialInstructions { if case .establishSyncStream(request: let request) = instruction { let serviceEvents = try await syncClient.fetchSyncLines(request: request) - controlArgs = AsyncAlgorithms.merge(serviceEvents, localEvents) + controlArgs = AsyncAlgorithms.merge(serviceEvents, localEvents.subscribe()) } else { try await self.execute(instr: instruction) } @@ -281,7 +275,20 @@ private struct ActiveSyncIteration: Sendable { } } - return SyncIterationResult() + // We use an immediately-awaited Task.detached here because running the stop command shouldn't + // get aborted. + return try await Task.detached { + let control = try await powersyncControl(.stop) + for instr in control { + if case let .closeSyncStream(hideDisconnect) = instr { + return SyncIterationResult(hideDisconnect: hideDisconnect) + } + + try await execute(instr: instr) + } + + return SyncIterationResult() + }.value } private func powersyncControl(_ args: PowerSyncControlArguments) async throws -> [Instruction] { @@ -314,9 +321,13 @@ private struct ActiveSyncIteration: Sendable { if didExpire { await syncClient.invalidateCredentials() } else { - signals.triggerAsyncFetchCredentials() + Task { + let _ = try await syncClient.connector.fetchCredentials(allowCached: false) + syncClient.db.logger.debug("Stopping because new credentials are available", tag: tag) + // Token has been refreshed, start another iteration + localEvents.dispatch(event: .didRefreshToken) + } } - fatalError("todo: fetchCredentials") case .flushFileSystem: // Noop on native platforms. break; @@ -375,15 +386,18 @@ private struct SyncIterationResult { } } -struct SyncSignals { +private struct SyncSignals { let signalCrudUpload = BroadcastStream() - let prefetchCredentials = BroadcastStream() func triggerAsyncCrudUpload() { self.signalCrudUpload.dispatch(event: ()) } - - func triggerAsyncFetchCredentials() { - self.prefetchCredentials.dispatch(event: ()) - } +} + +struct WriteCheckpointData: Codable { + let write_checkpoint: String +} + +struct WriteCheckpointResponse: Codable { + let data: WriteCheckpointData } diff --git a/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift b/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift index c892f65..a2ebf21 100644 --- a/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift +++ b/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift @@ -1,12 +1,13 @@ +/// Manages a connection task for a PowerSync database. actor SyncCoordinator { private var activeSync: Task? - func connect(db: KotlinPowerSyncDatabaseImpl, connector: PowerSyncBackendConnectorProtocol, options: ConnectOptions) async { + func connect(db: KotlinPowerSyncDatabaseImpl, connector: PowerSyncBackendConnectorProtocol, options: ConnectOptions, client: HttpClient) async { if let task = activeSync { await self.finishSyncTask(task: task) } - let sync = StreamingSyncClient(db: db, connector: connector) + let sync = StreamingSyncClient(db: db, connector: connector, httpClient: client, options: options) activeSync = sync.run() } @@ -18,6 +19,15 @@ actor SyncCoordinator { await self.finishSyncTask(task: task) } + /// Executes an inner function, but only if no connection is active or scheduled. + func guardNotConnected(inner: () async throws -> T, ifConnected: () throws -> Never) async rethrows -> T { + if activeSync == nil { + return try await inner(); + } else { + try ifConnected() + } + } + private func finishSyncTask(task: Task) async { self.activeSync = nil task.cancel() diff --git a/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift b/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift index 9ff03d2..4b9ef6a 100644 --- a/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift +++ b/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift @@ -12,11 +12,13 @@ final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, private let syncCoordinator = SyncCoordinator() internal let syncStatus = SwiftSyncStatus() private let dbFilename: String + private let httpClient: HttpClient init( kotlinDatabase: PowerSyncKotlin.PowerSyncDatabase, dbFilename: String, - logger: DatabaseLogger + logger: DatabaseLogger, + httpClient: HttpClient ) { self.logger = logger self.kotlinDatabase = kotlinDatabase @@ -24,6 +26,7 @@ final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, /// The kotlin PowerSyncDatabase.identifier currently prepends `null` to the dbFilename (for the directory). /// FIXME. Update this once we support database directory configuration. self.dbFilename = dbFilename + self.httpClient = httpClient } var currentStatus: any SyncStatus { @@ -35,8 +38,13 @@ final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, } func updateSchema(schema: any SchemaProtocol) async throws { - try await kotlinDatabase.updateSchema( - schema: KotlinAdapter.Schema.toKotlin(schema) + try await syncCoordinator.guardNotConnected( + inner: { + try await kotlinDatabase.updateSchema( + schema: KotlinAdapter.Schema.toKotlin(schema) + ) + }, + ifConnected: { throw PowerSyncError.operationFailed(message: "Cannot update schema while connected") } ) } @@ -54,7 +62,12 @@ final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, connector: PowerSyncBackendConnectorProtocol, options: ConnectOptions? ) async { - await syncCoordinator.connect(db: self, connector: connector, options: options ?? ConnectOptions()) + await syncCoordinator.connect( + db: self, + connector: connector, + options: options ?? ConnectOptions(), + client: httpClient + ) } func getCrudBatch(limit: Int32 = 100) async throws -> CrudBatch? { @@ -328,6 +341,7 @@ final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, } func close() async throws { + await disconnect() try await kotlinDatabase.close() } @@ -427,7 +441,8 @@ func openKotlinDBDefault( schema: Schema, dbFilename: String, logger: DatabaseLogger, - initialStatements: [String] = [] + initialStatements: [String] = [], + httpClient: HttpClient = PlatformHttpClient.shared ) -> PowerSyncDatabaseProtocol { let rc = sqlite3_initialize() if rc != 0 { @@ -453,7 +468,8 @@ func openKotlinDBDefault( return KotlinPowerSyncDatabaseImpl( kotlinDatabase: kotlinDatabase, dbFilename: dbFilename, - logger: logger + logger: logger, + httpClient: httpClient ) } @@ -471,7 +487,8 @@ func openKotlinDBWithPool( logger: logger.kLogger ), dbFilename: identifier, - logger: logger + logger: logger, + httpClient: PlatformHttpClient.shared ) } diff --git a/Sources/PowerSync/Protocol/sync/BucketPriority.swift b/Sources/PowerSync/Protocol/sync/BucketPriority.swift index 65627b1..e0140a9 100644 --- a/Sources/PowerSync/Protocol/sync/BucketPriority.swift +++ b/Sources/PowerSync/Protocol/sync/BucketPriority.swift @@ -1,7 +1,7 @@ import Foundation /// Represents the priority of a bucket, used for sorting and managing operations based on priority levels. -public struct BucketPriority: Comparable, Sendable, Decodable { +public struct BucketPriority: Comparable, Sendable, Codable { /// The priority code associated with the bucket. Higher values indicate lower priority. public let priorityCode: Int32 @@ -19,6 +19,11 @@ public struct BucketPriority: Comparable, Sendable, Decodable { self.init(code) } + public func encode(to encoder: any Encoder) throws { + var container = encoder.singleValueContainer() + try container.encode(priorityCode) + } + /// Compares two `BucketPriority` instances to determine their order. /// - Parameters: /// - lhs: The left-hand side `BucketPriority` instance. diff --git a/Tests/PowerSyncTests/SyncTests.swift b/Tests/PowerSyncTests/SyncTests.swift new file mode 100644 index 0000000..12654a3 --- /dev/null +++ b/Tests/PowerSyncTests/SyncTests.swift @@ -0,0 +1,355 @@ +import AsyncAlgorithms +@testable import PowerSync +import Synchronization +import Testing + +@Suite() +class InMemorySyncIntegrationTests { + @Test func useParameters() async throws { + let didConnect = Signal() + let db = openDatabase(MockHttpClient { request in + try #require(request["parameters"] == .object(["foo": .string("bar")])) + await didConnect.complete() + return AsyncThrowingChannel() + }) + + try await db.connect(connector: TestConnector(), options: ConnectOptions( + params: ["foo": .string("bar")] + )) + await didConnect.await() + try await db.disconnect() + } + + @Test func useAppMetadata() async throws { + let didConnect = Signal() + let db = openDatabase(MockHttpClient { request in + try #require(request["app_metadata"] == .object(["app_version": .string("1.0.0")])) + await didConnect.complete() + return AsyncThrowingChannel() + }) + + try await db.connect(connector: TestConnector(), options: ConnectOptions( + appMetadata: ["app_version": "1.0.0"] + )) + await didConnect.await() + try await db.disconnect() + } + + @Test func cannotUpdateSchemaWhileConnected() async throws { + let db = openDatabase(MockHttpClient { request in AsyncThrowingChannel() }) + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + + await #expect(throws: PowerSyncError.self) { + try await db.updateSchema(schema: Schema()) + } + + try await db.close() + } + + @Test func partialSync() async throws { + let channel = AsyncThrowingChannel() + let checksums = Array((0...3).map { prio in BucketChecksum(bucket: "bucket\(prio)", priority: .init(prio), checksum: 10 + prio) }) + var operationId = 1 + + func pushData(priority: Int32) async throws { + let id = operationId + operationId += 1 + + try await channel.pushLine(.syncDataBucket(SyncDataBucket(bucket: "bucket\(priority)", data: [ + OplogEntry( + checksum: priority + 10, + op_id: String(id), + object_id: String(id), + object_type: "users", + op: .put, + data: String(data: StreamingSyncClient.jsonEncoder.encode([ + "name": "user \(priority)" + ]), encoding: .utf8)! + ) + ]))) + } + + let db = openDatabase(MockHttpClient { request in channel }) + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + await waitForStatus(db.currentStatus) { $0.connected } + + try await expectUserCount(db, 0) + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "4", buckets: checksums))) + // Emit a partial sync complete for each priority but the last + for priorityNo in Int32(0)..<3 { + try await pushData(priority: priorityNo) + let priority = BucketPriority(priorityNo) + try await channel.pushLine(.checkpointPartiallyComplete(lastOpId: String(operationId), priority: priority)) + + await waitForStatus(db.currentStatus) { $0.statusForPriority(priority).hasSynced == true } + try await expectUserCount(db, priorityNo + 1) + } + + // Then complete the sync + try await pushData(priority: 3) + try await channel.pushLine(.checkpointComplete(lastOpId: String(operationId))) + try await db.waitForFirstSync() + try await expectUserCount(db, 4) + + try await db.disconnect() + } + + @Test func setsDownloadingState() async throws { + let channel = AsyncThrowingChannel() + let db = openDatabase(MockHttpClient { request in channel }) + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + await waitForStatus(db.currentStatus) { $0.connected } + + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "1", buckets: [.init(bucket: "bkt", checksum: 0)]))) + await waitForStatus(db.currentStatus) { $0.downloading } + + try await channel.pushLine(.checkpointComplete(lastOpId: "1")) + await waitForStatus(db.currentStatus) { !$0.downloading } + try await db.disconnect() + } + + @Test func setsConnectingState() async throws { + let didSeeConnecting = Signal() + + let db = openDatabase(MockHttpClient { request in + await didSeeConnecting.await() + return AsyncThrowingChannel() + }) + + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + await waitForStatus(db.currentStatus) { $0.connecting } + await didSeeConnecting.complete() + await waitForStatus(db.currentStatus) { $0.connected } + } + + @Test func reconnectsAfterDisconnecting() async throws { + let db = openDatabase(MockHttpClient { request in AsyncThrowingChannel() }) + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + await waitForStatus(db.currentStatus) { $0.connected } + + try await db.disconnect() + await waitForStatus(db.currentStatus) { !$0.connected && !$0.connecting } + + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + await waitForStatus(db.currentStatus) { $0.connected } + } + + @Test func reconnects() async throws { + let db = openDatabase(MockHttpClient { request in AsyncThrowingChannel() }) + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + await waitForStatus(db.currentStatus) { $0.connected } + + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + await waitForStatus(db.currentStatus) { !$0.connected } + await waitForStatus(db.currentStatus) { $0.connected } + } + + // TODO: "handles checkpoints during uploads" test + + // TODO: "handles write made while offline" test + + @Test func tokenExpired() async throws { + final class BackendConnector: PowerSyncBackendConnectorProtocol { + let fetchCredentialsCalls = Atomic(0) + + func fetchCredentials() async throws -> PowerSyncCredentials? { + fetchCredentialsCalls.add(1, ordering: .sequentiallyConsistent) + return testCredentials + } + + func uploadData(database: any PowerSyncDatabaseProtocol) async throws {} + } + + let channel = AsyncThrowingChannel() + let db = openDatabase(MockHttpClient { request in channel }) + let connector = BackendConnector() + try await db.connect(connector: connector, options: ConnectOptions(retryDelay: 0)) + + try await channel.pushLine(.keepAlive(tokenExpiresIn: 4000)) + await waitForStatus(db.currentStatus) { $0.connected } + try #require(connector.fetchCredentialsCalls.load(ordering: .sequentiallyConsistent) == 1) + + // Should invalidate credentials when token expires + try await channel.pushLine(.keepAlive(tokenExpiresIn: 0)) + await waitForStatus(db.currentStatus) { !$0.connected } + await waitForStatus(db.currentStatus) { $0.connected } + try #require(connector.fetchCredentialsCalls.load(ordering: .sequentiallyConsistent) == 2) + } + + @Test func tokenThrows() async throws { + actor BackendConnector: PowerSyncBackendConnectorProtocol { + var isFirstFetchCall = true + + func fetchCredentials() async throws -> PowerSyncCredentials? { + if isFirstFetchCall { + isFirstFetchCall = false + throw PowerSyncError.operationFailed(message: "error in connector") + } + return testCredentials + } + + func uploadData(database: any PowerSyncDatabaseProtocol) async throws {} + } + + let channel = AsyncThrowingChannel() + let db = openDatabase(MockHttpClient { request in channel }) + try await db.connect(connector: BackendConnector(), options: ConnectOptions(retryDelay: 0.2)) + await waitForStatus(db.currentStatus) { !$0.connected && $0.downloadError != nil } + + // Should retry, and the second fetchCredentials call will work + await waitForStatus(db.currentStatus) { $0.connected } + } + + @Test func tokenPrefetch() async throws { + actor BackendConnector: PowerSyncBackendConnectorProtocol { + let prefetchCalled = Signal() + let completePrefetch = Signal() + var fetchCredentialsCount = 0 + + func fetchCredentials() async throws -> PowerSyncCredentials? { + fetchCredentialsCount += 1 + if fetchCredentialsCount == 2 { + await prefetchCalled.complete() + await completePrefetch.await() + } + return testCredentials + } + + func uploadData(database: any PowerSyncDatabaseProtocol) async throws {} + } + + let connector = BackendConnector() + let channel = AsyncThrowingChannel() + let db = openDatabase(MockHttpClient { request in channel }) + try await db.connect(connector: connector, options: ConnectOptions()) + + try await channel.pushLine(.keepAlive(tokenExpiresIn: 4000)) + await waitForStatus(db.currentStatus) { $0.connected } + try #require(await connector.fetchCredentialsCount == 1) + + try await channel.pushLine(.keepAlive(tokenExpiresIn: 10)) + await connector.prefetchCalled.await() + // Should still be connected before prefetch completes + try #require(db.currentStatus.connected == true) + + // After the prefetch completes, we should reconnect. + await connector.completePrefetch.complete() + await waitForStatus(db.currentStatus) { !$0.connected } + await waitForStatus(db.currentStatus) { $0.connected } + try #require(await connector.fetchCredentialsCount == 2) + } + + // TODO: Raw table tests (requires serializable schema) + + @Test(.disabled("Blocked on https://github.com/powersync-ja/powersync-sqlite-core/pull/174")) func endsIterationOnHttpClose() async throws { + let channel = AsyncThrowingChannel() + let db = openDatabase(MockHttpClient { request in channel }) + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + await waitForStatus(db.currentStatus) { $0.connected } + + channel.finish() + await waitForStatus(db.currentStatus) { !$0.connected } + } + + @Test func syncProgress() async throws { + let channel = AsyncThrowingChannel() + let db = openDatabase(MockHttpClient { request in channel }) + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + await waitForStatus(db.currentStatus) { $0.connected } + var status = db.currentStatus.asFlow().makeAsyncIterator() + + // Send checkpoint with 10 ops, progress should be 0/10 + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "10", buckets: [BucketChecksum(bucket: "a", checksum: 0, count: 10)]))) + try (try #require(await status.next())).expectProgress(total: (0, 10)) + + try await channel.pushLine(.syncDataBucket(SyncDataBucket(bucket: "a", data: (0..<10).map { i in + .init(checksum: 0, op_id: String(i+1), object_id: String(i), object_type: "a", op: .put, data: "{}") + }))) + try (try #require(await status.next())).expectProgress(total: (10, 10)) + + // Emit new data, progress should be 0/2 instead of 2/2 + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "12", buckets: [ + BucketChecksum(bucket: "a", checksum: 0, count: 12), + ]))) + try (try #require(await status.next())).expectProgress(total: (10, 12)) + + try await channel.pushLine(.syncDataBucket(SyncDataBucket(bucket: "a", data: (10..<12).map { i in + .init(checksum: 0, op_id: String(i+1), object_id: String(i), object_type: "a", op: .put, data: "{}") + }))) + try (try #require(await status.next())).expectProgress(total: (12, 12)) + } +} + +private func openDatabase(_ client: MockHttpClient) -> PowerSyncDatabaseProtocol { + let schema = Schema(tables: [ + Table( + name: "users", + columns: [ + .text("name"), + ] + ), + ]) + + return openKotlinDBDefault( + schema: schema, + dbFilename: ":memory:", + logger: DatabaseLogger(DefaultLogger()), + httpClient: client + ) +} + +let testCredentials = PowerSyncCredentials( + endpoint: "https://powersynctest.example.org", + token: "test-token" +) + +private final class TestConnector: PowerSyncBackendConnectorProtocol { + func fetchCredentials() async throws -> PowerSyncCredentials? { + return testCredentials + } + + func uploadData(database _: any PowerSync.PowerSyncDatabaseProtocol) async throws {} +} + +private final class Signal: Sendable { + let completer = AsyncChannel() + + func complete() async { + await completer.send(()) + } + + func await() async { + await completer.first { true } + } +} + +private final class Box: Sendable { + let inner: T + + init(inner: consuming T) { + self.inner = inner + } +} + +func expectUserCount(_ db: PowerSyncDatabaseProtocol, _ amount: Int32) async throws { + let users = try await db.getAll("SELECT name FROM users") { $0.getStringOptional(index: 0) } + try #require(users.count == amount) +} + +func waitForStatus(_ status: SyncStatus, predicate: (borrowing SyncStatusData) -> Bool) async { + if predicate(status) { + return + } + + let _ = await status.asFlow().first(where: predicate) +} + +private extension SyncStatusData { + func expectProgress(total: (Int32, Int32)) throws { + let progress = try #require(self.downloadProgress) + try #require(self.downloading) + + try #require(progress.downloadedOperations == total.0) + try #require(progress.totalOperations == total.1) + } +} diff --git a/Tests/PowerSyncTests/test-utils/MockHttpClient.swift b/Tests/PowerSyncTests/test-utils/MockHttpClient.swift new file mode 100644 index 0000000..d0663fc --- /dev/null +++ b/Tests/PowerSyncTests/test-utils/MockHttpClient.swift @@ -0,0 +1,37 @@ +import AsyncAlgorithms +import Foundation +@testable import PowerSync +import Testing +import Synchronization + +final class MockHttpClient: HttpClient { + let writeCheckpoint: Atomic = Atomic(1000) + let handleSyncLines: @Sendable (_ body: JsonParam) async throws -> AsyncThrowingChannel + + init(handleSyncLines: @Sendable @escaping (_ body: JsonParam) async throws -> AsyncThrowingChannel) { + self.handleSyncLines = handleSyncLines + } + + func receiveSyncLines(request: URLRequest) async throws -> (HTTPURLResponse, any Sendable & AsyncSequence) { + try #require(request.url?.path() == "/sync/stream") + + let body = try StreamingSyncClient.jsonDecoder.decode(JsonParam.self, from: try #require(request.httpBody)) + let channel = try await handleSyncLines(body) + let response = HTTPURLResponse(url: request.url!, mimeType: "application/x-ndjson", expectedContentLength: 0, textEncodingName: "utf-8") + + return (response, channel) + } + + func readFully(request: URLRequest) async throws -> (HTTPURLResponse, Data) { + // The sync client only uses this method to get /write-checkpoint2.json. + try #require(request.url?.path() == "/write-checkpoint2.json") + + let checkpoint = writeCheckpoint.load(ordering: .sequentiallyConsistent) + let body = WriteCheckpointResponse(data: WriteCheckpointData(write_checkpoint: String(checkpoint))) + + let data = try StreamingSyncClient.jsonEncoder.encode(body) + let response = HTTPURLResponse(url: request.url!, mimeType: "application/json", expectedContentLength: data.count, textEncodingName: "utf-8") + + return (response, data) + } +} diff --git a/Tests/PowerSyncTests/test-utils/SyncProtocol.swift b/Tests/PowerSyncTests/test-utils/SyncProtocol.swift new file mode 100644 index 0000000..f99b1e2 --- /dev/null +++ b/Tests/PowerSyncTests/test-utils/SyncProtocol.swift @@ -0,0 +1,140 @@ +// Helpers to encode sync lines sent by the PowerSync service, used to test sync with a mocked HTTP client. +import AsyncAlgorithms +@testable import PowerSync + +enum SyncLine: Encodable { + case fullCheckpoint(Checkpoint) + case checkpointComplete(lastOpId: String) + case checkpointPartiallyComplete(lastOpId: String, priority: BucketPriority) + case syncDataBucket(SyncDataBucket) + case keepAlive(tokenExpiresIn: Int) + + enum CodingKeys: String, CodingKey { + case fullCheckpoint = "checkpoint" + case checkpointComplete = "checkpoint_complete" + case checkpointPartiallyComplete = "partial_checkpoint_complete" + case syncDataBucket = "data" + case keepAlive = "token_expires_in" + } + + enum CheckpointCompleteCodingKeys: String, CodingKey { + case lastOpId = "last_op_id" + } + + enum CheckpointPartiallyCompleteCodingKeys: String, CodingKey { + case lastOpId = "last_op_id" + case priority + } + + func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .fullCheckpoint(let checkpoint): + var nestedContainer = container.nestedContainer(keyedBy: Checkpoint.CodingKeys.self, forKey: .fullCheckpoint) + try checkpoint.encodeToContainer(&nestedContainer) + case .checkpointComplete(let lastOpId): + var nestedContainer = container.nestedContainer(keyedBy: SyncLine.CheckpointCompleteCodingKeys.self, forKey: .checkpointComplete) + try nestedContainer.encode(lastOpId, forKey: SyncLine.CheckpointCompleteCodingKeys.lastOpId) + case .checkpointPartiallyComplete(let lastOpId, let priority): + var nestedContainer = container.nestedContainer(keyedBy: SyncLine.CheckpointPartiallyCompleteCodingKeys.self, forKey: .checkpointPartiallyComplete) + try nestedContainer.encode(lastOpId, forKey: SyncLine.CheckpointPartiallyCompleteCodingKeys.lastOpId) + try nestedContainer.encode(priority, forKey: SyncLine.CheckpointPartiallyCompleteCodingKeys.priority) + case .syncDataBucket(let bucket): + var nestedContainer = container.nestedContainer(keyedBy: SyncDataBucket.CodingKeys.self, forKey: .syncDataBucket) + try bucket.encodeToContainer(&nestedContainer) + case .keepAlive(let tokenExpiresIn): + try container.encode(tokenExpiresIn, forKey: .keepAlive) + } + } +} + +struct SyncDataBucket { + var bucket: String + var data: [OplogEntry] + var hasMore: Bool = false + var after: String? = nil + var nextAfter: String? = nil + + enum CodingKeys: String, CodingKey { + case bucket + case data + case hasMore = "has_more" + case after + case nextAfter = "next_after" + } + + func encodeToContainer(_ container: inout KeyedEncodingContainer) throws { + try container.encode(self.bucket, forKey: .bucket) + try container.encode(self.data, forKey: .data) + try container.encode(self.hasMore, forKey: .hasMore) + try container.encode(self.after, forKey: .after) + try container.encode(self.nextAfter, forKey: .nextAfter) + } +} + +struct Checkpoint { + var last_op_id: String + var buckets: [BucketChecksum] + var writeCheckpoint: String? = nil + + enum CodingKeys: String, CodingKey { + case last_op_id + case buckets + case writeCheckpoint = "write_checkpoint" + } + + func encodeToContainer(_ container: inout KeyedEncodingContainer) throws { + try container.encode(self.last_op_id, forKey: .last_op_id) + try container.encode(self.buckets, forKey: .buckets) + try container.encode(self.writeCheckpoint, forKey: .writeCheckpoint) + } +} + +struct OplogEntry: Encodable { + var checksum: Int32 + var op_id: String + var object_id: String + var object_type: String + var op: OpType? = nil + var subkey: String? = nil + var data: String? = nil +} + +enum OpType: String, Codable { + case clear = "CLEAR" + case move = "MOVE" + case put = "PUT" + case remove = "REMOVE" +} + +struct BucketChecksum: Encodable { + var bucket: String + var priority: BucketPriority = .defaultPriority + var checksum: Int32 + var count: Int? = nil + var lastOpId: String? = nil + + enum CodingKeys: String, CodingKey { + case bucket + case priority + case checksum + case count + case last_op_id = "last_op_id" + } + + func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(self.bucket, forKey: .bucket) + try container.encode(self.priority, forKey: .priority) + try container.encode(self.checksum, forKey: .checksum) + try container.encode(self.count, forKey: .count) + try container.encode(self.lastOpId, forKey: .last_op_id) + } +} + +extension AsyncThrowingChannel { + func pushLine(_ line: SyncLine) async throws { + let encoded = try StreamingSyncClient.jsonEncoder.encode(line) + await send(.text(contents: String(data: encoded, encoding: .utf8)!)) + } +} From 0bf87448dcdea9c29f3a54b74eecaf59077a24dc Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 20 Apr 2026 14:08:15 +0200 Subject: [PATCH 07/40] Implement schema serialization in Swift --- .../PowerSync/Protocol/Schema/Column.swift | 16 +++- Sources/PowerSync/Protocol/Schema/Index.swift | 23 ++++- .../PowerSync/Protocol/Schema/RawTable.swift | 72 +++++++++++++-- .../PowerSync/Protocol/Schema/Schema.swift | 17 +++- Sources/PowerSync/Protocol/Schema/Table.swift | 29 +++--- .../Protocol/Schema/TableOptions.swift | 73 +++++++++++++++ Tests/PowerSyncTests/Schema/SchemaTests.swift | 17 +++- Tests/PowerSyncTests/Schema/TableTests.swift | 89 +++++++++++++++++++ 8 files changed, 315 insertions(+), 21 deletions(-) diff --git a/Sources/PowerSync/Protocol/Schema/Column.swift b/Sources/PowerSync/Protocol/Schema/Column.swift index f1d1bd0..feb6a32 100644 --- a/Sources/PowerSync/Protocol/Schema/Column.swift +++ b/Sources/PowerSync/Protocol/Schema/Column.swift @@ -15,14 +15,26 @@ public protocol ColumnProtocol: Equatable, Sendable { var type: ColumnData { get } } -public enum ColumnData: Sendable { +public enum ColumnData: Sendable, Encodable { case text case integer case real + + public func encode(to encoder: any Encoder) throws { + var container = encoder.singleValueContainer() + switch self { + case .text: + try container.encode("text") + case .integer: + try container.encode("integer") + case .real: + try container.encode("real") + } + } } /// A single column in a table schema. -public struct Column: ColumnProtocol { +public struct Column: ColumnProtocol, Encodable { public let name: String public let type: ColumnData diff --git a/Sources/PowerSync/Protocol/Schema/Index.swift b/Sources/PowerSync/Protocol/Schema/Index.swift index 955a27f..acfa7c3 100644 --- a/Sources/PowerSync/Protocol/Schema/Index.swift +++ b/Sources/PowerSync/Protocol/Schema/Index.swift @@ -12,7 +12,7 @@ public protocol IndexProtocol: Sendable { var columns: [IndexedColumnProtocol] { get } } -public struct Index: IndexProtocol { +public struct Index: IndexProtocol, Encodable { public let name: String public let columns: [IndexedColumnProtocol] @@ -47,4 +47,25 @@ public struct Index: IndexProtocol { ) -> Index { return ascending(name: name, columns: [column]) } + + public func encode(to encoder: any Encoder) throws { + enum CodingKeys: CodingKey { + case name + case columns + } + + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(name, forKey: .name) + var columnsContainer = container.nestedUnkeyedContainer(forKey: .columns) + for column in columns { + enum CodingKeys: CodingKey { + case name + case ascending + } + + var container = columnsContainer.nestedContainer(keyedBy: CodingKeys.self) + try container.encode(column.column, forKey: .name) + try container.encode(column.ascending, forKey: .ascending) + } + } } diff --git a/Sources/PowerSync/Protocol/Schema/RawTable.swift b/Sources/PowerSync/Protocol/Schema/RawTable.swift index 63ed9d9..209caf9 100644 --- a/Sources/PowerSync/Protocol/Schema/RawTable.swift +++ b/Sources/PowerSync/Protocol/Schema/RawTable.swift @@ -11,7 +11,7 @@ /// /// Note that raw tables are only supported when ``ConnectOptions/newClientImplementation`` /// is enabled. -public struct RawTable: BaseTableProtocol { +public struct RawTable: BaseTableProtocol, Encodable { /// The name of the table as it appears in sync rules. /// /// This doesn't necessarily have to match the statement that ``RawTable/put`` and ``RawTable/delete`` @@ -57,12 +57,43 @@ public struct RawTable: BaseTableProtocol { /// /// The output of this can be passed to the `powersync_create_raw_table_crud_trigger` SQL /// function to define triggers for this table. - public func jsonDescription() -> String { - return KotlinAdapter.Table.toKotlin(self).jsonDescription() + public func jsonDescription() throws -> String { + let serialized = try Schema.encoder.encode(self) + return String(data: serialized, encoding: .utf8)! + } + + internal func validate() throws(TableError) { + if let schema { + try schema.options.validate(tableName: name) + } + } + + public func encode(to encoder: any Encoder) throws { + enum CodingKeys: String, CodingKey { + case name + case put + case delete + case clear + // For schema + case tableName = "table_name" + case syncedColumns = "synced_columns" + } + typealias Keys = TableOptionsCodingKeys + + var container = encoder.container(keyedBy: Keys.self) + try container.encode(name, forKey: .outer(.name)) + try container.encodeIfPresent(put, forKey: .outer(.put)) + try container.encodeIfPresent(delete, forKey: .outer(.delete)) + try container.encodeIfPresent(clear, forKey: .outer(.clear)) + if let schema { + try container.encode(schema.tableName ?? name, forKey: .outer(.tableName)) + try container.encodeIfPresent(schema.syncedColumns, forKey: .outer(.syncedColumns)) + try schema.options.serializeTo(container) + } } } -/// THe schema of a ``RawTable`` in the local database. +/// The schema of a ``RawTable`` in the local database. /// /// This information is optional when declaring raw tables. However, providing it allows the sync /// client to infer ``RawTable/put`` and ``RawTable/delete`` statements automatically. @@ -95,7 +126,7 @@ public struct RawTableSchema: Sendable { } /// A statement to run to sync server-side changes into a local raw table. -public struct PendingStatement: Sendable { +public struct PendingStatement: Sendable, Encodable { /// The SQL statement to execute. public let sql: String /// For parameters in the prepared statement, the values to fill in. @@ -108,10 +139,21 @@ public struct PendingStatement: Sendable { self.sql = sql self.parameters = parameters } + + public func encode(to encoder: any Encoder) throws { + enum CodingKeys: String, CodingKey { + case sql + case parameters = "params" + } + + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(self.sql, forKey: .sql) + try container.encode(self.parameters, forKey: .parameters) + } } /// A parameter that can be used in a ``PendingStatement``. -public enum PendingStatementParameter: Sendable { +public enum PendingStatementParameter: Sendable, Encodable { /// A value that resolves to the textual id of the row to insert, update or delete. case id /// A value that resolves to the value of a column in a `PUT` operation for inserts or updates. @@ -122,4 +164,22 @@ public enum PendingStatementParameter: Sendable { /// Resolves to a JSON object containing all columns from the synced row that haven't been matched /// by a ``PendingStatementParameter/column`` value in the same statement. case rest + + public func encode(to encoder: any Encoder) throws { + switch self { + case .id: + var container = encoder.singleValueContainer() + try container.encode("Id") + case .column(let name): + enum CodingKeys: String, CodingKey { + case column = "Column" + } + + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(name, forKey: .column) + case .rest: + var container = encoder.singleValueContainer() + try container.encode("Rest") + } + } } diff --git a/Sources/PowerSync/Protocol/Schema/Schema.swift b/Sources/PowerSync/Protocol/Schema/Schema.swift index c1e93be..7c26de9 100644 --- a/Sources/PowerSync/Protocol/Schema/Schema.swift +++ b/Sources/PowerSync/Protocol/Schema/Schema.swift @@ -1,3 +1,5 @@ +import Foundation + public protocol SchemaProtocol: Sendable { /// /// Tables used in Schema @@ -12,7 +14,7 @@ public protocol SchemaProtocol: Sendable { func validate() throws } -public struct Schema: SchemaProtocol { +public struct Schema: SchemaProtocol, Encodable { public let tables: [Table] public let rawTables: [RawTable] @@ -50,7 +52,20 @@ public struct Schema: SchemaProtocol { } try table.validate() } + + for table in rawTables { + if let schema = table.schema { + let name = schema.tableName ?? table.name + if !tableNames.insert(name).inserted { + throw SchemaError.duplicateTableName(name) + } + } + + try table.validate() + } } + + internal static let encoder = JSONEncoder() } public enum SchemaError: Error { diff --git a/Sources/PowerSync/Protocol/Schema/Table.swift b/Sources/PowerSync/Protocol/Schema/Table.swift index 8ce60d3..323d566 100644 --- a/Sources/PowerSync/Protocol/Schema/Table.swift +++ b/Sources/PowerSync/Protocol/Schema/Table.swift @@ -30,7 +30,7 @@ private let MAX_AMOUNT_OF_COLUMNS = 63 /// /// A single table in the schema. /// -public struct Table: TableProtocol { +public struct Table: TableProtocol, Encodable { public let name: String public let columns: [Column] public let indexes: [Index] @@ -119,15 +119,7 @@ public struct Table: TableProtocol { throw TableError.invalidViewName(viewName: viewNameOverride) } - if localOnly { - if trackPreviousValues != nil { - throw TableError.trackPreviousForLocalTable(tableName: name) - } - if trackMetadata { - throw TableError.metadataForLocalTable(tableName: name) - } - } - + try options.validate(tableName: name) var columnNames = Set(["id"]) for column in columns { @@ -184,6 +176,23 @@ public struct Table: TableProtocol { indexNames.insert(index.name) } } + + public func encode(to encoder: any Encoder) throws { + enum CodingKeys: String, CodingKey { + case name + case viewName = "view_name" + case columns + case indexes + } + typealias Keys = TableOptionsCodingKeys + + var container = encoder.container(keyedBy: Keys.self) + try container.encode(name, forKey: .outer(.name)) + try container.encodeIfPresent(viewNameOverride, forKey: .outer(.viewName)) + try container.encode(columns, forKey: .outer(.columns)) + try container.encode(indexes, forKey: .outer(.indexes)) + try options.serializeTo(container) + } } public enum TableError: Error { diff --git a/Sources/PowerSync/Protocol/Schema/TableOptions.swift b/Sources/PowerSync/Protocol/Schema/TableOptions.swift index bc57105..3b5310b 100644 --- a/Sources/PowerSync/Protocol/Schema/TableOptions.swift +++ b/Sources/PowerSync/Protocol/Schema/TableOptions.swift @@ -71,6 +71,26 @@ public struct TableOptions: TableOptionsProtocol { self.trackPreviousValues = trackPreviousValues self.ignoreEmptyUpdates = ignoreEmptyUpdates } + + internal func validate(tableName: String) throws(TableError) { + if localOnly { + if trackPreviousValues != nil { + throw TableError.trackPreviousForLocalTable(tableName: tableName) + } + if trackMetadata { + throw TableError.metadataForLocalTable(tableName: tableName) + } + } + } + + internal func serializeTo(_ container: KeyedEncodingContainer>) throws { + var container = container + try container.encode(localOnly, forKey: .localOnly) + try container.encode(insertOnly, forKey: .insertOnly) + try container.encode(trackMetadata, forKey: .includeMetadata) + try container.encode(ignoreEmptyUpdates, forKey: .ignoreEmptyUpdate) + try trackPreviousValues?.serializeTo(container) + } } /// Options to include old values in ``CrudEntry/previousValues`` for update statements. @@ -91,4 +111,57 @@ public struct TrackPreviousValuesOptions: Sendable { self.columnFilter = columnFilter self.onlyWhenChanged = onlyWhenChanged } + + internal func serializeTo(_ container: KeyedEncodingContainer>) throws { + var container = container + if let columnFilter { + try container.encode(columnFilter, forKey: .diffIncludeOld) + } else { + try container.encode(true, forKey: .diffIncludeOld) + } + try container.encode(onlyWhenChanged, forKey: .includeOldOnlyWhenChanged) + } +} + +/// Coding keys for table options (which are always embedded into another outer object. +internal enum TableOptionsCodingKeys: CodingKey { + case outer(T) + case diffIncludeOld + case localOnly + case insertOnly + case includeMetadata + case includeOldOnlyWhenChanged + case ignoreEmptyUpdate + + // We don't use these for decoding, so we can return nil here. + init?(stringValue: String) { + return nil + } + init?(intValue: Int) { + return nil + } + + var stringValue: String { + switch self { + case .outer(let field): + return field.stringValue + case .diffIncludeOld: + return "include_old" + case .localOnly: + return "local_only" + case .insertOnly: + return "insert_only" + case .includeMetadata: + return "include_metadata" + case .includeOldOnlyWhenChanged: + return "include_old_only_when_changed" + case .ignoreEmptyUpdate: + return "ignore_empty_update" + } + } + + // We'll only encode into string-keyed dictionaries (JSON objects). + var intValue: Int? { + nil + } } diff --git a/Tests/PowerSyncTests/Schema/SchemaTests.swift b/Tests/PowerSyncTests/Schema/SchemaTests.swift index ebcbc22..cf1d844 100644 --- a/Tests/PowerSyncTests/Schema/SchemaTests.swift +++ b/Tests/PowerSyncTests/Schema/SchemaTests.swift @@ -66,7 +66,22 @@ final class SchemaTests: XCTestCase { XCTAssertEqual(tableName, "users") } } - + + func testDuplicateTableOneRaw() { + let schema = Schema( + makeValidTable(name: "users"), + RawTable(name: "source_from_sync_streams", schema: RawTableSchema(tableName: "users")) + ) + + XCTAssertThrowsError(try schema.validate()) { error in + guard case SchemaError.duplicateTableName(let tableName) = error else { + XCTFail("Expected duplicateTableName error") + return + } + XCTAssertEqual(tableName, "users") + } + } + func testCascadingTableValidation() { let schema = Schema( makeValidTable(name: "users"), diff --git a/Tests/PowerSyncTests/Schema/TableTests.swift b/Tests/PowerSyncTests/Schema/TableTests.swift index 89d36fe..94e2393 100644 --- a/Tests/PowerSyncTests/Schema/TableTests.swift +++ b/Tests/PowerSyncTests/Schema/TableTests.swift @@ -230,4 +230,93 @@ final class TableTests: XCTestCase { XCTAssertNoThrow(try table.validate()) } + + func testSerialize() throws { + let table = Table( + name: "users", + columns: makeValidColumns(), + indexes: [makeValidIndex()], + localOnly: false, + insertOnly: false, + trackPreviousValues: TrackPreviousValuesOptions(columnFilter: ["name"], onlyWhenChanged: true) + ) + let encoder = JSONEncoder() + encoder.outputFormatting.insert(.prettyPrinted) + encoder.outputFormatting.insert(.sortedKeys) + let serialized = String(data: try encoder.encode(table), encoding: .utf8) + + XCTAssertEqual(serialized, """ +{ + "columns" : [ + { + "name" : "name", + "type" : "text" + }, + { + "name" : "age", + "type" : "integer" + }, + { + "name" : "score", + "type" : "real" + } + ], + "ignore_empty_update" : false, + "include_metadata" : false, + "include_old" : [ + "name" + ], + "include_old_only_when_changed" : true, + "indexes" : [ + { + "columns" : [ + { + "ascending" : true, + "name" : "name" + } + ], + "name" : "test_index" + } + ], + "insert_only" : false, + "local_only" : false, + "name" : "users" +} +""") + } + + func testRawTableSerialize() throws { + let table = RawTable( + name: "users", + put: PendingStatement(sql: "SELECT 1", parameters: [.id]), + delete: PendingStatement(sql: "SELECT 2", parameters: [.column("a"), .rest]), + clear: "SELECT 3" + ) + let encoder = JSONEncoder() + encoder.outputFormatting.insert(.prettyPrinted) + encoder.outputFormatting.insert(.sortedKeys) + let serialized = String(data: try encoder.encode(table), encoding: .utf8) + + XCTAssertEqual(serialized, """ +{ + "clear" : "SELECT 3", + "delete" : { + "params" : [ + { + "Column" : "a" + }, + "Rest" + ], + "sql" : "SELECT 2" + }, + "name" : "users", + "put" : { + "params" : [ + "Id" + ], + "sql" : "SELECT 1" + } +} +""") + } } From 9aa2776b75498f59c40ff134b7d4ddb225e6f44d Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 20 Apr 2026 14:35:17 +0200 Subject: [PATCH 08/40] AI feedback --- Sources/PowerSync/Protocol/Schema/Index.swift | 4 +-- .../PowerSync/Protocol/Schema/RawTable.swift | 20 +++++++++--- .../PowerSync/Protocol/Schema/Schema.swift | 19 +++++++++--- .../Protocol/Schema/TableOptions.swift | 2 +- Tests/PowerSyncTests/Schema/SchemaTests.swift | 8 +++++ Tests/PowerSyncTests/Schema/TableTests.swift | 31 ++++++++++++++++++- 6 files changed, 72 insertions(+), 12 deletions(-) diff --git a/Sources/PowerSync/Protocol/Schema/Index.swift b/Sources/PowerSync/Protocol/Schema/Index.swift index acfa7c3..820af49 100644 --- a/Sources/PowerSync/Protocol/Schema/Index.swift +++ b/Sources/PowerSync/Protocol/Schema/Index.swift @@ -58,12 +58,12 @@ public struct Index: IndexProtocol, Encodable { try container.encode(name, forKey: .name) var columnsContainer = container.nestedUnkeyedContainer(forKey: .columns) for column in columns { - enum CodingKeys: CodingKey { + enum IndexedColumnCodingKeys: CodingKey { case name case ascending } - var container = columnsContainer.nestedContainer(keyedBy: CodingKeys.self) + var container = columnsContainer.nestedContainer(keyedBy: IndexedColumnCodingKeys.self) try container.encode(column.column, forKey: .name) try container.encode(column.ascending, forKey: .ascending) } diff --git a/Sources/PowerSync/Protocol/Schema/RawTable.swift b/Sources/PowerSync/Protocol/Schema/RawTable.swift index 209caf9..a4f6bc4 100644 --- a/Sources/PowerSync/Protocol/Schema/RawTable.swift +++ b/Sources/PowerSync/Protocol/Schema/RawTable.swift @@ -1,3 +1,5 @@ +import Foundation + /// A table that is managed by the user instead of being auto-created and migrated by the PowerSync SDK. /// /// These tables give application developers full control over the table (including table and column constraints). @@ -57,11 +59,21 @@ public struct RawTable: BaseTableProtocol, Encodable { /// /// The output of this can be passed to the `powersync_create_raw_table_crud_trigger` SQL /// function to define triggers for this table. - public func jsonDescription() throws -> String { - let serialized = try Schema.encoder.encode(self) - return String(data: serialized, encoding: .utf8)! + public func jsonDescription() -> String { + let encoder = JSONEncoder() + do { + let serialized = try encoder.encode(self) + return String(data: serialized, encoding: .utf8)! + } catch { + // An older version of the Swift SDK used to implement this method in Kotlin. + // That could also throw an exception (which would crash the process), but that + // was overlooked due to a missing @Throws annotation. + // Now, we can't mark this as throws without breaking backwards compatibility. + // We should conver this to be throwing in a future major release. + fatalError("Serializing a raw table failed: \(error)") + } } - + internal func validate() throws(TableError) { if let schema { try schema.options.validate(tableName: name) diff --git a/Sources/PowerSync/Protocol/Schema/Schema.swift b/Sources/PowerSync/Protocol/Schema/Schema.swift index 7c26de9..9d7e47e 100644 --- a/Sources/PowerSync/Protocol/Schema/Schema.swift +++ b/Sources/PowerSync/Protocol/Schema/Schema.swift @@ -1,5 +1,3 @@ -import Foundation - public protocol SchemaProtocol: Sendable { /// /// Tables used in Schema @@ -54,6 +52,10 @@ public struct Schema: SchemaProtocol, Encodable { } for table in rawTables { + // Only check for duplicate names if the raw table has a fixed local schema + // name. By default, the name in raw tables refers to the name of the table as + // defined in Sync Streams. The local table populated by put/delete statements + // might be different and we can't check that. if let schema = table.schema { let name = schema.tableName ?? table.name if !tableNames.insert(name).inserted { @@ -64,8 +66,17 @@ public struct Schema: SchemaProtocol, Encodable { try table.validate() } } - - internal static let encoder = JSONEncoder() + + public func encode(to encoder: any Encoder) throws { + enum CodingKeys: String, CodingKey { + case tables + case rawTables = "raw_tables" + } + + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(self.tables, forKey: .tables) + try container.encode(self.rawTables, forKey: .rawTables) + } } public enum SchemaError: Error { diff --git a/Sources/PowerSync/Protocol/Schema/TableOptions.swift b/Sources/PowerSync/Protocol/Schema/TableOptions.swift index 3b5310b..b959e03 100644 --- a/Sources/PowerSync/Protocol/Schema/TableOptions.swift +++ b/Sources/PowerSync/Protocol/Schema/TableOptions.swift @@ -132,7 +132,7 @@ internal enum TableOptionsCodingKeys: CodingKey { case includeMetadata case includeOldOnlyWhenChanged case ignoreEmptyUpdate - + // We don't use these for decoding, so we can return nil here. init?(stringValue: String) { return nil diff --git a/Tests/PowerSyncTests/Schema/SchemaTests.swift b/Tests/PowerSyncTests/Schema/SchemaTests.swift index cf1d844..f9f4fd4 100644 --- a/Tests/PowerSyncTests/Schema/SchemaTests.swift +++ b/Tests/PowerSyncTests/Schema/SchemaTests.swift @@ -122,4 +122,12 @@ final class SchemaTests: XCTestCase { XCTAssertEqual(schema.tables[0].name, users.name) XCTAssertEqual(schema.tables[1].name, posts.name) } + + func testEncode() throws { + let schema = Schema() + let encoder = JSONEncoder() + encoder.outputFormatting.insert(.sortedKeys) + let serialized = String(data: try encoder.encode(schema), encoding: .utf8) + XCTAssertEqual(serialized, #"{"raw_tables":[],"tables":[]}"#) + } } diff --git a/Tests/PowerSyncTests/Schema/TableTests.swift b/Tests/PowerSyncTests/Schema/TableTests.swift index 94e2393..53c91cf 100644 --- a/Tests/PowerSyncTests/Schema/TableTests.swift +++ b/Tests/PowerSyncTests/Schema/TableTests.swift @@ -285,7 +285,7 @@ final class TableTests: XCTestCase { """) } - func testRawTableSerialize() throws { + func testRawTableSerializeSimple() throws { let table = RawTable( name: "users", put: PendingStatement(sql: "SELECT 1", parameters: [.id]), @@ -319,4 +319,33 @@ final class TableTests: XCTestCase { } """) } + + func testRawTableSerializeWithOptions() throws { + let table = RawTable( + name: "users", + schema: RawTableSchema( + syncedColumns: ["foo"], + options: TableOptions(insertOnly: true) + ) + ) + let encoder = JSONEncoder() + encoder.outputFormatting.insert(.prettyPrinted) + encoder.outputFormatting.insert(.sortedKeys) + let serialized = String(data: try encoder.encode(table), encoding: .utf8) + + XCTAssertEqual(serialized, """ +{ + "ignore_empty_update" : false, + "include_metadata" : false, + "insert_only" : true, + "local_only" : false, + "name" : "users", + "synced_columns" : [ + "foo" + ], + "table_name" : "users" +} +""") + } + } From 7949c021341cc2341009a0fa9a0d7ba355d1d7d3 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 21 Apr 2026 10:38:23 +0200 Subject: [PATCH 09/40] Restrict CI timeouts --- .github/workflows/build_and_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 22e6506..4d2a163 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -8,6 +8,7 @@ jobs: build: name: Build and test runs-on: macos-latest + timeout-minutes: 30 steps: - uses: actions/checkout@v4 - name: Set up XCode From dbecd32ce818765c32d3c908425311ad68414fc9 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 21 Apr 2026 11:19:03 +0200 Subject: [PATCH 10/40] Support raw tables --- Package.resolved | 6 +- Package.swift | 2 +- .../sync/PowerSyncControlArguments.swift | 5 +- .../sync/StreamingSyncClient.swift | 5 +- .../Kotlin/KotlinPowerSyncDatabaseImpl.swift | 13 +- .../PowerSync/Protocol/Schema/Schema.swift | 5 + Sources/PowerSync/Utils/AsyncMutex.swift | 11 ++ Tests/PowerSyncTests/SyncTests.swift | 123 ++++++++++++++++-- 8 files changed, 144 insertions(+), 26 deletions(-) create mode 100644 Sources/PowerSync/Utils/AsyncMutex.swift diff --git a/Package.resolved b/Package.resolved index c61f4d2..8969295 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "3165e395f09f43046349a7f58470df039b67a3cc3e2ce3d2c3ee2635489c6fb7", + "originHash" : "ad5382ed9d817fca8fdec89f15357bf19a77421c7ea034496baf99710ab60846", "pins" : [ { "identity" : "csqlite", @@ -24,8 +24,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/powersync-ja/powersync-sqlite-core-swift.git", "state" : { - "revision" : "aa35758afca6cc97db836e1545ad28e4c5ddc50d", - "version" : "0.4.12" + "revision" : "05c2af384558011f0915d757b6677f5dcbbc5c54", + "version" : "0.4.13" } }, { diff --git a/Package.swift b/Package.swift index 81cf654..d5bdd4a 100644 --- a/Package.swift +++ b/Package.swift @@ -48,7 +48,7 @@ if let corePath = localCoreExtension { conditionalDependencies.append( .package( url: "https://github.com/powersync-ja/powersync-sqlite-core-swift.git", - exact: "0.4.12", + exact: "0.4.13", )) } diff --git a/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift b/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift index 97e25f9..5c5f8a5 100644 --- a/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift +++ b/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift @@ -61,15 +61,16 @@ enum PowerSyncControlArguments { } } -struct StartSyncIteration: Codable { +struct StartSyncIteration: Encodable { let parameters: JsonParam - // TODO: Schema + let schema: Schema let includeDefaults: Bool let activeStreams: [StreamKey] let appMetadata: [String: String] enum CodingKeys: String, CodingKey { case parameters + case schema case includeDefaults = "include_defaults" case activeStreams = "active_streams" case appMetadata = "app_metadata" diff --git a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift index 3b5ca9b..1cb7d18 100644 --- a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift +++ b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift @@ -35,10 +35,6 @@ final class StreamingSyncClient: Sendable { let _ = try await download } } - - private func watchPsCrudChanges(signals: SyncSignals) async throws { - - } private func uploadLoop(signals: SyncSignals) async throws { // TODO: Replace with better watch mechanism @@ -232,6 +228,7 @@ private struct ActiveSyncIteration: Sendable { func run() async throws -> SyncIterationResult { let initialInstructions = try await powersyncControl(.start(StartSyncIteration( parameters: syncClient.options.params, + schema: await syncClient.db.schema.inner, includeDefaults: true, activeStreams: [], appMetadata: syncClient.options.appMetadata, diff --git a/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift b/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift index 4b9ef6a..5958662 100644 --- a/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift +++ b/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift @@ -13,12 +13,14 @@ final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, internal let syncStatus = SwiftSyncStatus() private let dbFilename: String private let httpClient: HttpClient + let schema: AsyncMutex init( kotlinDatabase: PowerSyncKotlin.PowerSyncDatabase, dbFilename: String, logger: DatabaseLogger, - httpClient: HttpClient + httpClient: HttpClient, + schema: Schema ) { self.logger = logger self.kotlinDatabase = kotlinDatabase @@ -27,6 +29,7 @@ final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, /// FIXME. Update this once we support database directory configuration. self.dbFilename = dbFilename self.httpClient = httpClient + self.schema = AsyncMutex(schema) } var currentStatus: any SyncStatus { @@ -40,6 +43,8 @@ final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, func updateSchema(schema: any SchemaProtocol) async throws { try await syncCoordinator.guardNotConnected( inner: { + await self.schema.withMutex { $0 = Schema(other: schema) } + try await kotlinDatabase.updateSchema( schema: KotlinAdapter.Schema.toKotlin(schema) ) @@ -469,7 +474,8 @@ func openKotlinDBDefault( kotlinDatabase: kotlinDatabase, dbFilename: dbFilename, logger: logger, - httpClient: httpClient + httpClient: httpClient, + schema: schema ) } @@ -488,7 +494,8 @@ func openKotlinDBWithPool( ), dbFilename: identifier, logger: logger, - httpClient: PlatformHttpClient.shared + httpClient: PlatformHttpClient.shared, + schema: schema ) } diff --git a/Sources/PowerSync/Protocol/Schema/Schema.swift b/Sources/PowerSync/Protocol/Schema/Schema.swift index 9d7e47e..57acc32 100644 --- a/Sources/PowerSync/Protocol/Schema/Schema.swift +++ b/Sources/PowerSync/Protocol/Schema/Schema.swift @@ -20,6 +20,11 @@ public struct Schema: SchemaProtocol, Encodable { self.tables = tables self.rawTables = rawTables } + + init(other: SchemaProtocol) { + self.tables = other.tables + self.rawTables = other.rawTables + } /// /// Convenience initializer with variadic parameters diff --git a/Sources/PowerSync/Utils/AsyncMutex.swift b/Sources/PowerSync/Utils/AsyncMutex.swift new file mode 100644 index 0000000..bbb3c31 --- /dev/null +++ b/Sources/PowerSync/Utils/AsyncMutex.swift @@ -0,0 +1,11 @@ +actor AsyncMutex { + var inner: T + + init(_ inner: consuming sending T) { + self.inner = inner + } + + func withMutex(callback: (_ element: inout T) throws -> R) rethrows -> R { + try callback(&inner) + } +} diff --git a/Tests/PowerSyncTests/SyncTests.swift b/Tests/PowerSyncTests/SyncTests.swift index 12654a3..d1a1e90 100644 --- a/Tests/PowerSyncTests/SyncTests.swift +++ b/Tests/PowerSyncTests/SyncTests.swift @@ -231,17 +231,114 @@ class InMemorySyncIntegrationTests { await connector.prefetchCalled.await() // Should still be connected before prefetch completes try #require(db.currentStatus.connected == true) - + // After the prefetch completes, we should reconnect. await connector.completePrefetch.complete() await waitForStatus(db.currentStatus) { !$0.connected } await waitForStatus(db.currentStatus) { $0.connected } try #require(await connector.fetchCredentialsCount == 2) } + + @Test func rawTablesWithImplicitStatements() async throws { + struct List: Equatable { + let id: String + let name: String + } + + let channel = AsyncThrowingChannel() + let db = openDatabase(MockHttpClient { request in channel }, schema: Schema(RawTable(name: "lists", schema: RawTableSchema()))) + + try await db.execute("CREATE TABLE lists (id TEXT NOT NULL PRIMARY KEY, name TEXT)") + var query = try db.watch("SELECT * FROM lists") { cursor in + List(id: try cursor.getString(index: 0), name: try cursor.getString(index: 1)) + }.makeAsyncIterator() + try #require(try await query.next() == []) + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "1", buckets: [BucketChecksum(bucket: "a", checksum: 0)]))) + try await channel.pushLine(.syncDataBucket(SyncDataBucket(bucket: "a", data: [ + OplogEntry( + checksum: 0, + op_id: "1", + object_id: "my_list", + object_type: "lists", + op: .put, + data: #"{"name": "custom list"}"# + ) + ]))) + try await channel.pushLine(.checkpointComplete(lastOpId: "1")) + try #require(try await query.next() == [List(id: "my_list", name: "custom list")]) + + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "2", buckets: [BucketChecksum(bucket: "a", checksum: 0)]))) + try await channel.pushLine(.syncDataBucket(SyncDataBucket(bucket: "a", data: [ + OplogEntry( + checksum: 0, + op_id: "2", + object_id: "my_list", + object_type: "lists", + op: .remove, + ) + ]))) + try await channel.pushLine(.checkpointComplete(lastOpId: "2")) + try #require(try await query.next() == []) + } - // TODO: Raw table tests (requires serializable schema) + @Test func rawTablesWithExplicitStatements() async throws { + struct List: Equatable { + let id: String + let name: String + let rest: String + } + + let channel = AsyncThrowingChannel() + let db = openDatabase(MockHttpClient { request in channel }, schema: Schema(RawTable( + name: "lists", + put: PendingStatement(sql: "INSERT OR REPLACE INTO lists (id, name, _rest) VALUES (?, ?, ?)", parameters: [ + .id, + .column("name"), + .rest + ]), + delete: PendingStatement(sql: "DELETE FROM lists WHERE id = ?", parameters: [ + .id + ]), + ))) + + try await db.execute("CREATE TABLE lists (id TEXT NOT NULL PRIMARY KEY, name TEXT, _rest TEXT)") + var query = try db.watch("SELECT * FROM lists") { cursor in + List(id: try cursor.getString(index: 0), name: try cursor.getString(index: 1), rest: try cursor.getString(index: 2)) + }.makeAsyncIterator() + try #require(try await query.next() == []) + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "1", buckets: [BucketChecksum(bucket: "a", checksum: 0)]))) + try await channel.pushLine(.syncDataBucket(SyncDataBucket(bucket: "a", data: [ + OplogEntry( + checksum: 0, + op_id: "1", + object_id: "my_list", + object_type: "lists", + op: .put, + data: #"{"name": "custom list", "additional_column": "foo"}"# + ) + ]))) + try await channel.pushLine(.checkpointComplete(lastOpId: "1")) + try #require(try await query.next() == [List(id: "my_list", name: "custom list", rest: #"{"additional_column":"foo"}"#)]) + + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "2", buckets: [BucketChecksum(bucket: "a", checksum: 0)]))) + try await channel.pushLine(.syncDataBucket(SyncDataBucket(bucket: "a", data: [ + OplogEntry( + checksum: 0, + op_id: "2", + object_id: "my_list", + object_type: "lists", + op: .remove, + ) + ]))) + try await channel.pushLine(.checkpointComplete(lastOpId: "2")) + try #require(try await query.next() == []) + } - @Test(.disabled("Blocked on https://github.com/powersync-ja/powersync-sqlite-core/pull/174")) func endsIterationOnHttpClose() async throws { + @Test func endsIterationOnHttpClose() async throws { let channel = AsyncThrowingChannel() let db = openDatabase(MockHttpClient { request in channel }) try await db.connect(connector: TestConnector(), options: ConnectOptions()) @@ -250,7 +347,7 @@ class InMemorySyncIntegrationTests { channel.finish() await waitForStatus(db.currentStatus) { !$0.connected } } - + @Test func syncProgress() async throws { let channel = AsyncThrowingChannel() let db = openDatabase(MockHttpClient { request in channel }) @@ -280,16 +377,16 @@ class InMemorySyncIntegrationTests { } } -private func openDatabase(_ client: MockHttpClient) -> PowerSyncDatabaseProtocol { - let schema = Schema(tables: [ - Table( - name: "users", - columns: [ - .text("name"), - ] - ), - ]) +let defaultSchema = Schema(tables: [ + Table( + name: "users", + columns: [ + .text("name"), + ] + ), +]) +private func openDatabase(_ client: MockHttpClient, schema: Schema = defaultSchema) -> PowerSyncDatabaseProtocol { return openKotlinDBDefault( schema: schema, dbFilename: ":memory:", From 6a6bf2c945d4814a4c430e29c5f8522e8476394b Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 21 Apr 2026 11:31:41 +0200 Subject: [PATCH 11/40] Set request headers --- .../sync/StreamingSyncClient.swift | 1 + .../sync/UserAgent.swift | 0 Tests/PowerSyncTests/SyncTests.swift | 19 +++++++++++++++++-- .../test-utils/MockHttpClient.swift | 7 +++---- 4 files changed, 21 insertions(+), 6 deletions(-) rename Sources/PowerSync/{Kotlin => Implementation}/sync/UserAgent.swift (100%) diff --git a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift index 1cb7d18..9da940b 100644 --- a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift +++ b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift @@ -190,6 +190,7 @@ The next upload iteration will be delayed. let url = buildUrl(base) var request = URLRequest(url: url) request.setValue("Token \(credentials.token)", forHTTPHeaderField: "Authorization") + request.setValue(await userAgent(), forHTTPHeaderField: "User-Agent") return (url, request) } diff --git a/Sources/PowerSync/Kotlin/sync/UserAgent.swift b/Sources/PowerSync/Implementation/sync/UserAgent.swift similarity index 100% rename from Sources/PowerSync/Kotlin/sync/UserAgent.swift rename to Sources/PowerSync/Implementation/sync/UserAgent.swift diff --git a/Tests/PowerSyncTests/SyncTests.swift b/Tests/PowerSyncTests/SyncTests.swift index d1a1e90..9a40be6 100644 --- a/Tests/PowerSyncTests/SyncTests.swift +++ b/Tests/PowerSyncTests/SyncTests.swift @@ -5,10 +5,24 @@ import Testing @Suite() class InMemorySyncIntegrationTests { + @Test func setsHeaders() async throws { + let didConnect = Signal() + let db = openDatabase(MockHttpClient { request in + try #require(request.value(forHTTPHeaderField: "User-Agent")!.contains("powersync-swift/")) + try #require(request.value(forHTTPHeaderField: "Authorization") == "Token test-token") + await didConnect.complete() + return AsyncThrowingChannel() + }) + + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + await didConnect.await() + } + @Test func useParameters() async throws { let didConnect = Signal() let db = openDatabase(MockHttpClient { request in - try #require(request["parameters"] == .object(["foo": .string("bar")])) + let body = try StreamingSyncClient.jsonDecoder.decode(JsonParam.self, from: try #require(request.httpBody)) + try #require(body["parameters"] == .object(["foo": .string("bar")])) await didConnect.complete() return AsyncThrowingChannel() }) @@ -23,7 +37,8 @@ class InMemorySyncIntegrationTests { @Test func useAppMetadata() async throws { let didConnect = Signal() let db = openDatabase(MockHttpClient { request in - try #require(request["app_metadata"] == .object(["app_version": .string("1.0.0")])) + let body = try StreamingSyncClient.jsonDecoder.decode(JsonParam.self, from: try #require(request.httpBody)) + try #require(body["app_metadata"] == .object(["app_version": .string("1.0.0")])) await didConnect.complete() return AsyncThrowingChannel() }) diff --git a/Tests/PowerSyncTests/test-utils/MockHttpClient.swift b/Tests/PowerSyncTests/test-utils/MockHttpClient.swift index d0663fc..3dd623e 100644 --- a/Tests/PowerSyncTests/test-utils/MockHttpClient.swift +++ b/Tests/PowerSyncTests/test-utils/MockHttpClient.swift @@ -6,17 +6,16 @@ import Synchronization final class MockHttpClient: HttpClient { let writeCheckpoint: Atomic = Atomic(1000) - let handleSyncLines: @Sendable (_ body: JsonParam) async throws -> AsyncThrowingChannel + let handleSyncLines: @Sendable (_ request: URLRequest) async throws -> AsyncThrowingChannel - init(handleSyncLines: @Sendable @escaping (_ body: JsonParam) async throws -> AsyncThrowingChannel) { + init(handleSyncLines: @Sendable @escaping (_ request: URLRequest) async throws -> AsyncThrowingChannel) { self.handleSyncLines = handleSyncLines } func receiveSyncLines(request: URLRequest) async throws -> (HTTPURLResponse, any Sendable & AsyncSequence) { try #require(request.url?.path() == "/sync/stream") - let body = try StreamingSyncClient.jsonDecoder.decode(JsonParam.self, from: try #require(request.httpBody)) - let channel = try await handleSyncLines(body) + let channel = try await handleSyncLines(request) let response = HTTPURLResponse(url: request.url!, mimeType: "application/x-ndjson", expectedContentLength: 0, textEncodingName: "utf-8") return (response, channel) From d561a2a8d81fca49e6ce47314df64197884477c8 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 21 Apr 2026 12:03:23 +0200 Subject: [PATCH 12/40] Support network logging --- .../Implementation/sync/HttpClient.swift | 113 ++++++++++++++++++ .../Implementation/sync/SyncCoordinator.swift | 5 + .../Kotlin/KotlinNetworkLogger.swift | 29 ----- Tests/PowerSyncTests/SyncTests.swift | 20 ++++ 4 files changed, 138 insertions(+), 29 deletions(-) delete mode 100644 Sources/PowerSync/Kotlin/KotlinNetworkLogger.swift diff --git a/Sources/PowerSync/Implementation/sync/HttpClient.swift b/Sources/PowerSync/Implementation/sync/HttpClient.swift index 80ae471..21dd101 100644 --- a/Sources/PowerSync/Implementation/sync/HttpClient.swift +++ b/Sources/PowerSync/Implementation/sync/HttpClient.swift @@ -34,3 +34,116 @@ struct PlatformHttpClient: HttpClient { static let shared = PlatformHttpClient(session: .shared) } + +struct LoggingClient: HttpClient { + let inner: HttpClient + let logger: SyncRequestLoggerConfiguration + + fileprivate var shouldLogInfo: Bool { + logger.requestLevel != .none + } + + fileprivate var shouldLogHeaders: Bool { + logger.requestLevel == .all || logger.requestLevel == .headers + } + + fileprivate var shouldLogBody: Bool { + logger.requestLevel == .all || logger.requestLevel == .body + } + + func receiveSyncLines(request: URLRequest) async throws -> (HTTPURLResponse, any Sendable & AsyncSequence) { + logRequest(request: request) + do { + let (response, lines) = try await inner.receiveSyncLines(request: request) + logResponse(response: response) + + return (response, LogSyncLines(logger: self, inner: lines)) + } catch { + logError(error: error) + throw error + } + } + + func readFully(request: URLRequest) async throws -> (HTTPURLResponse, Data) { + logRequest(request: request) + do { + let (response, data) = try await inner.readFully(request: request) + logResponse(response: response) + if shouldLogBody, let content = String(data: data, encoding: .utf8) { + logger.log(" Response: \(content)") + } + return (response, data) + } catch { + logError(error: error) + throw error + } + } + + private func logRequest(request: URLRequest) { + if shouldLogInfo, let method = request.httpMethod, let url = request.url { + logger.log("Starting request to \(method) \(url)") + } + + if shouldLogHeaders, let headers = request.allHTTPHeaderFields { + for (key, value) in headers { + logger.log("with header \(key): \(value)") + } + } + + if shouldLogBody, let rawBody = request.httpBody, let body = String(data: rawBody, encoding: .utf8) { + logger.log("with body: \(body)") + } + + if shouldLogInfo { + logger.log("sending request") + } + } + + private func logResponse(response: HTTPURLResponse) { + if shouldLogInfo, let url = response.url { + logger.log("Got response code \(response.statusCode) on \(url)") + } + + if shouldLogHeaders { + for (key, value) in response.allHeaderFields { + logger.log("with header \(key): \(value)") + } + } + } + + private func logError(error: any Error) { + if shouldLogInfo { + logger.log("Error: \(error)") + } + } +} + +private struct LogSyncLines: AsyncSequence, Sendable { + typealias AsyncIterator = LogSyncLinesIterator + + let logger: LoggingClient + let inner: any AsyncSequence & Sendable + + func makeAsyncIterator() -> LogSyncLinesIterator { + LogSyncLinesIterator(logger: logger, inner: inner.makeAsyncIterator()) + } +} + +private struct LogSyncLinesIterator: AsyncIteratorProtocol { + let logger: LoggingClient + var inner: any AsyncIteratorProtocol + + mutating func next() async throws -> SyncLine? { + let line = try await self.inner.next() + if logger.shouldLogBody { + switch line { + case .none: + logger.logger.log("End of response") + case .some(.text(contents: let contents)): + logger.logger.log("Response line: \(contents)") + } + } + + return line + } +} diff --git a/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift b/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift index a2ebf21..c4d0b13 100644 --- a/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift +++ b/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift @@ -7,6 +7,11 @@ actor SyncCoordinator { await self.finishSyncTask(task: task) } + var client = client + if let logger = options.clientConfiguration?.requestLogger { + client = LoggingClient(inner: client, logger: logger) + } + let sync = StreamingSyncClient(db: db, connector: connector, httpClient: client, options: options) activeSync = sync.run() } diff --git a/Sources/PowerSync/Kotlin/KotlinNetworkLogger.swift b/Sources/PowerSync/Kotlin/KotlinNetworkLogger.swift deleted file mode 100644 index 767a550..0000000 --- a/Sources/PowerSync/Kotlin/KotlinNetworkLogger.swift +++ /dev/null @@ -1,29 +0,0 @@ -import PowerSyncKotlin - -extension SyncRequestLogLevel { - func toKotlin() -> SwiftSyncRequestLogLevel { - switch self { - case .all: - return SwiftSyncRequestLogLevel.all - case .headers: - return SwiftSyncRequestLogLevel.headers - case .body: - return SwiftSyncRequestLogLevel.body - case .info: - return SwiftSyncRequestLogLevel.info - case .none: - return SwiftSyncRequestLogLevel.none - } - } -} - -extension SyncRequestLoggerConfiguration { - func toKotlinConfig() -> SwiftRequestLoggerConfig { - return SwiftRequestLoggerConfig( - logLevel: self.requestLevel.toKotlin(), - log: { [log] message in - log(message) - } - ) - } -} diff --git a/Tests/PowerSyncTests/SyncTests.swift b/Tests/PowerSyncTests/SyncTests.swift index 9a40be6..e87ef0c 100644 --- a/Tests/PowerSyncTests/SyncTests.swift +++ b/Tests/PowerSyncTests/SyncTests.swift @@ -390,6 +390,26 @@ class InMemorySyncIntegrationTests { }))) try (try #require(await status.next())).expectProgress(total: (12, 12)) } + + @Test func requestLogger() async throws { + let channel = AsyncThrowingChannel() + let db = openDatabase(MockHttpClient { request in channel }) + let lines: Mutex<[String]> = Mutex([]) + + try await db.connect(connector: TestConnector(), options: ConnectOptions( + clientConfiguration: SyncClientConfiguration(requestLogger: SyncRequestLoggerConfiguration(requestLevel: .all, logHandler: { line in + lines.withLock { $0.append(line) } + })) + )) + await waitForStatus(db.currentStatus) { $0.connected } + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "0", buckets: [BucketChecksum(bucket: "a", checksum: 0)]))) + try await channel.pushLine(.checkpointComplete(lastOpId: "0")) + try await db.waitForFirstSync() + + let logEntries = lines.withLock { $0 } + try #require(logEntries.contains("Starting request to POST https://powersynctest.example.org/sync/stream")) + try #require(logEntries.contains(#"Response line: {"checkpoint_complete":{"last_op_id":"0"}}"#)) + } } let defaultSchema = Schema(tables: [ From 60f7fb254d07f04b076c30e921c675e6bbb016eb Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 21 Apr 2026 14:54:19 +0200 Subject: [PATCH 13/40] Support sync streams --- .../Implementation/SyncStreams.swift | 164 ++++++++++++++ .../sync/PowerSyncControlArguments.swift | 4 +- .../sync/StreamingSyncClient.swift | 13 +- .../Implementation/sync/SyncCoordinator.swift | 5 +- .../Kotlin/KotlinPowerSyncDatabaseImpl.swift | 20 +- .../PowerSync/Kotlin/db/KotlinJsonParam.swift | 49 ---- .../Protocol/PowerSyncDatabaseProtocol.swift | 12 +- Sources/PowerSync/Protocol/db/JsonParam.swift | 2 +- Tests/PowerSyncTests/SyncTests.swift | 213 +++++++++++++++++- .../test-utils/SyncProtocol.swift | 32 +++ 10 files changed, 452 insertions(+), 62 deletions(-) create mode 100644 Sources/PowerSync/Implementation/SyncStreams.swift delete mode 100644 Sources/PowerSync/Kotlin/db/KotlinJsonParam.swift diff --git a/Sources/PowerSync/Implementation/SyncStreams.swift b/Sources/PowerSync/Implementation/SyncStreams.swift new file mode 100644 index 0000000..264aa95 --- /dev/null +++ b/Sources/PowerSync/Implementation/SyncStreams.swift @@ -0,0 +1,164 @@ +import Foundation +import Synchronization + +final class StreamTracker: Sendable { + // For each active stream key, how many StreamSubscription instances are active in that key. + private let groups: Mutex> = Mutex([:]) + let streamsChanged = BroadcastStream<[StreamKey]>() + + var currentStreams: [StreamKey] { + groups.withLock { groups in Array(groups.keys) } + } + + private func markActiveStreamsHaveChanged() { + streamsChanged.dispatch(event: currentStreams) + } + + fileprivate func subscriptionsCommand(db: KotlinPowerSyncDatabaseImpl, request: RustSubscriptionChangeRequest) async throws { + let _ = try await db.writeTransaction { tx in + let payload = String(data: try StreamingSyncClient.jsonEncoder.encode(request), encoding: .utf8) + try tx.execute(sql: "SELECT powersync_control(?, ?)", parameters: [ + "subscriptions", + payload + ]) + } + + try await db.resolveOfflineSyncStatusIfNotConnected() + } + + fileprivate func subscribe(db: KotlinPowerSyncDatabaseImpl, stream: PendingSyncStream, ttl: TimeInterval?, priority: BucketPriority?) async throws -> SyncSubscriptionImplementation { + let key = stream.key + try await subscriptionsCommand( + db: db, + request: .subscribe( + stream: key, + ttl: ttl.map { Int64($0) }, + priority: priority + ) + ) + + let didCreateGroup = groups.withLock { groups in + if let existingCount = groups[key] { + groups[key] = existingCount + 1 + return false + } else { + groups[key] = 1 + return true + } + } + + if didCreateGroup { + markActiveStreamsHaveChanged() + } + + return SyncSubscriptionImplementation(db: db, key: key) + } + + fileprivate func removeStreamGroup(key: StreamKey) { + let _ = groups.withLock { groups in groups.removeValue(forKey: key) } + markActiveStreamsHaveChanged() + } + + fileprivate func decrementRefCount(key: StreamKey) { + let didChangeStreams = groups.withLock { groups in + if let count = groups[key] { + if count == 1 { + groups.removeValue(forKey: key) + return true + } else { + groups[key] = count - 1 + } + } + + return false + } + if didChangeStreams { + markActiveStreamsHaveChanged() + } + } +} + +/// A Sync Stream that can be subscribed to. +struct PendingSyncStream: SyncStream { + let db: KotlinPowerSyncDatabaseImpl + let name: String + let parameters: JsonParam? + + var key: StreamKey { + StreamKey(name: name, params: parameters) + } + + func subscribe(ttl: TimeInterval?, priority: BucketPriority?) async throws -> any SyncStreamSubscription { + return try await db.syncCoordinator.streams.subscribe(db: db, stream: self, ttl: ttl, priority: priority) + } + + func unsubscribeAll() async throws { + let tracker = db.syncCoordinator.streams + let key = self.key + tracker.removeStreamGroup(key: key) + try await tracker.subscriptionsCommand(db: db, request: .unsubscribe(key)) + } +} + +final class SyncSubscriptionImplementation: SyncStreamSubscription { + private let db: KotlinPowerSyncDatabaseImpl + private let key: StreamKey + + init(db: KotlinPowerSyncDatabaseImpl, key: StreamKey) { + self.db = db + self.key = key + } + + var name: String { + key.name + } + + var parameters: JsonParam? { + key.params + } + + func waitForFirstSync() async throws { + await db.syncStatus.waitFor { status in status.forStream(stream: self)?.subscription.hasSynced == true } + } + + func unsubscribe() async throws { + // We don't need to do anything here, we'll unsubscribe on deinit instead. + } + + deinit { + db.syncCoordinator.streams.decrementRefCount(key: key) + } +} + +private enum RustSubscriptionChangeRequest: Encodable { + case subscribe( + stream: StreamKey, + ttl: Int64? = nil, + priority: BucketPriority? = nil + ) + case unsubscribe(StreamKey) + + enum CodingKeys: CodingKey { + case subscribe + case unsubscribe + } + + enum SubscribeCodingKeys: CodingKey { + case stream + case ttl + case priority + } + + func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .subscribe(let stream, let ttl, let priority): + var nestedContainer = container.nestedContainer(keyedBy: RustSubscriptionChangeRequest.SubscribeCodingKeys.self, forKey: .subscribe) + try nestedContainer.encode(stream, forKey: RustSubscriptionChangeRequest.SubscribeCodingKeys.stream) + try nestedContainer.encodeIfPresent(ttl, forKey: RustSubscriptionChangeRequest.SubscribeCodingKeys.ttl) + try nestedContainer.encodeIfPresent(priority, forKey: RustSubscriptionChangeRequest.SubscribeCodingKeys.priority) + case .unsubscribe(let key): + try container.encode(key, forKey: .unsubscribe) + } + } +} diff --git a/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift b/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift index 5c5f8a5..ec203ce 100644 --- a/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift +++ b/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift @@ -77,7 +77,7 @@ struct StartSyncIteration: Encodable { } } -struct StreamKey: Codable { +struct StreamKey: Codable, Equatable, Hashable { let name: String - let params: JsonParam + let params: JsonParam? } diff --git a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift index 9da940b..18620df 100644 --- a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift +++ b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift @@ -227,11 +227,13 @@ private struct ActiveSyncIteration: Sendable { } func run() async throws -> SyncIterationResult { + async let _ = watchSyncStreams() + let initialInstructions = try await powersyncControl(.start(StartSyncIteration( parameters: syncClient.options.params, schema: await syncClient.db.schema.inner, - includeDefaults: true, - activeStreams: [], + includeDefaults: syncClient.options.includeDefaultStreams, + activeStreams: syncClient.db.syncCoordinator.streams.currentStreams, appMetadata: syncClient.options.appMetadata, ))) @@ -335,6 +337,13 @@ private struct ActiveSyncIteration: Sendable { } } } + + private func watchSyncStreams() async throws { + let changes = syncClient.db.syncCoordinator.streams.streamsChanged.subscribe() + for await change in changes { + self.localEvents.dispatch(event: .updateSubscriptions(streams: change)) + } + } } /// Wraps an HTTP response by mapping it to control invocations for lines. This also adds an "connection established" / "response ended" prefix and suffix. diff --git a/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift b/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift index c4d0b13..e9b1f96 100644 --- a/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift +++ b/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift @@ -1,5 +1,6 @@ /// Manages a connection task for a PowerSync database. actor SyncCoordinator { + nonisolated let streams = StreamTracker() private var activeSync: Task? func connect(db: KotlinPowerSyncDatabaseImpl, connector: PowerSyncBackendConnectorProtocol, options: ConnectOptions, client: HttpClient) async { @@ -25,11 +26,11 @@ actor SyncCoordinator { } /// Executes an inner function, but only if no connection is active or scheduled. - func guardNotConnected(inner: () async throws -> T, ifConnected: () throws -> Never) async rethrows -> T { + func guardNotConnected(inner: () async throws -> T, ifConnected: () async throws -> T) async rethrows -> T { if activeSync == nil { return try await inner(); } else { - try ifConnected() + return try await ifConnected() } } diff --git a/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift b/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift index 5958662..f22576f 100644 --- a/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift +++ b/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift @@ -9,7 +9,7 @@ final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, let logger: any LoggerProtocol private let kotlinDatabase: PowerSyncKotlin.PowerSyncDatabase private let encoder = JSONEncoder() - private let syncCoordinator = SyncCoordinator() + let syncCoordinator = SyncCoordinator() internal let syncStatus = SwiftSyncStatus() private let dbFilename: String private let httpClient: HttpClient @@ -53,14 +53,28 @@ final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, ) } + func resolveOfflineSyncStatusIfNotConnected() async throws { + try await syncCoordinator.guardNotConnected(inner: { + try await resolveOfflineSyncStatus() + }, ifConnected: {}) + } + + private func resolveOfflineSyncStatus() async throws { + let offlineSyncStatus = try await get("SELECT powersync_offline_sync_status()") { cursor in + let raw = try cursor.getString(index: 0) + return try StreamingSyncClient.jsonDecoder.decode(CoreDownloadSyncStatus.self, from: raw.data(using: .utf8)!) + } + + syncStatus.mutateStatus { $0 = MutableSyncStatus(core: offlineSyncStatus) } + } + func waitForFirstSync(priority: Int32) async { let priority = BucketPriority(priority) await syncStatus.waitFor { $0.statusForPriority(priority).hasSynced == true } } func syncStream(name: String, params: JsonParam?) -> any SyncStream { - let rawStream = kotlinDatabase.syncStream(name: name, parameters: params?.mapValues { $0.toKotlinMap() }) - fatalError("todo") + PendingSyncStream(db: self, name: name, parameters: params) } func connect( diff --git a/Sources/PowerSync/Kotlin/db/KotlinJsonParam.swift b/Sources/PowerSync/Kotlin/db/KotlinJsonParam.swift deleted file mode 100644 index b0751ad..0000000 --- a/Sources/PowerSync/Kotlin/db/KotlinJsonParam.swift +++ /dev/null @@ -1,49 +0,0 @@ -import PowerSyncKotlin - -/// Converts a Swift `JsonValue` to one accepted by the Kotlin SDK -extension JsonValue { - func toKotlinMap() -> PowerSyncKotlin.JsonParam { - switch self { - case .string(let value): - return PowerSyncKotlin.JsonParam.String(value: value) - case .int(let value): - return PowerSyncKotlin.JsonParam.Number(value: value) - case .double(let value): - return PowerSyncKotlin.JsonParam.Number(value: value) - case .bool(let value): - return PowerSyncKotlin.JsonParam.Boolean(value: value) - case .null: - return PowerSyncKotlin.JsonParam.Null() - case .array(let array): - return PowerSyncKotlin.JsonParam.Collection( - value: array.map { $0.toKotlinMap() } - ) - case .object(let dict): - var anyDict: [String: PowerSyncKotlin.JsonParam] = [:] - for (key, value) in dict { - anyDict[key] = value.toKotlinMap() - } - return PowerSyncKotlin.JsonParam.Map(value: anyDict) - } - } - - static func kotlinValueToJsonParam(raw: Any?) -> JsonValue { - if let string = raw as? String { - return Self.string(string) - } else if let bool = raw as? KotlinBoolean { - return Self.bool(bool.boolValue) - } else if let int = raw as? KotlinInt { - return Self.int(int.intValue) - } else if let double = raw as? KotlinDouble { - return Self.double(double.doubleValue) - } else if let array = raw as? [Any?] { - return Self.array(array.map(kotlinValueToJsonParam)) - } else if let object = raw as? [String: Any?] { - return Self.object(object.mapValues(kotlinValueToJsonParam)) - } else { - // fatalError is fine here because this function is internal, so this being reached - // is an SDK bug. - fatalError("fromValue must only be called on outputs of JsonValue.toValue()"); - } - } -} diff --git a/Sources/PowerSync/Protocol/PowerSyncDatabaseProtocol.swift b/Sources/PowerSync/Protocol/PowerSyncDatabaseProtocol.swift index a88020e..76c0d58 100644 --- a/Sources/PowerSync/Protocol/PowerSyncDatabaseProtocol.swift +++ b/Sources/PowerSync/Protocol/PowerSyncDatabaseProtocol.swift @@ -84,6 +84,10 @@ public struct ConnectOptions: Sendable { /// - SeeAlso: `SyncClientConfiguration` for available configuration options public var clientConfiguration: SyncClientConfiguration? + /// Whether streams that have been defined with `auto_subscribe: true` should be synced even + /// when they don't have an explicit subscription. + public var includeDefaultStreams: Bool + /// Initializes a `ConnectOptions` instance with optional values. /// /// - Parameters: @@ -96,7 +100,8 @@ public struct ConnectOptions: Sendable { retryDelay: TimeInterval = 5, params: JsonParam = [:], clientConfiguration: SyncClientConfiguration? = nil, - appMetadata: [String: String] = [:] + appMetadata: [String: String] = [:], + includeDefaultStreams: Bool = true ) { self.crudThrottle = crudThrottle self.retryDelay = retryDelay @@ -104,6 +109,7 @@ public struct ConnectOptions: Sendable { newClientImplementation = true self.clientConfiguration = clientConfiguration self.appMetadata = appMetadata + self.includeDefaultStreams = includeDefaultStreams } /// Initializes a ``ConnectOptions`` instance with optional values, including experimental options. @@ -118,7 +124,8 @@ public struct ConnectOptions: Sendable { params: JsonParam = [:], newClientImplementation: Bool = true, clientConfiguration: SyncClientConfiguration? = nil, - appMetadata: [String: String] = [:] + appMetadata: [String: String] = [:], + includeDefaultStreams: Bool = true ) { self.crudThrottle = crudThrottle self.retryDelay = retryDelay @@ -126,6 +133,7 @@ public struct ConnectOptions: Sendable { self.newClientImplementation = newClientImplementation self.clientConfiguration = clientConfiguration self.appMetadata = appMetadata + self.includeDefaultStreams = true } } diff --git a/Sources/PowerSync/Protocol/db/JsonParam.swift b/Sources/PowerSync/Protocol/db/JsonParam.swift index 87ea5a6..3c6fcb7 100644 --- a/Sources/PowerSync/Protocol/db/JsonParam.swift +++ b/Sources/PowerSync/Protocol/db/JsonParam.swift @@ -2,7 +2,7 @@ /// /// Supports all standard JSON types: string, number (integer and double), /// boolean, null, arrays, and nested objects. -public enum JsonValue: Codable, Sendable, Equatable { +public enum JsonValue: Codable, Sendable, Equatable, Hashable { /// A JSON string value. case string(String) diff --git a/Tests/PowerSyncTests/SyncTests.swift b/Tests/PowerSyncTests/SyncTests.swift index e87ef0c..dccddb6 100644 --- a/Tests/PowerSyncTests/SyncTests.swift +++ b/Tests/PowerSyncTests/SyncTests.swift @@ -410,6 +410,217 @@ class InMemorySyncIntegrationTests { try #require(logEntries.contains("Starting request to POST https://powersynctest.example.org/sync/stream")) try #require(logEntries.contains(#"Response line: {"checkpoint_complete":{"last_op_id":"0"}}"#)) } + + @Test func canDisableDefaultStreams() async throws { + let didConnect = Signal() + let db = openDatabase(MockHttpClient { request in + let body = try StreamingSyncClient.jsonDecoder.decode(JsonParam.self, from: try #require(request.httpBody)) + if case let .object(options) = body["streams"] { + try #require(options["include_defaults"] == .bool(false)) + } else { + Issue.record("Should have streams key in body") + } + + await didConnect.complete() + return AsyncThrowingChannel() + }) + + try await db.connect(connector: TestConnector(), options: ConnectOptions( + includeDefaultStreams: false + )) + await didConnect.await() + } + + @Test func subscribesWithStreams() async throws { + let channel = AsyncThrowingChannel() + let db = openDatabase(MockHttpClient { request in + let body = try StreamingSyncClient.jsonDecoder.decode(JsonParam.self, from: try #require(request.httpBody)) + if case let .object(streams) = body["streams"] { + try #require(streams["include_defaults"] == .bool(true)) + try #require(streams["subscriptions"] == .array([ + .object([ + "stream": .string("stream"), + "parameters": .object(["foo": .string("a")]), + "override_priority": .null + ]), + .object([ + "stream": .string("stream"), + "parameters": .object(["foo": .string("b")]), + "override_priority": .int(1) + ]) + ])) + } else { + Issue.record("Should have streams key in body") + } + + return channel + }) + + let a = try await db.syncStream(name:"stream", params: ["foo": .string("a")]).subscribe() + let b = try await db.syncStream(name: "stream", params: ["foo": .string("b")]).subscribe(ttl: nil, priority: .init(1)) + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + await waitForStatus(db.currentStatus) { $0.connected } + var statusUpdates = db.currentStatus.asFlow().makeAsyncIterator() + + // Without an initial checkpoint, sync streams should not be marked as active + try #require(db.currentStatus.forStream(stream: a)?.subscription.hasSynced == false) + try #require(db.currentStatus.forStream(stream: b)?.subscription.hasSynced == false) + + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "1", buckets: [ + BucketChecksum( + bucket: "a", + priority: BucketPriority(3), + checksum: 0, + subscriptions: [.explicitSubscription(0)] + ), + BucketChecksum( + bucket: "b", + priority: BucketPriority(1), + checksum: 0, + subscriptions: [.explicitSubscription(1)] + ) + ], streams: [StreamDescription(name: "stream", is_default: false)]))) + + // Subscriptions should be active now, but not marked as synced + do { + let status = try #require(await statusUpdates.next()) + for subscription in [a, b] { + let status = try #require(status.forStream(stream: subscription)) + try #require(status.subscription.active) + try #require(status.subscription.lastSyncedAt == nil) + try #require(status.subscription.hasExplicitSubscription) + } + } + + try await channel.pushLine(.checkpointPartiallyComplete(lastOpId: "0", priority: BucketPriority(1))) + do { + let status = try #require(await statusUpdates.next()) + try #require(status.forStream(stream: a)!.subscription.lastSyncedAt == nil) + try #require(status.forStream(stream: b)!.subscription.lastSyncedAt != nil) + try await b.waitForFirstSync() + } + + try await channel.pushLine(.checkpointComplete(lastOpId: "0")) + try await a.waitForFirstSync() + } + + @Test func reportsDefaultStreams() async throws { + let channel = AsyncThrowingChannel() + let db = openDatabase(MockHttpClient { request in channel }) + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + + await waitForStatus(db.currentStatus) { $0.connected } + var statusUpdates = db.currentStatus.asFlow().makeAsyncIterator() + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "0", buckets: [], streams: [StreamDescription(name: "default_stream", is_default: true)]))) + + let status = try #require(await statusUpdates.next()) + let stream = try #require(status.syncStreams?.first) + try #require(stream.subscription.name == "default_stream") + try #require(stream.subscription.parameters == nil) + try #require(stream.subscription.isDefault) + try #require(!stream.subscription.hasExplicitSubscription) + } + + @Test func changesSubscriptionsDynamically() async throws { + let lastRequest = AsyncMutex(nil) + let db = openDatabase(MockHttpClient { request in + let body = try StreamingSyncClient.jsonDecoder.decode(JsonParam.self, from: try #require(request.httpBody)) + await lastRequest.withMutex { $0 = body } + return AsyncThrowingChannel() + }) + + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + await waitForStatus(db.currentStatus) { $0.connected } + let request = try #require(await lastRequest.inner) + if case let .object(streams) = request["streams"] { + try #require(streams["subscriptions"] == .array([])) + } else { + Issue.record("Should have streams key in body") + } + + // Adding a new subscription should reconnect + let subscription = try await db.syncStream(name: "a", params: nil).subscribe() + await waitForStatus(db.currentStatus) { !$0.connected } + await waitForStatus(db.currentStatus) { $0.connected } + let secondRequest = try #require(await lastRequest.inner) + if case let .object(streams) = secondRequest["streams"] { + try #require(streams["subscriptions"] == .array([ + .object([ + "stream": .string("a"), + "parameters": .null, + "override_priority": .null, + ]) + ])) + } else { + Issue.record("Should have streams key in body") + } + let _ = consume subscription + } + + @Test func subscriptionsUpdateWhileOffline() async throws { + let db = openDatabase(PlatformHttpClient.shared) + var statusUpdates = db.currentStatus.asFlow().makeAsyncIterator() + + // Subscribing while offline should add the stream to subscriptions reported in the status. + let subscription = try await db.syncStream(name: "a", params: nil).subscribe() + let status = try #require(await statusUpdates.next()) + let _ = try #require(status.forStream(stream: subscription)) + } + + @Test func unsubscribingMultipleTimesHasNoEffect() async throws { + let db = openDatabase(MockHttpClient { request in + let body = try StreamingSyncClient.jsonDecoder.decode(JsonParam.self, from: try #require(request.httpBody)) + if case let .object(streams) = body["streams"] { + try #require(streams["subscriptions"] == .array([ + .object([ + "stream": .string("a"), + "parameters": .null, + "override_priority": .null + ]), + ])) + } else { + Issue.record("Should have streams key in body") + } + + return AsyncThrowingChannel() + }) + + let a = try await db.syncStream(name: "a", params: nil).subscribe() + let aAgain = try await db.syncStream(name: "a", params: nil).subscribe() + try await a.unsubscribe() + try await a.unsubscribe() + + // Pretend the streams are expired, they should still be requested because the + // core extension extends the lifetime of streams currently referenced before connecting + try await db.execute("UPDATE ps_stream_subscriptions SET expires_at = unixepoch() - 1000") + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + await waitForStatus(db.currentStatus) { $0.connected } + + let _ = consume aAgain + } + + @Test func unsubscribeAll() async throws { + let didConnect = Signal() + let db = openDatabase(MockHttpClient { request in + let body = try StreamingSyncClient.jsonDecoder.decode(JsonParam.self, from: try #require(request.httpBody)) + if case let .object(streams) = body["streams"] { + // While we did request a stream, we called unsubscribeAll() before connecting. So it should not + // be part of the request. + try #require(streams["subscriptions"] == .array([])) + } else { + Issue.record("Should have streams key in body") + } + + await didConnect.complete() + return AsyncThrowingChannel() + }) + + let a = try await db.syncStream(name: "a", params: nil).subscribe() + try await db.syncStream(name: "a", params: nil).unsubscribeAll() + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + await didConnect.await() + let _ = consume a + } } let defaultSchema = Schema(tables: [ @@ -421,7 +632,7 @@ let defaultSchema = Schema(tables: [ ), ]) -private func openDatabase(_ client: MockHttpClient, schema: Schema = defaultSchema) -> PowerSyncDatabaseProtocol { +private func openDatabase(_ client: any HttpClient, schema: Schema = defaultSchema) -> PowerSyncDatabaseProtocol { return openKotlinDBDefault( schema: schema, dbFilename: ":memory:", diff --git a/Tests/PowerSyncTests/test-utils/SyncProtocol.swift b/Tests/PowerSyncTests/test-utils/SyncProtocol.swift index f99b1e2..8e254bc 100644 --- a/Tests/PowerSyncTests/test-utils/SyncProtocol.swift +++ b/Tests/PowerSyncTests/test-utils/SyncProtocol.swift @@ -76,20 +76,29 @@ struct Checkpoint { var last_op_id: String var buckets: [BucketChecksum] var writeCheckpoint: String? = nil + var streams: [StreamDescription] = [] enum CodingKeys: String, CodingKey { case last_op_id case buckets case writeCheckpoint = "write_checkpoint" + case streams } func encodeToContainer(_ container: inout KeyedEncodingContainer) throws { try container.encode(self.last_op_id, forKey: .last_op_id) try container.encode(self.buckets, forKey: .buckets) try container.encode(self.writeCheckpoint, forKey: .writeCheckpoint) + try container.encode(self.streams, forKey: .streams) } } +struct StreamDescription: Encodable { + var name: String + var is_default: Bool + var errors: [Never] = [] +} + struct OplogEntry: Encodable { var checksum: Int32 var op_id: String @@ -113,6 +122,7 @@ struct BucketChecksum: Encodable { var checksum: Int32 var count: Int? = nil var lastOpId: String? = nil + var subscriptions: [BucketSubscriptionReason]? = nil enum CodingKeys: String, CodingKey { case bucket @@ -120,6 +130,7 @@ struct BucketChecksum: Encodable { case checksum case count case last_op_id = "last_op_id" + case subscriptions } func encode(to encoder: any Encoder) throws { @@ -129,6 +140,27 @@ struct BucketChecksum: Encodable { try container.encode(self.checksum, forKey: .checksum) try container.encode(self.count, forKey: .count) try container.encode(self.lastOpId, forKey: .last_op_id) + try container.encodeIfPresent(self.subscriptions, forKey: .subscriptions) + } +} + +enum BucketSubscriptionReason: Encodable { + case defaultStream(Int) + case explicitSubscription(Int) + + enum CodingKeys: String, CodingKey { + case defaultStream = "default" + case explicitSubscription = "sub" + } + + func encode(to encoder: any Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .defaultStream(let idx): + try container.encode(idx, forKey: .defaultStream) + case .explicitSubscription(let idx): + try container.encode(idx, forKey: .explicitSubscription) + } } } From e7cf25f59dfaa3d9ec2e67eaf0bad3883d35e350 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 21 Apr 2026 15:15:07 +0200 Subject: [PATCH 14/40] Fix tests --- .../Implementation/sync/Status.swift | 6 +- Sources/PowerSync/Utils/BroadcastStream.swift | 9 +- Tests/PowerSyncTests/ConnectTests.swift | 157 ------------------ Tests/PowerSyncTests/SyncTests.swift | 3 + 4 files changed, 12 insertions(+), 163 deletions(-) diff --git a/Sources/PowerSync/Implementation/sync/Status.swift b/Sources/PowerSync/Implementation/sync/Status.swift index bc7c66e..9989bb3 100644 --- a/Sources/PowerSync/Implementation/sync/Status.swift +++ b/Sources/PowerSync/Implementation/sync/Status.swift @@ -132,14 +132,10 @@ final class SwiftSyncStatus: SyncStatus { } func asFlow() -> AsyncStream { - self.listeners.subscribe() + self.listeners.subscribe(addInitial: self) } func waitFor(_ predicate: (borrowing SwiftSyncStatus) -> Bool) async { - if predicate(self) { - return - } - for await _ in self.asFlow() { if predicate(self) { return diff --git a/Sources/PowerSync/Utils/BroadcastStream.swift b/Sources/PowerSync/Utils/BroadcastStream.swift index cbc4690..63f3620 100644 --- a/Sources/PowerSync/Utils/BroadcastStream.swift +++ b/Sources/PowerSync/Utils/BroadcastStream.swift @@ -21,8 +21,15 @@ final class BroadcastStream: Sendable { } } - func subscribe(bufferingPolicy: AsyncStream.Continuation.BufferingPolicy = .unbounded) -> AsyncStream { + func subscribe( + bufferingPolicy: AsyncStream.Continuation.BufferingPolicy = .unbounded, + addInitial: T? = nil + ) -> AsyncStream { return AsyncStream(bufferingPolicy: bufferingPolicy) { continuation in + if let addInitial { + continuation.yield(addInitial) + } + self.register(continuation: continuation) } } diff --git a/Tests/PowerSyncTests/ConnectTests.swift b/Tests/PowerSyncTests/ConnectTests.swift index 991c3d4..6c7f2fe 100644 --- a/Tests/PowerSyncTests/ConnectTests.swift +++ b/Tests/PowerSyncTests/ConnectTests.swift @@ -33,163 +33,6 @@ final class ConnectTests: XCTestCase { try await super.tearDown() } - /// Tests passing basic JSON as client parameters - func testClientParameters() async throws { - /// This is an example of specifying JSON client params. - /// The test here just ensures that the Kotlin SDK accepts these params and does not crash - try await database.connect( - connector: MockConnector(), - params: [ - "foo": .string("bar"), - ] - ) - } - - func testSyncStatus() async throws { - XCTAssert(database.currentStatus.connected == false) - XCTAssert(database.currentStatus.connecting == false) - - try await database.connect( - connector: MockConnector() - ) - - try await waitFor(timeout: 10) { - guard database.currentStatus.connecting == true else { - throw WaitForMatchError.predicateFail(message: "Should be connecting") - } - } - - try await database.disconnect() - - try await waitFor(timeout: 10) { - guard database.currentStatus.connecting == false else { - throw WaitForMatchError.predicateFail(message: "Should not be connecting after disconnect") - } - } - } - - func testSyncStatusUpdates() async throws { - let expectation = XCTestExpectation( - description: "Watch Sync Status" - ) - - let watchTask = Task { [database] in - for try await _ in database!.currentStatus.asFlow() { - expectation.fulfill() - } - } - - // Do some connecting operations - try await database.connect( - connector: MockConnector() - ) - - // We should get an update - await fulfillment(of: [expectation], timeout: 5) - watchTask.cancel() - } - - func testSyncHTTPLogs() async throws { - let expectation = XCTestExpectation( - description: "Should log a request to the PowerSync endpoint" - ) - - let fakeUrl = "https://fakepowersyncinstance.fakepowersync.local" - - final class TestConnector: PowerSyncBackendConnectorProtocol { - let url: String - - init(url: String) { - self.url = url - } - - func fetchCredentials() async throws -> PowerSyncCredentials? { - PowerSyncCredentials( - endpoint: url, - token: "123" - ) - } - - func uploadData(database _: PowerSyncDatabaseProtocol) async throws {} - } - - try await database.connect( - connector: TestConnector(url: fakeUrl), - options: ConnectOptions( - /// Note that currently, HTTP logs are only supported with the old client implementation - /// which uses HTTP streams. - /// The new client implementation uses a WebSocket connection instead. - /// Which we don't get logs for currently. - newClientImplementation: false, - clientConfiguration: SyncClientConfiguration( - requestLogger: SyncRequestLoggerConfiguration( - requestLevel: .all - ) { message in - // We want to see a request to the specified instance - if message.contains(fakeUrl) { - expectation.fulfill() - } - } - ) - ) - ) - - await fulfillment(of: [expectation], timeout: 5) - - try await database.disconnectAndClear() - } - - func testAppMetadata() async throws { - let expectation = XCTestExpectation( - description: "Should log a request to the PowerSync endpoint with metadata" - ) - - let fakeUrl = "https://fakepowersyncinstance.fakepowersync.local" - let testAppName = "testAppName" - final class TestConnector: PowerSyncBackendConnectorProtocol { - let url: String - - init(url: String) { - self.url = url - } - - func fetchCredentials() async throws -> PowerSyncCredentials? { - PowerSyncCredentials( - endpoint: url, - token: "123" - ) - } - - func uploadData(database _: PowerSyncDatabaseProtocol) async throws {} - } - - try await database.connect( - connector: TestConnector(url: fakeUrl), - options: ConnectOptions( - /// Note that currently, HTTP logs are only supported with the old client implementation - /// which uses HTTP streams. - /// The new client implementation uses a WebSocket connection instead. - /// Which we don't get logs for currently. - newClientImplementation: false, - clientConfiguration: SyncClientConfiguration( - requestLogger: SyncRequestLoggerConfiguration( - requestLevel: .all - ) { message in - // We want to see a request to the specified instance with the app_metadata present - if message.contains("\"app_metadata\":{\"appName\":\"\(testAppName)\"}") { - expectation.fulfill() - } - } - ), - appMetadata: ["appName": testAppName] - ) - ) - - await fulfillment(of: [expectation], timeout: 5) - - try await database.disconnectAndClear() - } - func testSendableConnect() async throws { /// This is just a basic sanity check to confirm that these protocols are /// correctly defined as Sendable. diff --git a/Tests/PowerSyncTests/SyncTests.swift b/Tests/PowerSyncTests/SyncTests.swift index dccddb6..bdffb78 100644 --- a/Tests/PowerSyncTests/SyncTests.swift +++ b/Tests/PowerSyncTests/SyncTests.swift @@ -369,6 +369,7 @@ class InMemorySyncIntegrationTests { try await db.connect(connector: TestConnector(), options: ConnectOptions()) await waitForStatus(db.currentStatus) { $0.connected } var status = db.currentStatus.asFlow().makeAsyncIterator() + let _ = await status.next() // Skip initial // Send checkpoint with 10 ops, progress should be 0/10 try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "10", buckets: [BucketChecksum(bucket: "a", checksum: 0, count: 10)]))) @@ -461,6 +462,7 @@ class InMemorySyncIntegrationTests { try await db.connect(connector: TestConnector(), options: ConnectOptions()) await waitForStatus(db.currentStatus) { $0.connected } var statusUpdates = db.currentStatus.asFlow().makeAsyncIterator() + let _ = await statusUpdates.next() // Skip initial // Without an initial checkpoint, sync streams should not be marked as active try #require(db.currentStatus.forStream(stream: a)?.subscription.hasSynced == false) @@ -511,6 +513,7 @@ class InMemorySyncIntegrationTests { await waitForStatus(db.currentStatus) { $0.connected } var statusUpdates = db.currentStatus.asFlow().makeAsyncIterator() + let _ = await statusUpdates.next() // Skip initial try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "0", buckets: [], streams: [StreamDescription(name: "default_stream", is_default: true)]))) let status = try #require(await statusUpdates.next()) From 4b459969f7fa256fa28613785055171439fe5bb5 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 21 Apr 2026 15:53:39 +0200 Subject: [PATCH 15/40] Trigger crud uploads --- .../sync/StreamingSyncClient.swift | 28 +++++++++-- Tests/PowerSyncTests/SyncTests.swift | 47 ++++++++++++++++--- 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift index 18620df..033e191 100644 --- a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift +++ b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift @@ -31,8 +31,9 @@ final class StreamingSyncClient: Sendable { Task(name: "StreamingSyncClient.run") { let signals = SyncSignals() async let download = downloadLoop(signals: signals) + async let upload = uploadLoop(signals: signals) - let _ = try await download + let _ = try await (download, upload) } } @@ -45,6 +46,9 @@ final class StreamingSyncClient: Sendable { for try await _ in allTriggers { try await uploadAllCrud() + + db.logger.debug("crud upload: notify completion", tag: tag) + signals.notifyCrudUploadComplete() } } @@ -55,7 +59,7 @@ final class StreamingSyncClient: Sendable { defer { db.syncStatus.mutateStatus { $0.uploading = false } } do { - let nextItem = try await db.getOptional("SELECT id FROM ps_crud ORDER BY id LIMIT 1", mapper: { cursor in try cursor.getInt64(index: 1) }) + let nextItem = try await db.getOptional("SELECT id FROM ps_crud ORDER BY id LIMIT 1", mapper: { cursor in try cursor.getInt64(index: 0) }) if let nextItem { if nextItem == lastUploadItem { db.logger.warning(""" @@ -97,7 +101,7 @@ The next upload iteration will be delayed. private func uploadLocalTarget() async throws { guard let _ = try await db.getOptional( - sql: "SELECT 1 FROM ps_bucket WHERE name = '$local' AND target_op = ?", + sql: "SELECT 1 FROM ps_buckets WHERE name = '$local' AND target_op = ?", parameters: [KotlinPowerSyncDatabaseImpl.maxOpId], mapper: { cursor in () } ) else { @@ -227,7 +231,11 @@ private struct ActiveSyncIteration: Sendable { } func run() async throws -> SyncIterationResult { + // Notify the core extension for changed Sync Stream subscriptions, as we might have to reconnect. async let _ = watchSyncStreams() + // Notify the core extension for completed crud uploads, as we might want to retry applying a + // checkpoint in that case. + async let _ = watchCompletedCrudUploads() let initialInstructions = try await powersyncControl(.start(StartSyncIteration( parameters: syncClient.options.params, @@ -344,6 +352,13 @@ private struct ActiveSyncIteration: Sendable { self.localEvents.dispatch(event: .updateSubscriptions(streams: change)) } } + + private func watchCompletedCrudUploads() async throws { + let uploads = signals.signalCrudUploadComplete.subscribe() + for await _ in uploads { + self.localEvents.dispatch(event: .completedUpload) + } + } } /// Wraps an HTTP response by mapping it to control invocations for lines. This also adds an "connection established" / "response ended" prefix and suffix. @@ -395,10 +410,15 @@ private struct SyncIterationResult { private struct SyncSignals { let signalCrudUpload = BroadcastStream() - + let signalCrudUploadComplete = BroadcastStream() + func triggerAsyncCrudUpload() { self.signalCrudUpload.dispatch(event: ()) } + + func notifyCrudUploadComplete() { + self.signalCrudUploadComplete.dispatch(event: ()) + } } struct WriteCheckpointData: Codable { diff --git a/Tests/PowerSyncTests/SyncTests.swift b/Tests/PowerSyncTests/SyncTests.swift index bdffb78..5b5f325 100644 --- a/Tests/PowerSyncTests/SyncTests.swift +++ b/Tests/PowerSyncTests/SyncTests.swift @@ -158,10 +158,31 @@ class InMemorySyncIntegrationTests { await waitForStatus(db.currentStatus) { !$0.connected } await waitForStatus(db.currentStatus) { $0.connected } } - - // TODO: "handles checkpoints during uploads" test - - // TODO: "handles write made while offline" test + + @Test func uploadsOfflineWrites() async throws { + let channel = AsyncThrowingChannel() + let mockClient = MockHttpClient { request in channel } + let db = openDatabase(mockClient) + mockClient.writeCheckpoint.store(1, ordering: .sequentiallyConsistent) + + try await db.execute(sql: "INSERT INTO users (id, name) VALUES (uuid(), ?)", parameters: ["local write"]) + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + + var query = try db.watch("SELECT name FROM users") { try $0.getString(index: 0) }.makeAsyncIterator() + try #require(try await query.next() == ["local write"]) + + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "1", buckets: [BucketChecksum(bucket: "a", checksum: 0)], writeCheckpoint: "1"))) + try await channel.pushLine(.syncDataBucket(SyncDataBucket(bucket: "a", data: [OplogEntry( + checksum: 0, + op_id: "1", + object_id: "1", + object_type: "users", + op: .put, + data: #"{"id": "test1", "name": "from server"}"#, + )]))) + try await channel.pushLine(.checkpointComplete(lastOpId: "1")) + try #require(try await query.next() == ["from server"]) + } @Test func tokenExpired() async throws { final class BackendConnector: PowerSyncBackendConnectorProtocol { @@ -635,11 +656,11 @@ let defaultSchema = Schema(tables: [ ), ]) -private func openDatabase(_ client: any HttpClient, schema: Schema = defaultSchema) -> PowerSyncDatabaseProtocol { +private func openDatabase(_ client: any HttpClient, schema: Schema = defaultSchema, logger: any LoggerProtocol = DefaultLogger()) -> PowerSyncDatabaseProtocol { return openKotlinDBDefault( schema: schema, dbFilename: ":memory:", - logger: DatabaseLogger(DefaultLogger()), + logger: DatabaseLogger(logger), httpClient: client ) } @@ -650,11 +671,23 @@ let testCredentials = PowerSyncCredentials( ) private final class TestConnector: PowerSyncBackendConnectorProtocol { + private let uploadDataCallback: @Sendable (_ database: any PowerSyncDatabaseProtocol) async throws -> () + + init( + uploadDataCallback: @Sendable @escaping (_: any PowerSyncDatabaseProtocol) async throws -> Void = { db in + let tx = try await db.getNextCrudTransaction() + try await tx?.complete() + }) { + self.uploadDataCallback = uploadDataCallback + } + func fetchCredentials() async throws -> PowerSyncCredentials? { return testCredentials } - func uploadData(database _: any PowerSync.PowerSyncDatabaseProtocol) async throws {} + func uploadData(database: any PowerSync.PowerSyncDatabaseProtocol) async throws { + try await self.uploadDataCallback(database) + } } private final class Signal: Sendable { From bbf9027e536045aa8399da1797c0cc0d3c337d87 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 21 Apr 2026 18:02:14 +0200 Subject: [PATCH 16/40] Use latest xcode --- .github/workflows/build_and_test.yaml | 4 ++-- .github/workflows/docs.yaml | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 4d2a163..904b50f 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -7,10 +7,10 @@ on: jobs: build: name: Build and test - runs-on: macos-latest + runs-on: macos-26 timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up XCode uses: maxim-lobanov/setup-xcode@v1 with: diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 3d3434d..8556eca 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -11,9 +11,13 @@ permissions: jobs: build: name: Build - runs-on: macos-latest + runs-on: macos-26 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 + - name: Set up XCode + uses: maxim-lobanov/setup-xcode@v1 + with: + xcode-version: latest-stable - name: Build Docs run: | xcodebuild docbuild \ From 1ab3ab90e8187593d9c27da392787f9fa878c895 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 21 Apr 2026 18:14:48 +0200 Subject: [PATCH 17/40] Avoid updating status on end of upload attempt --- .../Implementation/sync/Status.swift | 24 +++++++++++++++---- .../sync/StreamingSyncClient.swift | 12 ++++++---- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/Sources/PowerSync/Implementation/sync/Status.swift b/Sources/PowerSync/Implementation/sync/Status.swift index 9989bb3..fe20e7f 100644 --- a/Sources/PowerSync/Implementation/sync/Status.swift +++ b/Sources/PowerSync/Implementation/sync/Status.swift @@ -123,14 +123,28 @@ final class SwiftSyncStatus: SyncStatus { } internal func mutateStatus(update: (_ status: inout MutableSyncStatus) -> Void) { - self.current.withLock { - update(&$0.inner) - $0.snapshot = SyncStatusDataImpl(status: $0.inner) + maybeMutateStatus(shouldUpdate: { _ in true }, apply: update) + } + + internal func maybeMutateStatus( + shouldUpdate: (_ status: borrowing MutableSyncStatus) -> Bool, + apply: (_ status: inout MutableSyncStatus) -> Void + ) { + let didUpdate = self.current.withLock { + if shouldUpdate($0.inner) { + apply(&$0.inner) + $0.snapshot = SyncStatusDataImpl(status: $0.inner) + return true + } else { + return false + } } - self.listeners.dispatch(event: self) + if didUpdate { + self.listeners.dispatch(event: self) + } } - + func asFlow() -> AsyncStream { self.listeners.subscribe(addInitial: self) } diff --git a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift index 033e191..7ad5918 100644 --- a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift +++ b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift @@ -56,7 +56,9 @@ final class StreamingSyncClient: Sendable { var lastUploadItem: Int64? = nil while (true) { - defer { db.syncStatus.mutateStatus { $0.uploading = false } } + defer { + db.syncStatus.maybeMutateStatus(shouldUpdate: { $0.uploading }, apply: { $0.uploading = false }) + } do { let nextItem = try await db.getOptional("SELECT id FROM ps_crud ORDER BY id LIMIT 1", mapper: { cursor in try cursor.getInt64(index: 0) }) @@ -79,14 +81,14 @@ The next upload iteration will be delayed. break } } catch { + if error is CancellationError { + return + } + db.syncStatus.mutateStatus { $0.uploading = false $0.internalUploadError = error } - - if error is CancellationError { - return - } db.logger.error("Error uploading crud: \(error)", tag: tag) do { From 3e0abdb5dec5abc4c38982cb5d61c83e4fc96ce7 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 21 Apr 2026 22:32:21 +0200 Subject: [PATCH 18/40] Support old platform versions again --- Package.swift | 6 +-- .../Implementation/sync/HttpClient.swift | 44 +++++++++++++----- .../sync/StreamingSyncClient.swift | 45 +++++++++++-------- Sources/PowerSync/Utils/Mutex.swift | 28 ++++++++++++ Tests/PowerSyncTests/SyncTests.swift | 13 +++--- .../test-utils/MockHttpClient.swift | 35 ++++++++++++--- 6 files changed, 127 insertions(+), 44 deletions(-) create mode 100644 Sources/PowerSync/Utils/Mutex.swift diff --git a/Package.swift b/Package.swift index d5bdd4a..e36d619 100644 --- a/Package.swift +++ b/Package.swift @@ -55,9 +55,9 @@ if let corePath = localCoreExtension { let package = Package( name: packageName, platforms: [ - .iOS(.v18), - .macOS(.v15), - .watchOS(.v11), + .iOS(.v15), + .macOS(.v12), + .watchOS(.v9), .tvOS(.v15), ], products: [ diff --git a/Sources/PowerSync/Implementation/sync/HttpClient.swift b/Sources/PowerSync/Implementation/sync/HttpClient.swift index 21dd101..5c2c760 100644 --- a/Sources/PowerSync/Implementation/sync/HttpClient.swift +++ b/Sources/PowerSync/Implementation/sync/HttpClient.swift @@ -2,10 +2,16 @@ import Foundation /// An internal protocol for HTTP clients, we use this to mock clients in tests. protocol HttpClient: Sendable { - func receiveSyncLines(request: URLRequest) async throws -> (HTTPURLResponse, any AsyncSequence & Sendable) + func receiveSyncLines(request: URLRequest) async throws -> (HTTPURLResponse, any SyncLineResponse) func readFully(request: URLRequest) async throws -> (HTTPURLResponse, Data) } +protocol SyncLineResponse: Sendable, AsyncSequence where AsyncIterator: SyncLineResponseIterator {} + +protocol SyncLineResponseIterator: AsyncIteratorProtocol { + mutating func next() async throws -> SyncLine? +} + enum SyncLine { case text(contents: String) // In the future, we might also want to support splitting BSON objects. Currently, we stream JSON only. @@ -15,16 +21,15 @@ enum SyncLine { struct PlatformHttpClient: HttpClient { let session: URLSession - func receiveSyncLines(request: URLRequest) async throws -> (HTTPURLResponse, any AsyncSequence & Sendable) { + func receiveSyncLines(request: URLRequest) async throws -> (HTTPURLResponse, any SyncLineResponse) { let (bytes, response) = try await session.bytes(for: request) let jsonStreamMimeType = "application/x-ndjson" if response.mimeType != jsonStreamMimeType { throw PowerSyncError.operationFailed(message: "Invalid sync lines response, (expected \(jsonStreamMimeType), got \(response.mimeType, default: "")") } - - let syncLines = bytes.lines.map { line in SyncLine.text(contents: line) } - return (response as! HTTPURLResponse, syncLines) + + return (response as! HTTPURLResponse, PlatformSyncLineResponse(lines: bytes.lines)) } func readFully(request: URLRequest) async throws -> (HTTPURLResponse, Data) { @@ -35,6 +40,25 @@ struct PlatformHttpClient: HttpClient { static let shared = PlatformHttpClient(session: .shared) } +private struct PlatformSyncLineResponse: SyncLineResponse where Base : AsyncSequence, Base.Element == UInt8, Base: Sendable { + let lines: AsyncLineSequence + + func makeAsyncIterator() -> some SyncLineResponseIterator { + return PlatformSyncLineResponseIterator(inner: lines.makeAsyncIterator()) + } +} + +private struct PlatformSyncLineResponseIterator: SyncLineResponseIterator where Base : AsyncSequence, Base.Element == UInt8, Base: Sendable { + typealias Element = SyncLine + + var inner: AsyncLineSequence.AsyncIterator + + mutating func next() async throws -> SyncLine? { + let line = try await inner.next() + return line.map { SyncLine.text(contents: $0) } + } +} + struct LoggingClient: HttpClient { let inner: HttpClient let logger: SyncRequestLoggerConfiguration @@ -51,7 +75,7 @@ struct LoggingClient: HttpClient { logger.requestLevel == .all || logger.requestLevel == .body } - func receiveSyncLines(request: URLRequest) async throws -> (HTTPURLResponse, any Sendable & AsyncSequence) { + func receiveSyncLines(request: URLRequest) async throws -> (HTTPURLResponse, any SyncLineResponse) { logRequest(request: request) do { let (response, lines) = try await inner.receiveSyncLines(request: request) @@ -118,20 +142,20 @@ struct LoggingClient: HttpClient { } } -private struct LogSyncLines: AsyncSequence, Sendable { +private struct LogSyncLines: SyncLineResponse, Sendable { typealias AsyncIterator = LogSyncLinesIterator let logger: LoggingClient - let inner: any AsyncSequence & Sendable + let inner: any SyncLineResponse func makeAsyncIterator() -> LogSyncLinesIterator { LogSyncLinesIterator(logger: logger, inner: inner.makeAsyncIterator()) } } -private struct LogSyncLinesIterator: AsyncIteratorProtocol { +private struct LogSyncLinesIterator: SyncLineResponseIterator { let logger: LoggingClient - var inner: any AsyncIteratorProtocol + var inner: any SyncLineResponseIterator mutating func next() async throws -> SyncLine? { let line = try await self.inner.next() diff --git a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift index 7ad5918..59f122e 100644 --- a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift +++ b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift @@ -92,7 +92,7 @@ The next upload iteration will be delayed. db.logger.error("Error uploading crud: \(error)", tag: tag) do { - try await Task.sleep(for: .seconds(self.options.retryDelay)) + try await sleepForSeconds(seconds: self.options.retryDelay) } catch { // Cancelled, abort return @@ -136,9 +136,8 @@ The next upload iteration will be delayed. private func getWriteCheckpoint() async throws -> String { let clientId = try await db.get("SELECT powersync_client_id()") { try $0.getString(index: 0) } let (_, request) = try await authenticatedRequest { endpoint in - endpoint - .appending(path: "write-checkpoint2.json") - .appending(queryItems: [.init(name: "client_id", value: clientId)]) + endpoint.path += "/write-checkpoint2.json" + endpoint.queryItems = [.init(name: "client_id", value: clientId)] } let (response, data) = try await httpClient.readFully(request: request) @@ -172,7 +171,7 @@ The next upload iteration will be delayed. if !result.hideDisconnect { do { - try await Task.sleep(for: .seconds(options.retryDelay)) + try await sleepForSeconds(seconds: options.retryDelay) } catch { // Cancelled break @@ -185,15 +184,19 @@ The next upload iteration will be delayed. await self.connector.invalidateCachedCredentials() } - private func authenticatedRequest(buildUrl: (URL) -> URL) async throws -> (URL, URLRequest) { + private func authenticatedRequest(buildUrl: (inout URLComponents) -> ()) async throws -> (URL, URLRequest) { guard let credentials = try await connector.fetchCredentials() else { throw PowerSyncError.operationFailed(message: "fetchCredentials() returned nil") } - guard let base = URL(string: credentials.endpoint) else { + guard var base = URLComponents(string: credentials.endpoint) else { throw PowerSyncError.operationFailed(message: "Invalid backend connector URL: \(credentials.endpoint)") } - let url = buildUrl(base) + buildUrl(&base) + guard let url = base.url else { + throw PowerSyncError.operationFailed(message: "Invalid resolved backend connector URL: \(base)") + } + var request = URLRequest(url: url) request.setValue("Token \(credentials.token)", forHTTPHeaderField: "Authorization") request.setValue(await userAgent(), forHTTPHeaderField: "User-Agent") @@ -201,7 +204,7 @@ The next upload iteration will be delayed. } fileprivate func fetchSyncLines(request: JsonParam) async throws -> ControlInvocationsFromStream { - var (url, httpRequest) = try await authenticatedRequest { endpoint in endpoint.appending(path: "sync/stream") } + var (url, httpRequest) = try await authenticatedRequest { endpoint in endpoint.path += "/sync/stream" } httpRequest.httpMethod = "POST" httpRequest.setValue("application/json", forHTTPHeaderField: "Content-Type") httpRequest.setValue("application/x-ndjson", forHTTPHeaderField: "Accept") @@ -246,9 +249,9 @@ private struct ActiveSyncIteration: Sendable { activeStreams: syncClient.db.syncCoordinator.streams.currentStreams, appMetadata: syncClient.options.appMetadata, ))) - - var controlArgs: (any AsyncSequence)? - + + var controlArgs: AsyncMerge2Sequence>? + for instruction in initialInstructions { if case .establishSyncStream(request: let request) = instruction { let serviceEvents = try await syncClient.fetchSyncLines(request: request) @@ -257,12 +260,12 @@ private struct ActiveSyncIteration: Sendable { try await self.execute(instr: instruction) } } - + guard let controlArgs else { // Rust client didn't ask for a connection?? Ok then, end the iteration and retry return SyncIterationResult() } - + var hadSyncLine = false for try await arg in controlArgs { let control = try await powersyncControl(arg) @@ -270,10 +273,10 @@ private struct ActiveSyncIteration: Sendable { if case let .closeSyncStream(hideDisconnect) = instr { return SyncIterationResult(hideDisconnect: hideDisconnect) } - + try await execute(instr: instr) } - + if !hadSyncLine && arg.isSyncLine() { // Trigger a crud upload when receiving the first sync line: We could have // pending local writes made while disconnected, so in addition to listening on @@ -368,7 +371,7 @@ fileprivate struct ControlInvocationsFromStream: AsyncSequence, Sendable { typealias AsyncIterator = ControlInvocationsFromStreamIterator typealias Element = PowerSyncControlArguments - let sequence: any AsyncSequence & Sendable + let sequence: any SyncLineResponse func makeAsyncIterator() -> ControlInvocationsFromStreamIterator { .beforeStart(self.sequence) @@ -378,8 +381,8 @@ fileprivate struct ControlInvocationsFromStream: AsyncSequence, Sendable { fileprivate enum ControlInvocationsFromStreamIterator: AsyncIteratorProtocol { typealias Element = PowerSyncControlArguments - case beforeStart(any AsyncSequence) - case isReceiving(any AsyncIteratorProtocol) + case beforeStart(any SyncLineResponse) + case isReceiving(any SyncLineResponseIterator) case eof mutating func next() async throws -> PowerSyncControlArguments? { @@ -430,3 +433,7 @@ struct WriteCheckpointData: Codable { struct WriteCheckpointResponse: Codable { let data: WriteCheckpointData } + +private func sleepForSeconds(seconds: TimeInterval) async throws { + try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_00)) +} diff --git a/Sources/PowerSync/Utils/Mutex.swift b/Sources/PowerSync/Utils/Mutex.swift new file mode 100644 index 0000000..d311317 --- /dev/null +++ b/Sources/PowerSync/Utils/Mutex.swift @@ -0,0 +1,28 @@ +import Darwin + +/// A backport of `Mutex` from the `Synchronization` module. +struct Mutex: @unchecked Sendable, ~Copyable { + // We have to use pointer indirection to ensure the os_unfair_lock_t has a stable address. + private let osLock = os_unfair_lock_t.allocate(capacity: 1) + // This is behind a pointer to silence a compiler error about mutating its contents in a non-mutating func. + private let value: UnsafeMutablePointer = .allocate(capacity: 1) + + init(_ value: consuming T) { + self.osLock.initialize(to: os_unfair_lock()) + self.value.initialize(to: value) + } + + deinit { + self.osLock.deinitialize(count: 1) + self.osLock.deallocate() + self.value.deinitialize(count: 1) + self.value.deallocate() + } + + func withLock(_ action: (_ value: inout T) throws -> R) rethrows -> R { + os_unfair_lock_lock(self.osLock) + defer { os_unfair_lock_unlock(self.osLock) } + + return try action(&value.pointee) + } +} diff --git a/Tests/PowerSyncTests/SyncTests.swift b/Tests/PowerSyncTests/SyncTests.swift index 5b5f325..6700310 100644 --- a/Tests/PowerSyncTests/SyncTests.swift +++ b/Tests/PowerSyncTests/SyncTests.swift @@ -1,6 +1,5 @@ import AsyncAlgorithms @testable import PowerSync -import Synchronization import Testing @Suite() @@ -163,7 +162,7 @@ class InMemorySyncIntegrationTests { let channel = AsyncThrowingChannel() let mockClient = MockHttpClient { request in channel } let db = openDatabase(mockClient) - mockClient.writeCheckpoint.store(1, ordering: .sequentiallyConsistent) + mockClient.writeCheckpoint = 1 try await db.execute(sql: "INSERT INTO users (id, name) VALUES (uuid(), ?)", parameters: ["local write"]) try await db.connect(connector: TestConnector(), options: ConnectOptions()) @@ -185,11 +184,11 @@ class InMemorySyncIntegrationTests { } @Test func tokenExpired() async throws { - final class BackendConnector: PowerSyncBackendConnectorProtocol { - let fetchCredentialsCalls = Atomic(0) + final actor BackendConnector: PowerSyncBackendConnectorProtocol { + var fetchCredentialsCalls = 0 func fetchCredentials() async throws -> PowerSyncCredentials? { - fetchCredentialsCalls.add(1, ordering: .sequentiallyConsistent) + fetchCredentialsCalls += 1 return testCredentials } @@ -203,13 +202,13 @@ class InMemorySyncIntegrationTests { try await channel.pushLine(.keepAlive(tokenExpiresIn: 4000)) await waitForStatus(db.currentStatus) { $0.connected } - try #require(connector.fetchCredentialsCalls.load(ordering: .sequentiallyConsistent) == 1) + try #require(await connector.fetchCredentialsCalls == 1) // Should invalidate credentials when token expires try await channel.pushLine(.keepAlive(tokenExpiresIn: 0)) await waitForStatus(db.currentStatus) { !$0.connected } await waitForStatus(db.currentStatus) { $0.connected } - try #require(connector.fetchCredentialsCalls.load(ordering: .sequentiallyConsistent) == 2) + try #require(await connector.fetchCredentialsCalls == 2) } @Test func tokenThrows() async throws { diff --git a/Tests/PowerSyncTests/test-utils/MockHttpClient.swift b/Tests/PowerSyncTests/test-utils/MockHttpClient.swift index 3dd623e..f5b2e0d 100644 --- a/Tests/PowerSyncTests/test-utils/MockHttpClient.swift +++ b/Tests/PowerSyncTests/test-utils/MockHttpClient.swift @@ -5,27 +5,36 @@ import Testing import Synchronization final class MockHttpClient: HttpClient { - let writeCheckpoint: Atomic = Atomic(1000) + private let _writeCheckpoint = PowerSync.Mutex(1000) let handleSyncLines: @Sendable (_ request: URLRequest) async throws -> AsyncThrowingChannel - + + var writeCheckpoint: Int { + get { + _writeCheckpoint.withLock { $0 } + } + set { + _writeCheckpoint.withLock { $0 = newValue } + } + } + init(handleSyncLines: @Sendable @escaping (_ request: URLRequest) async throws -> AsyncThrowingChannel) { self.handleSyncLines = handleSyncLines } - func receiveSyncLines(request: URLRequest) async throws -> (HTTPURLResponse, any Sendable & AsyncSequence) { + func receiveSyncLines(request: URLRequest) async throws -> (HTTPURLResponse, SyncLineResponse) { try #require(request.url?.path() == "/sync/stream") let channel = try await handleSyncLines(request) let response = HTTPURLResponse(url: request.url!, mimeType: "application/x-ndjson", expectedContentLength: 0, textEncodingName: "utf-8") - return (response, channel) + return (response, MockSyncLineResponse(inner: channel)) } func readFully(request: URLRequest) async throws -> (HTTPURLResponse, Data) { // The sync client only uses this method to get /write-checkpoint2.json. try #require(request.url?.path() == "/write-checkpoint2.json") - let checkpoint = writeCheckpoint.load(ordering: .sequentiallyConsistent) + let checkpoint = writeCheckpoint let body = WriteCheckpointResponse(data: WriteCheckpointData(write_checkpoint: String(checkpoint))) let data = try StreamingSyncClient.jsonEncoder.encode(body) @@ -34,3 +43,19 @@ final class MockHttpClient: HttpClient { return (response, data) } } + +private struct MockSyncLineResponse: SyncLineResponse { + let inner: AsyncThrowingChannel + + func makeAsyncIterator() -> MockSyncLineResponseIterator { + return MockSyncLineResponseIterator(inner: inner.makeAsyncIterator()) + } +} + +private struct MockSyncLineResponseIterator: SyncLineResponseIterator { + var inner: AsyncThrowingChannel.AsyncIterator + + mutating func next() async throws -> PowerSync.SyncLine? { + return try await inner.next() + } +} From 6384f611bbc1feacd8f4d2312f2a1fc9e3fcb1c2 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 21 Apr 2026 22:32:42 +0200 Subject: [PATCH 19/40] Test on old macOS --- .github/workflows/build_and_test.yaml | 5 ++--- .github/workflows/docs.yaml | 8 ++------ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 904b50f..22e6506 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -7,10 +7,9 @@ on: jobs: build: name: Build and test - runs-on: macos-26 - timeout-minutes: 30 + runs-on: macos-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - name: Set up XCode uses: maxim-lobanov/setup-xcode@v1 with: diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 8556eca..3d3434d 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -11,13 +11,9 @@ permissions: jobs: build: name: Build - runs-on: macos-26 + runs-on: macos-latest steps: - - uses: actions/checkout@v6 - - name: Set up XCode - uses: maxim-lobanov/setup-xcode@v1 - with: - xcode-version: latest-stable + - uses: actions/checkout@v4 - name: Build Docs run: | xcodebuild docbuild \ From 76ae91dccc4c90a5cf62f61641a078c496b80f47 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 21 Apr 2026 22:41:28 +0200 Subject: [PATCH 20/40] More fixes --- .github/workflows/docs.yaml | 4 ++++ .../PowerSync/Implementation/sync/StreamingSyncClient.swift | 4 ++-- Tests/PowerSyncTests/test-utils/MockHttpClient.swift | 6 +++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 3d3434d..907f4f8 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -14,6 +14,10 @@ jobs: runs-on: macos-latest steps: - uses: actions/checkout@v4 + - name: Set up XCode + uses: maxim-lobanov/setup-xcode@v1 + with: + xcode-version: latest-stable - name: Build Docs run: | xcodebuild docbuild \ diff --git a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift index 59f122e..1ac86e2 100644 --- a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift +++ b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift @@ -30,8 +30,8 @@ final class StreamingSyncClient: Sendable { func run() -> Task { Task(name: "StreamingSyncClient.run") { let signals = SyncSignals() - async let download = downloadLoop(signals: signals) - async let upload = uploadLoop(signals: signals) + async let download: () = downloadLoop(signals: signals) + async let upload: () = uploadLoop(signals: signals) let _ = try await (download, upload) } diff --git a/Tests/PowerSyncTests/test-utils/MockHttpClient.swift b/Tests/PowerSyncTests/test-utils/MockHttpClient.swift index f5b2e0d..1ca13e7 100644 --- a/Tests/PowerSyncTests/test-utils/MockHttpClient.swift +++ b/Tests/PowerSyncTests/test-utils/MockHttpClient.swift @@ -21,8 +21,8 @@ final class MockHttpClient: HttpClient { self.handleSyncLines = handleSyncLines } - func receiveSyncLines(request: URLRequest) async throws -> (HTTPURLResponse, SyncLineResponse) { - try #require(request.url?.path() == "/sync/stream") + func receiveSyncLines(request: URLRequest) async throws -> (HTTPURLResponse, any SyncLineResponse) { + try #require(request.url?.path == "/sync/stream") let channel = try await handleSyncLines(request) let response = HTTPURLResponse(url: request.url!, mimeType: "application/x-ndjson", expectedContentLength: 0, textEncodingName: "utf-8") @@ -32,7 +32,7 @@ final class MockHttpClient: HttpClient { func readFully(request: URLRequest) async throws -> (HTTPURLResponse, Data) { // The sync client only uses this method to get /write-checkpoint2.json. - try #require(request.url?.path() == "/write-checkpoint2.json") + try #require(request.url?.path == "/write-checkpoint2.json") let checkpoint = writeCheckpoint let body = WriteCheckpointResponse(data: WriteCheckpointData(write_checkpoint: String(checkpoint))) From 21a7ed67e4262f2f4ab0e1e1c2c6acba2e3ccff2 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 22 Apr 2026 17:20:28 +0200 Subject: [PATCH 21/40] Implement connection pool in Swift --- Package.swift | 38 +- .../Implementation/AsyncConnectionPool.swift | 173 +++++ .../PoolConnectionContext.swift | 38 ++ .../PowerSyncDatabaseImpl.swift | 343 ++++++++++ .../Implementation/SyncStreams.swift | 10 +- .../sqlite3/NativeConnectionContext.swift | 335 ++++++++++ .../sqlite3/NativeConnectionPool.swift | 121 ++++ .../sqlite3/TransactionImpl.swift | 37 ++ .../registerPowerSyncCoreExtension.swift | 17 + .../sqlite3/throwDatabaseError.swift | 17 + .../sync/StreamingSyncClient.swift | 6 +- .../Implementation/sync/SyncCoordinator.swift | 8 +- .../Kotlin/AllLeaseCallback+Sendable.swift | 18 - Sources/PowerSync/Kotlin/DatabaseLogger.swift | 101 --- Sources/PowerSync/Kotlin/KotlinAdapter.swift | 136 ---- .../Kotlin/KotlinPowerSyncDatabaseImpl.swift | 612 ------------------ .../Kotlin/KotlinSQLiteConnectionPool.swift | 109 ---- .../KotlinSuspendFunction1+Sendable.swift | 15 - Sources/PowerSync/Kotlin/KotlinTypes.swift | 15 - .../Kotlin/LeaseCallback+Sendable.swift | 15 - Sources/PowerSync/Kotlin/SafeCastError.swift | 30 - .../Kotlin/db/KotlinConnectionContext.swift | 107 --- .../PowerSync/Kotlin/db/KotlinSqlCursor.swift | 136 ---- .../db/PowerSyncDataTypeConvertible.swift | 32 - ...esolvePowerSyncLoadableExtensionPath.swift | 9 - .../PowerSync/Kotlin/kotlinWithSession.swift | 39 -- .../PowerSync/Kotlin/wrapQueryCursor.swift | 83 --- Sources/PowerSync/PowerSyncCredentials.swift | 5 - Sources/PowerSync/PowerSyncDatabase.swift | 24 +- .../PowerSync/Protocol/PowerSyncError.swift | 24 + .../PowerSync/Protocol/QueriesProtocol.swift | 64 +- .../PowerSync/Protocol/Schema/Column.swift | 1 - Sources/PowerSync/Protocol/Schema/Index.swift | 1 - .../Protocol/db/ConnectionContext.swift | 6 +- Sources/PowerSync/Protocol/db/CrudBatch.swift | 2 +- .../Protocol/db/DataConvertible.swift | 54 ++ Sources/PowerSync/Protocol/db/SqlCursor.swift | 142 ++-- Sources/PowerSync/Utils/AsyncMutex.swift | 233 +++++++ .../Utils/ThrottledAsyncSequence.swift | 108 ++++ Sources/PowerSync/Utils/withSession.swift | 5 +- ...esolvePowerSyncLoadableExtensionPath.swift | 72 +-- Sources/PowerSyncCoreShim/empty.c | 1 + .../include/powersync_core.h | 1 + Sources/PowerSyncCoreShim/module.modulemap | 4 + Tests/PowerSyncGRDBTests/BasicTest.swift | 2 +- Tests/PowerSyncTests/ConnectTests.swift | 4 +- Tests/PowerSyncTests/CrudTests.swift | 4 +- Tests/PowerSyncTests/EncryptionTests.swift | 6 +- .../KotlinPowerSyncDatabaseImplTests.swift | 15 +- .../Kotlin/SqlCursorTests.swift | 8 +- Tests/PowerSyncTests/SyncTests.swift | 9 +- .../Utils/AsyncMutexTests.swift | 98 +++ 52 files changed, 1810 insertions(+), 1683 deletions(-) create mode 100644 Sources/PowerSync/Implementation/AsyncConnectionPool.swift create mode 100644 Sources/PowerSync/Implementation/PoolConnectionContext.swift create mode 100644 Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift create mode 100644 Sources/PowerSync/Implementation/sqlite3/NativeConnectionContext.swift create mode 100644 Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift create mode 100644 Sources/PowerSync/Implementation/sqlite3/TransactionImpl.swift create mode 100644 Sources/PowerSync/Implementation/sqlite3/registerPowerSyncCoreExtension.swift create mode 100644 Sources/PowerSync/Implementation/sqlite3/throwDatabaseError.swift delete mode 100644 Sources/PowerSync/Kotlin/AllLeaseCallback+Sendable.swift delete mode 100644 Sources/PowerSync/Kotlin/DatabaseLogger.swift delete mode 100644 Sources/PowerSync/Kotlin/KotlinAdapter.swift delete mode 100644 Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift delete mode 100644 Sources/PowerSync/Kotlin/KotlinSQLiteConnectionPool.swift delete mode 100644 Sources/PowerSync/Kotlin/KotlinSuspendFunction1+Sendable.swift delete mode 100644 Sources/PowerSync/Kotlin/KotlinTypes.swift delete mode 100644 Sources/PowerSync/Kotlin/LeaseCallback+Sendable.swift delete mode 100644 Sources/PowerSync/Kotlin/SafeCastError.swift delete mode 100644 Sources/PowerSync/Kotlin/db/KotlinConnectionContext.swift delete mode 100644 Sources/PowerSync/Kotlin/db/KotlinSqlCursor.swift delete mode 100644 Sources/PowerSync/Kotlin/db/PowerSyncDataTypeConvertible.swift delete mode 100644 Sources/PowerSync/Kotlin/kotlinResolvePowerSyncLoadableExtensionPath.swift delete mode 100644 Sources/PowerSync/Kotlin/kotlinWithSession.swift delete mode 100644 Sources/PowerSync/Kotlin/wrapQueryCursor.swift create mode 100644 Sources/PowerSync/Protocol/db/DataConvertible.swift create mode 100644 Sources/PowerSync/Utils/ThrottledAsyncSequence.swift create mode 100644 Sources/PowerSyncCoreShim/empty.c create mode 100644 Sources/PowerSyncCoreShim/include/powersync_core.h create mode 100644 Sources/PowerSyncCoreShim/module.modulemap create mode 100644 Tests/PowerSyncTests/Utils/AsyncMutexTests.swift diff --git a/Package.swift b/Package.swift index e36d619..e20fd24 100644 --- a/Package.swift +++ b/Package.swift @@ -5,10 +5,6 @@ import PackageDescription let packageName = "PowerSync" -// Set this to the absolute path of your Kotlin SDK checkout if you want to use a local Kotlin -// build. Also see docs/LocalBuild.md for details -let localKotlinSdkOverride: String? = nil - // Set this to the absolute path of your powersync-sqlite-core checkout if you want to use a // local build of the core extension. let localCoreExtension: String? = nil @@ -20,24 +16,6 @@ let localCoreExtension: String? = nil // towards a local framework build var conditionalDependencies: [Package.Dependency] = [] var conditionalTargets: [Target] = [] -var kotlinTargetDependency = Target.Dependency.target(name: "PowerSyncKotlin") - -if let kotlinSdkPath = localKotlinSdkOverride { - // We can't depend on local XCFrameworks outside of this project's root, so there's a Package.swift - // in the PowerSyncKotlin project pointing towards a local build. - conditionalDependencies.append(.package(path: "\(kotlinSdkPath)/internal/PowerSyncKotlin")) - - kotlinTargetDependency = .product(name: "PowerSyncKotlin", package: "PowerSyncKotlin") -} else { - // Not using a local build, so download from releases - conditionalTargets.append( - .binaryTarget( - name: "PowerSyncKotlin", - url: - "https://github.com/powersync-ja/powersync-kotlin/releases/download/v1.11.2/PowersyncKotlinRelease.zip", - checksum: "bd4f9a4411a10a30bd67c3231d1d0d0dde42f0ec19161ccbd26d4e58b31efdfd" - )) -} var corePackageName = "powersync-sqlite-core-swift" if let corePath = localCoreExtension { @@ -84,7 +62,8 @@ let package = Package( dependencies: conditionalDependencies + [ .package(url: "https://github.com/groue/GRDB.swift.git", from: "7.9.0"), .package(url: "https://github.com/powersync-ja/CSQLite.git", exact: "3.51.2"), - .package(url: "https://github.com/apple/swift-async-algorithms.git", from: "1.1.0") + .package(url: "https://github.com/apple/swift-async-algorithms.git", from: "1.1.0"), + .package(url: "https://github.com/apple/swift-collections.git", from: "1.4.0") ], targets: [ // Targets are the basic building blocks of a package, defining a module or a test suite. @@ -92,12 +71,19 @@ let package = Package( .target( name: packageName, dependencies: [ - kotlinTargetDependency, - .product(name: "PowerSyncSQLiteCore", package: corePackageName), + .target(name: "PowerSyncCoreShim"), .product(name: "CSQLite", package: "CSQLite"), - .product(name: "AsyncAlgorithms", package: "swift-async-algorithms") + .product(name: "AsyncAlgorithms", package: "swift-async-algorithms"), + .product(name: "BasicContainers", package: "swift-collections"), + .product(name: "DequeModule", package: "swift-collections") ] ), + .target( + name: "PowerSyncCoreShim", + dependencies: [ + .product(name: "PowerSyncSQLiteCore", package: corePackageName), + ], + ), .target( name: "PowerSyncGRDB", dependencies: [ diff --git a/Sources/PowerSync/Implementation/AsyncConnectionPool.swift b/Sources/PowerSync/Implementation/AsyncConnectionPool.swift new file mode 100644 index 0000000..48bb461 --- /dev/null +++ b/Sources/PowerSync/Implementation/AsyncConnectionPool.swift @@ -0,0 +1,173 @@ +import CSQLite +import DequeModule +import Foundation + +enum DatabaseLocation { + case inMemory + case inDefaultDirectory(name: String) + + func openConnection(writer: Bool) throws -> RawSqliteConnection { + var db: OpaquePointer? + let rc: Int32 + let path: String + + switch self { + case .inMemory: + path = ":memory:" + rc = sqlite3_open_v2(path, &db, SQLITE_OPEN_READWRITE, nil) + case .inDefaultDirectory(let name): + let fileManager = FileManager.default + let databaseDirectory = (try DatabaseLocation.appleDefaultDatabaseDirectory()).path + + if !fileManager.fileExists(atPath: databaseDirectory) { + try fileManager.createDirectory(atPath: databaseDirectory, withIntermediateDirectories: true) + } + + path = "\(databaseDirectory)/\(name)" + let flags = if writer { + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE + } else { + SQLITE_OPEN_READONLY + } + rc = sqlite3_open_v2(path, &db, flags, nil) + } + + if rc != 0 { + throw PowerSyncError.sqliteError(extendedResultCode: rc, offset: nil, message: "Could not open database \(path)", errorString: nil, sql: nil) + } + + return RawSqliteConnection(connection: db!) + } + + /// This returns the default directory in which we store SQLite database files. + static func appleDefaultDatabaseDirectory() throws -> URL { + let fileManager = FileManager.default + + // Get the application support directory + guard let documentsDirectory = fileManager.urls( + for: .applicationSupportDirectory, + in: .userDomainMask + ).first else { + throw PowerSyncError.operationFailed(message: "Unable to find application support directory") + } + + return documentsDirectory.appendingPathComponent("databases") + } +} + +/// Wraps an ``NativeConnectionPool`` to handle opening connections and to dispatch database tasks in a suitable queue. +final class AsyncConnectionPool: SQLiteConnectionPoolProtocol { + private let location: DatabaseLocation + private let initialStatements: [String] + private let tableUpdatesStream = BroadcastStream>() + private let inner: AsyncSemaphore = AsyncSemaphore(singleElement: nil) + + init(location: DatabaseLocation, initialStatements: [String] = []) { + self.location = location + self.initialStatements = initialStatements + } + + var tableUpdates: AsyncStream> { + tableUpdatesStream.subscribe() + } + + private func runBlocking(action: @escaping @Sendable () throws -> T, qos: DispatchQoS.QoSClass = .userInitiated) async throws -> T { + return try await withCheckedThrowingContinuation { continuation in + DispatchQueue.global(qos: qos).async { + continuation.resume(with: Result(catching: { try action() })) + } + } + } + + private func configureConnection(connection: borrowing RawSqliteConnection, isWriter: Bool) throws { + let context = NativeConnectionContext(connection.asLease()) + for stmt in initialStatements { + let _ = try context.execute(sql: stmt, parameters: []) + } + + if isWriter { + let _ = try context.execute(sql: "pragma journal_mode = WAL", parameters: []) + } else { + let _ = try context.execute(sql: "pragma query_only = TRUE", parameters: []) + } + + let _ = try context.execute(sql: "pragma journal_size_limit = \(6 * 1024 * 1024)", parameters: []) + let _ = try context.execute(sql: "pragma busy_timeout = 30000", parameters: []) + let _ = try context.execute(sql: "pragma cache_size = -\(50 * 1024)", parameters: []) + + if isWriter { + // Older versions of the SDK used to set up an empty schema and raise the user version to 1. + // Keep doing that for consistency. + let version = try context.get(sql: "pragma user_version", parameters: []) { try $0.getInt(index: 0) } + if version < 1 { + let _ = try context.execute(sql: "pragma user_version = 1", parameters: []) + } + + let _ = try context.execute(sql: "select powersync_update_hooks('install')", parameters: []) + } + } + + private func obtainInner() async throws -> NativeConnectionPool { + var lease = try await inner.acquire(count: 1) + if let pool = lease.acquiredItems[0] { + return pool + } else { + try registerPowerSyncCoreExtension() + + @Sendable func handleUpdates(_ updates: Set) { + self.tableUpdatesStream.dispatch(event: updates) + } + + let pool = try await runBlocking { [self] in + let writer = try location.openConnection(writer: true) + try configureConnection(connection: writer, isWriter: true) + + if case .inMemory = location { + return NativeConnectionPool(singleConnection: writer, handleUpdates: handleUpdates) + } else { + let numReaders = 4 + var readers = RigidDeque(capacity: numReaders) + while !readers.isFull { + let connection = try location.openConnection(writer: false) + try configureConnection(connection: connection, isWriter: false) + readers.append(connection) + } + + return NativeConnectionPool(writer: writer, readers: readers, handleUpdates: handleUpdates) + } + } + + lease.acquiredItems[0] = pool + return pool + } + } + + func read(onConnection: @escaping @Sendable (any SQLiteConnectionLease) throws -> Void) async throws { + let pool = try await obtainInner() + try await pool.read { connection in + try await runBlocking { try onConnection(connection) } + } + } + + func write(onConnection: @escaping @Sendable (any SQLiteConnectionLease) throws -> Void) async throws { + let pool = try await obtainInner() + try await pool.write { connection in + try await runBlocking { try onConnection(connection) } + } + } + + func withAllConnections(onConnection: @escaping @Sendable (any SQLiteConnectionLease, [any SQLiteConnectionLease]) throws -> Void) async throws { + let pool = try await obtainInner() + try await pool.withAllConnections { writer, readers in + try await runBlocking { try onConnection(writer, readers) } + } + } + + func close() async throws { + var lease = try await inner.acquire(count: 1) + if let pool = lease.acquiredItems[0] { + try await pool.close() + lease.acquiredItems[0] = nil + } + } +} diff --git a/Sources/PowerSync/Implementation/PoolConnectionContext.swift b/Sources/PowerSync/Implementation/PoolConnectionContext.swift new file mode 100644 index 0000000..def6b40 --- /dev/null +++ b/Sources/PowerSync/Implementation/PoolConnectionContext.swift @@ -0,0 +1,38 @@ +func poolRead(_ pool: borrowing SQLiteConnectionPoolProtocol, action: @escaping @Sendable (_: any ConnectionContext) throws -> T) async throws -> T { + let result = UnsafeSendable() + try await pool.read { lease in + let context = NativeConnectionContext(lease) + result.resolve(value: try action(context)) + } + + return result.inner! +} + +func poolWrite(_ pool: borrowing SQLiteConnectionPoolProtocol, action: @escaping @Sendable (_: any ConnectionContext) throws -> T) async throws -> T { + let result = UnsafeSendable() + try await pool.read { lease in + let context = NativeConnectionContext(lease) + result.resolve(value: try action(context)) + } + + return result.inner! +} + +func poolWithAll(_ pool: borrowing SQLiteConnectionPoolProtocol, action: @escaping @Sendable (_ writer: any ConnectionContext, _ readers: [any ConnectionContext]) throws -> T) async throws -> T { + let result = UnsafeSendable() + try await pool.withAllConnections { writer, readers in + let writer = NativeConnectionContext(writer) + let readers = readers.map { NativeConnectionContext($0) } + + result.resolve(value: try action(writer, readers)) + } + return result.inner! +} + +private final class UnsafeSendable: @unchecked Sendable { + var inner: T? = nil + + func resolve(value: T) { + self.inner = value + } +} diff --git a/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift b/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift new file mode 100644 index 0000000..0db0995 --- /dev/null +++ b/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift @@ -0,0 +1,343 @@ +import AsyncAlgorithms +import Foundation + +final class PowerSyncDatabaseImpl: PowerSyncDatabaseProtocol { + let logger: any LoggerProtocol + let syncCoordinator = SyncCoordinator() + let syncStatus = SwiftSyncStatus() + private let dbFilename: String? + private let httpClient: HttpClient + private let initializer = DatabaseInitizalizationActor() + fileprivate let pool: any SQLiteConnectionPoolProtocol + let schema: AsyncMutex + + init( + dbFilename: String? = nil, + logger: any LoggerProtocol, + pool: any SQLiteConnectionPoolProtocol, + httpClient: HttpClient, + schema: Schema + ) { + self.dbFilename = dbFilename + self.logger = logger + self.schema = AsyncMutex(schema) + self.httpClient = httpClient + self.pool = pool + } + + var currentStatus: any SyncStatus { + syncStatus + } + + func resolveOfflineSyncStatusIfNotConnected() async throws { + try await syncCoordinator.guardNotConnected(inner: { + try await resolveOfflineSyncStatus() + }, ifConnected: {}) + } + + private func initialize() async throws { + try await initializer.ensureInitialized(db: self) + } + + fileprivate func resolveOfflineSyncStatus() async throws { + // We can't use get() here because it runs as part of the initialization step. + let offlineSyncStatus = try await poolRead(pool) { connection in + try connection.get(sql: "SELECT powersync_offline_sync_status()", parameters: []) { cursor in + let raw = try cursor.getString(index: 0) + return try StreamingSyncClient.jsonDecoder.decode(CoreDownloadSyncStatus.self, from: raw.data(using: .utf8)!) + } + } + + syncStatus.mutateStatus { $0 = MutableSyncStatus(core: offlineSyncStatus) } + } + + func updateSchema(schema: any SchemaProtocol) async throws { + try await initializer.ensureInitialized(db: self) + try await syncCoordinator.guardNotConnected( + inner: { + let schema = Schema(other: schema) + await self.schema.withMutex { $0 = schema } + try await applySchema(schema: schema) + }, + ifConnected: { throw PowerSyncError.operationFailed(message: "Cannot update schema while connected") } + ) + } + + fileprivate func applySchema(schema: Schema) async throws { + try await poolWithAll(pool) { writer, readers in + let encoded = try StreamingSyncClient.jsonEncoder.encode(schema) + guard let asString = String(data: encoded, encoding: .utf8) else { + throw PowerSyncError.operationFailed(message: "Could not serialize schema") + } + try writer.execute(sql: "SELECT powersync_replace_schema(?)", parameters: [asString]) + + for reader in readers { + // Update the schema on all read connections + try reader.execute(sql: "pragma table_info('sqlite_master')", parameters: []) + } + } + } + + func waitForFirstSync() async throws { + try await initialize() + await syncStatus.waitFor { $0.hasSynced == true } + } + + func waitForFirstSync(priority: Int32) async throws { + try await initialize() + let priority = BucketPriority(priority) + await syncStatus.waitFor { $0.statusForPriority(priority).hasSynced == true } + } + + func getCrudTransactions() -> CrudTransactions { + CrudTransactions(db: self) + } + + func getCrudBatch(limit: Int32) async throws -> CrudBatch? { + + } + + func getPowerSyncVersion() async throws -> String { + try await initialize() + // Set during initialization + return await initializer.powerSyncVersion! + } + + func disconnect() async throws { + await syncCoordinator.disconnect() + } + + func syncStream(name: String, params: JsonParam?) -> any SyncStream { + PendingSyncStream(db: self, name: name, parameters: params) + } + + func close() async throws { + try await initialize() + try await initializer.close { + await syncCoordinator.disconnect() + try await pool.close() + } + } + + func close(deleteDatabase: Bool) async throws { + try await close() + if deleteDatabase { + try await self.deleteDatabase() + } + } + + private func deleteDatabase() async throws { + if let dbFilename { + // We can use the supplied dbLocation when we support that in future + let directory = try DatabaseLocation.appleDefaultDatabaseDirectory() + let fileManager = FileManager.default + + // SQLite files to delete: + // 1. Main database file: dbFilename + // 2. WAL file: dbFilename-wal + // 3. SHM file: dbFilename-shm + // 4. Journal file: dbFilename-journal (for rollback journal mode, though WAL mode typically doesn't use it) + + let filesToDelete = [ + dbFilename, + "\(dbFilename)-wal", + "\(dbFilename)-shm", + "\(dbFilename)-journal" + ] + + for filename in filesToDelete { + let fileUrl = directory.appendingPathComponent(filename) + if fileManager.fileExists(atPath: fileUrl.path) { + try fileManager.removeItem(at: fileUrl) + } + } + } + } + + func connect(connector: any PowerSyncBackendConnectorProtocol, options: ConnectOptions?) async throws { + await syncCoordinator.connect(db: self, connector: connector, options: options ?? ConnectOptions(), client: httpClient) + } + + func watch(options: WatchOptions) throws -> AsyncThrowingStream<[RowType], any Error> { + AsyncThrowingStream { continuation in + // Create an outer task to monitor cancellation + let task = Task { + do { + try await initialize() + let watchedTables = try await self.getQuerySourceTables( + sql: options.sql, + parameters: options.parameters + ) + + let updateNotifications = pool.tableUpdates.filter { changedTables in + changedTables.contains(where: watchedTables.contains) + }.map { _ in () } + // Allows emitting the first result even if there aren't changes + let withInitial = AsyncAlgorithms.merge([()].async, updateNotifications) + let throttled = AsyncThrottleSequence(inner: withInitial, duration: options.throttle) + + for try await _ in throttled { + // Check if the outer task is cancelled + try Task.checkCancellation() + + try continuation.yield(await self.getAll( + sql: options.sql, + parameters: options.parameters, + mapper: options.mapper + )) + } + + continuation.finish() + } catch { + if error is CancellationError { + continuation.finish() + } else { + continuation.finish(throwing: error) + } + } + } + + // Propagate cancellation from the outer task to the inner task + continuation.onTermination = { @Sendable _ in + task.cancel() // This cancels the inner task when the stream is terminated + } + } + } + + func watch(sql: String, parameters: [(any Sendable)?]?, mapper: @escaping @Sendable (any SqlCursor) throws -> RowType) throws -> AsyncThrowingStream<[RowType], any Error> { + return try watch(options: WatchOptions(sql: sql, parameters: parameters, mapper: mapper)) + } + + func disconnectAndClear(clearLocal: Bool, soft: Bool) async throws { + try await initialize() + try await syncCoordinator.disconnectAndThen { + var flags = 0 + if clearLocal { + flags |= 1 + } + if soft { + flags |= 2 + } + + do { + let flags = flags + let _ = try await poolWrite(pool) { ctx in try ctx.execute(sql: "SELECT powersync_clear(?)", parameters: [flags]) } + } + } + } + + func writeLock(callback: @escaping @Sendable (any ConnectionContext) throws -> R) async throws -> R { + try await initialize() + return try await poolWrite(pool, action: callback) + } + + func readLock(callback: @escaping @Sendable (any ConnectionContext) throws -> R) async throws -> R { + try await initialize() + return try await poolRead(pool, action: callback) + } + + func writeTransaction(callback: @escaping @Sendable (any Transaction) throws -> R) async throws -> R { + return try await writeLock { ctx in try TransactionImpl.run(conn: ctx, callback: callback) } + } + + func readTransaction(callback: @escaping @Sendable (any Transaction) throws -> R) async throws -> R { + return try await readLock { ctx in try TransactionImpl.run(conn: ctx, callback: callback) } + } + + private func getQuerySourceTables( + sql: String, + parameters: [Sendable?] + ) async throws -> Set { + let rows = try await getAll( + sql: "EXPLAIN \(sql)", + parameters: parameters, + mapper: { cursor in + try ExplainQueryResult( + addr: cursor.getString(index: 0), + opcode: cursor.getString(index: 1), + p1: cursor.getInt64(index: 2), + p2: cursor.getInt64(index: 3), + p3: cursor.getInt64(index: 4) + ) + } + ) + + let rootPages = rows.compactMap { row in + if (row.opcode == "OpenRead" || row.opcode == "OpenWrite") && + row.p3 == 0 && row.p2 != 0 + { + return row.p2 + } + return nil + } + + do { + let pagesData = try StreamingSyncClient.jsonEncoder.encode(rootPages) + guard let pagesString = String(data: pagesData, encoding: .utf8) else { + throw PowerSyncError.operationFailed( + message: "Failed to convert pages data to UTF-8 string" + ) + } + + let tableRows = try await getAll( + sql: "SELECT tbl_name FROM sqlite_master WHERE rootpage IN (SELECT json_each.value FROM json_each(?))", + parameters: [ + pagesString, + ] + ) { try $0.getString(index: 0) } + + return Set(tableRows) + } catch { + throw PowerSyncError.operationFailed( + message: "Could not determine watched query tables", + underlyingError: error + ) + } + } + + static let maxOpId = Int64.max +} + +private struct ExplainQueryResult { + let addr: String + let opcode: String + let p1: Int64 + let p2: Int64 + let p3: Int64 +} + +private actor DatabaseInitizalizationActor { + private var isInitialized = false + var powerSyncVersion: String? + private var closed = false + + func ensureInitialized(db: PowerSyncDatabaseImpl) async throws { + if closed { + throw PowerSyncError.operationFailed(message: "Attempted to use closed PowerSync database") + } + if isInitialized { + return + } + + powerSyncVersion = try await poolWrite(db.pool) { conn in + let sqliteVersion = try conn.get(sql: "SELECT sqlite_version()", parameters: []) { try $0.getString(index: 0) } + let powerSyncVersion = try conn.get(sql: "SELECT powersync_rs_version()", parameters: []) { try $0.getString(index: 0) } + + db.logger.debug("Opened connection. SQLite version \(sqliteVersion), PowerSync SQLite core extension \(powerSyncVersion)", tag: "PowerSyncDatabase") + + try conn.execute(sql: "SELECT powersync_init()", parameters: []) + return powerSyncVersion + } + + try await db.applySchema(schema: db.schema.withMutex { $0 }) + try await db.resolveOfflineSyncStatus() + isInitialized = true + } + + func close(action: () async throws -> ()) async rethrows { + if !closed { + closed = true + try await action() + } + } +} diff --git a/Sources/PowerSync/Implementation/SyncStreams.swift b/Sources/PowerSync/Implementation/SyncStreams.swift index 264aa95..54d9b47 100644 --- a/Sources/PowerSync/Implementation/SyncStreams.swift +++ b/Sources/PowerSync/Implementation/SyncStreams.swift @@ -14,7 +14,7 @@ final class StreamTracker: Sendable { streamsChanged.dispatch(event: currentStreams) } - fileprivate func subscriptionsCommand(db: KotlinPowerSyncDatabaseImpl, request: RustSubscriptionChangeRequest) async throws { + fileprivate func subscriptionsCommand(db: PowerSyncDatabaseImpl, request: RustSubscriptionChangeRequest) async throws { let _ = try await db.writeTransaction { tx in let payload = String(data: try StreamingSyncClient.jsonEncoder.encode(request), encoding: .utf8) try tx.execute(sql: "SELECT powersync_control(?, ?)", parameters: [ @@ -26,7 +26,7 @@ final class StreamTracker: Sendable { try await db.resolveOfflineSyncStatusIfNotConnected() } - fileprivate func subscribe(db: KotlinPowerSyncDatabaseImpl, stream: PendingSyncStream, ttl: TimeInterval?, priority: BucketPriority?) async throws -> SyncSubscriptionImplementation { + fileprivate func subscribe(db: PowerSyncDatabaseImpl, stream: PendingSyncStream, ttl: TimeInterval?, priority: BucketPriority?) async throws -> SyncSubscriptionImplementation { let key = stream.key try await subscriptionsCommand( db: db, @@ -80,7 +80,7 @@ final class StreamTracker: Sendable { /// A Sync Stream that can be subscribed to. struct PendingSyncStream: SyncStream { - let db: KotlinPowerSyncDatabaseImpl + let db: PowerSyncDatabaseImpl let name: String let parameters: JsonParam? @@ -101,10 +101,10 @@ struct PendingSyncStream: SyncStream { } final class SyncSubscriptionImplementation: SyncStreamSubscription { - private let db: KotlinPowerSyncDatabaseImpl + private let db: PowerSyncDatabaseImpl private let key: StreamKey - init(db: KotlinPowerSyncDatabaseImpl, key: StreamKey) { + init(db: PowerSyncDatabaseImpl, key: StreamKey) { self.db = db self.key = key } diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionContext.swift b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionContext.swift new file mode 100644 index 0000000..6bd4931 --- /dev/null +++ b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionContext.swift @@ -0,0 +1,335 @@ +import CSQLite +import Darwin +import Synchronization + +private struct NativeConnectionState { + let lease: SQLiteConnectionLease + var closed: Bool = false + + func checkNotClosed() throws(PowerSyncError) { + if self.closed { + throw .operationFailed(message: "Attempted to use a connection context after it was closed") + } + } +} + +final class NativeConnectionContext: ConnectionContext { + private let state: Mutex; + + init(_ lease: consuming SQLiteConnectionLease) { + self.state = Mutex(NativeConnectionState(lease: lease)); + } + + func invalidateLease() { + state.withLock { $0.closed = true } + } + + func execute(sql: String, parameters: [(any Sendable)?]?) throws -> Int64 { + return try state.withLock { + try $0.checkNotClosed() + + var stmt = try SqliteStatement(db: $0.lease, sql: sql) + try stmt.bind_values(parameters) + while try stmt.step() { + // Iterate through the statement. + } + + let _ = consume stmt + return sqlite3_changes64($0.lease.pointer); + } + } + + func getOptional(sql: String, parameters: [(any Sendable)?]?, mapper: @Sendable (SqlCursor) throws -> RowType) throws -> RowType? { + return try state.withLock { + try $0.checkNotClosed() + + var stmt = try SqliteStatement(db: $0.lease, sql: sql) + try stmt.bind_values(parameters) + if try stmt.step() { + return try NativeConnectionContext.invokeMapper(stmt, mapper) + } else { + return nil + } + } + } + + func getAll(sql: String, parameters: [(any Sendable)?]?, mapper: @Sendable (SqlCursor) throws -> RowType) throws -> [RowType] { + return try state.withLock { + try $0.checkNotClosed() + + var stmt = try SqliteStatement(db: $0.lease, sql: sql) + try stmt.bind_values(parameters) + var rows: [RowType] = [] + while try stmt.step() { + rows.append(try NativeConnectionContext.invokeMapper(stmt, mapper)) + } + return rows + } + } + + func get(sql: String, parameters: [(any Sendable)?]?, mapper: @Sendable (SqlCursor) throws -> RowType) throws -> RowType { + return try state.withLock { + try $0.checkNotClosed() + + var stmt = try SqliteStatement(db: $0.lease, sql: sql) + try stmt.bind_values(parameters) + if try stmt.step() { + return try NativeConnectionContext.invokeMapper(stmt, mapper) + } else { + throw PowerSyncError.operationFailed(message: "Called get(\(sql), which did not return any row") + } + } + } + + private static func invokeMapper(_ stmt: borrowing SqliteStatement, _ mapper: (SqlCursor) throws -> RowType) rethrows -> RowType { + return try withUnsafePointer(to: stmt) { ptr in + let cursor = StatementCursor(ptr) + defer { + cursor.invalidate() + } + + return try mapper(cursor) + } + } +} + +struct SqliteStatement: ~Copyable { + private var resolvedColumnNames: [String : Int]? = nil + private let db: SQLiteConnectionLease + let stmt: OpaquePointer + private let sql: String + + init(db: SQLiteConnectionLease, sql: String) throws(PowerSyncError) { + self.db = db + var stmt: OpaquePointer? + var sql = sql + let rc = sql.withUTF8 { sqlBytes in + return sqlite3_prepare_v2( + db.pointer, + sqlBytes.baseAddress, + Int32(sqlBytes.count), + &stmt, + nil + ) + } + if (rc != 0) { + try throwDatabaseError(db: db, sql: sql) + } + + self.stmt = stmt! + self.sql = sql + } + + deinit { + sqlite3_finalize(stmt) + } + + var columnCount: Int { + return Int(sqlite3_column_count(self.stmt)) + } + + + var columnNames: [String : Int] { + return resolvedColumnNames! + } + + borrowing func bind_values(_ parameters: [Any?]?) throws (PowerSyncError) { + if let parameters { + for (i, parameter) in parameters.enumerated() { + let index = Int32(i + 1) + + if parameter == nil { + try bind_value(index, nil) + } else { + try bind_value(index, try PowerSyncDataType(from: parameter!)) + } + } + } + } + + borrowing func bind_value(_ index: Int32, _ parameter: PowerSyncDataType?) throws (PowerSyncError) { + let rc: Int32 + + switch parameter { + case nil: + rc = sqlite3_bind_null(self.stmt, index) + case .bool(let value): + rc = sqlite3_bind_int(self.stmt, index, value ? 1 : 0) + case .string(let value): + var str = value + rc = str.withUTF8 { buffer in + sqlite3_bind_text( + self.stmt, + index, + buffer.baseAddress, + Int32(buffer.count), + // SQLITE_TRANSIENT + unsafeBitCast(-1, to: (@convention(c) (UnsafeMutableRawPointer?) -> Void).self), + ) + } + case .int64(let value): + rc = sqlite3_bind_int64(self.stmt, index, value) + case .int32(let value): + rc = sqlite3_bind_int(self.stmt, index, value) + case .double(let value): + rc = sqlite3_bind_double(self.stmt, index, value) + case .data(let value): + // Data object can be made up of multiple memory regions, so copy once. + let buffer = malloc(value.count)! + value.copyBytes(to: buffer.assumingMemoryBound(to: UInt8.self), count: value.count) + + rc = sqlite3_bind_blob( + self.stmt, + index, + buffer, + Int32(value.count), + free, + ) + + if rc != 0 { + free(buffer) + } + } + + if rc != 0 { + try throwDatabaseError(db: self.db, sql: self.sql) + } + } + + mutating func step() throws (PowerSyncError) -> Bool { + let rc = sqlite3_step(self.stmt) + if rc == SQLITE_DONE { + return false + } else if rc == SQLITE_ROW { + if resolvedColumnNames == nil { + let count = self.columnCount + var nameToIndex = Dictionary(minimumCapacity: count) + for i in 0..? + + init(_ stmtPtr: UnsafePointer) { + self.stmtPtr = stmtPtr + } + + func invalidate() { + stmtPtr = nil + } + + private func withStatement(_ body: (borrowing SqliteStatement) throws -> R) rethrows -> R { + if let stmtPtr { + return try body(stmtPtr.pointee) + } + + fatalError("Cursor used outside of callback") + } + + var columnCount: Int { + return withStatement { stmt in stmt.columnCount } + } + + var columnNames: [String : Int] { + return withStatement { stmt in stmt.columnNames } + } + + func checkColumnNotNull(stmt: borrowing SqliteStatement, index: Int) throws(SqlCursorError) { + if index < 0 || index >= stmt.columnCount { + throw SqlCursorError.nullValueFound("invalid index \(index)") + } + + let type = sqlite3_column_type(stmt.stmt, Int32(index)) + if type == SQLITE_NULL { + throw SqlCursorError.nullValueFound("at index \(index)") + } + } + + func getBoolean(index: Int) throws -> Bool { + return try getInt(index: index) == 0 ? false : true + } + + func getBooleanOptional(index: Int) -> Bool? { + do { + return try getBoolean(index: index) + } catch { + return nil + } + } + + func getDouble(index: Int) throws -> Double { + return try withStatement { stmt in + try self.checkColumnNotNull(stmt: stmt, index: index) + return sqlite3_column_double(stmt.stmt, Int32(index)) + } + } + + func getDoubleOptional(index: Int) -> Double? { + do { + return try getDouble(index: index) + } catch { + return nil + } + } + + func getInt(index: Int) throws -> Int { + return Int(try getInt64(index: index)) + } + + func getIntOptional(index: Int) -> Int? { + do { + return try getInt(index: index) + } catch { + return nil + } + } + + func getInt64(index: Int) throws -> Int64 { + return try withStatement { stmt in + try self.checkColumnNotNull(stmt: stmt, index: index) + return sqlite3_column_int64(stmt.stmt, Int32(index)) + } + } + + func getInt64Optional(index: Int) -> Int64? { + do { + return try getInt64(index: index) + } catch { + return nil + } + } + + func getString(index: Int) throws -> String { + return try withStatement { stmt in + try self.checkColumnNotNull(stmt: stmt, index: index) + let length = sqlite3_column_bytes(stmt.stmt, Int32(index)) + if length == 0 { + return "" + } + + let ptr = sqlite3_column_text(stmt.stmt, Int32(index)) + return String(decoding: UnsafeBufferPointer(start: ptr, count: Int(length)), as: UTF8.self) + } + } + + func getStringOptional(index: Int) -> String? { + do { + return try getString(index: index) + } catch { + return nil + } + } +} diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift new file mode 100644 index 0000000..35b0811 --- /dev/null +++ b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift @@ -0,0 +1,121 @@ +import CSQLite +import Foundation +import DequeModule + +/// A helper implementing a SQLite connection pool from opened and configured connections. +/// +/// This class does not configure or open connections (that is the responsibility of ``AsyncConnectionPool``). +final class NativeConnectionPool: Sendable { + private let writer: AsyncSemaphore + private let readers: AsyncSemaphore? + private let handleUpdates: @Sendable (_: Set) -> () + + init(writer: consuming RawSqliteConnection, readers: consuming RigidDeque, handleUpdates: @escaping @Sendable (_: Set) -> ()) { + self.writer = AsyncSemaphore(singleElement: writer) + self.readers = AsyncSemaphore(readers) + self.handleUpdates = handleUpdates + } + + init(singleConnection: consuming RawSqliteConnection, handleUpdates: @escaping @Sendable (_: Set) -> ()) { + self.writer = AsyncSemaphore(singleElement: singleConnection) + self.readers = nil + self.handleUpdates = handleUpdates + } + + private func dispatchWrites(lease: RawConnectionLease) throws { + let ctx = NativeConnectionContext(lease) + let affectedTables = try ctx.get(sql: "SELECT powersync_update_hooks('get')", parameters: []) { + let decoder = JSONDecoder() + return try decoder.decode(Set.self, from: try $0.getString(index: 0).data(using: .utf8)!) + } + + if !affectedTables.isEmpty { + self.handleUpdates(affectedTables) + } + } + + func read(onConnection: @Sendable (RawConnectionLease) async throws -> Void) async throws { + // No dedicated readers? Acquire write connection for this then + let semaphore = readers ?? writer + let connection = try await semaphore.acquire(count: 1) + let lease = connection.acquiredItems[0].asLease() + try await onConnection(lease) + } + + func write(onConnection: @Sendable (RawConnectionLease) async throws -> Void) async throws { + let connection = try await writer.acquire(count: 1) + let lease = connection.acquiredItems[0].asLease() + try await onConnection(lease) + try dispatchWrites(lease: lease) + } + + func withAllConnections(onConnection: @Sendable (RawConnectionLease, [RawConnectionLease]) async throws -> Void) async throws { + let write = try await writer.acquire(count: 1) + let writeLease = write.acquiredItems[0].asLease() + if let readers { + let acquiredReaders = try await readers.acquire(count: readers.count) + var readerLeases: [RawConnectionLease] = [] + var i = 0 + while i < acquiredReaders.acquiredItems.count { + readerLeases.append(write.acquiredItems[i].asLease()) + i += 1 + } + + try await onConnection(writeLease, readerLeases) + } else { + try await onConnection(writeLease, []) + } + + try dispatchWrites(lease: writeLease) + } + + func close() async throws { + // First, lock all connections + var write = try await writer.acquire(count: 1) + var acquiredReaders: SemaphoreGrant? = nil + if let readers { + acquiredReaders = try await readers.acquire(count: readers.count) + } + + // Close the write connection first + write.acquiredItems[0].close() + if var acquiredReaders { + var i = 0 + while i < acquiredReaders.acquiredItems.count { + acquiredReaders.acquiredItems[i].close() + i += 1 + } + } + } +} + +struct RawSqliteConnection: ~Copyable { + let connection: OpaquePointer + var closed = false + + deinit { + if !closed { + closeInner() + } + } + + mutating func close() { + if !closed { + closeInner() + closed = true + } + } + + private func closeInner() { + sqlite3_close_v2(connection) + } + + func asLease() -> RawConnectionLease { + precondition(!closed) + return RawConnectionLease(pointer: self.connection) + } +} + +struct RawConnectionLease: SQLiteConnectionLease, @unchecked Sendable { + let pointer: OpaquePointer +} diff --git a/Sources/PowerSync/Implementation/sqlite3/TransactionImpl.swift b/Sources/PowerSync/Implementation/sqlite3/TransactionImpl.swift new file mode 100644 index 0000000..0da486b --- /dev/null +++ b/Sources/PowerSync/Implementation/sqlite3/TransactionImpl.swift @@ -0,0 +1,37 @@ +struct TransactionImpl: Transaction { + let inner: ConnectionContext + + func execute(sql: String, parameters: [(any Sendable)?]?) throws -> Int64 { + return try self.inner.execute(sql: sql, parameters: parameters) + } + + func getOptional(sql: String, parameters: [(any Sendable)?]?, mapper: @escaping @Sendable (any SqlCursor) throws -> RowType) throws -> RowType? { + return try self.inner.getOptional(sql: sql, parameters: parameters, mapper: mapper) + } + + func getAll(sql: String, parameters: [(any Sendable)?]?, mapper: @escaping @Sendable (any SqlCursor) throws -> RowType) throws -> [RowType] { + return try self.inner.getAll(sql: sql, parameters: parameters, mapper: mapper) + } + + func get(sql: String, parameters: [(any Sendable)?]?, mapper: @escaping @Sendable (any SqlCursor) throws -> RowType) throws -> RowType { + return try self.inner.get(sql: sql, parameters: parameters, mapper: mapper) + } + + static func run(conn: any ConnectionContext, callback: @Sendable (any Transaction) throws -> R) throws -> R { + let _ = try conn.execute(sql: "BEGIN IMMEDIATE", parameters: nil) + + do { + let result = try callback(TransactionImpl(inner: conn)) + let _ = try conn.execute(sql: "COMMIT", parameters: nil) + return result + } catch { + do { + let _ = try conn.execute(sql: "ROLLBACK", parameters: nil) + } catch { + // Failed rollback, probably an INSERT OR ROLLBACK statement that rolled the transaction back already. Ignore. + } + + throw error + } + } +} diff --git a/Sources/PowerSync/Implementation/sqlite3/registerPowerSyncCoreExtension.swift b/Sources/PowerSync/Implementation/sqlite3/registerPowerSyncCoreExtension.swift new file mode 100644 index 0000000..19ad259 --- /dev/null +++ b/Sources/PowerSync/Implementation/sqlite3/registerPowerSyncCoreExtension.swift @@ -0,0 +1,17 @@ +import CSQLite +import PowerSyncCoreShim + +func registerPowerSyncCoreExtension() throws(PowerSyncError) { + let rc = sqlite3_auto_extension(sqlite3_powersync_init) + if rc != 0 { + let errStr = String(cString: sqlite3_errstr(rc)) + + throw .sqliteError( + extendedResultCode: rc, + offset: nil, + message: "Could not load PowerSync SQLite core extension", + errorString: errStr, + sql: nil + ) + } +} diff --git a/Sources/PowerSync/Implementation/sqlite3/throwDatabaseError.swift b/Sources/PowerSync/Implementation/sqlite3/throwDatabaseError.swift new file mode 100644 index 0000000..8eb0f34 --- /dev/null +++ b/Sources/PowerSync/Implementation/sqlite3/throwDatabaseError.swift @@ -0,0 +1,17 @@ +import CSQLite + +func throwDatabaseError(db: SQLiteConnectionLease, sql: String?) throws(PowerSyncError) -> Never { + let extended = sqlite3_extended_errcode(db.pointer) + let errStr = String(cString: sqlite3_errstr(extended)) + + let offset = sqlite3_error_offset(db.pointer) + let rawMessage = sqlite3_errmsg(db.pointer) + + throw PowerSyncError.sqliteError( + extendedResultCode: extended, + offset: offset >= 0 ? offset : nil, + message: rawMessage.map { ptr in String(cString: ptr) }, + errorString: errStr, + sql: sql, + ) +} diff --git a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift index 1ac86e2..c0b297d 100644 --- a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift +++ b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift @@ -5,13 +5,13 @@ import Synchronization fileprivate let tag = "StreamingSyncClient" final class StreamingSyncClient: Sendable { - let db: KotlinPowerSyncDatabaseImpl + let db: PowerSyncDatabaseImpl let options: ConnectOptions let connector: CachingCredentialsConnector let httpClient: any HttpClient init( - db: KotlinPowerSyncDatabaseImpl, + db: PowerSyncDatabaseImpl, connector: PowerSyncBackendConnectorProtocol, httpClient: any HttpClient, options: ConnectOptions, @@ -104,7 +104,7 @@ The next upload iteration will be delayed. private func uploadLocalTarget() async throws { guard let _ = try await db.getOptional( sql: "SELECT 1 FROM ps_buckets WHERE name = '$local' AND target_op = ?", - parameters: [KotlinPowerSyncDatabaseImpl.maxOpId], + parameters: [PowerSyncDatabaseImpl.maxOpId], mapper: { cursor in () } ) else { return // Nothing to update diff --git a/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift b/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift index e9b1f96..b54dbee 100644 --- a/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift +++ b/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift @@ -3,7 +3,7 @@ actor SyncCoordinator { nonisolated let streams = StreamTracker() private var activeSync: Task? - func connect(db: KotlinPowerSyncDatabaseImpl, connector: PowerSyncBackendConnectorProtocol, options: ConnectOptions, client: HttpClient) async { + func connect(db: PowerSyncDatabaseImpl, connector: PowerSyncBackendConnectorProtocol, options: ConnectOptions, client: HttpClient) async { if let task = activeSync { await self.finishSyncTask(task: task) } @@ -25,6 +25,12 @@ actor SyncCoordinator { await self.finishSyncTask(task: task) } + /// Runs ``Self/disconnect`` and `action` in a single actor message lock. + func disconnectAndThen(action: () async throws -> T) async rethrows -> T { + await disconnect() + return try await action() + } + /// Executes an inner function, but only if no connection is active or scheduled. func guardNotConnected(inner: () async throws -> T, ifConnected: () async throws -> T) async rethrows -> T { if activeSync == nil { diff --git a/Sources/PowerSync/Kotlin/AllLeaseCallback+Sendable.swift b/Sources/PowerSync/Kotlin/AllLeaseCallback+Sendable.swift deleted file mode 100644 index 5095d46..0000000 --- a/Sources/PowerSync/Kotlin/AllLeaseCallback+Sendable.swift +++ /dev/null @@ -1,18 +0,0 @@ -import PowerSyncKotlin - -// Since AllLeaseCallback is a protocol from PowerSyncKotlin, we need to use a wrapper class -// to make it Sendable since we can't extend the protocol directly with Sendable. -final class SendableAllLeaseCallback: @unchecked Sendable { - private let wrapped: any PowerSyncKotlin.AllLeaseCallback - - init(_ callback: any PowerSyncKotlin.AllLeaseCallback) { - wrapped = callback - } - - func execute( - writeLease: PowerSyncKotlin.SwiftLeaseAdapter, - readLeases: [PowerSyncKotlin.SwiftLeaseAdapter] - ) throws { - try wrapped.execute(writeLease: writeLease, readLeases: readLeases) - } -} diff --git a/Sources/PowerSync/Kotlin/DatabaseLogger.swift b/Sources/PowerSync/Kotlin/DatabaseLogger.swift deleted file mode 100644 index ed49d5a..0000000 --- a/Sources/PowerSync/Kotlin/DatabaseLogger.swift +++ /dev/null @@ -1,101 +0,0 @@ -import PowerSyncKotlin - -/// Adapts a Swift `LoggerProtocol` to Kermit's `LogWriter` interface. -/// -/// This allows Kotlin logging (via Kermit) to call into the Swift logging implementation. -private class KermitLogWriterAdapter: Kermit_coreLogWriter { - /// The underlying Swift log writer to forward log messages to. - let logger: any LoggerProtocol - - /// Initializes a new adapter. - /// - /// - Parameter logger: A Swift log writer that will handle log output. - init(logger: any LoggerProtocol) { - self.logger = logger - super.init() - } - - /// Called by Kermit to log a message. - /// - /// - Parameters: - /// - severity: The severity level of the log. - /// - message: The content of the log message. - /// - tag: A string categorizing the log. - /// - throwable: An optional Kotlin exception (ignored here). - override func log(severity: Kermit_coreSeverity, message: String, tag: String, throwable _: KotlinThrowable?) { - switch severity { - case PowerSyncKotlin.Kermit_coreSeverity.verbose: - return logger.debug(message, tag: tag) - case PowerSyncKotlin.Kermit_coreSeverity.debug: - return logger.debug(message, tag: tag) - case PowerSyncKotlin.Kermit_coreSeverity.info: - return logger.info(message, tag: tag) - case PowerSyncKotlin.Kermit_coreSeverity.warn: - return logger.warning(message, tag: tag) - case PowerSyncKotlin.Kermit_coreSeverity.error: - return logger.error(message, tag: tag) - case PowerSyncKotlin.Kermit_coreSeverity.assert: - return logger.fault(message, tag: tag) - } - } -} - -class KotlinKermitLoggerConfig: PowerSyncKotlin.Kermit_coreLoggerConfig { - var logWriterList: [Kermit_coreLogWriter] - var minSeverity: PowerSyncKotlin.Kermit_coreSeverity - - init(logWriterList: [Kermit_coreLogWriter], minSeverity: PowerSyncKotlin.Kermit_coreSeverity) { - self.logWriterList = logWriterList - self.minSeverity = minSeverity - } -} - -/// A logger implementation that integrates with PowerSync's Kotlin core using Kermit. -/// -/// This class bridges Swift log writers with the Kotlin logging system and supports -/// runtime configuration of severity levels and writer lists. -final class DatabaseLogger: LoggerProtocol { - /// The underlying Kermit logger instance provided by the PowerSyncKotlin SDK. - public let kLogger: PowerSyncKotlin.KermitLogger - public let logger: any LoggerProtocol - - /// Initializes a new logger with an optional list of writers. - /// - /// - Parameter logger: A logger which will be called for each internal log operation - init(_ logger: any LoggerProtocol) { - self.logger = logger - // Set to the lowest severity. The provided logger should filter by severity - kLogger = PowerSyncKotlin.KermitLogger( - config: KotlinKermitLoggerConfig( - logWriterList: [KermitLogWriterAdapter(logger: logger)], - minSeverity: Kermit_coreSeverity.verbose - ), - tag: "PowerSync" - ) - } - - /// Logs a debug-level message. - public func debug(_ message: String, tag: String?) { - logger.debug(message, tag: tag) - } - - /// Logs an info-level message. - public func info(_ message: String, tag: String?) { - logger.info(message, tag: tag) - } - - /// Logs a warning-level message. - public func warning(_ message: String, tag: String?) { - logger.warning(message, tag: tag) - } - - /// Logs an error-level message. - public func error(_ message: String, tag: String?) { - logger.error(message, tag: tag) - } - - /// Logs a fault (assert-level) message, typically used for critical issues. - public func fault(_ message: String, tag: String?) { - logger.fault(message, tag: tag) - } -} diff --git a/Sources/PowerSync/Kotlin/KotlinAdapter.swift b/Sources/PowerSync/Kotlin/KotlinAdapter.swift deleted file mode 100644 index fd88c78..0000000 --- a/Sources/PowerSync/Kotlin/KotlinAdapter.swift +++ /dev/null @@ -1,136 +0,0 @@ -import PowerSyncKotlin - -enum KotlinAdapter { - struct Index { - static func toKotlin(_ index: IndexProtocol) -> PowerSyncKotlin.Index { - PowerSyncKotlin.Index( - name: index.name, - columns: index.columns.map { IndexedColumn.toKotlin($0) } - ) - } - } - - struct IndexedColumn { - static func toKotlin(_ column: IndexedColumnProtocol) -> PowerSyncKotlin.IndexedColumn { - return PowerSyncKotlin.IndexedColumn( - column: column.column, - ascending: column.ascending, - columnDefinition: nil, - type: nil - ) - } - } - - struct Table { - static func toKotlin(_ table: TableProtocol) -> PowerSyncKotlin.Table { - return PowerSyncKotlin.Table( - name: table.name, - columns: table.columns.map { Column.toKotlin($0) }, - indexes: table.indexes.map { Index.toKotlin($0) }, - options: translateTableOptions(table), - viewNameOverride: table.viewNameOverride, - ) - } - - static func toKotlin(_ table: RawTable) -> PowerSyncKotlin.RawTable { - let translatedPut = table.put.map(translateStatement) - let translatedDelete = table.delete.map(translateStatement) - - if let schema = table.schema { - return PowerSyncKotlin.RawTable( - name: table.name, - schema: translateRawTableSchema(schema), - put: translatedPut, - delete: translatedDelete, - clear: table.clear, - ) - } - - // If we have no schema, put and delete are required. The constructor overloads on RawTable - // should ensure that, but it's better to be defensive here. - guard let put = translatedPut, let delete = translatedDelete else { - fatalError("RawTable '\(table.name)' has no schema and must provide both put and delete statements") - } - - return PowerSyncKotlin.RawTable( - name: table.name, - put: put, - delete: delete, - clear: table.clear - ); - } - - private static func translateTableOptions(_ options: TableOptionsProtocol) -> PowerSyncKotlin.TableOptions { - return PowerSyncKotlin.TableOptions( - localOnly: options.localOnly, - insertOnly: options.insertOnly, - trackMetadata: options.trackMetadata, - trackPreviousValues: options.trackPreviousValues.map { - PowerSyncKotlin.TrackPreviousValuesOptions( - columnFilter: $0.columnFilter, - onlyWhenChanged: $0.onlyWhenChanged - ) - }, - ignoreEmptyUpdates: options.ignoreEmptyUpdates, - ) - } - - private static func translateRawTableSchema(_ schema: RawTableSchema) -> PowerSyncKotlin.RawTableSchema { - return PowerSyncKotlin.RawTableSchema.init( - tableName: schema.tableName, - syncedColumns: schema.syncedColumns, - options: translateTableOptions(schema.options) - ) - } - - private static func translateStatement(_ stmt: PendingStatement) -> PowerSyncKotlin.PendingStatement { - return PowerSyncKotlin.PendingStatement( - sql: stmt.sql, - parameters: stmt.parameters.map(translateParameter) - ) - } - - private static func translateParameter(_ param: PendingStatementParameter) -> PowerSyncKotlin.PendingStatementParameter { - switch param { - case .id: - return PowerSyncKotlin.PendingStatementParameterId.shared - case .column(let name): - return PowerSyncKotlin.PendingStatementParameterColumn(name: name) - case .rest: - return PowerSyncKotlin.PendingStatementParameterRest.shared - } - } - } - - struct Column { - static func toKotlin(_ column: any ColumnProtocol) -> PowerSyncKotlin.Column { - PowerSyncKotlin.Column( - name: column.name, - type: columnType(from: column.type) - ) - } - - private static func columnType(from swiftType: ColumnData) -> PowerSyncKotlin.ColumnType { - switch swiftType { - case .text: - return PowerSyncKotlin.ColumnType.text - case .integer: - return PowerSyncKotlin.ColumnType.integer - case .real: - return PowerSyncKotlin.ColumnType.real - } - } - } - - struct Schema { - static func toKotlin(_ schema: SchemaProtocol) -> PowerSyncKotlin.Schema { - var mappedTables: [PowerSyncKotlin.BaseTable] = [] - mappedTables.append(contentsOf: schema.tables.map(Table.toKotlin)) - mappedTables.append(contentsOf: schema.rawTables.map(Table.toKotlin)) - - return PowerSyncKotlin.Schema( - tables: mappedTables - ) - } - } -} diff --git a/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift b/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift deleted file mode 100644 index f22576f..0000000 --- a/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift +++ /dev/null @@ -1,612 +0,0 @@ -import CSQLite -import Foundation -import PowerSyncKotlin - -final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, - // `PowerSyncKotlin.PowerSyncDatabase` cannot be marked as Sendable - @unchecked Sendable -{ - let logger: any LoggerProtocol - private let kotlinDatabase: PowerSyncKotlin.PowerSyncDatabase - private let encoder = JSONEncoder() - let syncCoordinator = SyncCoordinator() - internal let syncStatus = SwiftSyncStatus() - private let dbFilename: String - private let httpClient: HttpClient - let schema: AsyncMutex - - init( - kotlinDatabase: PowerSyncKotlin.PowerSyncDatabase, - dbFilename: String, - logger: DatabaseLogger, - httpClient: HttpClient, - schema: Schema - ) { - self.logger = logger - self.kotlinDatabase = kotlinDatabase - /// We currently use the dbFilename to delete the database files when the database is closed - /// The kotlin PowerSyncDatabase.identifier currently prepends `null` to the dbFilename (for the directory). - /// FIXME. Update this once we support database directory configuration. - self.dbFilename = dbFilename - self.httpClient = httpClient - self.schema = AsyncMutex(schema) - } - - var currentStatus: any SyncStatus { - syncStatus - } - - func waitForFirstSync() async { - await syncStatus.waitFor { $0.hasSynced == true } - } - - func updateSchema(schema: any SchemaProtocol) async throws { - try await syncCoordinator.guardNotConnected( - inner: { - await self.schema.withMutex { $0 = Schema(other: schema) } - - try await kotlinDatabase.updateSchema( - schema: KotlinAdapter.Schema.toKotlin(schema) - ) - }, - ifConnected: { throw PowerSyncError.operationFailed(message: "Cannot update schema while connected") } - ) - } - - func resolveOfflineSyncStatusIfNotConnected() async throws { - try await syncCoordinator.guardNotConnected(inner: { - try await resolveOfflineSyncStatus() - }, ifConnected: {}) - } - - private func resolveOfflineSyncStatus() async throws { - let offlineSyncStatus = try await get("SELECT powersync_offline_sync_status()") { cursor in - let raw = try cursor.getString(index: 0) - return try StreamingSyncClient.jsonDecoder.decode(CoreDownloadSyncStatus.self, from: raw.data(using: .utf8)!) - } - - syncStatus.mutateStatus { $0 = MutableSyncStatus(core: offlineSyncStatus) } - } - - func waitForFirstSync(priority: Int32) async { - let priority = BucketPriority(priority) - await syncStatus.waitFor { $0.statusForPriority(priority).hasSynced == true } - } - - func syncStream(name: String, params: JsonParam?) -> any SyncStream { - PendingSyncStream(db: self, name: name, parameters: params) - } - - func connect( - connector: PowerSyncBackendConnectorProtocol, - options: ConnectOptions? - ) async { - await syncCoordinator.connect( - db: self, - connector: connector, - options: options ?? ConnectOptions(), - client: httpClient - ) - } - - func getCrudBatch(limit: Int32 = 100) async throws -> CrudBatch? { - var entries = try await getAll( - sql: "SELECT id, tx_id, data FROM ps_crud ORDER BY id ASC LIMIT ?", - parameters: [Int64(limit + 1)], - mapper: CrudEntry.fromCursor - ) - - if entries.isEmpty { - return nil - } - - let hasMore = entries.count > limit - if hasMore { - entries.removeLast() - } - - return CrudBatch( - hasMore: hasMore, - crud: entries, - db: self - ) - } - - func getCrudTransactions() -> CrudTransactions { - return CrudTransactions(db: self) - } - - func getPowerSyncVersion() async throws -> String { - try await kotlinDatabase.getPowerSyncVersion() - } - - func disconnect() async { - await syncCoordinator.disconnect() - } - - func disconnectAndClear(clearLocal: Bool, soft: Bool) async throws { - await disconnect() - - try await kotlinDatabase.disconnectAndClear( - clearLocal: clearLocal, - soft: soft - ) - } - - @discardableResult - func execute(sql: String, parameters: [Sendable?]?) async throws -> Int64 { - try await writeTransaction { ctx in - try ctx.execute( - sql: sql, - parameters: parameters - ) - } - } - - func get( - sql: String, - parameters: [Sendable?]?, - mapper: @Sendable @escaping (SqlCursor) -> RowType - ) async throws -> RowType { - try await readLock { ctx in - try ctx.get( - sql: sql, - parameters: parameters, - mapper: mapper - ) - } - } - - func get( - sql: String, - parameters: [Sendable?]?, - mapper: @Sendable @escaping (SqlCursor) throws -> RowType - ) async throws -> RowType { - try await readLock { ctx in - try ctx.get( - sql: sql, - parameters: parameters, - mapper: mapper - ) - } - } - - func getAll( - sql: String, - parameters: [Sendable?]?, - mapper: @Sendable @escaping (SqlCursor) -> RowType - ) async throws -> [RowType] { - try await readLock { ctx in - try ctx.getAll( - sql: sql, - parameters: parameters, - mapper: mapper - ) - } - } - - func getAll( - sql: String, - parameters: [Sendable?]?, - mapper: @Sendable @escaping (SqlCursor) throws -> RowType - ) async throws -> [RowType] { - try await readLock { ctx in - try ctx.getAll( - sql: sql, - parameters: parameters, - mapper: mapper - ) - } - } - - func getOptional( - sql: String, - parameters: [Sendable?]?, - mapper: @Sendable @escaping (SqlCursor) -> RowType - ) async throws -> RowType? { - try await readLock { ctx in - try ctx.getOptional( - sql: sql, - parameters: parameters, - mapper: mapper - ) - } - } - - func getOptional( - sql: String, - parameters: [Sendable?]?, - mapper: @Sendable @escaping (SqlCursor) throws -> RowType - ) async throws -> RowType? { - try await readLock { ctx in - try ctx.getOptional( - sql: sql, - parameters: parameters, - mapper: mapper - ) - } - } - - func watch( - sql: String, - parameters: [Sendable?]?, - mapper: @Sendable @escaping (SqlCursor) -> RowType - ) throws -> AsyncThrowingStream<[RowType], any Error> { - try watch( - options: WatchOptions( - sql: sql, - parameters: parameters, - mapper: mapper - ) - ) - } - - func watch( - sql: String, - parameters: [Sendable?]?, - mapper: @Sendable @escaping (SqlCursor) throws -> RowType - ) throws -> AsyncThrowingStream<[RowType], any Error> { - try watch( - options: WatchOptions( - sql: sql, - parameters: parameters, - mapper: mapper - ) - ) - } - - func watch( - options: WatchOptions - ) throws -> AsyncThrowingStream<[RowType], Error> { - AsyncThrowingStream { continuation in - // Create an outer task to monitor cancellation - let task = Task { - do { - let watchedTables = try await self.getQuerySourceTables( - sql: options.sql, - parameters: options.parameters - ) - - // Watching for changes in the database - for try await _ in try self.kotlinDatabase.onChange( - tables: Set(watchedTables), - throttleMs: Int64(options.throttle * 1000), - triggerImmediately: true // Allows emitting the first result even if there aren't changes - ) { - // Check if the outer task is cancelled - try Task.checkCancellation() - - try continuation.yield( - safeCast( - await self.getAll( - sql: options.sql, - parameters: options.parameters, - mapper: options.mapper - ), - to: [RowType].self - ) - ) - } - - continuation.finish() - } catch { - if error is CancellationError { - continuation.finish() - } else { - continuation.finish(throwing: error) - } - } - } - - // Propagate cancellation from the outer task to the inner task - continuation.onTermination = { @Sendable _ in - task.cancel() // This cancels the inner task when the stream is terminated - } - } - } - - func writeLock( - callback: @Sendable @escaping (any ConnectionContext) throws -> R - ) async throws -> R { - return try await wrapPowerSyncException { - try safeCast( - await kotlinDatabase.writeLock( - callback: wrapLockContext(callback: callback) - ), - to: R.self - ) - } - } - - func writeTransaction( - callback: @Sendable @escaping (any Transaction) throws -> R - ) async throws -> R { - return try await wrapPowerSyncException { - try safeCast( - await kotlinDatabase.writeTransaction( - callback: wrapTransactionContext(callback: callback) - ), - to: R.self - ) - } - } - - func readLock( - callback: @Sendable @escaping (any ConnectionContext) throws -> R - ) - async throws -> R - { - return try await wrapPowerSyncException { - try safeCast( - await kotlinDatabase.readLock( - callback: wrapLockContext(callback: callback) - ), - to: R.self - ) - } - } - - func readTransaction( - callback: @Sendable @escaping (any Transaction) throws -> R - ) async throws -> R { - return try await wrapPowerSyncException { - try safeCast( - await kotlinDatabase.readTransaction( - callback: wrapTransactionContext(callback: callback) - ), - to: R.self - ) - } - } - - func close() async throws { - await disconnect() - try await kotlinDatabase.close() - } - - func close(deleteDatabase: Bool = false) async throws { - // Close the SQLite connections - try await close() - - if deleteDatabase { - try await self.deleteDatabase() - } - } - - private func deleteDatabase() async throws { - // We can use the supplied dbLocation when we support that in future - let directory = try appleDefaultDatabaseDirectory() - try deleteSQLiteFiles(dbFilename: dbFilename, in: directory) - } - - /// Tries to convert Kotlin PowerSyncExceptions to Swift Exceptions - private func wrapPowerSyncException( - handler: () async throws -> R) - async throws -> R - { - do { - return try await handler() - } catch { - if error is CancellationError { - throw error - } - // Try and parse errors back from the Kotlin side - if let mapperError = SqlCursorError.fromDescription(error.localizedDescription) { - throw mapperError - } - - throw PowerSyncError.operationFailed( - underlyingError: error - ) - } - } - - private func getQuerySourceTables( - sql: String, - parameters: [Sendable?] - ) async throws -> Set { - let rows = try await getAll( - sql: "EXPLAIN \(sql)", - parameters: parameters, - mapper: { cursor in - try ExplainQueryResult( - addr: cursor.getString(index: 0), - opcode: cursor.getString(index: 1), - p1: cursor.getInt64(index: 2), - p2: cursor.getInt64(index: 3), - p3: cursor.getInt64(index: 4) - ) - } - ) - - let rootPages = rows.compactMap { row in - if (row.opcode == "OpenRead" || row.opcode == "OpenWrite") && - row.p3 == 0 && row.p2 != 0 - { - return row.p2 - } - return nil - } - - do { - let pagesData = try encoder.encode(rootPages) - - guard let pagesString = String(data: pagesData, encoding: .utf8) else { - throw PowerSyncError.operationFailed( - message: "Failed to convert pages data to UTF-8 string" - ) - } - - let tableRows = try await getAll( - sql: "SELECT tbl_name FROM sqlite_master WHERE rootpage IN (SELECT json_each.value FROM json_each(?))", - parameters: [ - pagesString, - ] - ) { try $0.getString(index: 0) } - - return Set(tableRows) - } catch { - throw PowerSyncError.operationFailed( - message: "Could not determine watched query tables", - underlyingError: error - ) - } - } - - static let maxOpId = Int64.max -} - -func openKotlinDBDefault( - schema: Schema, - dbFilename: String, - logger: DatabaseLogger, - initialStatements: [String] = [], - httpClient: HttpClient = PlatformHttpClient.shared -) -> PowerSyncDatabaseProtocol { - let rc = sqlite3_initialize() - if rc != 0 { - fatalError("Call to sqlite3_initialize() failed with \(rc)") - } - - let factory = sqlite3DatabaseFactory(initialStatements: initialStatements) - let kotlinDatabase = if dbFilename == ":memory:" { - openPowerSyncInMemory( - factory: factory, - schema: KotlinAdapter.Schema.toKotlin(schema), - logger: logger.kLogger - ) - } else { - PowerSyncDatabase( - factory: factory, - schema: KotlinAdapter.Schema.toKotlin(schema), - dbFilename: dbFilename, - logger: logger.kLogger - ) - } - - return KotlinPowerSyncDatabaseImpl( - kotlinDatabase: kotlinDatabase, - dbFilename: dbFilename, - logger: logger, - httpClient: httpClient, - schema: schema - ) -} - -func openKotlinDBWithPool( - schema: Schema, - pool: SQLiteConnectionPoolProtocol, - identifier: String, - logger: DatabaseLogger -) -> PowerSyncDatabaseProtocol { - return KotlinPowerSyncDatabaseImpl( - kotlinDatabase: openPowerSyncWithPool( - pool: pool.toKotlin(), - identifier: identifier, - schema: KotlinAdapter.Schema.toKotlin(schema), - logger: logger.kLogger - ), - dbFilename: identifier, - logger: logger, - httpClient: PlatformHttpClient.shared, - schema: schema - ) -} - -private struct ExplainQueryResult { - let addr: String - let opcode: String - let p1: Int64 - let p2: Int64 - let p3: Int64 -} - -extension Error { - func toPowerSyncError() -> PowerSyncKotlin.PowerSyncException { - return PowerSyncKotlin.PowerSyncException( - message: localizedDescription, - cause: PowerSyncKotlin.KotlinThrowable(message: localizedDescription) - ) - } -} - -func wrapLockContext( - callback: @Sendable @escaping (any ConnectionContext) throws -> Any -) throws -> PowerSyncKotlin.ThrowableLockCallback { - PowerSyncKotlin.wrapContextHandler { kotlinContext in - do { - return try PowerSyncKotlin.PowerSyncResult.Success( - value: callback( - KotlinConnectionContext( - ctx: kotlinContext - ) - )) - } catch { - return PowerSyncKotlin.PowerSyncResult.Failure( - exception: error.toPowerSyncError() - ) - } - } -} - -func wrapTransactionContext( - callback: @Sendable @escaping (any Transaction) throws -> Any -) throws -> PowerSyncKotlin.ThrowableTransactionCallback { - PowerSyncKotlin.wrapTransactionContextHandler { kotlinContext in - do { - return try PowerSyncKotlin.PowerSyncResult.Success( - value: callback( - KotlinTransactionContext( - ctx: kotlinContext - ) - )) - } catch { - return PowerSyncKotlin.PowerSyncResult.Failure( - exception: error.toPowerSyncError() - ) - } - } -} - -/// This returns the default directory in which we store SQLite database files. -func appleDefaultDatabaseDirectory() throws -> URL { - let fileManager = FileManager.default - - // Get the application support directory - guard let documentsDirectory = fileManager.urls( - for: .applicationSupportDirectory, - in: .userDomainMask - ).first else { - throw PowerSyncError.operationFailed(message: "Unable to find application support directory") - } - - return documentsDirectory.appendingPathComponent("databases") -} - -/// Deletes all SQLite files for a given database filename in the specified directory. -/// This includes the main database file and WAL mode files (.wal, .shm, and .journal if present). -/// Throws an error if a file exists but could not be deleted. Files that don't exist are ignored. -func deleteSQLiteFiles(dbFilename: String, in directory: URL) throws { - let fileManager = FileManager.default - - // SQLite files to delete: - // 1. Main database file: dbFilename - // 2. WAL file: dbFilename-wal - // 3. SHM file: dbFilename-shm - // 4. Journal file: dbFilename-journal (for rollback journal mode, though WAL mode typically doesn't use it) - - let filesToDelete = [ - dbFilename, - "\(dbFilename)-wal", - "\(dbFilename)-shm", - "\(dbFilename)-journal" - ] - - for filename in filesToDelete { - let fileURL = directory.appendingPathComponent(filename) - if fileManager.fileExists(atPath: fileURL.path) { - try fileManager.removeItem(at: fileURL) - } - // If file doesn't exist, we ignore it and continue - } -} diff --git a/Sources/PowerSync/Kotlin/KotlinSQLiteConnectionPool.swift b/Sources/PowerSync/Kotlin/KotlinSQLiteConnectionPool.swift deleted file mode 100644 index c113659..0000000 --- a/Sources/PowerSync/Kotlin/KotlinSQLiteConnectionPool.swift +++ /dev/null @@ -1,109 +0,0 @@ -import PowerSyncKotlin - -class KotlinLeaseAdapter: PowerSyncKotlin.SwiftLeaseAdapter { - let pointer: UnsafeMutableRawPointer - - init( - lease: SQLiteConnectionLease - ) { - pointer = UnsafeMutableRawPointer(lease.pointer) - } -} - -final class SwiftSQLiteConnectionPoolAdapter: PowerSyncKotlin.SwiftPoolAdapter { - let pool: SQLiteConnectionPoolProtocol - var updateTrackingTask: Task? - - init( - pool: SQLiteConnectionPoolProtocol - ) { - self.pool = pool - } - - func linkExternalUpdates(callback: any KotlinSuspendFunction1) { - let sendableCallback = SendableSuspendFunction1(callback) - updateTrackingTask = Task { [pool] in - do { - for try await updates in pool.tableUpdates { - _ = try await sendableCallback.invoke(p1: updates) - } - } catch { - // none of these calls should actually throw - } - } - } - - func __dispose() async throws { - return try await wrapExceptions { - updateTrackingTask?.cancel() - updateTrackingTask = nil - } - } - - func __leaseRead(callback: any LeaseCallback) async throws { - return try await wrapExceptions { - let sendableCallback = SendableLeaseCallback(callback) - try await pool.read { lease in - try sendableCallback.execute( - lease: KotlinLeaseAdapter( - lease: lease - ) - ) - } - } - } - - func __leaseWrite(callback: any LeaseCallback) async throws { - return try await wrapExceptions { - let sendableCallback = SendableLeaseCallback(callback) - try await pool.write { lease in - try sendableCallback.execute( - lease: KotlinLeaseAdapter( - lease: lease - ) - ) - } - } - } - - func __leaseAll(callback: any AllLeaseCallback) async throws { - // FIXME, actually use all connections - // We currently only use this for schema updates - return try await wrapExceptions { - let sendableCallback = SendableAllLeaseCallback(callback) - try await pool.withAllConnections { writer, readers in - try sendableCallback.execute( - writeLease: KotlinLeaseAdapter( - lease: writer - ), - readLeases: readers.map { KotlinLeaseAdapter(lease: $0) } - ) - } - } - } - - private func wrapExceptions( - _ callback: () async throws -> Result - ) async throws -> Result { - do { - return try await callback() - } catch { - try? PowerSyncKotlin.throwPowerSyncException( - exception: PowerSyncException( - message: error.localizedDescription, - cause: nil - ) - ) - // Won't reach here - throw error - } - } -} - -extension SQLiteConnectionPoolProtocol { - func toKotlin() -> PowerSyncKotlin.SwiftSQLiteConnectionPool { - return PowerSyncKotlin.SwiftSQLiteConnectionPool( - adapter: SwiftSQLiteConnectionPoolAdapter(pool: self) - ) - } -} diff --git a/Sources/PowerSync/Kotlin/KotlinSuspendFunction1+Sendable.swift b/Sources/PowerSync/Kotlin/KotlinSuspendFunction1+Sendable.swift deleted file mode 100644 index d02d3cf..0000000 --- a/Sources/PowerSync/Kotlin/KotlinSuspendFunction1+Sendable.swift +++ /dev/null @@ -1,15 +0,0 @@ -import PowerSyncKotlin - -// Since SendableSuspendFunction1 is a protocol from PowerSyncKotlin, we need to use a wrapper class -// to make it Sendable since we can't extend the protocol directly with Sendable. -final class SendableSuspendFunction1: @unchecked Sendable { - private let wrapped: any PowerSyncKotlin.KotlinSuspendFunction1 - - init(_ function: any PowerSyncKotlin.KotlinSuspendFunction1) { - wrapped = function - } - - func invoke(p1: Any?) async throws -> Any? { - return try await wrapped.invoke(p1: p1) - } -} diff --git a/Sources/PowerSync/Kotlin/KotlinTypes.swift b/Sources/PowerSync/Kotlin/KotlinTypes.swift deleted file mode 100644 index 9c256a4..0000000 --- a/Sources/PowerSync/Kotlin/KotlinTypes.swift +++ /dev/null @@ -1,15 +0,0 @@ -import PowerSyncKotlin - -typealias KotlinSwiftBackendConnector = PowerSyncKotlin.SwiftBackendConnector -typealias KotlinPowerSyncBackendConnector = PowerSyncKotlin.PowerSyncBackendConnector -typealias KotlinPowerSyncCredentials = PowerSyncKotlin.PowerSyncCredentials -typealias KotlinPowerSyncDatabase = PowerSyncKotlin.PowerSyncDatabase - -extension KotlinPowerSyncBackendConnector: @retroactive @unchecked Sendable {} -extension KotlinPowerSyncCredentials: @retroactive @unchecked Sendable {} -extension PowerSyncKotlin.KermitLogger: @retroactive @unchecked Sendable {} -extension PowerSyncKotlin.SyncStatus: @retroactive @unchecked Sendable {} - -extension PowerSyncKotlin.CrudEntry: @retroactive @unchecked Sendable {} -extension PowerSyncKotlin.CrudBatch: @retroactive @unchecked Sendable {} -extension PowerSyncKotlin.CrudTransaction: @retroactive @unchecked Sendable {} diff --git a/Sources/PowerSync/Kotlin/LeaseCallback+Sendable.swift b/Sources/PowerSync/Kotlin/LeaseCallback+Sendable.swift deleted file mode 100644 index db2f51f..0000000 --- a/Sources/PowerSync/Kotlin/LeaseCallback+Sendable.swift +++ /dev/null @@ -1,15 +0,0 @@ -import PowerSyncKotlin - -// Since LeaseCallback is a protocol from PowerSyncKotlin, we need to use a wrapper class -// to make it Sendable since we can't extend the protocol directly with Sendable. -final class SendableLeaseCallback: @unchecked Sendable { - private let wrapped: any PowerSyncKotlin.LeaseCallback - - init(_ callback: any PowerSyncKotlin.LeaseCallback) { - self.wrapped = callback - } - - func execute(lease: PowerSyncKotlin.SwiftLeaseAdapter) throws { - try wrapped.execute(lease: lease) - } -} \ No newline at end of file diff --git a/Sources/PowerSync/Kotlin/SafeCastError.swift b/Sources/PowerSync/Kotlin/SafeCastError.swift deleted file mode 100644 index bd18664..0000000 --- a/Sources/PowerSync/Kotlin/SafeCastError.swift +++ /dev/null @@ -1,30 +0,0 @@ -import Foundation - -enum SafeCastError: Error, CustomStringConvertible { - case typeMismatch(expected: String, actual: String?) - - var description: String { - switch self { - case let .typeMismatch(expected, actual): - let actualType = actual.map { String(describing: type(of: $0)) } ?? "nil" - return "Type mismatch: Expected \(expected), but got \(actualType)." - } - } -} - -func safeCast(_ value: Any?, to type: T.Type) throws -> T { - // Special handling for nil when T is an optional type - if value == nil || value is NSNull { - // Check if T is an optional type that can accept nil - let nilValue: Any? = nil - if let nilAsT = nilValue as? T { - return nilAsT - } - } - - if let castedValue = value as? T { - return castedValue - } else { - throw SafeCastError.typeMismatch(expected: "\(type)", actual: "\(value ?? "nil")") - } -} diff --git a/Sources/PowerSync/Kotlin/db/KotlinConnectionContext.swift b/Sources/PowerSync/Kotlin/db/KotlinConnectionContext.swift deleted file mode 100644 index 668b3a1..0000000 --- a/Sources/PowerSync/Kotlin/db/KotlinConnectionContext.swift +++ /dev/null @@ -1,107 +0,0 @@ -import Foundation -import PowerSyncKotlin - -/// Extension of the `ConnectionContext` protocol which allows mixin of common logic required for Kotlin adapters -protocol KotlinConnectionContextProtocol: ConnectionContext { - /// Implementations should provide access to a Kotlin context. - /// The protocol extension will use this to provide shared implementation. - var ctx: PowerSyncKotlin.ConnectionContext { get } -} - -/// Implements most of `ConnectionContext` using the `ctx` provided. -extension KotlinConnectionContextProtocol { - func execute(sql: String, parameters: [Sendable?]?) throws -> Int64 { - try ctx.execute( - sql: sql, - parameters: mapParameters(parameters) - ) - } - - func getOptional( - sql: String, - parameters: [Sendable?]?, - mapper: @Sendable @escaping (any SqlCursor) throws -> RowType - ) throws -> RowType? { - return try wrapQueryCursorTyped( - mapper: mapper, - executor: { wrappedMapper in - try self.ctx.getOptional( - sql: sql, - parameters: mapParameters(parameters), - mapper: wrappedMapper - ) - }, - resultType: RowType?.self - ) - } - - func getAll( - sql: String, - parameters: [Sendable?]?, - mapper: @Sendable @escaping (any SqlCursor) throws -> RowType - ) throws -> [RowType] { - return try wrapQueryCursorTyped( - mapper: mapper, - executor: { wrappedMapper in - try self.ctx.getAll( - sql: sql, - parameters: mapParameters(parameters), - mapper: wrappedMapper - ) - }, - resultType: [RowType].self - ) - } - - func get( - sql: String, - parameters: [Sendable?]?, - mapper: @Sendable @escaping (any SqlCursor) throws -> RowType - ) throws -> RowType { - return try wrapQueryCursorTyped( - mapper: mapper, - executor: { wrappedMapper in - try self.ctx.get( - sql: sql, - parameters: mapParameters(parameters), - mapper: wrappedMapper - ) - }, - resultType: RowType.self - ) - } -} - -final class KotlinConnectionContext: KotlinConnectionContextProtocol, - // The Kotlin ConnectionContext is technically sendable, but we cannot annotate that - @unchecked Sendable -{ - let ctx: PowerSyncKotlin.ConnectionContext - - init(ctx: PowerSyncKotlin.ConnectionContext) { - self.ctx = ctx - } -} - -final class KotlinTransactionContext: Transaction, KotlinConnectionContextProtocol, - // The Kotlin ConnectionContext is technically sendable, but we cannot annotate that - @unchecked Sendable -{ - let ctx: PowerSyncKotlin.ConnectionContext - - init(ctx: PowerSyncKotlin.PowerSyncTransaction) { - self.ctx = ctx - } -} - -// Allows nil values to be passed to the Kotlin [Any] params -func mapParameters(_ parameters: [Any?]?) -> [Any] { - parameters?.map { item in - switch item { - case .none: NSNull() - case let item as PowerSyncDataTypeConvertible: - item.psDataType?.unwrap() ?? NSNull() - default: item as Any - } - } ?? [] -} diff --git a/Sources/PowerSync/Kotlin/db/KotlinSqlCursor.swift b/Sources/PowerSync/Kotlin/db/KotlinSqlCursor.swift deleted file mode 100644 index 7b57c4a..0000000 --- a/Sources/PowerSync/Kotlin/db/KotlinSqlCursor.swift +++ /dev/null @@ -1,136 +0,0 @@ -import PowerSyncKotlin - -/// Implements `SqlCursor` using the Kotlin SDK -class KotlinSqlCursor: SqlCursor { - let base: PowerSyncKotlin.SqlCursor - - var columnCount: Int - - var columnNames: [String: Int] - - init(base: PowerSyncKotlin.SqlCursor) { - self.base = base - self.columnCount = Int(base.columnCount) - self.columnNames = base.columnNames.mapValues { input in input.intValue } - } - - func getBoolean(index: Int) throws -> Bool { - guard let result = getBooleanOptional(index: index) else { - throw SqlCursorError.nullValueFound(String(index)) - } - return result - } - - func getBooleanOptional(index: Int) -> Bool? { - base.getBoolean( - index: Int32(index) - )?.boolValue - } - - func getBoolean(name: String) throws -> Bool { - guard let result = try getBooleanOptional(name: name) else { - throw SqlCursorError.nullValueFound(name) - } - return result - } - - func getBooleanOptional(name: String) throws -> Bool? { - return getBooleanOptional(index: try guardColumnName(name)) - } - - func getDouble(index: Int) throws -> Double { - guard let result = getDoubleOptional(index: index) else { - throw SqlCursorError.nullValueFound(String(index)) - } - return result - } - - func getDoubleOptional(index: Int) -> Double? { - base.getDouble(index: Int32(index))?.doubleValue - } - - func getDouble(name: String) throws -> Double { - guard let result = try getDoubleOptional(name: name) else { - throw SqlCursorError.nullValueFound(name) - } - return result - } - - func getDoubleOptional(name: String) throws -> Double? { - return getDoubleOptional(index: try guardColumnName(name)) - } - - func getInt(index: Int) throws -> Int { - guard let result = getIntOptional(index: index) else { - throw SqlCursorError.nullValueFound(String(index)) - } - return result - } - - func getIntOptional(index: Int) -> Int? { - base.getLong(index: Int32(index))?.intValue - } - - func getInt(name: String) throws -> Int { - guard let result = try getIntOptional(name: name) else { - throw SqlCursorError.nullValueFound(name) - } - return result - } - - func getIntOptional(name: String) throws -> Int? { - return getIntOptional(index: try guardColumnName(name)) - } - - func getInt64(index: Int) throws -> Int64 { - guard let result = getInt64Optional(index: index) else { - throw SqlCursorError.nullValueFound(String(index)) - } - return result - } - - func getInt64Optional(index: Int) -> Int64? { - base.getLong(index: Int32(index))?.int64Value - } - - func getInt64(name: String) throws -> Int64 { - guard let result = try getInt64Optional(name: name) else { - throw SqlCursorError.nullValueFound(name) - } - return result - } - - func getInt64Optional(name: String) throws -> Int64? { - return getInt64Optional(index: try guardColumnName(name)) - } - - func getString(index: Int) throws -> String { - guard let result = getStringOptional(index: index) else { - throw SqlCursorError.nullValueFound(String(index)) - } - return result - } - - func getStringOptional(index: Int) -> String? { - base.getString(index: Int32(index)) - } - - func getString(name: String) throws -> String { - guard let result = try getStringOptional(name: name) else { - throw SqlCursorError.nullValueFound(name) - } - return result - } - - func getStringOptional(name: String) throws -> String? { - return getStringOptional(index: try guardColumnName(name)) - } - - @discardableResult - private func guardColumnName(_ name: String) throws -> Int { - guard let index = columnNames[name] else { - throw SqlCursorError.columnNotFound(name) - } - return index - } -} diff --git a/Sources/PowerSync/Kotlin/db/PowerSyncDataTypeConvertible.swift b/Sources/PowerSync/Kotlin/db/PowerSyncDataTypeConvertible.swift deleted file mode 100644 index bfe6d05..0000000 --- a/Sources/PowerSync/Kotlin/db/PowerSyncDataTypeConvertible.swift +++ /dev/null @@ -1,32 +0,0 @@ -import struct Foundation.Data - -/// Represents the set of types that are supported -/// by the PowerSync Kotlin Multiplatform SDK -public enum PowerSyncDataType { - case bool(Bool) - case string(String) - case int64(Int64) - case int32(Int32) - case double(Double) - case data(Data) -} - -/// Types conforming to this protocol will be -/// mapped to the specified ``PowerSyncDataType`` -/// before use by SQLite -public protocol PowerSyncDataTypeConvertible { - var psDataType: PowerSyncDataType? { get } -} - -extension PowerSyncDataType { - func unwrap() -> Any { - switch self { - case let .bool(bool): bool - case let .string(string): string - case let .int32(int32): int32 - case let .int64(int64): int64 - case let .double(double): double - case let .data(data): data - } - } -} \ No newline at end of file diff --git a/Sources/PowerSync/Kotlin/kotlinResolvePowerSyncLoadableExtensionPath.swift b/Sources/PowerSync/Kotlin/kotlinResolvePowerSyncLoadableExtensionPath.swift deleted file mode 100644 index 12af0ef..0000000 --- a/Sources/PowerSync/Kotlin/kotlinResolvePowerSyncLoadableExtensionPath.swift +++ /dev/null @@ -1,9 +0,0 @@ -import PowerSyncKotlin - -func kotlinResolvePowerSyncLoadableExtensionPath() throws -> String? { - do { - return try PowerSyncKotlin.resolvePowerSyncLoadableExtensionPath() - } catch { - throw PowerSyncError.operationFailed(message: "Failed to resolve PowerSync loadable extension path: \(error.localizedDescription)") - } -} \ No newline at end of file diff --git a/Sources/PowerSync/Kotlin/kotlinWithSession.swift b/Sources/PowerSync/Kotlin/kotlinWithSession.swift deleted file mode 100644 index 6ec03d0..0000000 --- a/Sources/PowerSync/Kotlin/kotlinWithSession.swift +++ /dev/null @@ -1,39 +0,0 @@ -import PowerSyncKotlin - -func kotlinWithSession( - db: OpaquePointer, - action: @escaping () throws -> ReturnType, -) throws -> WithSessionResult { - var innerResult: ReturnType? - let baseResult = try withSession( - db: UnsafeMutableRawPointer(db), - block: { - do { - innerResult = try action() - // We'll use the innerResult closure above to return the result - return PowerSyncResult.Success(value: nil) - } catch { - return PowerSyncResult.Failure(exception: error.toPowerSyncError()) - } - } - ) - - var outputResult: Result - if let failure = baseResult.blockResult as? PowerSyncResult.Failure { - outputResult = .failure(failure.exception.asError()) - } else if let result = innerResult { - outputResult = .success(result) - } else { - // The return type is not nullable, so we should have a result - outputResult = .failure( - PowerSyncError.operationFailed( - message: "Unknown error encountered when processing session", - ) - ) - } - - return WithSessionResult( - blockResult: outputResult, - affectedTables: baseResult.affectedTables - ) -} diff --git a/Sources/PowerSync/Kotlin/wrapQueryCursor.swift b/Sources/PowerSync/Kotlin/wrapQueryCursor.swift deleted file mode 100644 index ebdd33a..0000000 --- a/Sources/PowerSync/Kotlin/wrapQueryCursor.swift +++ /dev/null @@ -1,83 +0,0 @@ -import PowerSyncKotlin - -/// The Kotlin SDK does not gracefully handle exceptions thrown from Swift callbacks. -/// If a Swift callback throws an exception, it results in a `BAD ACCESS` crash. -/// -/// This approach is a workaround. Ideally, we should introduce an internal mechanism -/// in the Kotlin SDK to handle errors from Swift more robustly. -/// -/// This hoists any exceptions thrown in a cursor mapper in order for the error to propagate correctly. -/// -/// Currently, we wrap the public `PowerSyncDatabase` class in Kotlin, which limits our -/// ability to handle exceptions cleanly. Instead, we should expose an internal implementation -/// from a "core" package in Kotlin that provides better control over exception handling -/// and other functionality—without modifying the public `PowerSyncDatabase` API to include -/// Swift-specific logic. -func wrapQueryCursor( - mapper: @Sendable @escaping (SqlCursor) throws -> RowType, - // The Kotlin APIs return the results as Any, we can explicitly cast internally - executor: @Sendable @escaping (_ wrappedMapper: @escaping (PowerSyncKotlin.SqlCursor) -> RowType?) throws -> ReturnType -) throws -> ReturnType { - var mapperException: Error? - - // Wrapped version of the mapper that catches exceptions and sets `mapperException` - // In the case of an exception this will return an empty result. - let wrappedMapper: (PowerSyncKotlin.SqlCursor) -> RowType? = { cursor in - do { - return try mapper(KotlinSqlCursor( - base: cursor - )) - } catch { - // Store the error in order to propagate it - mapperException = error - // Return nothing here. Kotlin should handle this as an empty object/row - return nil - } - } - - let executionResult = try executor(wrappedMapper) - - if let mapperException { - // Allow propagating the error - throw mapperException - } - - return executionResult -} - -func wrapQueryCursorTyped( - mapper: @Sendable @escaping (SqlCursor) throws -> RowType, - // The Kotlin APIs return the results as Any, we can explicitly cast internally - executor: @Sendable @escaping (_ wrappedMapper: @escaping (PowerSyncKotlin.SqlCursor) -> Any?) throws -> Any?, - resultType: ReturnType.Type -) throws -> ReturnType { - return try safeCast( - wrapQueryCursor( - mapper: mapper, - executor: executor - ), to: - resultType - ) -} - -/// Throws a `PowerSyncException` using a helper provided by the Kotlin SDK. -/// We can't directly throw Kotlin `PowerSyncException`s from Swift, but we can delegate the throwing -/// to the Kotlin implementation. -/// Our Kotlin SDK methods handle thrown Kotlin `PowerSyncException` correctly. -/// The flow of events is as follows -/// Swift code calls `throwKotlinPowerSyncError` -/// This method calls the Kotlin helper `throwPowerSyncException` which is annotated as being able to throw `PowerSyncException` -/// The Kotlin helper throws the provided `PowerSyncException`. Since the method is annotated the exception propagates back to Swift, but in a form which can propagate back -/// to any calling Kotlin stack. -/// This only works for SKIEE methods which have an associated completion handler which handles annotated errors. -/// This seems to only apply for Kotlin suspending function bindings. -func throwKotlinPowerSyncError(message: String, cause: String? = nil) throws { - try throwPowerSyncException( - exception: PowerSyncKotlin.PowerSyncException( - message: message, - cause: PowerSyncKotlin.KotlinThrowable( - message: cause ?? message - ) - ) - ) -} diff --git a/Sources/PowerSync/PowerSyncCredentials.swift b/Sources/PowerSync/PowerSyncCredentials.swift index e1497e1..46a54c7 100644 --- a/Sources/PowerSync/PowerSyncCredentials.swift +++ b/Sources/PowerSync/PowerSyncCredentials.swift @@ -34,11 +34,6 @@ public struct PowerSyncCredentials: Codable, Sendable { self.token = token } - init(kotlin: KotlinPowerSyncCredentials) { - endpoint = kotlin.endpoint - token = kotlin.token - } - public func endpointUri(path: String) -> String { return "\(endpoint)/\(path)" } diff --git a/Sources/PowerSync/PowerSyncDatabase.swift b/Sources/PowerSync/PowerSyncDatabase.swift index c199cba..ca205c7 100644 --- a/Sources/PowerSync/PowerSyncDatabase.swift +++ b/Sources/PowerSync/PowerSyncDatabase.swift @@ -16,11 +16,17 @@ public func PowerSyncDatabase( logger: (any LoggerProtocol) = DefaultLogger(), initialStatements: [String] = [] ) -> PowerSyncDatabaseProtocol { - return openKotlinDBDefault( - schema: schema, - dbFilename: dbFilename, - logger: DatabaseLogger(logger), - initialStatements: initialStatements + let location = if dbFilename == ":memory:" { + DatabaseLocation.inMemory + } else { + DatabaseLocation.inDefaultDirectory(name: dbFilename) + } + let pool = AsyncConnectionPool(location: location, initialStatements: initialStatements) + return PowerSyncDatabaseImpl( + logger: logger, + pool: pool, + httpClient: PlatformHttpClient.shared, + schema: schema ) } @@ -45,10 +51,10 @@ public func OpenedPowerSyncDatabase( identifier: String, logger: (any LoggerProtocol) = DefaultLogger() ) -> PowerSyncDatabaseProtocol { - return openKotlinDBWithPool( - schema: schema, + return PowerSyncDatabaseImpl( + logger: logger, pool: pool, - identifier: identifier, - logger: DatabaseLogger(logger) + httpClient: PlatformHttpClient.shared, + schema: schema ) } diff --git a/Sources/PowerSync/Protocol/PowerSyncError.swift b/Sources/PowerSync/Protocol/PowerSyncError.swift index a33c26e..757c2be 100644 --- a/Sources/PowerSync/Protocol/PowerSyncError.swift +++ b/Sources/PowerSync/Protocol/PowerSyncError.swift @@ -6,6 +6,15 @@ public enum PowerSyncError: Error, LocalizedError { /// Represents a failure in an operation, potentially with a custom message and an underlying error. case operationFailed(message: String? = nil, underlyingError: Error? = nil) + /// An error reported by SQLite. + case sqliteError( + extendedResultCode: Int32, + offset: Int32? = nil, + message: String? = nil, + errorString: String? = nil, + sql: String? = nil, + ) + /// A localized description of the error, providing details about the failure. public var errorDescription: String? { switch self { @@ -23,6 +32,21 @@ public enum PowerSyncError: Error, LocalizedError { // Fallback to a generic error description if neither message nor underlying error is provided return "An unknown error occurred." } + case let .sqliteError(extendedResultCode, offset, message, errorString, sql): + var msg = "SQLite failure (code \(extendedResultCode))" + if let errorString { + msg += ": \(errorString)" + } + if let offset { + msg += " at offset \(offset)" + } + if let message { + msg += ", \(message)" + } + if let sql { + msg += " for SQL: \(sql)" + } + return msg } } } diff --git a/Sources/PowerSync/Protocol/QueriesProtocol.swift b/Sources/PowerSync/Protocol/QueriesProtocol.swift index 0f315f4..a14f839 100644 --- a/Sources/PowerSync/Protocol/QueriesProtocol.swift +++ b/Sources/PowerSync/Protocol/QueriesProtocol.swift @@ -22,34 +22,6 @@ public struct WatchOptions: Sendable { } public protocol Queries { - /// Execute a write query (INSERT, UPDATE, DELETE) - /// Using `RETURNING *` will result in an error. - @discardableResult - func execute(sql: String, parameters: [Sendable?]?) async throws -> Int64 - - /// Execute a read-only (SELECT) query and return a single result. - /// If there is no result, throws an IllegalArgumentException. - /// See `getOptional` for queries where the result might be empty. - func get( - sql: String, - parameters: [Sendable?]?, - mapper: @Sendable @escaping (SqlCursor) throws -> RowType - ) async throws -> RowType - - /// Execute a read-only (SELECT) query and return the results. - func getAll( - sql: String, - parameters: [Sendable?]?, - mapper: @Sendable @escaping (SqlCursor) throws -> RowType - ) async throws -> [RowType] - - /// Execute a read-only (SELECT) query and return a single optional result. - func getOptional( - sql: String, - parameters: [Sendable?]?, - mapper: @Sendable @escaping (SqlCursor) throws -> RowType - ) async throws -> RowType? - /// Execute a read-only (SELECT) query every time the source tables are modified /// and return the results as an array in a Publisher. func watch( @@ -89,6 +61,42 @@ public protocol Queries { } public extension Queries { + /// Execute a write query (INSERT, UPDATE, DELETE) + /// Using `RETURNING *` will result in an error. + @discardableResult + func execute(sql: String, parameters: [Sendable?]?) async throws -> Int64 { + return try await self.writeLock { ctx in try ctx.execute(sql: sql, parameters: parameters) } + } + + /// Execute a read-only (SELECT) query and return a single result. + /// If there is no result, throws an IllegalArgumentException. + /// See `getOptional` for queries where the result might be empty. + func get( + sql: String, + parameters: [Sendable?]?, + mapper: @Sendable @escaping (SqlCursor) throws -> RowType + ) async throws -> RowType { + return try await self.readLock { ctx in try ctx.get(sql: sql, parameters: parameters, mapper: mapper) } + } + + /// Execute a read-only (SELECT) query and return the results. + func getAll( + sql: String, + parameters: [Sendable?]?, + mapper: @Sendable @escaping (SqlCursor) throws -> RowType + ) async throws -> [RowType] { + return try await self.readLock { ctx in try ctx.getAll(sql: sql, parameters: parameters, mapper: mapper) } + } + + /// Execute a read-only (SELECT) query and return a single optional result. + func getOptional( + sql: String, + parameters: [Sendable?]?, + mapper: @Sendable @escaping (SqlCursor) throws -> RowType + ) async throws -> RowType? { + return try await self.readLock { ctx in try ctx.getOptional(sql: sql, parameters: parameters, mapper: mapper) } + } + @discardableResult func execute(_ sql: String) async throws -> Int64 { return try await execute(sql: sql, parameters: []) diff --git a/Sources/PowerSync/Protocol/Schema/Column.swift b/Sources/PowerSync/Protocol/Schema/Column.swift index feb6a32..aaf2528 100644 --- a/Sources/PowerSync/Protocol/Schema/Column.swift +++ b/Sources/PowerSync/Protocol/Schema/Column.swift @@ -1,5 +1,4 @@ import Foundation -import PowerSyncKotlin public protocol ColumnProtocol: Equatable, Sendable { /// Name of the column. diff --git a/Sources/PowerSync/Protocol/Schema/Index.swift b/Sources/PowerSync/Protocol/Schema/Index.swift index 820af49..eeae7cf 100644 --- a/Sources/PowerSync/Protocol/Schema/Index.swift +++ b/Sources/PowerSync/Protocol/Schema/Index.swift @@ -1,5 +1,4 @@ import Foundation -import PowerSyncKotlin public protocol IndexProtocol: Sendable { /// diff --git a/Sources/PowerSync/Protocol/db/ConnectionContext.swift b/Sources/PowerSync/Protocol/db/ConnectionContext.swift index 4f904d1..ed54a14 100644 --- a/Sources/PowerSync/Protocol/db/ConnectionContext.swift +++ b/Sources/PowerSync/Protocol/db/ConnectionContext.swift @@ -27,7 +27,7 @@ public protocol ConnectionContext: Sendable { - Throws: PowerSyncError if the query fails */ - func getOptional( + func getOptional( sql: String, parameters: [Sendable?]?, mapper: @Sendable @escaping (SqlCursor) throws -> RowType @@ -45,7 +45,7 @@ public protocol ConnectionContext: Sendable { - Throws: PowerSyncError if the query fails */ - func getAll( + func getAll( sql: String, parameters: [Sendable?]?, mapper: @Sendable @escaping (SqlCursor) throws -> RowType @@ -63,7 +63,7 @@ public protocol ConnectionContext: Sendable { - Throws: PowerSyncError if the query fails or no result is found */ - func get( + func get( sql: String, parameters: [Sendable?]?, mapper: @Sendable @escaping (SqlCursor) throws -> RowType diff --git a/Sources/PowerSync/Protocol/db/CrudBatch.swift b/Sources/PowerSync/Protocol/db/CrudBatch.swift index b1c30dc..15ecbd9 100644 --- a/Sources/PowerSync/Protocol/db/CrudBatch.swift +++ b/Sources/PowerSync/Protocol/db/CrudBatch.swift @@ -42,6 +42,6 @@ internal func completeCrudItems(_ db: any PowerSyncDatabaseProtocol, _ lastItemI return } } - try tx.execute(sql: "UPDATE ps_buckets SET target_op = ? WHERE name = '$local'", parameters: [KotlinPowerSyncDatabaseImpl.maxOpId]) + try tx.execute(sql: "UPDATE ps_buckets SET target_op = ? WHERE name = '$local'", parameters: [PowerSyncDatabaseImpl.maxOpId]) } } diff --git a/Sources/PowerSync/Protocol/db/DataConvertible.swift b/Sources/PowerSync/Protocol/db/DataConvertible.swift new file mode 100644 index 0000000..90593a9 --- /dev/null +++ b/Sources/PowerSync/Protocol/db/DataConvertible.swift @@ -0,0 +1,54 @@ +import Foundation + +// Represents the set of types that are supported +/// by the PowerSync Kotlin Multiplatform SDK +public enum PowerSyncDataType { + case bool(Bool) + case string(String) + case int64(Int64) + case int32(Int32) + case double(Double) + case data(Data) +} + +extension PowerSyncDataType { + init(from: Any) throws (PowerSyncError) { + if let bool = from as? Bool { + self = .bool(bool) + return + } + if let string = from as? String { + self = .string(string) + return + } + if let int = from as? Int64 { + self = .int64(int) + return + } + if let int = from as? Int { + self = .int64(Int64(int)) + return + } + if let int = from as? Int32 { + self = .int32(int) + return + } + if let double = from as? Double { + self = .double(double) + return + } + if let data = from as? Data { + self = .data(data) + return + } + + throw .operationFailed(message: "Invalid parameter, expected Bool, String, Int64, Int32, Double or Data") + } +} + +/// Types conforming to this protocol will be +/// mapped to the specified ``PowerSyncDataType`` +/// before use by SQLite +public protocol PowerSyncDataTypeConvertible { + var psDataType: PowerSyncDataType? { get } +} diff --git a/Sources/PowerSync/Protocol/db/SqlCursor.swift b/Sources/PowerSync/Protocol/db/SqlCursor.swift index 46d85d2..bb3085b 100644 --- a/Sources/PowerSync/Protocol/db/SqlCursor.swift +++ b/Sources/PowerSync/Protocol/db/SqlCursor.swift @@ -13,18 +13,6 @@ public protocol SqlCursor { /// - Returns: The `Bool` value if present, or `nil` if the value is null. func getBooleanOptional(index: Int) -> Bool? - /// Retrieves a `Bool` value from the specified column name. - /// - Parameter name: The name of the column. - /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. - /// - Returns: The `Bool` value. - func getBoolean(name: String) throws -> Bool - - /// Retrieves an optional `Bool` value from the specified column name. - /// - Parameter name: The name of the column. - /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist. - /// - Returns: The `Bool` value if present, or `nil` if the value is null. - func getBooleanOptional(name: String) throws -> Bool? - /// Retrieves a `Double` value from the specified column name. /// - Parameter name: The name of the column. /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. @@ -36,18 +24,6 @@ public protocol SqlCursor { /// - Returns: The `Double` value if present, or `nil` if the value is null. func getDoubleOptional(index: Int) -> Double? - /// Retrieves a `Double` value from the specified column name. - /// - Parameter name: The name of the column. - /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. - /// - Returns: The `Double` value. - func getDouble(name: String) throws -> Double - - /// Retrieves an optional `Double` value from the specified column name. - /// - Parameter name: The name of the column. - /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist. - /// - Returns: The `Double` value if present, or `nil` if the value is null. - func getDoubleOptional(name: String) throws -> Double? - /// Retrieves an `Int` value from the specified column name. /// - Parameter name: The name of the column. /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. @@ -59,18 +35,6 @@ public protocol SqlCursor { /// - Returns: The `Int` value if present, or `nil` if the value is null. func getIntOptional(index: Int) -> Int? - /// Retrieves an `Int` value from the specified column name. - /// - Parameter name: The name of the column. - /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. - /// - Returns: The `Int` value. - func getInt(name: String) throws -> Int - - /// Retrieves an optional `Int` value from the specified column name. - /// - Parameter name: The name of the column. - /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist. - /// - Returns: The `Int` value if present, or `nil` if the value is null. - func getIntOptional(name: String) throws -> Int? - /// Retrieves an `Int64` value from the specified column name. /// - Parameter name: The name of the column. /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. @@ -82,18 +46,6 @@ public protocol SqlCursor { /// - Returns: The `Int64` value if present, or `nil` if the value is null. func getInt64Optional(index: Int) -> Int64? - /// Retrieves an `Int64` value from the specified column name. - /// - Parameter name: The name of the column. - /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. - /// - Returns: The `Int64` value. - func getInt64(name: String) throws -> Int64 - - /// Retrieves an optional `Int64` value from the specified column name. - /// - Parameter name: The name of the column. - /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist. - /// - Returns: The `Int64` value if present, or `nil` if the value is null. - func getInt64Optional(name: String) throws -> Int64? - /// Retrieves a `String` value from the specified column name. /// - Parameter name: The name of the column. /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. @@ -105,23 +57,101 @@ public protocol SqlCursor { /// - Returns: The `String` value if present, or `nil` if the value is null. func getStringOptional(index: Int) -> String? + /// The number of columns in the result set. + var columnCount: Int { get } + + /// A dictionary mapping column names to their zero-based indices. + var columnNames: [String: Int] { get } +} + +extension SqlCursor { + private func resolveIndex(name: String) throws(SqlCursorError) -> Int { + if let index = self.columnNames[name] { + return index + } else { + throw .columnNotFound(name) + } + } + + /// Retrieves a `Bool` value from the specified column name. + /// - Parameter name: The name of the column. + /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. + /// - Returns: The `Bool` value. + func getBoolean(name: String) throws -> Bool { + return try self.getBoolean(index: try self.resolveIndex(name: name)) + } + + /// Retrieves an optional `Bool` value from the specified column name. + /// - Parameter name: The name of the column. + /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist. + /// - Returns: The `Bool` value if present, or `nil` if the value is null. + func getBooleanOptional(name: String) throws -> Bool? { + return self.getBooleanOptional(index: try self.resolveIndex(name: name)) + } + + /// Retrieves a `Double` value from the specified column name. + /// - Parameter name: The name of the column. + /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. + /// - Returns: The `Double` value. + func getDouble(name: String) throws -> Double { + return try self.getDouble(index: try self.resolveIndex(name: name)) + } + + /// Retrieves an optional `Double` value from the specified column name. + /// - Parameter name: The name of the column. + /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist. + /// - Returns: The `Double` value if present, or `nil` if the value is null. + func getDoubleOptional(name: String) throws -> Double? { + return self.getDoubleOptional(index: try self.resolveIndex(name: name)) + } + + /// Retrieves an `Int` value from the specified column name. + /// - Parameter name: The name of the column. + /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. + /// - Returns: The `Int` value. + func getInt(name: String) throws -> Int { + return try self.getInt(index: try self.resolveIndex(name: name)) + } + + /// Retrieves an optional `Int` value from the specified column name. + /// - Parameter name: The name of the column. + /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist. + /// - Returns: The `Int` value if present, or `nil` if the value is null. + func getIntOptional(name: String) throws -> Int? { + return self.getIntOptional(index: try self.resolveIndex(name: name)) + } + + /// Retrieves an `Int64` value from the specified column name. + /// - Parameter name: The name of the column. + /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. + /// - Returns: The `Int64` value. + func getInt64(name: String) throws -> Int64 { + return try self.getInt64(index: try self.resolveIndex(name: name)) + } + + /// Retrieves an optional `Int64` value from the specified column name. + /// - Parameter name: The name of the column. + /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist. + /// - Returns: The `Int64` value if present, or `nil` if the value is null. + func getInt64Optional(name: String) throws -> Int64? { + return self.getInt64Optional(index: try self.resolveIndex(name: name)) + } + /// Retrieves a `String` value from the specified column name. /// - Parameter name: The name of the column. /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. /// - Returns: The `String` value. - func getString(name: String) throws -> String + func getString(name: String) throws -> String { + return try self.getString(index: try self.resolveIndex(name: name)) + } /// Retrieves an optional `String` value from the specified column name. /// - Parameter name: The name of the column. /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist. /// - Returns: The `String` value if present, or `nil` if the value is null. - func getStringOptional(name: String) throws -> String? - - /// The number of columns in the result set. - var columnCount: Int { get } - - /// A dictionary mapping column names to their zero-based indices. - var columnNames: [String: Int] { get } + func getStringOptional(name: String) throws -> String? { + return self.getStringOptional(index: try self.resolveIndex(name: name)) + } } /// An error type representing issues encountered while working with a `SqlCursor`. diff --git a/Sources/PowerSync/Utils/AsyncMutex.swift b/Sources/PowerSync/Utils/AsyncMutex.swift index bbb3c31..a9d2c98 100644 --- a/Sources/PowerSync/Utils/AsyncMutex.swift +++ b/Sources/PowerSync/Utils/AsyncMutex.swift @@ -1,3 +1,7 @@ +import BasicContainers +import DequeModule + +/// A simple async mutex implemented through actors. actor AsyncMutex { var inner: T @@ -9,3 +13,232 @@ actor AsyncMutex { try callback(&inner) } } + +/// A serialized asynchronous semaphore with associated items. +/// +/// Heavily inspired from https://github.com/powersync-ja/powersync-js/blob/main/packages/common/src/utils/mutex.ts. +final class AsyncSemaphore: Sendable { + let count: Int + private let state: Mutex> + + init (_ values: consuming RigidDeque) { + self.count = values.count + state = Mutex(SemaphoreState( + available: values + )) + } + + convenience init(singleElement: consuming T) { + var queue = RigidDeque(capacity: 1) + queue.append(singleElement) + self.init(queue) + } + + fileprivate func returnItems(items: consuming RigidArray) { + state.withLock { state in + while !items.isEmpty { + state.returnItem(item: items.removeLast()) + } + } + } + + func acquire(count: Int) async throws -> SemaphoreGrant { + precondition(count > 0 && count <= self.count) + try Task.checkCancellation() + + let waiter = Mutex(nil) + try await withTaskCancellationHandler(operation: { + try await withCheckedThrowingContinuation { continuation in + let node = state.withLock { state in + state.addWaiter(requestedItems: count, continuation: continuation) + } + + waiter.withLock { $0 = node } + } + }, onCancel: { + if let waiter = waiter.withLock({ $0 }) { + state.withLock { state in + state.abortWaiter(waiter: waiter) + } + } + }) + + let node = waiter.withLock { $0! } + let items: RigidArray? = node.consumeItems() + if let items, node.isFull { + return SemaphoreGrant(semaphore: self, items: items) + } else { + throw CancellationError() + } + } +} + +extension AsyncSemaphore where T: Copyable { + convenience init(from: [T]) { + var queue = RigidDeque(capacity: from.count) + for item in from { + let _ = queue.pushLast(item) + } + self.init(queue) + } +} + +struct SemaphoreGrant: ~Copyable { + private let semaphore: AsyncSemaphore + var acquiredItems: RigidArray + + fileprivate init(semaphore: AsyncSemaphore, items: consuming RigidArray) { + self.semaphore = semaphore + self.acquiredItems = items + } + + deinit { + semaphore.returnItems(items: acquiredItems) + } +} + +private struct SemaphoreState: ~Copyable { + // Available items that are not currently assigned to a waiter. + var available: RigidDeque + var firstWaiter: SemaphoreWaitNode? + var lastWaiter: SemaphoreWaitNode? + + var size: Int { + available.capacity + } + + deinit { + // Clean up reference cycle in double-linked list. + var currentNode = firstWaiter + while let node = currentNode { + currentNode = node.next + node.next = nil + node.prev = nil + } + } + + private mutating func deactivateWaiter(waiter: SemaphoreWaitNode) { + let prev = waiter.prev + let next = waiter.next + + if let prev { + prev.next = next + } + if let next { + next.prev = prev + } + + if waiter === firstWaiter { + firstWaiter = next + } + if waiter === lastWaiter { + lastWaiter = prev + } + + waiter.continuation.resume(returning: ()) + } + + mutating func returnItem(item: consuming T) { + // Give it to the next waiter, if possible. + if let firstWaiter { + firstWaiter.pushItem(item: item) + if firstWaiter.isFull { + self.deactivateWaiter(waiter: firstWaiter) + } + } else { + // No pending waiter, return lease into pool. + available.append(item) + } + } + + mutating func returnItems(items: consuming RigidArray) { + while !items.isEmpty { + returnItem(item: items.removeLast()) + } + } + + mutating func abortWaiter(waiter: SemaphoreWaitNode) { + let items: RigidArray? = waiter.consumeItems() + deactivateWaiter(waiter: waiter) + if let items { + returnItems(items: items) + } + } + + mutating func addWaiter(requestedItems: Int, continuation: CheckedContinuation<(), any Error>) -> SemaphoreWaitNode { + let node = SemaphoreWaitNode(requestedItems: requestedItems, continuation: continuation) + if let lastWaiter { + lastWaiter.next = node + self.lastWaiter = node + } else { + // First waiter + firstWaiter = node + lastWaiter = node + } + + // If there are items in the pool that haven't been assigned, we can pull them into this waiter. Note that this is + // only the case if we're the first waiter (otherwise, items would have been assigned to an earlier waiter). + while !available.isEmpty && !node.isFull { + node.pushItem(item: available.removeFirst()) + } + + if node.isFull { + self.deactivateWaiter(waiter: node) + } + return node + } +} + +// This isn't actually sendable, but we don't use it concurrently: While waiting, it's only mutated with a lock on SemaphoreState. +// Afterwards, it's sent to acquire() where it's only used in a single async context. +private final class SemaphoreWaitNode: @unchecked Sendable { + let requestedItems: Int + var acquiredItems: Int + var itemsBuffer: UnsafeMutableRawPointer? // pointer to [T; requestedItems] + var continuation: CheckedContinuation<(), any Error> + var prev: SemaphoreWaitNode? + var next: SemaphoreWaitNode? + + init(requestedItems: Int, continuation: CheckedContinuation<(), any Error>) { + self.requestedItems = requestedItems + self.continuation = continuation + self.acquiredItems = 0 + } + + var isFull: Bool { + acquiredItems == requestedItems + } + + func pushItem(item: consuming T) { + precondition(!isFull) + + if let items = itemsBuffer { + items.assumingMemoryBound(to: T.self).advanced(by: acquiredItems).initialize(to: item) + } else { + let buffer: UnsafeMutablePointer = .allocate(capacity: requestedItems) + buffer.initialize(to: item) + + itemsBuffer = UnsafeMutableRawPointer(buffer) + } + + acquiredItems += 1 + } + + func consumeItems() -> RigidArray? { + if let itemsBuffer { + var array = RigidArray(capacity: acquiredItems) + let ptr = UnsafeBufferPointer(start: itemsBuffer.assumingMemoryBound(to: T.self), count: acquiredItems) + array.insert(moving: UnsafeMutableBufferPointer(mutating: ptr), at: 0) + // We don't have to deinitialize, array.insert moves elements out of the buffer. + itemsBuffer.deallocate() + self.itemsBuffer = nil + return array + } else { + return nil + } + } + + deinit { + precondition(itemsBuffer == nil, "Wait node leaked items buffer") + } +} diff --git a/Sources/PowerSync/Utils/ThrottledAsyncSequence.swift b/Sources/PowerSync/Utils/ThrottledAsyncSequence.swift new file mode 100644 index 0000000..732e587 --- /dev/null +++ b/Sources/PowerSync/Utils/ThrottledAsyncSequence.swift @@ -0,0 +1,108 @@ +import Foundation + +// Throttled async sequences that drop events emitted during a timeout. +// Inspired from https://github.com/apple/swift-async-algorithms/blob/main/Sources/AsyncAlgorithms/AsyncThrottleSequence.swift, +// but with changes to support older platforms. +struct AsyncThrottleSequence: AsyncSequence where Base.Element == () { + typealias AsyncIterator = IteratorImpl + typealias Element = () + + private let inner: Base + private let duration: TimeInterval + + init(inner: Base, duration: TimeInterval) { + self.inner = inner + self.duration = duration + } + + func makeAsyncIterator() -> IteratorImpl { + IteratorImpl(duration: duration, inner: inner) + } + + final class IteratorImpl: AsyncIteratorProtocol, Sendable { + fileprivate let duration: TimeInterval + private let state: LockedThrottleSequenceState + let pollTask: Task<(), any Error> + + init(duration: TimeInterval, inner: Base) { + self.duration = duration + let state = LockedThrottleSequenceState() + self.pollTask = Task { + defer { state.state.withLock { $0.transitionToDone() } } + + do { + for try await event in inner { + state.state.withLock { $0.markHasEvent(event: .success(event)) } + } + } catch { + state.state.withLock { $0.markHasEvent(event: .failure(error)) } + } + } + + self.state = state + } + + func next() async throws -> ()? { + try await withTaskCancellationHandler( + operation: { + try await withCheckedThrowingContinuation { continuation in + state.state.withLock { $0.registerListener(continuation) } + } + }, + onCancel: { + pollTask.cancel() + state.state.withLock { + if case .waitingForUpstream(let continuation) = $0 { + continuation.resume(returning: nil) + } + $0 = .done + } + } + ) + } + } +} + +private final class LockedThrottleSequenceState: Sendable { + let state = Mutex(ThrottleSequenceState.idle) +} + +private enum ThrottleSequenceState { + /// No one waiting on next(), no pending emit either. + case idle + /// We're waiting in next() for an upstream emission. + case waitingForUpstream(CheckedContinuation<()?, any Error>) + /// We have an upstream emission that has not yet been sent (due to backpressure or throttle). + case hasPendingEvent(Result<(), any Error>) + case done + + mutating func registerListener(_ continuation: CheckedContinuation<()?, any Error>) { + switch self { + case .idle: + self = .waitingForUpstream(continuation) + case .waitingForUpstream(_): + fatalError("Async throttle sequence has two concurrent listeners?!") + case .hasPendingEvent(let pending): + continuation.resume(with: pending.map { _ in () }) + self = .idle + case .done: + continuation.resume(returning: nil) + } + } + + mutating func markHasEvent(event: Result<(), any Error>) { + if case let .waitingForUpstream(continuation) = self { + continuation.resume(with: event.map { _ in () }) + self = .idle + } else { + self = .hasPendingEvent(event) + } + } + + mutating func transitionToDone() { + if case let .waitingForUpstream(continuation) = self { + continuation.resume(returning: nil) + } + self = .done + } +} diff --git a/Sources/PowerSync/Utils/withSession.swift b/Sources/PowerSync/Utils/withSession.swift index f80de29..c2c2467 100644 --- a/Sources/PowerSync/Utils/withSession.swift +++ b/Sources/PowerSync/Utils/withSession.swift @@ -47,8 +47,5 @@ public func withSession( db: OpaquePointer, action: @escaping () throws -> ReturnType ) throws -> WithSessionResult { - return try kotlinWithSession( - db: db, - action: action - ) + fatalError("todo") } diff --git a/Sources/PowerSync/resolvePowerSyncLoadableExtensionPath.swift b/Sources/PowerSync/resolvePowerSyncLoadableExtensionPath.swift index 99a3fb1..ae46371 100644 --- a/Sources/PowerSync/resolvePowerSyncLoadableExtensionPath.swift +++ b/Sources/PowerSync/resolvePowerSyncLoadableExtensionPath.swift @@ -1,62 +1,12 @@ -/// Resolves the PowerSync SQLite extension path. -/// -/// This function returns the file system path to the PowerSync SQLite extension library. -/// For use with extension loading APIs or SQLite queries. -/// -/// ## Platform Behavior -/// -/// ### watchOS -/// On watchOS, the extension needs to be loaded statically. This function returns `nil` -/// on watchOS because the extension is statically linked and doesn't require a path. -/// Calling this function will auto-register the extension in watchOS. The static -/// initialization ensures the extension is available without requiring dynamic loading. -/// -/// ### Other Platforms -/// In other environments (iOS, macOS, tvOS, etc.), the extension needs to be loaded -/// dynamically using the path returned by this function. You'll need to: -/// 1. Enable extension loading on your SQLite connection -/// 2. Load the extension using the returned path -/// -/// ## Example Usage -/// -/// ### Loading with SQLite API -/// ```swift -/// guard let extensionPath = try resolvePowerSyncLoadableExtensionPath() else { -/// // On watchOS, extension is statically loaded, no path needed -/// return -/// } -/// -/// // Enable extension loading -/// sqlite3_enable_load_extension(db, 1) -/// -/// // Load the extension -/// var errorMsg: UnsafeMutablePointer? -/// let result = sqlite3_load_extension( -/// db, -/// extensionPath, -/// "sqlite3_powersync_init", -/// &errorMsg -/// ) -/// if result != SQLITE_OK { -/// // Handle error -/// } -/// ``` -/// -/// ### Loading with SQL Query -/// ```swift -/// guard let extensionPath = try resolvePowerSyncLoadableExtensionPath() else { -/// // On watchOS, extension is statically loaded, no path needed -/// return -/// } -/// let escapedPath = extensionPath.replacingOccurrences(of: "'", with: "''") -/// let query = "SELECT load_extension('\(escapedPath)', 'sqlite3_powersync_init')" -/// try db.execute(sql: query) -/// ``` -/// -/// - Returns: The file system path to the PowerSync SQLite extension, or `nil` on watchOS -/// (where the extension is statically loaded and doesn't require a path) -/// - Throws: An error if the extension path cannot be resolved on platforms that require it or -/// if the extension could not be registered on watchOS. -public func resolvePowerSyncLoadableExtensionPath() throws -> String? { - return try kotlinResolvePowerSyncLoadableExtensionPath() +/// Loads the PowerSync SQLite core extension. +/// +/// In older versions of the Swift SDK, this used to return a file path that would have to +/// be loaded with `sqlite3_load_extension`. This is no longer relevant: Calling this +/// function invokes `sqlite3_auto_extension` to load the core extension automatically. +/// +/// - Returns: `nil` +/// - Throws: An error if the extension could not be registered watchOS. +public func resolvePowerSyncLoadableExtensionPath() throws(PowerSyncError) -> String? { + try registerPowerSyncCoreExtension() + return nil } diff --git a/Sources/PowerSyncCoreShim/empty.c b/Sources/PowerSyncCoreShim/empty.c new file mode 100644 index 0000000..03f9cb9 --- /dev/null +++ b/Sources/PowerSyncCoreShim/empty.c @@ -0,0 +1 @@ +// Intentionally left empty to ensure this target generates an object file diff --git a/Sources/PowerSyncCoreShim/include/powersync_core.h b/Sources/PowerSyncCoreShim/include/powersync_core.h new file mode 100644 index 0000000..b24bbd5 --- /dev/null +++ b/Sources/PowerSyncCoreShim/include/powersync_core.h @@ -0,0 +1 @@ +void sqlite3_powersync_init(void); diff --git a/Sources/PowerSyncCoreShim/module.modulemap b/Sources/PowerSyncCoreShim/module.modulemap new file mode 100644 index 0000000..fc46926 --- /dev/null +++ b/Sources/PowerSyncCoreShim/module.modulemap @@ -0,0 +1,4 @@ +module PowerSyncCoreShim { + header "include/powersync_core.h" + export * +} diff --git a/Tests/PowerSyncGRDBTests/BasicTest.swift b/Tests/PowerSyncGRDBTests/BasicTest.swift index db8bd18..914480d 100644 --- a/Tests/PowerSyncGRDBTests/BasicTest.swift +++ b/Tests/PowerSyncGRDBTests/BasicTest.swift @@ -90,7 +90,7 @@ final class GRDBTests: XCTestCase { pool: pool, schema: schema, identifier: dbIdentifier, - logger: DatabaseLogger(logger) + logger: logger ) try await database.disconnectAndClear() diff --git a/Tests/PowerSyncTests/ConnectTests.swift b/Tests/PowerSyncTests/ConnectTests.swift index 6c7f2fe..b959c8b 100644 --- a/Tests/PowerSyncTests/ConnectTests.swift +++ b/Tests/PowerSyncTests/ConnectTests.swift @@ -18,10 +18,10 @@ final class ConnectTests: XCTestCase { ), ]) - database = openKotlinDBDefault( + database = PowerSyncDatabase( schema: schema, dbFilename: ":memory:", - logger: DatabaseLogger(DefaultLogger()) + logger: DefaultLogger() ) try await database.disconnectAndClear() } diff --git a/Tests/PowerSyncTests/CrudTests.swift b/Tests/PowerSyncTests/CrudTests.swift index 5eb2872..00403b6 100644 --- a/Tests/PowerSyncTests/CrudTests.swift +++ b/Tests/PowerSyncTests/CrudTests.swift @@ -19,10 +19,10 @@ final class CrudTests: XCTestCase { ), ]) - database = openKotlinDBDefault( + database = PowerSyncDatabase( schema: schema, dbFilename: ":memory:", - logger: DatabaseLogger(DefaultLogger()) + logger: DefaultLogger() ) try await database.disconnectAndClear() } diff --git a/Tests/PowerSyncTests/EncryptionTests.swift b/Tests/PowerSyncTests/EncryptionTests.swift index 4dbc74b..f0b3ec0 100644 --- a/Tests/PowerSyncTests/EncryptionTests.swift +++ b/Tests/PowerSyncTests/EncryptionTests.swift @@ -14,7 +14,7 @@ struct EncryptionTests { let database = PowerSyncDatabase( schema: Schema(), dbFilename: "linkSqlite3Mc", - logger: DatabaseLogger(DefaultLogger()) + logger: DefaultLogger() ) let cipher = try await database.get("pragma cipher", mapper: {cursor in @@ -37,7 +37,7 @@ struct EncryptionTests { ), ]), dbFilename: "encrypted.db", - logger: DatabaseLogger(DefaultLogger()), + logger: DefaultLogger(), initialStatements: [ "pragma key = 'foobar'" ], @@ -56,7 +56,7 @@ struct EncryptionTests { ), ]), dbFilename: "encrypted.db", - logger: DatabaseLogger(DefaultLogger()), + logger: DefaultLogger(), initialStatements: [ "pragma key = 'wrong password'", ], diff --git a/Tests/PowerSyncTests/Kotlin/KotlinPowerSyncDatabaseImplTests.swift b/Tests/PowerSyncTests/Kotlin/KotlinPowerSyncDatabaseImplTests.swift index 03f0131..ea66ccd 100644 --- a/Tests/PowerSyncTests/Kotlin/KotlinPowerSyncDatabaseImplTests.swift +++ b/Tests/PowerSyncTests/Kotlin/KotlinPowerSyncDatabaseImplTests.swift @@ -22,7 +22,7 @@ final class KotlinPowerSyncDatabaseImplTests: XCTestCase { database = PowerSyncDatabase( schema: schema, dbFilename: ":memory:", - logger: DatabaseLogger(DefaultLogger()) + logger: DefaultLogger() ) try await database.disconnectAndClear() } @@ -85,7 +85,7 @@ final class KotlinPowerSyncDatabaseImplTests: XCTestCase { XCTFail("Expected an error to be thrown") } catch { XCTAssertEqual(error.localizedDescription, """ - SqliteException(1): SQL logic error, no such table: usersfail for SQL: SELECT id, name, email FROM usersfail WHERE id = ? + SQLite failure (code 1): SQL logic error, no such table: usersfail for SQL: SELECT id, name, email FROM usersfail WHERE id = ? """) } } @@ -525,10 +525,10 @@ final class KotlinPowerSyncDatabaseImplTests: XCTestCase { let testWriter = TestLogWriterAdapter() let logger = DefaultLogger(minSeverity: LogSeverity.debug, writers: [testWriter]) - let db2 = openKotlinDBDefault( + let db2 = PowerSyncDatabase( schema: schema, dbFilename: ":memory:", - logger: DatabaseLogger(logger) + logger: logger ) try await db2.execute("SELECT 1") @@ -547,10 +547,10 @@ final class KotlinPowerSyncDatabaseImplTests: XCTestCase { let testWriter = TestLogWriterAdapter() let logger = DefaultLogger(minSeverity: LogSeverity.error, writers: [testWriter]) - let db2 = openKotlinDBDefault( + let db2 = PowerSyncDatabase( schema: schema, dbFilename: ":memory:", - logger: DatabaseLogger(logger) + logger: logger ) try await db2.close() @@ -646,6 +646,7 @@ final class KotlinPowerSyncDatabaseImplTests: XCTestCase { XCTAssertEqual(result[1], JoinOutput(name: "Test User", description: "task 2", comment: "comment 2")) } + /* func testCloseWithDeleteDatabase() async throws { let fileManager = FileManager.default let testDbFilename = "test_delete_\(UUID().uuidString).db" @@ -717,7 +718,7 @@ final class KotlinPowerSyncDatabaseImplTests: XCTestCase { // Clean up: delete all SQLite files using the helper function try deleteSQLiteFiles(dbFilename: testDbFilename, in: databaseDirectory) - } + }*/ func testSubscriptionsUpdateStateWhileOffline() async throws { var streams = database.currentStatus.asFlow().makeAsyncIterator() diff --git a/Tests/PowerSyncTests/Kotlin/SqlCursorTests.swift b/Tests/PowerSyncTests/Kotlin/SqlCursorTests.swift index 501b522..dbb156b 100644 --- a/Tests/PowerSyncTests/Kotlin/SqlCursorTests.swift +++ b/Tests/PowerSyncTests/Kotlin/SqlCursorTests.swift @@ -65,10 +65,12 @@ final class SqlCursorTests: XCTestCase { ]) ]) - database = openKotlinDBDefault( - schema: schema, + database = PowerSyncDatabaseImpl( dbFilename: ":memory:", - logger: DatabaseLogger(DefaultLogger()) + logger: DefaultLogger(), + pool: AsyncConnectionPool(location: .inMemory), + httpClient: PlatformHttpClient.shared, + schema: schema, ) try await database.disconnectAndClear() } diff --git a/Tests/PowerSyncTests/SyncTests.swift b/Tests/PowerSyncTests/SyncTests.swift index 6700310..cc2e6d6 100644 --- a/Tests/PowerSyncTests/SyncTests.swift +++ b/Tests/PowerSyncTests/SyncTests.swift @@ -656,11 +656,12 @@ let defaultSchema = Schema(tables: [ ]) private func openDatabase(_ client: any HttpClient, schema: Schema = defaultSchema, logger: any LoggerProtocol = DefaultLogger()) -> PowerSyncDatabaseProtocol { - return openKotlinDBDefault( - schema: schema, + return PowerSyncDatabaseImpl( dbFilename: ":memory:", - logger: DatabaseLogger(logger), - httpClient: client + logger: logger, + pool: AsyncConnectionPool(location: .inMemory), + httpClient: client, + schema: schema, ) } diff --git a/Tests/PowerSyncTests/Utils/AsyncMutexTests.swift b/Tests/PowerSyncTests/Utils/AsyncMutexTests.swift new file mode 100644 index 0000000..f72fc5f --- /dev/null +++ b/Tests/PowerSyncTests/Utils/AsyncMutexTests.swift @@ -0,0 +1,98 @@ +@testable import PowerSync +import Testing + +@Suite +struct AsyncSemaphoreTests { + @Test func dispatchesItemsInOrder() async throws { + let semaphore = AsyncSemaphore(from: ["a", "b", "c"]) + + let grant1 = try await semaphore.acquire(count: 1) + let grant2 = try await semaphore.acquire(count: 1) + let grant3 = try await semaphore.acquire(count: 1) + + try #require(grant1.acquiredItems[0] == "a") + try #require(grant2.acquiredItems[0] == "b") + try #require(grant3.acquiredItems[0] == "c") + } + + @Test @MainActor func returnsReleasedItemsToWaiters() async throws { + let semaphore = AsyncSemaphore(from: ["x"]) + + let grant1 = try await semaphore.acquire(count: 1) + var hasSecond = false + + let grant2 = Task { + let grant = try await semaphore.acquire(count: 1) + hasSecond = true + return grant.acquiredItems[0] + } + + try #require(!hasSecond) + let _ = consume grant1 + try #require(try await grant2.value == "x") + try #require(hasSecond) + } + + @Test @MainActor func canAcquireMultiple() async throws { + let semaphore = AsyncSemaphore(from: ["a", "b", "c"]) + let grant1 = try await semaphore.acquire(count: 1) + let grant2 = try await semaphore.acquire(count: 1) + + var hasAll = false + let acquireAllTask = Task { + let _ = try await semaphore.acquire(count: 3) + hasAll = false + } + + await Task.yield() + try #require(!hasAll) + + let _ = consume grant1 + await Task.yield() + try #require(!hasAll) // Still waiting for item2 + + let _ = consume grant2 + let _ = await acquireAllTask.result + } + + @Test func canReturnMultiple() async throws { + let semaphore = AsyncSemaphore(from: ["a", "b"]) + + let grantAll = try await semaphore.acquire(count: 2) + + let hasOther = Task { + let grant = try await semaphore.acquire(count: 1) + try #require(grant.acquiredItems[0] == "b") // We return the last item first + + let anotherGrant = try await semaphore.acquire(count: 1) + try #require(anotherGrant.acquiredItems[0] == "a") + return true + } + + let _ = consume grantAll + let _ = try await hasOther.value + } + + @Test func canAbort() async throws { + let semaphore = AsyncSemaphore(from: ["a"]) + + let grant1 = try await semaphore.acquire(count: 1) + + let second = Task { + await #expect(throws: CancellationError.self) { + let _ = try await semaphore.acquire(count: 1) + } + } + let third = Task { + let grant = try await semaphore.acquire(count: 1) + try #require(grant.acquiredItems[0] == "a") + return + } + + await Task.yield() + second.cancel() + + let _ = consume grant1 + let _ = await (second.result, third.result) + } +} From 72e99f0d3fc7ff66fdbd432b4e072b03da9d1bba Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 22 Apr 2026 17:24:11 +0200 Subject: [PATCH 22/40] Move to protocol extension --- .../Kotlin/KotlinPowerSyncDatabaseImpl.swift | 27 ------ .../Protocol/PowerSyncDatabaseProtocol.swift | 89 +++++++++++-------- 2 files changed, 53 insertions(+), 63 deletions(-) diff --git a/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift b/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift index 5a729ce..1fc57ac 100644 --- a/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift +++ b/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift @@ -72,33 +72,6 @@ final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, ) } - func getCrudBatch(limit: Int32 = 100) async throws -> CrudBatch? { - var entries = try await getAll( - sql: "SELECT id, tx_id, data FROM ps_crud ORDER BY id ASC LIMIT ?", - parameters: [Int64(limit + 1)], - mapper: CrudEntry.fromCursor - ) - - if entries.isEmpty { - return nil - } - - let hasMore = entries.count > limit - if hasMore { - entries.removeLast() - } - - return CrudBatch( - hasMore: hasMore, - crud: entries, - db: self - ) - } - - func getCrudTransactions() -> CrudTransactions { - return CrudTransactions(db: self) - } - func getPowerSyncVersion() async throws -> String { try await kotlinDatabase.getPowerSyncVersion() } diff --git a/Sources/PowerSync/Protocol/PowerSyncDatabaseProtocol.swift b/Sources/PowerSync/Protocol/PowerSyncDatabaseProtocol.swift index a88020e..392cf7a 100644 --- a/Sources/PowerSync/Protocol/PowerSyncDatabaseProtocol.swift +++ b/Sources/PowerSync/Protocol/PowerSyncDatabaseProtocol.swift @@ -191,40 +191,6 @@ public protocol PowerSyncDatabaseProtocol: Queries, Sendable { options: ConnectOptions? ) async throws - /// Get a batch of crud data to upload. - /// - /// Returns nil if there is no data to upload. - /// - /// Use this from the `PowerSyncBackendConnector.uploadData` callback. - /// - /// Once the data have been successfully uploaded, call `CrudBatch.complete` before - /// requesting the next batch. - /// - /// - Parameter limit: Maximum number of updates to return in a single batch. Default is 100. - /// - /// This method does include transaction ids in the result, but does not group - /// data by transaction. One batch may contain data from multiple transactions, - /// and a single transaction may be split over multiple batches. - func getCrudBatch(limit: Int32) async throws -> CrudBatch? - - /// Obtains an async iterator of completed transactions with local writes against the database. - /// - /// This is typically used from the ``PowerSyncBackendConnectorProtocol/uploadData(database:)`` callback. - /// Each entry emitted by teh returned flow is a full transaction containing all local writes made while that transaction was - /// active. - /// - /// Unlike ``getNextCrudTransaction()``, which always returns the oldest transaction that hasn't been - /// ``CrudTransaction/complete()``d yet, this iterator can be used to upload multiple transactions. - /// Calling ``CrudTransaction/complete()`` will mark that and all prior transactions returned by this iterator as - /// completed. - /// - /// This can be used to upload multiple transactions in a single batch, e.g. with - /// - /// ```Swift - /// - /// ``` - func getCrudTransactions() -> CrudTransactions - /// Convenience method to get the current version of PowerSync. func getPowerSyncVersion() async throws -> String @@ -343,9 +309,60 @@ public extension PowerSyncDatabaseProtocol { try await disconnectAndClear(clearLocal: true, soft: soft) } + /// Get a batch of crud data to upload. + /// + /// Returns nil if there is no data to upload. + /// + /// Use this from the `PowerSyncBackendConnector.uploadData` callback. + /// + /// Once the data have been successfully uploaded, call `CrudBatch.complete` before + /// requesting the next batch. + /// + /// - Parameter limit: Maximum number of updates to return in a single batch. Default is 100. + /// + /// This method does include transaction ids in the result, but does not group + /// data by transaction. One batch may contain data from multiple transactions, + /// and a single transaction may be split over multiple batches. func getCrudBatch(limit: Int32 = 100) async throws -> CrudBatch? { - try await getCrudBatch( - limit: limit + var entries = try await getAll( + sql: "SELECT id, tx_id, data FROM ps_crud ORDER BY id ASC LIMIT ?", + parameters: [Int64(limit + 1)], + mapper: CrudEntry.fromCursor ) + + if entries.isEmpty { + return nil + } + + let hasMore = entries.count > limit + if hasMore { + entries.removeLast() + } + + return CrudBatch( + hasMore: hasMore, + crud: entries, + db: self + ) + } + + /// Obtains an async iterator of completed transactions with local writes against the database. + /// + /// This is typically used from the ``PowerSyncBackendConnectorProtocol/uploadData(database:)`` callback. + /// Each entry emitted by teh returned flow is a full transaction containing all local writes made while that transaction was + /// active. + /// + /// Unlike ``getNextCrudTransaction()``, which always returns the oldest transaction that hasn't been + /// ``CrudTransaction/complete()``d yet, this iterator can be used to upload multiple transactions. + /// Calling ``CrudTransaction/complete()`` will mark that and all prior transactions returned by this iterator as + /// completed. + /// + /// This can be used to upload multiple transactions in a single batch, e.g. with + /// + /// ```Swift + /// + /// ``` + func getCrudTransactions() -> CrudTransactions { + CrudTransactions(db: self) } } From c04af11c0d9508d3ebd43cb3197f87fcc944773f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 22 Apr 2026 20:58:11 +0200 Subject: [PATCH 23/40] Get tests to work --- .../PoolConnectionContext.swift | 2 +- .../PowerSyncDatabaseImpl.swift | 54 +++++++------ .../sqlite3/NativeConnectionContext.swift | 78 +++++++++++-------- .../sqlite3/NativeConnectionPool.swift | 17 ++-- Sources/PowerSync/PowerSyncDatabase.swift | 1 + .../Protocol/db/DataConvertible.swift | 2 +- Sources/PowerSync/Protocol/db/SqlCursor.swift | 44 +++++++---- .../KotlinPowerSyncDatabaseImplTests.swift | 25 +++--- 8 files changed, 120 insertions(+), 103 deletions(-) diff --git a/Sources/PowerSync/Implementation/PoolConnectionContext.swift b/Sources/PowerSync/Implementation/PoolConnectionContext.swift index def6b40..7ad63f9 100644 --- a/Sources/PowerSync/Implementation/PoolConnectionContext.swift +++ b/Sources/PowerSync/Implementation/PoolConnectionContext.swift @@ -10,7 +10,7 @@ func poolRead(_ pool: borrowing SQLiteConnectionPoolProtocol, action: @escapi func poolWrite(_ pool: borrowing SQLiteConnectionPoolProtocol, action: @escaping @Sendable (_: any ConnectionContext) throws -> T) async throws -> T { let result = UnsafeSendable() - try await pool.read { lease in + try await pool.write { lease in let context = NativeConnectionContext(lease) result.resolve(value: try action(context)) } diff --git a/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift b/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift index acfa43b..8e5116d 100644 --- a/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift +++ b/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift @@ -113,36 +113,10 @@ final class PowerSyncDatabaseImpl: PowerSyncDatabaseProtocol { func close(deleteDatabase: Bool) async throws { try await close() - if deleteDatabase { - try await self.deleteDatabase() - } - } - - private func deleteDatabase() async throws { - if let dbFilename { + if deleteDatabase, let dbFilename { // We can use the supplied dbLocation when we support that in future let directory = try DatabaseLocation.appleDefaultDatabaseDirectory() - let fileManager = FileManager.default - - // SQLite files to delete: - // 1. Main database file: dbFilename - // 2. WAL file: dbFilename-wal - // 3. SHM file: dbFilename-shm - // 4. Journal file: dbFilename-journal (for rollback journal mode, though WAL mode typically doesn't use it) - - let filesToDelete = [ - dbFilename, - "\(dbFilename)-wal", - "\(dbFilename)-shm", - "\(dbFilename)-journal" - ] - - for filename in filesToDelete { - let fileUrl = directory.appendingPathComponent(filename) - if fileManager.fileExists(atPath: fileUrl.path) { - try fileManager.removeItem(at: fileUrl) - } - } + try deleteSQLiteFiles(dbFilename: dbFilename, in: directory) } } @@ -333,3 +307,27 @@ private actor DatabaseInitizalizationActor { } } } + +func deleteSQLiteFiles(dbFilename: String, in directory: URL) throws { + let fileManager = FileManager.default + + // SQLite files to delete: + // 1. Main database file: dbFilename + // 2. WAL file: dbFilename-wal + // 3. SHM file: dbFilename-shm + // 4. Journal file: dbFilename-journal (for rollback journal mode, though WAL mode typically doesn't use it) + + let filesToDelete = [ + dbFilename, + "\(dbFilename)-wal", + "\(dbFilename)-shm", + "\(dbFilename)-journal" + ] + + for filename in filesToDelete { + let fileUrl = directory.appendingPathComponent(filename) + if fileManager.fileExists(atPath: fileUrl.path) { + try fileManager.removeItem(at: fileUrl) + } + } +} diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionContext.swift b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionContext.swift index 6bd4931..56b4e13 100644 --- a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionContext.swift +++ b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionContext.swift @@ -29,7 +29,7 @@ final class NativeConnectionContext: ConnectionContext { try $0.checkNotClosed() var stmt = try SqliteStatement(db: $0.lease, sql: sql) - try stmt.bind_values(parameters) + try stmt.bindValues(parameters) while try stmt.step() { // Iterate through the statement. } @@ -44,7 +44,7 @@ final class NativeConnectionContext: ConnectionContext { try $0.checkNotClosed() var stmt = try SqliteStatement(db: $0.lease, sql: sql) - try stmt.bind_values(parameters) + try stmt.bindValues(parameters) if try stmt.step() { return try NativeConnectionContext.invokeMapper(stmt, mapper) } else { @@ -58,7 +58,7 @@ final class NativeConnectionContext: ConnectionContext { try $0.checkNotClosed() var stmt = try SqliteStatement(db: $0.lease, sql: sql) - try stmt.bind_values(parameters) + try stmt.bindValues(parameters) var rows: [RowType] = [] while try stmt.step() { rows.append(try NativeConnectionContext.invokeMapper(stmt, mapper)) @@ -72,7 +72,7 @@ final class NativeConnectionContext: ConnectionContext { try $0.checkNotClosed() var stmt = try SqliteStatement(db: $0.lease, sql: sql) - try stmt.bind_values(parameters) + try stmt.bindValues(parameters) if try stmt.step() { return try NativeConnectionContext.invokeMapper(stmt, mapper) } else { @@ -133,21 +133,29 @@ struct SqliteStatement: ~Copyable { return resolvedColumnNames! } - borrowing func bind_values(_ parameters: [Any?]?) throws (PowerSyncError) { + borrowing func bindValues(_ parameters: [Any?]?) throws (PowerSyncError) { if let parameters { for (i, parameter) in parameters.enumerated() { let index = Int32(i + 1) - if parameter == nil { - try bind_value(index, nil) + let psDataType: PowerSyncDataType? = if let convertible = parameter as? PowerSyncDataTypeConvertible { + convertible.psDataType + } else if let parameter { + try PowerSyncDataType(from: parameter) } else { - try bind_value(index, try PowerSyncDataType(from: parameter!)) + nil + } + + if let psDataType { + try bindValue(index, psDataType) + } else { + try bindValue(index, nil) } } } } - borrowing func bind_value(_ index: Int32, _ parameter: PowerSyncDataType?) throws (PowerSyncError) { + borrowing func bindValue(_ index: Int32, _ parameter: PowerSyncDataType?) throws (PowerSyncError) { let rc: Int32 switch parameter { @@ -231,34 +239,43 @@ class StatementCursor: SqlCursor { stmtPtr = nil } - private func withStatement(_ body: (borrowing SqliteStatement) throws -> R) rethrows -> R { + private func withStatement(_ body: (borrowing SqliteStatement) -> R) -> R { if let stmtPtr { - return try body(stmtPtr.pointee) + return body(stmtPtr.pointee) } fatalError("Cursor used outside of callback") } - var columnCount: Int { - return withStatement { stmt in stmt.columnCount } - } - - var columnNames: [String : Int] { - return withStatement { stmt in stmt.columnNames } - } - - func checkColumnNotNull(stmt: borrowing SqliteStatement, index: Int) throws(SqlCursorError) { + private func checkColumnNotNull(stmt: borrowing SqliteStatement, index: Int) throws(SqlCursorError) { if index < 0 || index >= stmt.columnCount { throw SqlCursorError.nullValueFound("invalid index \(index)") } let type = sqlite3_column_type(stmt.stmt, Int32(index)) if type == SQLITE_NULL { - throw SqlCursorError.nullValueFound("at index \(index)") + throw SqlCursorError.nullValueFound("\(index)") } } - func getBoolean(index: Int) throws -> Bool { + private func withStatementCheckNotNull(_ index: Int, body: (borrowing SqliteStatement) throws (SqlCursorError) -> R) throws (SqlCursorError) -> R { + if let stmtPtr { + try self.checkColumnNotNull(stmt: stmtPtr.pointee, index: index) + return try body(stmtPtr.pointee) + } + + fatalError("Cursor used outside of callback") + } + + var columnCount: Int { + return withStatement { stmt in stmt.columnCount } + } + + var columnNames: [String : Int] { + return withStatement { stmt in stmt.columnNames } + } + + func getBoolean(index: Int) throws(SqlCursorError) -> Bool { return try getInt(index: index) == 0 ? false : true } @@ -270,9 +287,8 @@ class StatementCursor: SqlCursor { } } - func getDouble(index: Int) throws -> Double { - return try withStatement { stmt in - try self.checkColumnNotNull(stmt: stmt, index: index) + func getDouble(index: Int) throws(SqlCursorError) -> Double { + return try withStatementCheckNotNull(index) { stmt in return sqlite3_column_double(stmt.stmt, Int32(index)) } } @@ -285,7 +301,7 @@ class StatementCursor: SqlCursor { } } - func getInt(index: Int) throws -> Int { + func getInt(index: Int) throws(SqlCursorError) -> Int { return Int(try getInt64(index: index)) } @@ -297,9 +313,8 @@ class StatementCursor: SqlCursor { } } - func getInt64(index: Int) throws -> Int64 { - return try withStatement { stmt in - try self.checkColumnNotNull(stmt: stmt, index: index) + func getInt64(index: Int) throws(SqlCursorError) -> Int64 { + return try withStatementCheckNotNull(index) { stmt in return sqlite3_column_int64(stmt.stmt, Int32(index)) } } @@ -312,9 +327,8 @@ class StatementCursor: SqlCursor { } } - func getString(index: Int) throws -> String { - return try withStatement { stmt in - try self.checkColumnNotNull(stmt: stmt, index: index) + func getString(index: Int) throws(SqlCursorError) -> String { + return try withStatementCheckNotNull(index) { stmt in let length = sqlite3_column_bytes(stmt.stmt, Int32(index)) if length == 0 { return "" diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift index 35b0811..d962e32 100644 --- a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift +++ b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift @@ -55,12 +55,11 @@ final class NativeConnectionPool: Sendable { if let readers { let acquiredReaders = try await readers.acquire(count: readers.count) var readerLeases: [RawConnectionLease] = [] - var i = 0 - while i < acquiredReaders.acquiredItems.count { - readerLeases.append(write.acquiredItems[i].asLease()) - i += 1 + + let span = acquiredReaders.acquiredItems.span + for idx in span.indices { + readerLeases.append(span[idx].asLease()) } - try await onConnection(writeLease, readerLeases) } else { try await onConnection(writeLease, []) @@ -79,11 +78,9 @@ final class NativeConnectionPool: Sendable { // Close the write connection first write.acquiredItems[0].close() - if var acquiredReaders { - var i = 0 - while i < acquiredReaders.acquiredItems.count { - acquiredReaders.acquiredItems[i].close() - i += 1 + if var span = acquiredReaders?.acquiredItems.mutableSpan { + for idx in span.indices { + span[idx].close() } } } diff --git a/Sources/PowerSync/PowerSyncDatabase.swift b/Sources/PowerSync/PowerSyncDatabase.swift index ca205c7..5da4587 100644 --- a/Sources/PowerSync/PowerSyncDatabase.swift +++ b/Sources/PowerSync/PowerSyncDatabase.swift @@ -23,6 +23,7 @@ public func PowerSyncDatabase( } let pool = AsyncConnectionPool(location: location, initialStatements: initialStatements) return PowerSyncDatabaseImpl( + dbFilename: dbFilename, logger: logger, pool: pool, httpClient: PlatformHttpClient.shared, diff --git a/Sources/PowerSync/Protocol/db/DataConvertible.swift b/Sources/PowerSync/Protocol/db/DataConvertible.swift index 90593a9..cb7d6c6 100644 --- a/Sources/PowerSync/Protocol/db/DataConvertible.swift +++ b/Sources/PowerSync/Protocol/db/DataConvertible.swift @@ -41,7 +41,7 @@ extension PowerSyncDataType { self = .data(data) return } - + throw .operationFailed(message: "Invalid parameter, expected Bool, String, Int64, Int32, Double or Data") } } diff --git a/Sources/PowerSync/Protocol/db/SqlCursor.swift b/Sources/PowerSync/Protocol/db/SqlCursor.swift index bb3085b..52b0b21 100644 --- a/Sources/PowerSync/Protocol/db/SqlCursor.swift +++ b/Sources/PowerSync/Protocol/db/SqlCursor.swift @@ -6,7 +6,7 @@ public protocol SqlCursor { /// - Parameter name: The name of the column. /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. /// - Returns: The `Bool` value. - func getBoolean(index: Int) throws -> Bool + func getBoolean(index: Int) throws(SqlCursorError) -> Bool /// Retrieves a `Bool` value from the specified column index. /// - Parameter index: The zero-based index of the column. @@ -17,7 +17,7 @@ public protocol SqlCursor { /// - Parameter name: The name of the column. /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. /// - Returns: The `Double` value. - func getDouble(index: Int) throws -> Double + func getDouble(index: Int) throws(SqlCursorError) -> Double /// Retrieves a `Double` value from the specified column index. /// - Parameter index: The zero-based index of the column. @@ -28,7 +28,7 @@ public protocol SqlCursor { /// - Parameter name: The name of the column. /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. /// - Returns: The `Int` value. - func getInt(index: Int) throws -> Int + func getInt(index: Int) throws(SqlCursorError) -> Int /// Retrieves an `Int` value from the specified column index. /// - Parameter index: The zero-based index of the column. @@ -39,7 +39,7 @@ public protocol SqlCursor { /// - Parameter name: The name of the column. /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. /// - Returns: The `Int64` value. - func getInt64(index: Int) throws -> Int64 + func getInt64(index: Int) throws(SqlCursorError) -> Int64 /// Retrieves an `Int64` value from the specified column index. /// - Parameter index: The zero-based index of the column. @@ -50,7 +50,7 @@ public protocol SqlCursor { /// - Parameter name: The name of the column. /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. /// - Returns: The `String` value. - func getString(index: Int) throws -> String + func getString(index: Int) throws(SqlCursorError) -> String /// Retrieves a `String` value from the specified column index. /// - Parameter index: The zero-based index of the column. @@ -65,9 +65,19 @@ public protocol SqlCursor { } extension SqlCursor { - private func resolveIndex(name: String) throws(SqlCursorError) -> Int { + private func withResolvedIndex(name: String, read: (_ index: Int) throws(SqlCursorError) -> T) throws(SqlCursorError) -> T { if let index = self.columnNames[name] { - return index + do { + return try read(index) + } catch { + // Add correct column name instead of index + switch error { + case .columnNotFound(_): + throw .columnNotFound(name) + case .nullValueFound(_): + throw .nullValueFound(name) + } + } } else { throw .columnNotFound(name) } @@ -78,7 +88,7 @@ extension SqlCursor { /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. /// - Returns: The `Bool` value. func getBoolean(name: String) throws -> Bool { - return try self.getBoolean(index: try self.resolveIndex(name: name)) + return try withResolvedIndex(name: name, read: self.getBoolean) } /// Retrieves an optional `Bool` value from the specified column name. @@ -86,7 +96,7 @@ extension SqlCursor { /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist. /// - Returns: The `Bool` value if present, or `nil` if the value is null. func getBooleanOptional(name: String) throws -> Bool? { - return self.getBooleanOptional(index: try self.resolveIndex(name: name)) + return try withResolvedIndex(name: name, read: self.getBooleanOptional) } /// Retrieves a `Double` value from the specified column name. @@ -94,7 +104,7 @@ extension SqlCursor { /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. /// - Returns: The `Double` value. func getDouble(name: String) throws -> Double { - return try self.getDouble(index: try self.resolveIndex(name: name)) + return try withResolvedIndex(name: name, read: self.getDouble) } /// Retrieves an optional `Double` value from the specified column name. @@ -102,7 +112,7 @@ extension SqlCursor { /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist. /// - Returns: The `Double` value if present, or `nil` if the value is null. func getDoubleOptional(name: String) throws -> Double? { - return self.getDoubleOptional(index: try self.resolveIndex(name: name)) + return try withResolvedIndex(name: name, read: self.getDoubleOptional) } /// Retrieves an `Int` value from the specified column name. @@ -110,7 +120,7 @@ extension SqlCursor { /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. /// - Returns: The `Int` value. func getInt(name: String) throws -> Int { - return try self.getInt(index: try self.resolveIndex(name: name)) + return try withResolvedIndex(name: name, read: self.getInt) } /// Retrieves an optional `Int` value from the specified column name. @@ -118,7 +128,7 @@ extension SqlCursor { /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist. /// - Returns: The `Int` value if present, or `nil` if the value is null. func getIntOptional(name: String) throws -> Int? { - return self.getIntOptional(index: try self.resolveIndex(name: name)) + return try withResolvedIndex(name: name, read: self.getIntOptional) } /// Retrieves an `Int64` value from the specified column name. @@ -126,7 +136,7 @@ extension SqlCursor { /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. /// - Returns: The `Int64` value. func getInt64(name: String) throws -> Int64 { - return try self.getInt64(index: try self.resolveIndex(name: name)) + return try withResolvedIndex(name: name, read: self.getInt64) } /// Retrieves an optional `Int64` value from the specified column name. @@ -134,7 +144,7 @@ extension SqlCursor { /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist. /// - Returns: The `Int64` value if present, or `nil` if the value is null. func getInt64Optional(name: String) throws -> Int64? { - return self.getInt64Optional(index: try self.resolveIndex(name: name)) + return try withResolvedIndex(name: name, read: self.getInt64Optional) } /// Retrieves a `String` value from the specified column name. @@ -142,7 +152,7 @@ extension SqlCursor { /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist, or `SqlCursorError.nullValueFound` if the value is null. /// - Returns: The `String` value. func getString(name: String) throws -> String { - return try self.getString(index: try self.resolveIndex(name: name)) + return try withResolvedIndex(name: name, read: self.getString) } /// Retrieves an optional `String` value from the specified column name. @@ -150,7 +160,7 @@ extension SqlCursor { /// - Throws: `SqlCursorError.columnNotFound` if the column does not exist. /// - Returns: The `String` value if present, or `nil` if the value is null. func getStringOptional(name: String) throws -> String? { - return self.getStringOptional(index: try self.resolveIndex(name: name)) + return try withResolvedIndex(name: name, read: self.getStringOptional) } } diff --git a/Tests/PowerSyncTests/Kotlin/KotlinPowerSyncDatabaseImplTests.swift b/Tests/PowerSyncTests/Kotlin/KotlinPowerSyncDatabaseImplTests.swift index ea66ccd..c9b0a6c 100644 --- a/Tests/PowerSyncTests/Kotlin/KotlinPowerSyncDatabaseImplTests.swift +++ b/Tests/PowerSyncTests/Kotlin/KotlinPowerSyncDatabaseImplTests.swift @@ -43,7 +43,7 @@ final class KotlinPowerSyncDatabaseImplTests: XCTestCase { XCTFail("Expected an error to be thrown") } catch { XCTAssertEqual(error.localizedDescription, """ - SqliteException(1): SQL logic error, no such table: usersfail for SQL: INSERT INTO usersfail (id, name, email) VALUES (?, ?, ?) + SQLite failure (code 1): SQL logic error, no such table: usersfail for SQL: INSERT INTO usersfail (id, name, email) VALUES (?, ?, ?) """) } } @@ -130,7 +130,7 @@ final class KotlinPowerSyncDatabaseImplTests: XCTestCase { XCTFail("Expected an error to be thrown") } catch { XCTAssertEqual(error.localizedDescription, """ - SqliteException(1): SQL logic error, no such table: usersfail for SQL: SELECT id, name, email FROM usersfail WHERE id = ? + SQLite failure (code 1): SQL logic error, no such table: usersfail for SQL: SELECT id, name, email FROM usersfail WHERE id = ? """) } } @@ -214,7 +214,7 @@ final class KotlinPowerSyncDatabaseImplTests: XCTestCase { XCTFail("Expected an error to be thrown") } catch { XCTAssertEqual(error.localizedDescription, """ - SqliteException(1): SQL logic error, no such table: usersfail for SQL: SELECT id, name, email FROM usersfail WHERE id = ? + SQLite failure (code 1): SQL logic error, no such table: usersfail for SQL: SELECT id, name, email FROM usersfail WHERE id = ? """) } } @@ -297,7 +297,7 @@ final class KotlinPowerSyncDatabaseImplTests: XCTestCase { XCTFail("Expected an error to be thrown") } catch { XCTAssertEqual(error.localizedDescription, """ - SqliteException(1): SQL logic error, no such table: usersfail for SQL: EXPLAIN SELECT name FROM usersfail ORDER BY id + SQLite failure (code 1): SQL logic error, no such table: usersfail for SQL: EXPLAIN SELECT name FROM usersfail ORDER BY id """) } } @@ -386,7 +386,7 @@ final class KotlinPowerSyncDatabaseImplTests: XCTestCase { } } catch { XCTAssertEqual(error.localizedDescription, """ - SqliteException(1): SQL logic error, no such table: usersfail for SQL: INSERT INTO usersfail (id, name, email) VALUES (?, ?, ?) + SQLite failure (code 1): SQL logic error, no such table: usersfail for SQL: INSERT INTO usersfail (id, name, email) VALUES (?, ?, ?) """) } } @@ -406,7 +406,7 @@ final class KotlinPowerSyncDatabaseImplTests: XCTestCase { } } catch { XCTAssertEqual(error.localizedDescription, """ - SqliteException(1): SQL logic error, no such table: usersfail for SQL: INSERT INTO usersfail (id, name, email) VALUES (?, ?, ?) + SQLite failure (code 1): SQL logic error, no such table: usersfail for SQL: INSERT INTO usersfail (id, name, email) VALUES (?, ?, ?) """) } @@ -449,7 +449,7 @@ final class KotlinPowerSyncDatabaseImplTests: XCTestCase { } } catch { XCTAssertEqual(error.localizedDescription, """ - SqliteException(1): SQL logic error, no such table: usersfail for SQL: SELECT COUNT(*) FROM usersfail + SQLite failure (code 1): SQL logic error, no such table: usersfail for SQL: SELECT COUNT(*) FROM usersfail """) } } @@ -536,7 +536,7 @@ final class KotlinPowerSyncDatabaseImplTests: XCTestCase { let warningIndex = testWriter.getLogs().firstIndex( where: { value in - value.contains("debug: PowerSyncVersion") + value.contains("debug: Opened connection. SQLite version") } ) @@ -646,19 +646,17 @@ final class KotlinPowerSyncDatabaseImplTests: XCTestCase { XCTAssertEqual(result[1], JoinOutput(name: "Test User", description: "task 2", comment: "comment 2")) } - /* func testCloseWithDeleteDatabase() async throws { let fileManager = FileManager.default let testDbFilename = "test_delete_\(UUID().uuidString).db" // Get the database directory using the helper function - let databaseDirectory = try appleDefaultDatabaseDirectory() + let databaseDirectory = try DatabaseLocation.appleDefaultDatabaseDirectory() // Create a database with a real file let testDatabase = PowerSyncDatabase( schema: schema, dbFilename: testDbFilename, - logger: DatabaseLogger(DefaultLogger()) ) // Perform some operations to ensure the database file is created @@ -691,13 +689,12 @@ final class KotlinPowerSyncDatabaseImplTests: XCTestCase { let testDbFilename = "test_no_delete_\(UUID().uuidString).db" // Get the database directory using the helper function - let databaseDirectory = try appleDefaultDatabaseDirectory() + let databaseDirectory = try DatabaseLocation.appleDefaultDatabaseDirectory() // Create a database with a real file let testDatabase = PowerSyncDatabase( schema: schema, dbFilename: testDbFilename, - logger: DatabaseLogger(DefaultLogger()) ) // Perform some operations to ensure the database file is created @@ -718,7 +715,7 @@ final class KotlinPowerSyncDatabaseImplTests: XCTestCase { // Clean up: delete all SQLite files using the helper function try deleteSQLiteFiles(dbFilename: testDbFilename, in: databaseDirectory) - }*/ + } func testSubscriptionsUpdateStateWhileOffline() async throws { var streams = database.currentStatus.asFlow().makeAsyncIterator() From d067ced383956a9902f7607108d62d833ad7a087 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 23 Apr 2026 13:19:55 +0200 Subject: [PATCH 24/40] Use high-level GRDB APIs instead of pointer --- .../Implementation/AsyncConnectionPool.swift | 24 +- .../PoolConnectionContext.swift | 38 -- .../PowerSyncDatabaseImpl.swift | 147 +------- .../queries/ConnectionPoolQueries.swift | 131 +++++++ .../Implementation/queries/LeaseContext.swift | 62 ++++ .../TransactionImpl.swift | 0 .../sqlite3/NativeConnectionContext.swift | 349 ------------------ .../sqlite3/NativeConnectionPool.swift | 78 +++- .../sqlite3/NativeStatement.swift | 126 +++++++ .../sqlite3/StatementCursor.swift | 122 ++++++ .../sqlite3/throwDatabaseError.swift | 8 +- .../PowerSync/Protocol/QueriesProtocol.swift | 34 +- .../Protocol/SQLiteConnectionPool.swift | 29 +- .../Protocol/db/DataConvertible.swift | 3 +- Sources/PowerSync/Utils/AsyncMutex.swift | 6 + .../Config/Configuration+PowerSync.swift | 29 +- .../Connections/GRDBConnectionLease.swift | 42 ++- .../Connections/GRDBConnectionPool.swift | 57 +-- .../PowerSyncGRDB/Connections/RowCursor.swift | 92 +++++ .../PowerSyncGRDB/SQLite/SQLite+Utils.swift | 24 -- Tests/PowerSyncGRDBTests/BasicTest.swift | 2 +- 21 files changed, 733 insertions(+), 670 deletions(-) delete mode 100644 Sources/PowerSync/Implementation/PoolConnectionContext.swift create mode 100644 Sources/PowerSync/Implementation/queries/ConnectionPoolQueries.swift create mode 100644 Sources/PowerSync/Implementation/queries/LeaseContext.swift rename Sources/PowerSync/Implementation/{sqlite3 => queries}/TransactionImpl.swift (100%) delete mode 100644 Sources/PowerSync/Implementation/sqlite3/NativeConnectionContext.swift create mode 100644 Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift create mode 100644 Sources/PowerSync/Implementation/sqlite3/StatementCursor.swift create mode 100644 Sources/PowerSyncGRDB/Connections/RowCursor.swift delete mode 100644 Sources/PowerSyncGRDB/SQLite/SQLite+Utils.swift diff --git a/Sources/PowerSync/Implementation/AsyncConnectionPool.swift b/Sources/PowerSync/Implementation/AsyncConnectionPool.swift index 48bb461..1b3fee7 100644 --- a/Sources/PowerSync/Implementation/AsyncConnectionPool.swift +++ b/Sources/PowerSync/Implementation/AsyncConnectionPool.swift @@ -80,7 +80,7 @@ final class AsyncConnectionPool: SQLiteConnectionPoolProtocol { } private func configureConnection(connection: borrowing RawSqliteConnection, isWriter: Bool) throws { - let context = NativeConnectionContext(connection.asLease()) + let context = connection.asLease() for stmt in initialStatements { let _ = try context.execute(sql: stmt, parameters: []) } @@ -98,8 +98,10 @@ final class AsyncConnectionPool: SQLiteConnectionPoolProtocol { if isWriter { // Older versions of the SDK used to set up an empty schema and raise the user version to 1. // Keep doing that for consistency. - let version = try context.get(sql: "pragma user_version", parameters: []) { try $0.getInt(index: 0) } - if version < 1 { + let version = try context.withIterator(sql: "pragma user_version", parameters: []) { rows in + try rows.next { try $0.getInt(index: 0) } + } + if let version, version < 1 { let _ = try context.execute(sql: "pragma user_version = 1", parameters: []) } @@ -142,23 +144,23 @@ final class AsyncConnectionPool: SQLiteConnectionPoolProtocol { } } - func read(onConnection: @escaping @Sendable (any SQLiteConnectionLease) throws -> Void) async throws { + func read(onConnection: @escaping @Sendable (any SQLiteConnectionLease) throws -> T) async throws -> T { let pool = try await obtainInner() - try await pool.read { connection in - try await runBlocking { try onConnection(connection) } + return try await pool.read { connection in + return try await runBlocking { try onConnection(connection) } } } - - func write(onConnection: @escaping @Sendable (any SQLiteConnectionLease) throws -> Void) async throws { + + func write(onConnection: @escaping @Sendable (any SQLiteConnectionLease) throws -> T) async throws -> T{ let pool = try await obtainInner() - try await pool.write { connection in + return try await pool.write { connection in try await runBlocking { try onConnection(connection) } } } - func withAllConnections(onConnection: @escaping @Sendable (any SQLiteConnectionLease, [any SQLiteConnectionLease]) throws -> Void) async throws { + func withAllConnections(onConnection: @escaping @Sendable (any SQLiteConnectionLease, [any SQLiteConnectionLease]) throws -> T) async throws -> T { let pool = try await obtainInner() - try await pool.withAllConnections { writer, readers in + return try await pool.withAllConnections { writer, readers in try await runBlocking { try onConnection(writer, readers) } } } diff --git a/Sources/PowerSync/Implementation/PoolConnectionContext.swift b/Sources/PowerSync/Implementation/PoolConnectionContext.swift deleted file mode 100644 index 7ad63f9..0000000 --- a/Sources/PowerSync/Implementation/PoolConnectionContext.swift +++ /dev/null @@ -1,38 +0,0 @@ -func poolRead(_ pool: borrowing SQLiteConnectionPoolProtocol, action: @escaping @Sendable (_: any ConnectionContext) throws -> T) async throws -> T { - let result = UnsafeSendable() - try await pool.read { lease in - let context = NativeConnectionContext(lease) - result.resolve(value: try action(context)) - } - - return result.inner! -} - -func poolWrite(_ pool: borrowing SQLiteConnectionPoolProtocol, action: @escaping @Sendable (_: any ConnectionContext) throws -> T) async throws -> T { - let result = UnsafeSendable() - try await pool.write { lease in - let context = NativeConnectionContext(lease) - result.resolve(value: try action(context)) - } - - return result.inner! -} - -func poolWithAll(_ pool: borrowing SQLiteConnectionPoolProtocol, action: @escaping @Sendable (_ writer: any ConnectionContext, _ readers: [any ConnectionContext]) throws -> T) async throws -> T { - let result = UnsafeSendable() - try await pool.withAllConnections { writer, readers in - let writer = NativeConnectionContext(writer) - let readers = readers.map { NativeConnectionContext($0) } - - result.resolve(value: try action(writer, readers)) - } - return result.inner! -} - -private final class UnsafeSendable: @unchecked Sendable { - var inner: T? = nil - - func resolve(value: T) { - self.inner = value - } -} diff --git a/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift b/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift index 8e5116d..270e299 100644 --- a/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift +++ b/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift @@ -8,7 +8,7 @@ final class PowerSyncDatabaseImpl: PowerSyncDatabaseProtocol { private let dbFilename: String? private let httpClient: HttpClient private let initializer = DatabaseInitizalizationActor() - fileprivate let pool: any SQLiteConnectionPoolProtocol + fileprivate let queries: ConnectionPoolQueries let schema: AsyncMutex init( @@ -22,7 +22,7 @@ final class PowerSyncDatabaseImpl: PowerSyncDatabaseProtocol { self.logger = logger self.schema = AsyncMutex(schema) self.httpClient = httpClient - self.pool = pool + self.queries = ConnectionPoolQueries(pool: pool) } var currentStatus: any SyncStatus { @@ -41,7 +41,7 @@ final class PowerSyncDatabaseImpl: PowerSyncDatabaseProtocol { fileprivate func resolveOfflineSyncStatus() async throws { // We can't use get() here because it runs as part of the initialization step. - let offlineSyncStatus = try await poolRead(pool) { connection in + let offlineSyncStatus = try await queries.readLock { connection in try connection.get(sql: "SELECT powersync_offline_sync_status()", parameters: []) { cursor in let raw = try cursor.getString(index: 0) return try StreamingSyncClient.jsonDecoder.decode(CoreDownloadSyncStatus.self, from: raw.data(using: .utf8)!) @@ -64,7 +64,7 @@ final class PowerSyncDatabaseImpl: PowerSyncDatabaseProtocol { } fileprivate func applySchema(schema: Schema) async throws { - try await poolWithAll(pool) { writer, readers in + try await queries.withAll { writer, readers in let encoded = try StreamingSyncClient.jsonEncoder.encode(schema) guard let asString = String(data: encoded, encoding: .utf8) else { throw PowerSyncError.operationFailed(message: "Could not serialize schema") @@ -107,7 +107,7 @@ final class PowerSyncDatabaseImpl: PowerSyncDatabaseProtocol { try await initialize() try await initializer.close { await syncCoordinator.disconnect() - try await pool.close() + try await queries.pool.close() } } @@ -119,61 +119,11 @@ final class PowerSyncDatabaseImpl: PowerSyncDatabaseProtocol { try deleteSQLiteFiles(dbFilename: dbFilename, in: directory) } } - + func connect(connector: any PowerSyncBackendConnectorProtocol, options: ConnectOptions?) async throws { await syncCoordinator.connect(db: self, connector: connector, options: options ?? ConnectOptions(), client: httpClient) } - - func watch(options: WatchOptions) throws -> AsyncThrowingStream<[RowType], any Error> { - AsyncThrowingStream { continuation in - // Create an outer task to monitor cancellation - let task = Task { - do { - try await initialize() - let watchedTables = try await self.getQuerySourceTables( - sql: options.sql, - parameters: options.parameters - ) - - let updateNotifications = pool.tableUpdates.filter { changedTables in - changedTables.contains(where: watchedTables.contains) - }.map { _ in () } - // Allows emitting the first result even if there aren't changes - let withInitial = AsyncAlgorithms.merge([()].async, updateNotifications) - let throttled = AsyncThrottleSequence(inner: withInitial, duration: options.throttle) - - for try await _ in throttled { - // Check if the outer task is cancelled - try Task.checkCancellation() - try continuation.yield(await self.getAll( - sql: options.sql, - parameters: options.parameters, - mapper: options.mapper - )) - } - - continuation.finish() - } catch { - if error is CancellationError { - continuation.finish() - } else { - continuation.finish(throwing: error) - } - } - } - - // Propagate cancellation from the outer task to the inner task - continuation.onTermination = { @Sendable _ in - task.cancel() // This cancels the inner task when the stream is terminated - } - } - } - - func watch(sql: String, parameters: [(any Sendable)?]?, mapper: @escaping @Sendable (any SqlCursor) throws -> RowType) throws -> AsyncThrowingStream<[RowType], any Error> { - return try watch(options: WatchOptions(sql: sql, parameters: parameters, mapper: mapper)) - } - func disconnectAndClear(clearLocal: Bool, soft: Bool) async throws { try await initialize() try await syncCoordinator.disconnectAndThen { @@ -187,89 +137,26 @@ final class PowerSyncDatabaseImpl: PowerSyncDatabaseProtocol { do { let flags = flags - let _ = try await poolWrite(pool) { ctx in try ctx.execute(sql: "SELECT powersync_clear(?)", parameters: [flags]) } + let _ = try await queries.writeLock { ctx in try ctx.execute(sql: "SELECT powersync_clear(?)", parameters: [flags]) } } } } - - func writeLock(callback: @escaping @Sendable (any ConnectionContext) throws -> R) async throws -> R { + + func writeLock(callback: @escaping @Sendable (any ConnectionContext) throws -> R) async throws -> R { try await initialize() - return try await poolWrite(pool, action: callback) + return try await queries.writeLock(callback: callback) } - - func readLock(callback: @escaping @Sendable (any ConnectionContext) throws -> R) async throws -> R { + + func readLock(callback: @escaping @Sendable (any ConnectionContext) throws -> R) async throws -> R { try await initialize() - return try await poolRead(pool, action: callback) + return try await queries.readLock(callback: callback) } - - func writeTransaction(callback: @escaping @Sendable (any Transaction) throws -> R) async throws -> R { - return try await writeLock { ctx in try TransactionImpl.run(conn: ctx, callback: callback) } - } - - func readTransaction(callback: @escaping @Sendable (any Transaction) throws -> R) async throws -> R { - return try await readLock { ctx in try TransactionImpl.run(conn: ctx, callback: callback) } - } - - private func getQuerySourceTables( - sql: String, - parameters: [Sendable?] - ) async throws -> Set { - let rows = try await getAll( - sql: "EXPLAIN \(sql)", - parameters: parameters, - mapper: { cursor in - try ExplainQueryResult( - addr: cursor.getString(index: 0), - opcode: cursor.getString(index: 1), - p1: cursor.getInt64(index: 2), - p2: cursor.getInt64(index: 3), - p3: cursor.getInt64(index: 4) - ) - } - ) - - let rootPages = rows.compactMap { row in - if (row.opcode == "OpenRead" || row.opcode == "OpenWrite") && - row.p3 == 0 && row.p2 != 0 - { - return row.p2 - } - return nil - } - - do { - let pagesData = try StreamingSyncClient.jsonEncoder.encode(rootPages) - guard let pagesString = String(data: pagesData, encoding: .utf8) else { - throw PowerSyncError.operationFailed( - message: "Failed to convert pages data to UTF-8 string" - ) - } - let tableRows = try await getAll( - sql: "SELECT tbl_name FROM sqlite_master WHERE rootpage IN (SELECT json_each.value FROM json_each(?))", - parameters: [ - pagesString, - ] - ) { try $0.getString(index: 0) } - - return Set(tableRows) - } catch { - throw PowerSyncError.operationFailed( - message: "Could not determine watched query tables", - underlyingError: error - ) - } + func watch(options: WatchOptions) throws -> AsyncThrowingStream<[RowType], any Error> { + return try queries.watch(options: options) } - - static let maxOpId = Int64.max -} -private struct ExplainQueryResult { - let addr: String - let opcode: String - let p1: Int64 - let p2: Int64 - let p3: Int64 + static let maxOpId = Int64.max } private actor DatabaseInitizalizationActor { @@ -285,7 +172,7 @@ private actor DatabaseInitizalizationActor { return } - powerSyncVersion = try await poolWrite(db.pool) { conn in + powerSyncVersion = try await db.queries.writeLock { conn in let sqliteVersion = try conn.get(sql: "SELECT sqlite_version()", parameters: []) { try $0.getString(index: 0) } let powerSyncVersion = try conn.get(sql: "SELECT powersync_rs_version()", parameters: []) { try $0.getString(index: 0) } diff --git a/Sources/PowerSync/Implementation/queries/ConnectionPoolQueries.swift b/Sources/PowerSync/Implementation/queries/ConnectionPoolQueries.swift new file mode 100644 index 0000000..d9ec650 --- /dev/null +++ b/Sources/PowerSync/Implementation/queries/ConnectionPoolQueries.swift @@ -0,0 +1,131 @@ +import AsyncAlgorithms + +/// Implements ``Queries`` by delegating to a ``SQLiteConnectionPoolProtocol``. +struct ConnectionPoolQueries: Sendable, Queries { + let pool: SQLiteConnectionPoolProtocol + + func writeLock(callback: @escaping @Sendable (any ConnectionContext) throws -> R) async throws -> R { + try await pool.write { connection in + try callback(ConnectionLeaseContext(lease: connection)) + } + } + + func readLock(callback: @escaping @Sendable (any ConnectionContext) throws -> R) async throws -> R { + try await pool.read { connection in + try callback(ConnectionLeaseContext(lease: connection)) + } + } + + func withAll(callback: @escaping @Sendable (_ writer: any ConnectionContext, _ readers: [any ConnectionContext]) throws -> R) async throws -> R { + try await pool.withAllConnections { writer, readers in + let writer = ConnectionLeaseContext(lease: writer) + let readers = readers.map { ConnectionLeaseContext(lease: $0) } + + return try callback(writer, readers) + } + } + + func watch(options: WatchOptions) throws -> AsyncThrowingStream<[RowType], any Error> { + AsyncThrowingStream { continuation in + // Create an outer task to monitor cancellation + let task = Task { + do { + let watchedTables = try await self.getQuerySourceTables( + sql: options.sql, + parameters: options.parameters + ) + + let updateNotifications = pool.tableUpdates.filter { changedTables in + changedTables.contains(where: watchedTables.contains) + }.map { _ in () } + // Allows emitting the first result even if there aren't changes + let withInitial = AsyncAlgorithms.merge([()].async, updateNotifications) + let throttled = AsyncThrottleSequence(inner: withInitial, duration: options.throttle) + + for try await _ in throttled { + // Check if the outer task is cancelled + try Task.checkCancellation() + + try continuation.yield(await self.getAll( + sql: options.sql, + parameters: options.parameters, + mapper: options.mapper + )) + } + + continuation.finish() + } catch { + if error is CancellationError { + continuation.finish() + } else { + continuation.finish(throwing: error) + } + } + } + + // Propagate cancellation from the outer task to the inner task + continuation.onTermination = { @Sendable _ in + task.cancel() // This cancels the inner task when the stream is terminated + } + } + } + + private func getQuerySourceTables( + sql: String, + parameters: [Sendable?] + ) async throws -> Set { + let rows = try await getAll( + sql: "EXPLAIN \(sql)", + parameters: parameters, + mapper: { cursor in + try ExplainQueryResult( + addr: cursor.getString(index: 0), + opcode: cursor.getString(index: 1), + p1: cursor.getInt64(index: 2), + p2: cursor.getInt64(index: 3), + p3: cursor.getInt64(index: 4) + ) + } + ) + + let rootPages = rows.compactMap { row in + if (row.opcode == "OpenRead" || row.opcode == "OpenWrite") && + row.p3 == 0 && row.p2 != 0 + { + return row.p2 + } + return nil + } + + do { + let pagesData = try StreamingSyncClient.jsonEncoder.encode(rootPages) + guard let pagesString = String(data: pagesData, encoding: .utf8) else { + throw PowerSyncError.operationFailed( + message: "Failed to convert pages data to UTF-8 string" + ) + } + + let tableRows = try await getAll( + sql: "SELECT tbl_name FROM sqlite_master WHERE rootpage IN (SELECT json_each.value FROM json_each(?))", + parameters: [ + pagesString, + ] + ) { try $0.getString(index: 0) } + + return Set(tableRows) + } catch { + throw PowerSyncError.operationFailed( + message: "Could not determine watched query tables", + underlyingError: error + ) + } + } +} + +private struct ExplainQueryResult { + let addr: String + let opcode: String + let p1: Int64 + let p2: Int64 + let p3: Int64 +} diff --git a/Sources/PowerSync/Implementation/queries/LeaseContext.swift b/Sources/PowerSync/Implementation/queries/LeaseContext.swift new file mode 100644 index 0000000..29f0958 --- /dev/null +++ b/Sources/PowerSync/Implementation/queries/LeaseContext.swift @@ -0,0 +1,62 @@ +/// An implementation of ``ConnectionContext`` based on a raw ``SQLiteConnectionLease``. +final class ConnectionLeaseContext: ConnectionContext { + private let lease: Mutex + + init(lease: consuming SQLiteConnectionLease) { + self.lease = Mutex(lease) + } + + /// Maps any parameter array to typed SQLite values. + private func mapMarameters(_ parameters: [(any Sendable)?]?) throws -> [PowerSyncDataType?] { + guard let parameters else { + return [] + } + + return try parameters.map { parameter in + if let convertible = parameter as? PowerSyncDataTypeConvertible { + return convertible.psDataType + } else if let parameter { + return try PowerSyncDataType(from: parameter) + } else { + return nil + } + } + } + + func execute(sql: String, parameters: [(any Sendable)?]?) throws -> Int64 { + try lease.withLock { lease in + return try lease.execute(sql: sql, parameters: mapMarameters(parameters)) + } + } + + func getOptional(sql: String, parameters: [(any Sendable)?]?, mapper: @escaping @Sendable (any SqlCursor) throws -> RowType) throws -> RowType? { + try lease.withLock { lease in + try lease.withIterator(sql: sql, parameters: mapMarameters(parameters)) { rows in + return try rows.next(callback: mapper) + } + } + } + + func getAll(sql: String, parameters: [(any Sendable)?]?, mapper: @escaping @Sendable (any SqlCursor) throws -> RowType) throws -> [RowType] { + try lease.withLock { lease in + try lease.withIterator(sql: sql, parameters: mapMarameters(parameters)) { rows in + var result: [RowType] = [] + while let row = try rows.next(callback: mapper) { + result.append(row) + } + return result + } + } + } + + func get(sql: String, parameters: [(any Sendable)?]?, mapper: @escaping @Sendable (any SqlCursor) throws -> RowType) throws -> RowType { + try lease.withLock { lease in + try lease.withIterator(sql: sql, parameters: mapMarameters(parameters)) { rows in + guard let cursor = try rows.next(callback: mapper) else { + throw PowerSyncError.operationFailed(message: "Expected \(sql) to return a row, but got an empty result set.") + } + return cursor + } + } + } +} diff --git a/Sources/PowerSync/Implementation/sqlite3/TransactionImpl.swift b/Sources/PowerSync/Implementation/queries/TransactionImpl.swift similarity index 100% rename from Sources/PowerSync/Implementation/sqlite3/TransactionImpl.swift rename to Sources/PowerSync/Implementation/queries/TransactionImpl.swift diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionContext.swift b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionContext.swift deleted file mode 100644 index 56b4e13..0000000 --- a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionContext.swift +++ /dev/null @@ -1,349 +0,0 @@ -import CSQLite -import Darwin -import Synchronization - -private struct NativeConnectionState { - let lease: SQLiteConnectionLease - var closed: Bool = false - - func checkNotClosed() throws(PowerSyncError) { - if self.closed { - throw .operationFailed(message: "Attempted to use a connection context after it was closed") - } - } -} - -final class NativeConnectionContext: ConnectionContext { - private let state: Mutex; - - init(_ lease: consuming SQLiteConnectionLease) { - self.state = Mutex(NativeConnectionState(lease: lease)); - } - - func invalidateLease() { - state.withLock { $0.closed = true } - } - - func execute(sql: String, parameters: [(any Sendable)?]?) throws -> Int64 { - return try state.withLock { - try $0.checkNotClosed() - - var stmt = try SqliteStatement(db: $0.lease, sql: sql) - try stmt.bindValues(parameters) - while try stmt.step() { - // Iterate through the statement. - } - - let _ = consume stmt - return sqlite3_changes64($0.lease.pointer); - } - } - - func getOptional(sql: String, parameters: [(any Sendable)?]?, mapper: @Sendable (SqlCursor) throws -> RowType) throws -> RowType? { - return try state.withLock { - try $0.checkNotClosed() - - var stmt = try SqliteStatement(db: $0.lease, sql: sql) - try stmt.bindValues(parameters) - if try stmt.step() { - return try NativeConnectionContext.invokeMapper(stmt, mapper) - } else { - return nil - } - } - } - - func getAll(sql: String, parameters: [(any Sendable)?]?, mapper: @Sendable (SqlCursor) throws -> RowType) throws -> [RowType] { - return try state.withLock { - try $0.checkNotClosed() - - var stmt = try SqliteStatement(db: $0.lease, sql: sql) - try stmt.bindValues(parameters) - var rows: [RowType] = [] - while try stmt.step() { - rows.append(try NativeConnectionContext.invokeMapper(stmt, mapper)) - } - return rows - } - } - - func get(sql: String, parameters: [(any Sendable)?]?, mapper: @Sendable (SqlCursor) throws -> RowType) throws -> RowType { - return try state.withLock { - try $0.checkNotClosed() - - var stmt = try SqliteStatement(db: $0.lease, sql: sql) - try stmt.bindValues(parameters) - if try stmt.step() { - return try NativeConnectionContext.invokeMapper(stmt, mapper) - } else { - throw PowerSyncError.operationFailed(message: "Called get(\(sql), which did not return any row") - } - } - } - - private static func invokeMapper(_ stmt: borrowing SqliteStatement, _ mapper: (SqlCursor) throws -> RowType) rethrows -> RowType { - return try withUnsafePointer(to: stmt) { ptr in - let cursor = StatementCursor(ptr) - defer { - cursor.invalidate() - } - - return try mapper(cursor) - } - } -} - -struct SqliteStatement: ~Copyable { - private var resolvedColumnNames: [String : Int]? = nil - private let db: SQLiteConnectionLease - let stmt: OpaquePointer - private let sql: String - - init(db: SQLiteConnectionLease, sql: String) throws(PowerSyncError) { - self.db = db - var stmt: OpaquePointer? - var sql = sql - let rc = sql.withUTF8 { sqlBytes in - return sqlite3_prepare_v2( - db.pointer, - sqlBytes.baseAddress, - Int32(sqlBytes.count), - &stmt, - nil - ) - } - if (rc != 0) { - try throwDatabaseError(db: db, sql: sql) - } - - self.stmt = stmt! - self.sql = sql - } - - deinit { - sqlite3_finalize(stmt) - } - - var columnCount: Int { - return Int(sqlite3_column_count(self.stmt)) - } - - - var columnNames: [String : Int] { - return resolvedColumnNames! - } - - borrowing func bindValues(_ parameters: [Any?]?) throws (PowerSyncError) { - if let parameters { - for (i, parameter) in parameters.enumerated() { - let index = Int32(i + 1) - - let psDataType: PowerSyncDataType? = if let convertible = parameter as? PowerSyncDataTypeConvertible { - convertible.psDataType - } else if let parameter { - try PowerSyncDataType(from: parameter) - } else { - nil - } - - if let psDataType { - try bindValue(index, psDataType) - } else { - try bindValue(index, nil) - } - } - } - } - - borrowing func bindValue(_ index: Int32, _ parameter: PowerSyncDataType?) throws (PowerSyncError) { - let rc: Int32 - - switch parameter { - case nil: - rc = sqlite3_bind_null(self.stmt, index) - case .bool(let value): - rc = sqlite3_bind_int(self.stmt, index, value ? 1 : 0) - case .string(let value): - var str = value - rc = str.withUTF8 { buffer in - sqlite3_bind_text( - self.stmt, - index, - buffer.baseAddress, - Int32(buffer.count), - // SQLITE_TRANSIENT - unsafeBitCast(-1, to: (@convention(c) (UnsafeMutableRawPointer?) -> Void).self), - ) - } - case .int64(let value): - rc = sqlite3_bind_int64(self.stmt, index, value) - case .int32(let value): - rc = sqlite3_bind_int(self.stmt, index, value) - case .double(let value): - rc = sqlite3_bind_double(self.stmt, index, value) - case .data(let value): - // Data object can be made up of multiple memory regions, so copy once. - let buffer = malloc(value.count)! - value.copyBytes(to: buffer.assumingMemoryBound(to: UInt8.self), count: value.count) - - rc = sqlite3_bind_blob( - self.stmt, - index, - buffer, - Int32(value.count), - free, - ) - - if rc != 0 { - free(buffer) - } - } - - if rc != 0 { - try throwDatabaseError(db: self.db, sql: self.sql) - } - } - - mutating func step() throws (PowerSyncError) -> Bool { - let rc = sqlite3_step(self.stmt) - if rc == SQLITE_DONE { - return false - } else if rc == SQLITE_ROW { - if resolvedColumnNames == nil { - let count = self.columnCount - var nameToIndex = Dictionary(minimumCapacity: count) - for i in 0..? - - init(_ stmtPtr: UnsafePointer) { - self.stmtPtr = stmtPtr - } - - func invalidate() { - stmtPtr = nil - } - - private func withStatement(_ body: (borrowing SqliteStatement) -> R) -> R { - if let stmtPtr { - return body(stmtPtr.pointee) - } - - fatalError("Cursor used outside of callback") - } - - private func checkColumnNotNull(stmt: borrowing SqliteStatement, index: Int) throws(SqlCursorError) { - if index < 0 || index >= stmt.columnCount { - throw SqlCursorError.nullValueFound("invalid index \(index)") - } - - let type = sqlite3_column_type(stmt.stmt, Int32(index)) - if type == SQLITE_NULL { - throw SqlCursorError.nullValueFound("\(index)") - } - } - - private func withStatementCheckNotNull(_ index: Int, body: (borrowing SqliteStatement) throws (SqlCursorError) -> R) throws (SqlCursorError) -> R { - if let stmtPtr { - try self.checkColumnNotNull(stmt: stmtPtr.pointee, index: index) - return try body(stmtPtr.pointee) - } - - fatalError("Cursor used outside of callback") - } - - var columnCount: Int { - return withStatement { stmt in stmt.columnCount } - } - - var columnNames: [String : Int] { - return withStatement { stmt in stmt.columnNames } - } - - func getBoolean(index: Int) throws(SqlCursorError) -> Bool { - return try getInt(index: index) == 0 ? false : true - } - - func getBooleanOptional(index: Int) -> Bool? { - do { - return try getBoolean(index: index) - } catch { - return nil - } - } - - func getDouble(index: Int) throws(SqlCursorError) -> Double { - return try withStatementCheckNotNull(index) { stmt in - return sqlite3_column_double(stmt.stmt, Int32(index)) - } - } - - func getDoubleOptional(index: Int) -> Double? { - do { - return try getDouble(index: index) - } catch { - return nil - } - } - - func getInt(index: Int) throws(SqlCursorError) -> Int { - return Int(try getInt64(index: index)) - } - - func getIntOptional(index: Int) -> Int? { - do { - return try getInt(index: index) - } catch { - return nil - } - } - - func getInt64(index: Int) throws(SqlCursorError) -> Int64 { - return try withStatementCheckNotNull(index) { stmt in - return sqlite3_column_int64(stmt.stmt, Int32(index)) - } - } - - func getInt64Optional(index: Int) -> Int64? { - do { - return try getInt64(index: index) - } catch { - return nil - } - } - - func getString(index: Int) throws(SqlCursorError) -> String { - return try withStatementCheckNotNull(index) { stmt in - let length = sqlite3_column_bytes(stmt.stmt, Int32(index)) - if length == 0 { - return "" - } - - let ptr = sqlite3_column_text(stmt.stmt, Int32(index)) - return String(decoding: UnsafeBufferPointer(start: ptr, count: Int(length)), as: UTF8.self) - } - } - - func getStringOptional(index: Int) -> String? { - do { - return try getString(index: index) - } catch { - return nil - } - } -} diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift index d962e32..f66d050 100644 --- a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift +++ b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift @@ -22,50 +22,55 @@ final class NativeConnectionPool: Sendable { self.handleUpdates = handleUpdates } - private func dispatchWrites(lease: RawConnectionLease) throws { - let ctx = NativeConnectionContext(lease) - let affectedTables = try ctx.get(sql: "SELECT powersync_update_hooks('get')", parameters: []) { - let decoder = JSONDecoder() - return try decoder.decode(Set.self, from: try $0.getString(index: 0).data(using: .utf8)!) - } + private func dispatchWrites(lease: NativeConnectionLease) throws { + try lease.withIterator(sql: "SELECT powersync_update_hooks('get')", parameters: []) { rows in + var rows = rows + let affectedTables = try rows.next { + let decoder = JSONDecoder() + return try decoder.decode(Set.self, from: try $0.getString(index: 0).data(using: .utf8)!) + } - if !affectedTables.isEmpty { - self.handleUpdates(affectedTables) + if let affectedTables, !affectedTables.isEmpty { + self.handleUpdates(affectedTables) + } } } - func read(onConnection: @Sendable (RawConnectionLease) async throws -> Void) async throws { + func read(onConnection: (NativeConnectionLease) async throws -> T) async throws -> T { // No dedicated readers? Acquire write connection for this then let semaphore = readers ?? writer let connection = try await semaphore.acquire(count: 1) let lease = connection.acquiredItems[0].asLease() - try await onConnection(lease) + return try await onConnection(lease) } - func write(onConnection: @Sendable (RawConnectionLease) async throws -> Void) async throws { + func write(onConnection: (NativeConnectionLease) async throws -> T) async throws -> T { let connection = try await writer.acquire(count: 1) let lease = connection.acquiredItems[0].asLease() - try await onConnection(lease) + let result = try await onConnection(lease) try dispatchWrites(lease: lease) + return result } - func withAllConnections(onConnection: @Sendable (RawConnectionLease, [RawConnectionLease]) async throws -> Void) async throws { + func withAllConnections(onConnection: (NativeConnectionLease, [NativeConnectionLease]) async throws -> T) async throws -> T{ let write = try await writer.acquire(count: 1) let writeLease = write.acquiredItems[0].asLease() + let result: T if let readers { let acquiredReaders = try await readers.acquire(count: readers.count) - var readerLeases: [RawConnectionLease] = [] + var readerLeases: [NativeConnectionLease] = [] let span = acquiredReaders.acquiredItems.span for idx in span.indices { readerLeases.append(span[idx].asLease()) } - try await onConnection(writeLease, readerLeases) + result = try await onConnection(writeLease, readerLeases) } else { - try await onConnection(writeLease, []) + result = try await onConnection(writeLease, []) } try dispatchWrites(lease: writeLease) + return result } func close() async throws { @@ -107,12 +112,47 @@ struct RawSqliteConnection: ~Copyable { sqlite3_close_v2(connection) } - func asLease() -> RawConnectionLease { + func asLease() -> NativeConnectionLease { precondition(!closed) - return RawConnectionLease(pointer: self.connection) + return NativeConnectionLease(pointer: self.connection) } } -struct RawConnectionLease: SQLiteConnectionLease, @unchecked Sendable { +struct NativeConnectionLease: SQLiteConnectionLease, @unchecked Sendable { let pointer: OpaquePointer + + func execute(sql: String, parameters: [PowerSyncDataType?]) throws -> Int64 { + do { + var stmt = try NativeSqliteStatement(db: pointer, sql: sql) + try stmt.bindValues(parameters) + while try stmt.step() { + // Iterate through the statement. + } + } + + return sqlite3_changes64(pointer) + } + + func withIterator(sql: String, parameters: [PowerSyncDataType?], callback: (SQLiteStatementIteratorProtocol) throws -> T) throws -> T { + var stmt = try NativeSqliteStatement(db: pointer, sql: sql) + try stmt.bindValues(parameters) + return try withUnsafeMutablePointer(to: &stmt) { ptr in + let iterator = NativeStatementIterator(stmt: ptr) + return try callback(iterator) + } + } +} + +private struct NativeStatementIterator: SQLiteStatementIteratorProtocol { + var stmt: UnsafeMutablePointer + + func next(callback: (any SqlCursor) throws -> T) throws -> T? { + if try stmt.pointee.step() { + let cursor = StatementCursor(stmt) + defer { cursor.invalidate() } + return try callback(cursor) + } else { + return nil + } + } } diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift b/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift new file mode 100644 index 0000000..36ad7f0 --- /dev/null +++ b/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift @@ -0,0 +1,126 @@ +import CSQLite +import Darwin + +struct NativeSqliteStatement: ~Copyable { + private var resolvedColumnNames: [String : Int]? = nil + private let db: OpaquePointer + let stmt: OpaquePointer + private let sql: String + + init(db: OpaquePointer, sql: String) throws(PowerSyncError) { + self.db = db + var stmt: OpaquePointer? + var sql = sql + let rc = sql.withUTF8 { sqlBytes in + return sqlite3_prepare_v2( + db, + sqlBytes.baseAddress, + Int32(sqlBytes.count), + &stmt, + nil + ) + } + if (rc != 0) { + try throwDatabaseError(db: db, sql: sql) + } + + self.stmt = stmt! + self.sql = sql + } + + deinit { + sqlite3_finalize(stmt) + } + + var columnCount: Int { + return Int(sqlite3_column_count(self.stmt)) + } + + + var columnNames: [String : Int] { + return resolvedColumnNames! + } + + borrowing func bindValues(_ parameters: [PowerSyncDataType?]) throws (PowerSyncError) { + for (i, parameter) in parameters.enumerated() { + let index = Int32(i + 1) + + if let parameter { + try bindValue(index, parameter) + } else { + try bindValue(index, nil) + } + } + } + + borrowing func bindValue(_ index: Int32, _ parameter: PowerSyncDataType?) throws (PowerSyncError) { + let rc: Int32 + + switch parameter { + case nil: + rc = sqlite3_bind_null(self.stmt, index) + case .bool(let value): + rc = sqlite3_bind_int(self.stmt, index, value ? 1 : 0) + case .string(let value): + var str = value + rc = str.withUTF8 { buffer in + sqlite3_bind_text( + self.stmt, + index, + buffer.baseAddress, + Int32(buffer.count), + // SQLITE_TRANSIENT + unsafeBitCast(-1, to: (@convention(c) (UnsafeMutableRawPointer?) -> Void).self), + ) + } + case .int64(let value): + rc = sqlite3_bind_int64(self.stmt, index, value) + case .int32(let value): + rc = sqlite3_bind_int(self.stmt, index, value) + case .double(let value): + rc = sqlite3_bind_double(self.stmt, index, value) + case .data(let value): + // Data object can be made up of multiple memory regions, so copy once. + let buffer = malloc(value.count)! + value.copyBytes(to: buffer.assumingMemoryBound(to: UInt8.self), count: value.count) + + rc = sqlite3_bind_blob( + self.stmt, + index, + buffer, + Int32(value.count), + free, + ) + + if rc != 0 { + free(buffer) + } + } + + if rc != 0 { + try throwDatabaseError(db: self.db, sql: self.sql) + } + } + + mutating func step() throws (PowerSyncError) -> Bool { + let rc = sqlite3_step(self.stmt) + if rc == SQLITE_DONE { + return false + } else if rc == SQLITE_ROW { + if resolvedColumnNames == nil { + let count = self.columnCount + var nameToIndex = Dictionary(minimumCapacity: count) + for i in 0..? + + init(_ stmtPtr: UnsafePointer) { + self.stmtPtr = stmtPtr + } + + func invalidate() { + stmtPtr = nil + } + + private func withStatement(_ body: (borrowing NativeSqliteStatement) -> R) -> R { + if let stmtPtr { + return body(stmtPtr.pointee) + } + + fatalError("Cursor used outside of callback") + } + + private func checkColumnNotNull(stmt: borrowing NativeSqliteStatement, index: Int) throws(SqlCursorError) { + if index < 0 || index >= stmt.columnCount { + throw SqlCursorError.nullValueFound("invalid index \(index)") + } + + let type = sqlite3_column_type(stmt.stmt, Int32(index)) + if type == SQLITE_NULL { + throw SqlCursorError.nullValueFound("\(index)") + } + } + + private func withStatementCheckNotNull(_ index: Int, body: (borrowing NativeSqliteStatement) throws (SqlCursorError) -> R) throws (SqlCursorError) -> R { + if let stmtPtr { + try self.checkColumnNotNull(stmt: stmtPtr.pointee, index: index) + return try body(stmtPtr.pointee) + } + + fatalError("Cursor used outside of callback") + } + + var columnCount: Int { + return withStatement { stmt in stmt.columnCount } + } + + var columnNames: [String : Int] { + return withStatement { stmt in stmt.columnNames } + } + + func getBoolean(index: Int) throws(SqlCursorError) -> Bool { + return try getInt(index: index) == 0 ? false : true + } + + func getBooleanOptional(index: Int) -> Bool? { + do { + return try getBoolean(index: index) + } catch { + return nil + } + } + + func getDouble(index: Int) throws(SqlCursorError) -> Double { + return try withStatementCheckNotNull(index) { stmt in + return sqlite3_column_double(stmt.stmt, Int32(index)) + } + } + + func getDoubleOptional(index: Int) -> Double? { + do { + return try getDouble(index: index) + } catch { + return nil + } + } + + func getInt(index: Int) throws(SqlCursorError) -> Int { + return Int(try getInt64(index: index)) + } + + func getIntOptional(index: Int) -> Int? { + do { + return try getInt(index: index) + } catch { + return nil + } + } + + func getInt64(index: Int) throws(SqlCursorError) -> Int64 { + return try withStatementCheckNotNull(index) { stmt in + return sqlite3_column_int64(stmt.stmt, Int32(index)) + } + } + + func getInt64Optional(index: Int) -> Int64? { + do { + return try getInt64(index: index) + } catch { + return nil + } + } + + func getString(index: Int) throws(SqlCursorError) -> String { + return try withStatementCheckNotNull(index) { stmt in + let length = sqlite3_column_bytes(stmt.stmt, Int32(index)) + if length == 0 { + return "" + } + + let ptr = sqlite3_column_text(stmt.stmt, Int32(index)) + return String(decoding: UnsafeBufferPointer(start: ptr, count: Int(length)), as: UTF8.self) + } + } + + func getStringOptional(index: Int) -> String? { + do { + return try getString(index: index) + } catch { + return nil + } + } +} diff --git a/Sources/PowerSync/Implementation/sqlite3/throwDatabaseError.swift b/Sources/PowerSync/Implementation/sqlite3/throwDatabaseError.swift index 8eb0f34..d089aab 100644 --- a/Sources/PowerSync/Implementation/sqlite3/throwDatabaseError.swift +++ b/Sources/PowerSync/Implementation/sqlite3/throwDatabaseError.swift @@ -1,11 +1,11 @@ import CSQLite -func throwDatabaseError(db: SQLiteConnectionLease, sql: String?) throws(PowerSyncError) -> Never { - let extended = sqlite3_extended_errcode(db.pointer) +func throwDatabaseError(db: OpaquePointer, sql: String?) throws(PowerSyncError) -> Never { + let extended = sqlite3_extended_errcode(db) let errStr = String(cString: sqlite3_errstr(extended)) - let offset = sqlite3_error_offset(db.pointer) - let rawMessage = sqlite3_errmsg(db.pointer) + let offset = sqlite3_error_offset(db) + let rawMessage = sqlite3_errmsg(db) throw PowerSyncError.sqliteError( extendedResultCode: extended, diff --git a/Sources/PowerSync/Protocol/QueriesProtocol.swift b/Sources/PowerSync/Protocol/QueriesProtocol.swift index a14f839..9c5e746 100644 --- a/Sources/PowerSync/Protocol/QueriesProtocol.swift +++ b/Sources/PowerSync/Protocol/QueriesProtocol.swift @@ -22,14 +22,6 @@ public struct WatchOptions: Sendable { } public protocol Queries { - /// Execute a read-only (SELECT) query every time the source tables are modified - /// and return the results as an array in a Publisher. - func watch( - sql: String, - parameters: [Sendable?]?, - mapper: @Sendable @escaping (SqlCursor) throws -> RowType - ) throws -> AsyncThrowingStream<[RowType], Error> - func watch( options: WatchOptions ) throws -> AsyncThrowingStream<[RowType], Error> @@ -48,19 +40,23 @@ public protocol Queries { func readLock( callback: @Sendable @escaping (any ConnectionContext) throws -> R ) async throws -> R +} +public extension Queries { /// Execute a write transaction with the given callback func writeTransaction( callback: @Sendable @escaping (any Transaction) throws -> R - ) async throws -> R + ) async throws -> R { + try await writeLock { ctx in try TransactionImpl.run(conn: ctx, callback: callback) } + } /// Execute a read transaction with the given callback - func readTransaction( + func readTransaction( callback: @Sendable @escaping (any Transaction) throws -> R - ) async throws -> R -} - -public extension Queries { + ) async throws -> R { + try await readLock { ctx in try TransactionImpl.run(conn: ctx, callback: callback) } + } + /// Execute a write query (INSERT, UPDATE, DELETE) /// Using `RETURNING *` will result in an error. @discardableResult @@ -123,6 +119,16 @@ public extension Queries { return try await getOptional(sql: sql, parameters: [], mapper: mapper) } + /// Execute a read-only (SELECT) query every time the source tables are modified + /// and return the results as an array in a Publisher. + func watch( + sql: String, + parameters: [Sendable?]?, + mapper: @Sendable @escaping (SqlCursor) throws -> RowType + ) throws -> AsyncThrowingStream<[RowType], Error> { + return try watch(options: WatchOptions(sql: sql, parameters: parameters, mapper: mapper)) + } + func watch( _ sql: String, mapper: @Sendable @escaping (SqlCursor) throws -> RowType diff --git a/Sources/PowerSync/Protocol/SQLiteConnectionPool.swift b/Sources/PowerSync/Protocol/SQLiteConnectionPool.swift index 7257064..9b632d2 100644 --- a/Sources/PowerSync/Protocol/SQLiteConnectionPool.swift +++ b/Sources/PowerSync/Protocol/SQLiteConnectionPool.swift @@ -4,7 +4,16 @@ import Foundation public protocol SQLiteConnectionLease { /// Pointer to the underlying SQLite connection. /// This pointer should not be used outside of the closure which provided the lease. - var pointer: OpaquePointer { get } + var pointer: OpaquePointer { borrowing get } + + /// Executes an SQL statement and returns the amount of rows affected. + func execute(sql: String, parameters: [PowerSyncDataType?]) throws -> Int64 + + func withIterator(sql: String, parameters: [PowerSyncDataType?], callback: (_: SQLiteStatementIteratorProtocol) throws -> T) throws -> T +} + +public protocol SQLiteStatementIteratorProtocol { + func next(callback: (_ cursor: SqlCursor) throws -> T) throws -> T? } /// An implementation of a connection pool providing asynchronous access to a single writer and multiple readers. @@ -13,22 +22,22 @@ public protocol SQLiteConnectionPoolProtocol: Sendable { var tableUpdates: AsyncStream> { get } /// Calls the callback with a read-only connection temporarily leased from the pool. - func read( - onConnection: @Sendable @escaping (SQLiteConnectionLease) throws -> Void, - ) async throws + func read( + onConnection: @Sendable @escaping (SQLiteConnectionLease) throws -> T, + ) async throws -> T /// Calls the callback with a read-write connection temporarily leased from the pool. - func write( - onConnection: @Sendable @escaping (SQLiteConnectionLease) throws -> Void, - ) async throws + func write( + onConnection: @Sendable @escaping (SQLiteConnectionLease) throws -> T, + ) async throws -> T /// Invokes the callback with all connections leased from the pool. - func withAllConnections( + func withAllConnections( onConnection: @Sendable @escaping ( _ writer: SQLiteConnectionLease, _ readers: [SQLiteConnectionLease] - ) throws -> Void, - ) async throws + ) throws -> T, + ) async throws -> T /// Closes the connection pool and associated resources. func close() async throws diff --git a/Sources/PowerSync/Protocol/db/DataConvertible.swift b/Sources/PowerSync/Protocol/db/DataConvertible.swift index cb7d6c6..9a963d9 100644 --- a/Sources/PowerSync/Protocol/db/DataConvertible.swift +++ b/Sources/PowerSync/Protocol/db/DataConvertible.swift @@ -1,7 +1,6 @@ import Foundation -// Represents the set of types that are supported -/// by the PowerSync Kotlin Multiplatform SDK +// Represents the set of types that are supported as parameters for SQlite statements. public enum PowerSyncDataType { case bool(Bool) case string(String) diff --git a/Sources/PowerSync/Utils/AsyncMutex.swift b/Sources/PowerSync/Utils/AsyncMutex.swift index a9d2c98..c234d38 100644 --- a/Sources/PowerSync/Utils/AsyncMutex.swift +++ b/Sources/PowerSync/Utils/AsyncMutex.swift @@ -118,6 +118,11 @@ private struct SemaphoreState: ~Copyable { } private mutating func deactivateWaiter(waiter: SemaphoreWaitNode) { + if !waiter.isActive { + return + } + + waiter.isActive = false let prev = waiter.prev let next = waiter.next @@ -196,6 +201,7 @@ private final class SemaphoreWaitNode: @unchecked Sendable { var acquiredItems: Int var itemsBuffer: UnsafeMutableRawPointer? // pointer to [T; requestedItems] var continuation: CheckedContinuation<(), any Error> + var isActive = true var prev: SemaphoreWaitNode? var next: SemaphoreWaitNode? diff --git a/Sources/PowerSyncGRDB/Config/Configuration+PowerSync.swift b/Sources/PowerSyncGRDB/Config/Configuration+PowerSync.swift index 6f9d948..c08ee1a 100644 --- a/Sources/PowerSyncGRDB/Config/Configuration+PowerSync.swift +++ b/Sources/PowerSyncGRDB/Config/Configuration+PowerSync.swift @@ -26,33 +26,10 @@ public extension Configuration { mutating func configurePowerSync( schema: Schema ) throws { - // Handles the case on WatchOS where the extension is statically loaded. - // For WatchOS: We need to statically register the extension before SQLite connections are established. - // This should only throw on non-WatchOS platforms if the extension path cannot be resolved. + // This calls sqlite3_auto_extension and enables the PowerSync core extension for all + // new connections. let extensionPath = try resolvePowerSyncLoadableExtensionPath() - - // Register the PowerSync core extension - prepareDatabase { database in - if let extensionPath = extensionPath { - /// The extension is loaded as an automatic extension if resolvePowerSyncLoadableExtensionPath returns nil - /// We should dynamically load the extension if we received an extensionPath - let extensionLoadResult = sqlite3_enable_load_extension(database.sqliteConnection, 1) - if extensionLoadResult != SQLITE_OK { - throw PowerSyncGRDBError.extensionLoadFailed("Could not enable extension loading") - } - var errorMsg: UnsafeMutablePointer? - let loadResult = sqlite3_load_extension(database.sqliteConnection, extensionPath, "sqlite3_powersync_init", &errorMsg) - if loadResult != SQLITE_OK { - if let errorMsg = errorMsg { - let message = String(cString: errorMsg) - sqlite3_free(errorMsg) - throw PowerSyncGRDBError.extensionLoadFailed(message) - } else { - throw PowerSyncGRDBError.unknownExtensionLoadError - } - } - } - } + assert(extensionPath == nil) // Supply the PowerSync views as a SchemaSource let powerSyncSchemaSource = PowerSyncSchemaSource( diff --git a/Sources/PowerSyncGRDB/Connections/GRDBConnectionLease.swift b/Sources/PowerSyncGRDB/Connections/GRDBConnectionLease.swift index 7104f24..a79614f 100644 --- a/Sources/PowerSyncGRDB/Connections/GRDBConnectionLease.swift +++ b/Sources/PowerSyncGRDB/Connections/GRDBConnectionLease.swift @@ -4,15 +4,51 @@ import PowerSync /// Internal lease object that exposes the raw GRDB SQLite connection pointer. /// -/// This is used to bridge GRDB's managed database connection with the Kotlin SDK, +/// This is used to bridge GRDB's managed database connection with the Swift SDK, /// allowing direct access to the underlying SQLite connection for PowerSync operations. final class GRDBConnectionLease: SQLiteConnectionLease { - var pointer: OpaquePointer + let pointer: OpaquePointer + var database: Database init(database: Database) throws { guard let connection = database.sqliteConnection else { throw PowerSyncGRDBError.connectionUnavailable } - pointer = connection + self.pointer = connection + self.database = database + } + + func execute(sql: String, parameters: [PowerSync.PowerSyncDataType?]) throws -> Int64 { + try database.execute(sql: sql, arguments: StatementArguments(parameters)) + return Int64(database.changesCount) + } + + func withIterator(sql: String, parameters: [PowerSync.PowerSyncDataType?], callback: (any PowerSync.SQLiteStatementIteratorProtocol) throws -> T) throws -> T { + let statement = try database.makeStatement(sql: sql) + let rows = try Row.fetchCursor(statement, arguments: StatementArguments(parameters)) + return try callback(RowIterator(rows: rows)) + } +} + +extension PowerSync.PowerSyncDataType: DatabaseValueConvertible { + public var databaseValue: GRDB.DatabaseValue { + switch self { + case .bool(let value): + return value.databaseValue + case .string(let value): + return value.databaseValue + case .int64(let value): + return value.databaseValue + case .int32(let value): + return value.databaseValue + case .double(let value): + return value.databaseValue + case .data(let value): + return value.databaseValue + } + } + + public static func fromDatabaseValue(_ dbValue: GRDB.DatabaseValue) -> PowerSync.PowerSyncDataType? { + nil } } diff --git a/Sources/PowerSyncGRDB/Connections/GRDBConnectionPool.swift b/Sources/PowerSyncGRDB/Connections/GRDBConnectionPool.swift index ea954d4..59df99e 100644 --- a/Sources/PowerSyncGRDB/Connections/GRDBConnectionPool.swift +++ b/Sources/PowerSyncGRDB/Connections/GRDBConnectionPool.swift @@ -37,60 +37,39 @@ actor GRDBConnectionPool: SQLiteConnectionPoolProtocol { tableUpdatesContinuation = tempContinuation } - func read( - onConnection: @Sendable @escaping (SQLiteConnectionLease) throws -> Void - ) async throws { - try await pool.read { database in + func read( + onConnection: @Sendable @escaping (SQLiteConnectionLease) throws -> T + ) async throws -> T { + return try await pool.read { database in try onConnection( GRDBConnectionLease(database: database) ) } } - func write( - onConnection: @Sendable @escaping (SQLiteConnectionLease) throws -> Void - ) async throws { + func write( + onConnection: @Sendable @escaping (SQLiteConnectionLease) throws -> T + ) async throws -> T { // Don't start an explicit transaction, we do this internally - let result = try await pool.writeWithoutTransaction { database in - guard let pointer = database.sqliteConnection else { - throw PowerSyncGRDBError.connectionUnavailable - } - - let sessionResult = try withSession( - db: pointer, - ) { - try onConnection( - GRDBConnectionLease(database: database) - ) - } - - return sessionResult - } - // Notify PowerSync of these changes - tableUpdatesContinuation?.yield(result.affectedTables) - // Notify GRDB, this needs to be a write (transaction) - try await pool.write { database in - // Notify GRDB about these changes - for table in result.affectedTables { - try database.notifyChanges(in: Table(table)) - } - } - - if case let .failure(error) = result.blockResult { - throw error + return try await pool.writeWithoutTransaction { database in + return try onConnection( + GRDBConnectionLease(database: database) + ) } } - func withAllConnections( - onConnection: @Sendable @escaping (SQLiteConnectionLease, [SQLiteConnectionLease]) throws -> Void - ) async throws { + func withAllConnections( + onConnection: @Sendable @escaping (SQLiteConnectionLease, [SQLiteConnectionLease]) throws -> T + ) async throws -> T { // FIXME, we currently don't support updating the schema - try await pool.writeWithoutTransaction { database in + let result = try await pool.writeWithoutTransaction { database in let lease = try GRDBConnectionLease(database: database) - try onConnection(lease, []) + let result = try onConnection(lease, []) database.clearSchemaCache() + return result } pool.invalidateReadOnlyConnections() + return result } func close() async throws { diff --git a/Sources/PowerSyncGRDB/Connections/RowCursor.swift b/Sources/PowerSyncGRDB/Connections/RowCursor.swift new file mode 100644 index 0000000..ad95d41 --- /dev/null +++ b/Sources/PowerSyncGRDB/Connections/RowCursor.swift @@ -0,0 +1,92 @@ +import GRDB +import PowerSync + +final class RowIterator: PowerSync.SQLiteStatementIteratorProtocol { + let rows: RowCursor + var columnNames: [String: Int]? = nil + + init(rows: RowCursor) { + self.rows = rows + } + + func next(callback: (any SqlCursor) throws -> T) throws -> T? { + guard let row = try rows.next() else { + return nil + } + + return try callback(RowSqlCursor(columnNames: resolveColumnNames(), row: row)) + } + + private func resolveColumnNames() -> [String: Int] { + if let columnNames { + return columnNames + } + + var names: [String: Int] = [:] + for (i, name) in rows.columnNames.enumerated() { + names[name] = i + } + columnNames = names + return names + } +} + +struct RowSqlCursor: PowerSync.SqlCursor { + let columnNames: [String: Int] + let row: Row + + private func checkNotNull(index: Int) throws(PowerSync.SqlCursorError) { + if row.hasNull(atIndex: index) { + throw .nullValueFound("\(index)") + } + } + + public func getBoolean(index: Int) throws(PowerSync.SqlCursorError) -> Bool { + try checkNotNull(index: index) + return row.self[index] + } + + public func getBooleanOptional(index: Int) -> Bool? { + return row.self[index] + } + + public func getDouble(index: Int) throws(PowerSync.SqlCursorError) -> Double { + try checkNotNull(index: index) + return row.self[index] + } + + public func getDoubleOptional(index: Int) -> Double? { + return row.self[index] + } + + public func getInt(index: Int) throws(PowerSync.SqlCursorError) -> Int { + try checkNotNull(index: index) + return row.self[index] + } + + public func getIntOptional(index: Int) -> Int? { + return row.self[index] + } + + public func getInt64(index: Int) throws(PowerSync.SqlCursorError) -> Int64 { + try self.checkNotNull(index: index) + return row.self[index] + } + + public func getInt64Optional(index: Int) -> Int64? { + return row.self[index] + } + + public func getString(index: Int) throws(PowerSync.SqlCursorError) -> String { + try self.checkNotNull(index: index) + return row.self[index] + } + + public func getStringOptional(index: Int) -> String? { + return row.self[index] + } + + public var columnCount: Int { + row.count + } +} diff --git a/Sources/PowerSyncGRDB/SQLite/SQLite+Utils.swift b/Sources/PowerSyncGRDB/SQLite/SQLite+Utils.swift deleted file mode 100644 index 0830463..0000000 --- a/Sources/PowerSyncGRDB/SQLite/SQLite+Utils.swift +++ /dev/null @@ -1,24 +0,0 @@ -// The system SQLite does not expose this, -// linking CSQLite (or another implementation of SQLite) provides them -// Declare the missing function manually -@_silgen_name("sqlite3_enable_load_extension") -func sqlite3_enable_load_extension( - _ db: OpaquePointer?, - _ onoff: Int32 -) -> Int32 - -@_silgen_name("sqlite3_powersync_init") -func sqlite3_powersync_init( - _ db: OpaquePointer?, - _: OpaquePointer?, - _: OpaquePointer? -) -> Int32 - -// Similarly for sqlite3_load_extension if needed: -@_silgen_name("sqlite3_load_extension") -func sqlite3_load_extension( - _ db: OpaquePointer?, - _ fileName: UnsafePointer?, - _ procName: UnsafePointer?, - _ errMsg: UnsafeMutablePointer?>? -) -> Int32 diff --git a/Tests/PowerSyncGRDBTests/BasicTest.swift b/Tests/PowerSyncGRDBTests/BasicTest.swift index 914480d..1150b62 100644 --- a/Tests/PowerSyncGRDBTests/BasicTest.swift +++ b/Tests/PowerSyncGRDBTests/BasicTest.swift @@ -427,7 +427,7 @@ final class GRDBTests: XCTestCase { let warningIndex = logs.getLogs().firstIndex( where: { value in - value.contains("debug: PowerSyncVersion") + value.contains("debug: Opened connection. SQLite version") } ) From f4cdd3ce1abb4821a416936d9831db0234c158d3 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 27 Apr 2026 09:47:25 +0200 Subject: [PATCH 25/40] Cleanup --- Package.swift | 1 - .../sync/CachingCredentialsConnector.swift | 2 + .../Implementation/sync/CoreSyncStatus.swift | 11 +- .../Implementation/sync/HttpClient.swift | 50 +++--- .../sync/PowerSyncControlArguments.swift | 2 +- .../Implementation/sync/Status.swift | 62 ++++--- .../sync/StreamingSyncClient.swift | 15 +- Sources/PowerSync/Utils/AsyncMutex.swift | 5 +- Sources/PowerSync/Utils/BroadcastStream.swift | 7 +- Sources/SyncPlayground/main.swift | 35 ---- Tests/PowerSyncTests/SyncTests.swift | 170 +++++++++--------- 11 files changed, 180 insertions(+), 180 deletions(-) delete mode 100644 Sources/SyncPlayground/main.swift diff --git a/Package.swift b/Package.swift index e36d619..b067840 100644 --- a/Package.swift +++ b/Package.swift @@ -105,7 +105,6 @@ let package = Package( .product(name: "GRDB", package: "GRDB.swift") ] ), - .executableTarget(name: "SyncPlayground", dependencies: [.target(name: "PowerSync")]), .testTarget( name: "PowerSyncTests", dependencies: ["PowerSync"] diff --git a/Sources/PowerSync/Implementation/sync/CachingCredentialsConnector.swift b/Sources/PowerSync/Implementation/sync/CachingCredentialsConnector.swift index 110418a..48aab8c 100644 --- a/Sources/PowerSync/Implementation/sync/CachingCredentialsConnector.swift +++ b/Sources/PowerSync/Implementation/sync/CachingCredentialsConnector.swift @@ -1,3 +1,4 @@ +/// Wraps a ``PowerSyncBackendConnectorProtocol`` to cache and invalidate credentials. actor CachingCredentialsConnector { private let inner: PowerSyncBackendConnectorProtocol private var cachedCredentials: PowerSyncCredentials? = nil @@ -21,6 +22,7 @@ actor CachingCredentialsConnector { } nonisolated func uploadData(database: any PowerSyncDatabaseProtocol) async throws { + // Nonisolated because we don't want this to block fetching credentials. try await self.inner.uploadData(database: database) } } diff --git a/Sources/PowerSync/Implementation/sync/CoreSyncStatus.swift b/Sources/PowerSync/Implementation/sync/CoreSyncStatus.swift index 049f41d..15e67ad 100644 --- a/Sources/PowerSync/Implementation/sync/CoreSyncStatus.swift +++ b/Sources/PowerSync/Implementation/sync/CoreSyncStatus.swift @@ -1,10 +1,11 @@ +/// A raw sync status snapshot received from the core extension. struct CoreDownloadSyncStatus: Decodable, Sendable { let connected: Bool let connecting: Bool let priorityStatus: [PriorityStatusEntry] let downloading: CoreSyncDownloadProgress? let streams: [SyncStreamStatus] - + enum CodingKeys: String, CodingKey { case connected case connecting @@ -12,7 +13,7 @@ struct CoreDownloadSyncStatus: Decodable, Sendable { case downloading case streams } - + init() { self.connected = false self.connecting = false @@ -20,7 +21,7 @@ struct CoreDownloadSyncStatus: Decodable, Sendable { self.downloading = nil self.streams = [] } - + init(from decoder: any Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) self.connected = try container.decode(Bool.self, forKey: .connected) @@ -42,7 +43,7 @@ struct BucketProgress: Decodable { let atLast: Int64 let sinceLast: Int64 let targetCount: Int64 - + enum CodingKeys: String, CodingKey { case priority case atLast = "at_last" @@ -68,6 +69,8 @@ struct ProgressCounters: Decodable, ProgressWithOperations { } } +/// Wrapper to make ``SyncStreamStatus`` decodable without us having to implement that protocol +/// publicly. private struct DecodableSyncStreamStatus: Decodable { let inner: SyncStreamStatus diff --git a/Sources/PowerSync/Implementation/sync/HttpClient.swift b/Sources/PowerSync/Implementation/sync/HttpClient.swift index 5c2c760..9090ac5 100644 --- a/Sources/PowerSync/Implementation/sync/HttpClient.swift +++ b/Sources/PowerSync/Implementation/sync/HttpClient.swift @@ -1,8 +1,17 @@ import Foundation -/// An internal protocol for HTTP clients, we use this to mock clients in tests. +/// An internal protocol for HTTP clients. +/// +/// Outside of tests, this is implemented by ``PlatformHttpClient`` as a thin wrapper around +/// ``URLSession``. In tests, we can use a mock implementation to test the sync client instead. +/// +/// This is an internal protocol and tailored towards what the sync client actually needs. It is not +/// a general-purpose HTTP client. protocol HttpClient: Sendable { + /// Start streaming a `/sync/stream` response body, emitting individual lines. func receiveSyncLines(request: URLRequest) async throws -> (HTTPURLResponse, any SyncLineResponse) + + /// Read a full response body. func readFully(request: URLRequest) async throws -> (HTTPURLResponse, Data) } @@ -29,6 +38,25 @@ struct PlatformHttpClient: HttpClient { throw PowerSyncError.operationFailed(message: "Invalid sync lines response, (expected \(jsonStreamMimeType), got \(response.mimeType, default: "")") } + struct PlatformSyncLineResponse: SyncLineResponse where Base : AsyncSequence, Base.Element == UInt8, Base: Sendable { + let lines: AsyncLineSequence + + func makeAsyncIterator() -> some SyncLineResponseIterator { + return PlatformSyncLineResponseIterator(inner: lines.makeAsyncIterator()) + } + } + + struct PlatformSyncLineResponseIterator: SyncLineResponseIterator where Base : AsyncSequence, Base.Element == UInt8, Base: Sendable { + typealias Element = SyncLine + + var inner: AsyncLineSequence.AsyncIterator + + mutating func next() async throws -> SyncLine? { + let line = try await inner.next() + return line.map { SyncLine.text(contents: $0) } + } + } + return (response as! HTTPURLResponse, PlatformSyncLineResponse(lines: bytes.lines)) } @@ -40,25 +68,7 @@ struct PlatformHttpClient: HttpClient { static let shared = PlatformHttpClient(session: .shared) } -private struct PlatformSyncLineResponse: SyncLineResponse where Base : AsyncSequence, Base.Element == UInt8, Base: Sendable { - let lines: AsyncLineSequence - - func makeAsyncIterator() -> some SyncLineResponseIterator { - return PlatformSyncLineResponseIterator(inner: lines.makeAsyncIterator()) - } -} - -private struct PlatformSyncLineResponseIterator: SyncLineResponseIterator where Base : AsyncSequence, Base.Element == UInt8, Base: Sendable { - typealias Element = SyncLine - - var inner: AsyncLineSequence.AsyncIterator - - mutating func next() async throws -> SyncLine? { - let line = try await inner.next() - return line.map { SyncLine.text(contents: $0) } - } -} - +/// A wrapper around a ``HttpClient`` emitting log events for responses and sync lines. struct LoggingClient: HttpClient { let inner: HttpClient let logger: SyncRequestLoggerConfiguration diff --git a/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift b/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift index ec203ce..76d0054 100644 --- a/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift +++ b/Sources/PowerSync/Implementation/sync/PowerSyncControlArguments.swift @@ -1,6 +1,6 @@ /// Arguments to the `powersync_control()` SQL function driving the sync process. enum PowerSyncControlArguments { - case start(_ start: StartSyncIteration) + case start(start: StartSyncIteration) case stop case textLine(line: String) case binaryLine(line: ContiguousArray) diff --git a/Sources/PowerSync/Implementation/sync/Status.swift b/Sources/PowerSync/Implementation/sync/Status.swift index fe20e7f..1ed8fe1 100644 --- a/Sources/PowerSync/Implementation/sync/Status.swift +++ b/Sources/PowerSync/Implementation/sync/Status.swift @@ -1,6 +1,10 @@ import Foundation import Synchronization +/// The internal struct backing all sync status fields. +/// +/// The core extension drives most of the sync status through ``CoreDownloadSyncStatus``. +/// Additionally, we track upload status and Swift errors. struct MutableSyncStatus: ~Copyable { var core: CoreDownloadSyncStatus = CoreDownloadSyncStatus() var uploading: Bool = false @@ -8,6 +12,7 @@ struct MutableSyncStatus: ~Copyable { var internalUploadError: (any Error & Sendable)? } +/// An immutable snapshot of ``MutableSyncStatus``. fileprivate struct SyncStatusDataImpl: SyncStatusData { let core: CoreDownloadSyncStatus let downloadProgress: (any SyncDownloadProgress)? @@ -15,7 +20,7 @@ fileprivate struct SyncStatusDataImpl: SyncStatusData { let internalDownloadError: (any Error & Sendable)? let internalUploadError: (any Error & Sendable)? - + init(status: borrowing MutableSyncStatus) { self.core = status.core self.uploading = status.uploading @@ -28,24 +33,24 @@ fileprivate struct SyncStatusDataImpl: SyncStatusData { self.downloadProgress = nil } } - + var connected: Bool { core.connected } - + var connecting: Bool { core.connecting } - + var downloading: Bool { core.downloading != nil } - + var lastSyncedAt: Date? { let completeSyncStatus = core.priorityStatus.first { $0.priority == .fullSyncPriority } return completeSyncStatus?.lastSyncedAt } - + var hasSynced: Bool? { lastSyncedAt != nil } @@ -53,19 +58,19 @@ fileprivate struct SyncStatusDataImpl: SyncStatusData { var downloadError: Any? { internalDownloadError } - + var uploadError: Any? { internalUploadError } - + var anyError: Any? { downloadError ?? uploadError } - + var priorityStatusEntries: [PriorityStatusEntry] { core.priorityStatus } - + var syncStreams: [SyncStreamStatus]? { if downloadProgress != nil { return core.streams @@ -75,7 +80,7 @@ fileprivate struct SyncStatusDataImpl: SyncStatusData { return core.streams.map { stream in SyncStreamStatus.init(subscription: stream.subscription) } } } - + func statusForPriority(_ priority: BucketPriority) -> PriorityStatusEntry { for known in priorityStatusEntries { // Lower-priority buckets are synced after higher-priority buckets, and since priorityStatusEntries @@ -88,7 +93,7 @@ fileprivate struct SyncStatusDataImpl: SyncStatusData { // Fallback, report status for complete sync (which necessarily includes all priorities) return PriorityStatusEntry(priority: priority, lastSyncedAt: lastSyncedAt, hasSynced: hasSynced) } - + func forStream(stream: any SyncStreamDescription) -> SyncStreamStatus? { for found in syncStreams! { if found.subscription.name == stream.name && found.subscription.parameters == stream.parameters { @@ -103,7 +108,7 @@ fileprivate struct SyncStatusDataImpl: SyncStatusData { fileprivate struct SyncStatusContainer: ~Copyable { var inner: MutableSyncStatus var snapshot: SyncStatusDataImpl - + init(inner: consuming MutableSyncStatus) { self.snapshot = SyncStatusDataImpl(status: inner) self.inner = inner @@ -117,11 +122,11 @@ final class SwiftSyncStatus: SyncStatus { init() { self.current = Mutex(SyncStatusContainer(inner: MutableSyncStatus())) } - + private func readStatus(status: (borrowing SyncStatusDataImpl) -> T) -> T { return self.current.withLock { status($0.snapshot) } } - + internal func mutateStatus(update: (_ status: inout MutableSyncStatus) -> Void) { maybeMutateStatus(shouldUpdate: { _ in true }, apply: update) } @@ -148,7 +153,8 @@ final class SwiftSyncStatus: SyncStatus { func asFlow() -> AsyncStream { self.listeners.subscribe(addInitial: self) } - + + /// Waits for the first sync status matching a predicate. func waitFor(_ predicate: (borrowing SwiftSyncStatus) -> Bool) async { for await _ in self.asFlow() { if predicate(self) { @@ -160,15 +166,15 @@ final class SwiftSyncStatus: SyncStatus { var connected: Bool { self.readStatus { current in current.connected } } - + var connecting: Bool { self.readStatus { current in current.connecting } } - + var downloading: Bool { self.readStatus { current in current.downloading } } - + var downloadProgress: (any SyncDownloadProgress)? { self.readStatus { current in current.downloadProgress } } @@ -176,39 +182,39 @@ final class SwiftSyncStatus: SyncStatus { var uploading: Bool { self.readStatus { current in current.uploading } } - + var lastSyncedAt: Date? { self.readStatus { current in current.lastSyncedAt } } - + var hasSynced: Bool? { self.readStatus { current in current.hasSynced } } - + var downloadError: Any? { self.readStatus { current in current.downloadError } } - + var uploadError: Any? { self.readStatus { current in current.uploadError } } - + var anyError: Any? { self.readStatus { current in current.anyError } } - + var priorityStatusEntries: [PriorityStatusEntry] { self.readStatus { current in current.priorityStatusEntries } } - + var syncStreams: [SyncStreamStatus]? { self.readStatus { current in current.syncStreams } } - + func statusForPriority(_ priority: BucketPriority) -> PriorityStatusEntry { self.readStatus { current in current.statusForPriority(priority) } } - + func forStream(stream: any SyncStreamDescription) -> SyncStreamStatus? { self.readStatus { current in current.forStream(stream: stream) } } diff --git a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift index 1ac86e2..ef84e1d 100644 --- a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift +++ b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift @@ -26,7 +26,7 @@ final class StreamingSyncClient: Sendable { /// managing tokens and CRUD uploads. /// /// There should at most be one such task per database, but this internal method performs no concurrency - /// control for that. + /// control for that (that's the responsibility of a ``SyncCoordinator``). func run() -> Task { Task(name: "StreamingSyncClient.run") { let signals = SyncSignals() @@ -38,7 +38,7 @@ final class StreamingSyncClient: Sendable { } private func uploadLoop(signals: SyncSignals) async throws { - // TODO: Replace with better watch mechanism + // TODO: Replace with better watch mechanism once we've dropped the Kotlin dependency and can use onChange. let watch = try db.watch(sql: "SELECT 1 FROM ps_crud LIMIT 1", parameters: [], mapper: { _ in () }) .dropFirst() // Skip initial result, we just want to watch changes .map { _ in () } @@ -242,7 +242,7 @@ private struct ActiveSyncIteration: Sendable { // checkpoint in that case. async let _ = watchCompletedCrudUploads() - let initialInstructions = try await powersyncControl(.start(StartSyncIteration( + let initialInstructions = try await powersyncControl(.start(start: StartSyncIteration( parameters: syncClient.options.params, schema: await syncClient.db.schema.inner, includeDefaults: syncClient.options.includeDefaultStreams, @@ -327,9 +327,9 @@ private struct ActiveSyncIteration: Sendable { $0.core = status } case .establishSyncStream(request: _): - fatalError("There can only be one establishSyncStream instruction per sync iteration") + throw PowerSyncError.operationFailed(message: "There can only be one establishSyncStream instruction per sync iteration") case .closeSyncStream(hideDisconnect: _): - fatalError("Must be handled in run() loop") + throw PowerSyncError.operationFailed(message: "CloseSyncStream must be handled in run() loop") case .fetchCredentials(didExpire: let didExpire): if didExpire { await syncClient.invalidateCredentials() @@ -413,6 +413,11 @@ private struct SyncIterationResult { } } +/// Allows the concurrent upload and download tasks to communicate. +/// +/// The download task might request a CRUD upload (when we run into a checkpoint that couldn't +/// be applied due to local data), and the upload task needs to signal completions to the download +/// task (so that we can retry applying a checkpoint). private struct SyncSignals { let signalCrudUpload = BroadcastStream() let signalCrudUploadComplete = BroadcastStream() diff --git a/Sources/PowerSync/Utils/AsyncMutex.swift b/Sources/PowerSync/Utils/AsyncMutex.swift index bbb3c31..7867957 100644 --- a/Sources/PowerSync/Utils/AsyncMutex.swift +++ b/Sources/PowerSync/Utils/AsyncMutex.swift @@ -1,10 +1,11 @@ +/// An asynchronous mutex implemented as a simple actor. actor AsyncMutex { var inner: T - + init(_ inner: consuming sending T) { self.inner = inner } - + func withMutex(callback: (_ element: inout T) throws -> R) rethrows -> R { try callback(&inner) } diff --git a/Sources/PowerSync/Utils/BroadcastStream.swift b/Sources/PowerSync/Utils/BroadcastStream.swift index 63f3620..58986e1 100644 --- a/Sources/PowerSync/Utils/BroadcastStream.swift +++ b/Sources/PowerSync/Utils/BroadcastStream.swift @@ -1,8 +1,9 @@ import Synchronization +/// Dispatches events to a number of listeners as an ``AsyncStream``. final class BroadcastStream: Sendable { private let listeners: Mutex>> = Mutex([]) - + private func register(continuation: AsyncStream.Continuation) { let listener = BroadcastStreamListener(continuation: continuation) let _ = listeners.withLock { $0.insert(listener) } @@ -13,7 +14,7 @@ final class BroadcastStream: Sendable { } } } - + func dispatch(event: T) { let listeners = self.listeners.withLock { Array($0) } for listener in listeners { @@ -44,7 +45,7 @@ final private class BroadcastStreamListener: Sendable, Hashable { func hash(into hasher: inout Hasher) { hasher.combine(ObjectIdentifier(self)) } - + static func == (lhs: BroadcastStreamListener, rhs: BroadcastStreamListener) -> Bool { lhs === rhs } diff --git a/Sources/SyncPlayground/main.swift b/Sources/SyncPlayground/main.swift deleted file mode 100644 index b7ef35b..0000000 --- a/Sources/SyncPlayground/main.swift +++ /dev/null @@ -1,35 +0,0 @@ -import Foundation -import PowerSync - -func start() async throws { - let schema = Schema(tables: []) - let db = PowerSyncDatabase(schema: schema) - - try await db.connect(connector: TestConnector(), options: ConnectOptions()) - print("Is connected!") - - try await db.waitForFirstSync() -} - -final class TestConnector: PowerSyncBackendConnectorProtocol { - func fetchCredentials() async throws -> PowerSync.PowerSyncCredentials? { - let url = URL(string: "http://localhost:6060/api/auth/token")! - let (data, _) = try await URLSession.shared.data(from: url) - - struct Response: Decodable { - let token: String - } - - let response = try JSONDecoder().decode(Response.self, from: data) - return PowerSyncCredentials( - endpoint: "http://localhost:8080", - token: response.token - ) - } - - func uploadData(database: any PowerSync.PowerSyncDatabaseProtocol) async throws { - throw PowerSyncError.operationFailed(message: "todo: uploadData") - } -} - -let _ = try await start() diff --git a/Tests/PowerSyncTests/SyncTests.swift b/Tests/PowerSyncTests/SyncTests.swift index 6700310..cb1fea4 100644 --- a/Tests/PowerSyncTests/SyncTests.swift +++ b/Tests/PowerSyncTests/SyncTests.swift @@ -16,7 +16,7 @@ class InMemorySyncIntegrationTests { try await db.connect(connector: TestConnector(), options: ConnectOptions()) await didConnect.await() } - + @Test func useParameters() async throws { let didConnect = Signal() let db = openDatabase(MockHttpClient { request in @@ -25,14 +25,14 @@ class InMemorySyncIntegrationTests { await didConnect.complete() return AsyncThrowingChannel() }) - + try await db.connect(connector: TestConnector(), options: ConnectOptions( params: ["foo": .string("bar")] )) await didConnect.await() try await db.disconnect() } - + @Test func useAppMetadata() async throws { let didConnect = Signal() let db = openDatabase(MockHttpClient { request in @@ -41,34 +41,34 @@ class InMemorySyncIntegrationTests { await didConnect.complete() return AsyncThrowingChannel() }) - + try await db.connect(connector: TestConnector(), options: ConnectOptions( appMetadata: ["app_version": "1.0.0"] )) await didConnect.await() try await db.disconnect() } - + @Test func cannotUpdateSchemaWhileConnected() async throws { let db = openDatabase(MockHttpClient { request in AsyncThrowingChannel() }) try await db.connect(connector: TestConnector(), options: ConnectOptions()) - + await #expect(throws: PowerSyncError.self) { try await db.updateSchema(schema: Schema()) } try await db.close() } - + @Test func partialSync() async throws { let channel = AsyncThrowingChannel() let checksums = Array((0...3).map { prio in BucketChecksum(bucket: "bucket\(prio)", priority: .init(prio), checksum: 10 + prio) }) var operationId = 1 - + func pushData(priority: Int32) async throws { let id = operationId operationId += 1 - + try await channel.pushLine(.syncDataBucket(SyncDataBucket(bucket: "bucket\(priority)", data: [ OplogEntry( checksum: priority + 10, @@ -82,11 +82,11 @@ class InMemorySyncIntegrationTests { ) ]))) } - + let db = openDatabase(MockHttpClient { request in channel }) try await db.connect(connector: TestConnector(), options: ConnectOptions()) await waitForStatus(db.currentStatus) { $0.connected } - + try await expectUserCount(db, 0) try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "4", buckets: checksums))) // Emit a partial sync complete for each priority but the last @@ -98,25 +98,25 @@ class InMemorySyncIntegrationTests { await waitForStatus(db.currentStatus) { $0.statusForPriority(priority).hasSynced == true } try await expectUserCount(db, priorityNo + 1) } - + // Then complete the sync try await pushData(priority: 3) try await channel.pushLine(.checkpointComplete(lastOpId: String(operationId))) try await db.waitForFirstSync() try await expectUserCount(db, 4) - + try await db.disconnect() } - + @Test func setsDownloadingState() async throws { let channel = AsyncThrowingChannel() let db = openDatabase(MockHttpClient { request in channel }) try await db.connect(connector: TestConnector(), options: ConnectOptions()) await waitForStatus(db.currentStatus) { $0.connected } - + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "1", buckets: [.init(bucket: "bkt", checksum: 0)]))) await waitForStatus(db.currentStatus) { $0.downloading } - + try await channel.pushLine(.checkpointComplete(lastOpId: "1")) await waitForStatus(db.currentStatus) { !$0.downloading } try await db.disconnect() @@ -124,18 +124,34 @@ class InMemorySyncIntegrationTests { @Test func setsConnectingState() async throws { let didSeeConnecting = Signal() - + let db = openDatabase(MockHttpClient { request in await didSeeConnecting.await() return AsyncThrowingChannel() }) - + try await db.connect(connector: TestConnector(), options: ConnectOptions()) await waitForStatus(db.currentStatus) { $0.connecting } await didSeeConnecting.complete() await waitForStatus(db.currentStatus) { $0.connected } } - + + @Test func staysConnectedAfterCancellingConnectionTask() async throws { + let channel = AsyncThrowingChannel() + let mockClient = MockHttpClient { request in channel } + let db = openDatabase(mockClient) + let task = Task { + try await db.connect(connector: TestConnector(), options: ConnectOptions()) + } + + await waitForStatus(db.currentStatus) { $0.connected } + task.cancel() + let _ = await task.result + + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "1", buckets: [BucketChecksum(bucket: "a", checksum: 0)], writeCheckpoint: "1"))) + await waitForStatus(db.currentStatus) { $0.downloading } + } + @Test func reconnectsAfterDisconnecting() async throws { let db = openDatabase(MockHttpClient { request in AsyncThrowingChannel() }) try await db.connect(connector: TestConnector(), options: ConnectOptions()) @@ -143,16 +159,16 @@ class InMemorySyncIntegrationTests { try await db.disconnect() await waitForStatus(db.currentStatus) { !$0.connected && !$0.connecting } - + try await db.connect(connector: TestConnector(), options: ConnectOptions()) await waitForStatus(db.currentStatus) { $0.connected } } - + @Test func reconnects() async throws { let db = openDatabase(MockHttpClient { request in AsyncThrowingChannel() }) try await db.connect(connector: TestConnector(), options: ConnectOptions()) await waitForStatus(db.currentStatus) { $0.connected } - + try await db.connect(connector: TestConnector(), options: ConnectOptions()) await waitForStatus(db.currentStatus) { !$0.connected } await waitForStatus(db.currentStatus) { $0.connected } @@ -166,10 +182,10 @@ class InMemorySyncIntegrationTests { try await db.execute(sql: "INSERT INTO users (id, name) VALUES (uuid(), ?)", parameters: ["local write"]) try await db.connect(connector: TestConnector(), options: ConnectOptions()) - + var query = try db.watch("SELECT name FROM users") { try $0.getString(index: 0) }.makeAsyncIterator() try #require(try await query.next() == ["local write"]) - + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "1", buckets: [BucketChecksum(bucket: "a", checksum: 0)], writeCheckpoint: "1"))) try await channel.pushLine(.syncDataBucket(SyncDataBucket(bucket: "a", data: [OplogEntry( checksum: 0, @@ -182,11 +198,11 @@ class InMemorySyncIntegrationTests { try await channel.pushLine(.checkpointComplete(lastOpId: "1")) try #require(try await query.next() == ["from server"]) } - + @Test func tokenExpired() async throws { final actor BackendConnector: PowerSyncBackendConnectorProtocol { var fetchCredentialsCalls = 0 - + func fetchCredentials() async throws -> PowerSyncCredentials? { fetchCredentialsCalls += 1 return testCredentials @@ -194,12 +210,12 @@ class InMemorySyncIntegrationTests { func uploadData(database: any PowerSyncDatabaseProtocol) async throws {} } - + let channel = AsyncThrowingChannel() let db = openDatabase(MockHttpClient { request in channel }) let connector = BackendConnector() try await db.connect(connector: connector, options: ConnectOptions(retryDelay: 0)) - + try await channel.pushLine(.keepAlive(tokenExpiresIn: 4000)) await waitForStatus(db.currentStatus) { $0.connected } try #require(await connector.fetchCredentialsCalls == 1) @@ -214,7 +230,7 @@ class InMemorySyncIntegrationTests { @Test func tokenThrows() async throws { actor BackendConnector: PowerSyncBackendConnectorProtocol { var isFirstFetchCall = true - + func fetchCredentials() async throws -> PowerSyncCredentials? { if isFirstFetchCall { isFirstFetchCall = false @@ -222,25 +238,25 @@ class InMemorySyncIntegrationTests { } return testCredentials } - + func uploadData(database: any PowerSyncDatabaseProtocol) async throws {} } - + let channel = AsyncThrowingChannel() let db = openDatabase(MockHttpClient { request in channel }) try await db.connect(connector: BackendConnector(), options: ConnectOptions(retryDelay: 0.2)) await waitForStatus(db.currentStatus) { !$0.connected && $0.downloadError != nil } - + // Should retry, and the second fetchCredentials call will work await waitForStatus(db.currentStatus) { $0.connected } } - + @Test func tokenPrefetch() async throws { actor BackendConnector: PowerSyncBackendConnectorProtocol { let prefetchCalled = Signal() let completePrefetch = Signal() var fetchCredentialsCount = 0 - + func fetchCredentials() async throws -> PowerSyncCredentials? { fetchCredentialsCount += 1 if fetchCredentialsCount == 2 { @@ -252,12 +268,12 @@ class InMemorySyncIntegrationTests { func uploadData(database: any PowerSyncDatabaseProtocol) async throws {} } - + let connector = BackendConnector() let channel = AsyncThrowingChannel() let db = openDatabase(MockHttpClient { request in channel }) try await db.connect(connector: connector, options: ConnectOptions()) - + try await channel.pushLine(.keepAlive(tokenExpiresIn: 4000)) await waitForStatus(db.currentStatus) { $0.connected } try #require(await connector.fetchCredentialsCount == 1) @@ -279,17 +295,17 @@ class InMemorySyncIntegrationTests { let id: String let name: String } - + let channel = AsyncThrowingChannel() let db = openDatabase(MockHttpClient { request in channel }, schema: Schema(RawTable(name: "lists", schema: RawTableSchema()))) - + try await db.execute("CREATE TABLE lists (id TEXT NOT NULL PRIMARY KEY, name TEXT)") var query = try db.watch("SELECT * FROM lists") { cursor in List(id: try cursor.getString(index: 0), name: try cursor.getString(index: 1)) }.makeAsyncIterator() try #require(try await query.next() == []) try await db.connect(connector: TestConnector(), options: ConnectOptions()) - + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "1", buckets: [BucketChecksum(bucket: "a", checksum: 0)]))) try await channel.pushLine(.syncDataBucket(SyncDataBucket(bucket: "a", data: [ OplogEntry( @@ -303,7 +319,7 @@ class InMemorySyncIntegrationTests { ]))) try await channel.pushLine(.checkpointComplete(lastOpId: "1")) try #require(try await query.next() == [List(id: "my_list", name: "custom list")]) - + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "2", buckets: [BucketChecksum(bucket: "a", checksum: 0)]))) try await channel.pushLine(.syncDataBucket(SyncDataBucket(bucket: "a", data: [ OplogEntry( @@ -317,14 +333,14 @@ class InMemorySyncIntegrationTests { try await channel.pushLine(.checkpointComplete(lastOpId: "2")) try #require(try await query.next() == []) } - + @Test func rawTablesWithExplicitStatements() async throws { struct List: Equatable { let id: String let name: String let rest: String } - + let channel = AsyncThrowingChannel() let db = openDatabase(MockHttpClient { request in channel }, schema: Schema(RawTable( name: "lists", @@ -337,14 +353,14 @@ class InMemorySyncIntegrationTests { .id ]), ))) - + try await db.execute("CREATE TABLE lists (id TEXT NOT NULL PRIMARY KEY, name TEXT, _rest TEXT)") var query = try db.watch("SELECT * FROM lists") { cursor in List(id: try cursor.getString(index: 0), name: try cursor.getString(index: 1), rest: try cursor.getString(index: 2)) }.makeAsyncIterator() try #require(try await query.next() == []) try await db.connect(connector: TestConnector(), options: ConnectOptions()) - + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "1", buckets: [BucketChecksum(bucket: "a", checksum: 0)]))) try await channel.pushLine(.syncDataBucket(SyncDataBucket(bucket: "a", data: [ OplogEntry( @@ -358,7 +374,7 @@ class InMemorySyncIntegrationTests { ]))) try await channel.pushLine(.checkpointComplete(lastOpId: "1")) try #require(try await query.next() == [List(id: "my_list", name: "custom list", rest: #"{"additional_column":"foo"}"#)]) - + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "2", buckets: [BucketChecksum(bucket: "a", checksum: 0)]))) try await channel.pushLine(.syncDataBucket(SyncDataBucket(bucket: "a", data: [ OplogEntry( @@ -372,7 +388,7 @@ class InMemorySyncIntegrationTests { try await channel.pushLine(.checkpointComplete(lastOpId: "2")) try #require(try await query.next() == []) } - + @Test func endsIterationOnHttpClose() async throws { let channel = AsyncThrowingChannel() let db = openDatabase(MockHttpClient { request in channel }) @@ -390,7 +406,7 @@ class InMemorySyncIntegrationTests { await waitForStatus(db.currentStatus) { $0.connected } var status = db.currentStatus.asFlow().makeAsyncIterator() let _ = await status.next() // Skip initial - + // Send checkpoint with 10 ops, progress should be 0/10 try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "10", buckets: [BucketChecksum(bucket: "a", checksum: 0, count: 10)]))) try (try #require(await status.next())).expectProgress(total: (0, 10)) @@ -399,19 +415,19 @@ class InMemorySyncIntegrationTests { .init(checksum: 0, op_id: String(i+1), object_id: String(i), object_type: "a", op: .put, data: "{}") }))) try (try #require(await status.next())).expectProgress(total: (10, 10)) - + // Emit new data, progress should be 0/2 instead of 2/2 try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "12", buckets: [ BucketChecksum(bucket: "a", checksum: 0, count: 12), ]))) try (try #require(await status.next())).expectProgress(total: (10, 12)) - + try await channel.pushLine(.syncDataBucket(SyncDataBucket(bucket: "a", data: (10..<12).map { i in .init(checksum: 0, op_id: String(i+1), object_id: String(i), object_type: "a", op: .put, data: "{}") }))) try (try #require(await status.next())).expectProgress(total: (12, 12)) } - + @Test func requestLogger() async throws { let channel = AsyncThrowingChannel() let db = openDatabase(MockHttpClient { request in channel }) @@ -426,12 +442,12 @@ class InMemorySyncIntegrationTests { try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "0", buckets: [BucketChecksum(bucket: "a", checksum: 0)]))) try await channel.pushLine(.checkpointComplete(lastOpId: "0")) try await db.waitForFirstSync() - + let logEntries = lines.withLock { $0 } try #require(logEntries.contains("Starting request to POST https://powersynctest.example.org/sync/stream")) try #require(logEntries.contains(#"Response line: {"checkpoint_complete":{"last_op_id":"0"}}"#)) } - + @Test func canDisableDefaultStreams() async throws { let didConnect = Signal() let db = openDatabase(MockHttpClient { request in @@ -445,13 +461,13 @@ class InMemorySyncIntegrationTests { await didConnect.complete() return AsyncThrowingChannel() }) - + try await db.connect(connector: TestConnector(), options: ConnectOptions( includeDefaultStreams: false )) await didConnect.await() } - + @Test func subscribesWithStreams() async throws { let channel = AsyncThrowingChannel() let db = openDatabase(MockHttpClient { request in @@ -476,18 +492,18 @@ class InMemorySyncIntegrationTests { return channel }) - + let a = try await db.syncStream(name:"stream", params: ["foo": .string("a")]).subscribe() let b = try await db.syncStream(name: "stream", params: ["foo": .string("b")]).subscribe(ttl: nil, priority: .init(1)) try await db.connect(connector: TestConnector(), options: ConnectOptions()) await waitForStatus(db.currentStatus) { $0.connected } var statusUpdates = db.currentStatus.asFlow().makeAsyncIterator() let _ = await statusUpdates.next() // Skip initial - + // Without an initial checkpoint, sync streams should not be marked as active try #require(db.currentStatus.forStream(stream: a)?.subscription.hasSynced == false) try #require(db.currentStatus.forStream(stream: b)?.subscription.hasSynced == false) - + try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "1", buckets: [ BucketChecksum( bucket: "a", @@ -502,7 +518,7 @@ class InMemorySyncIntegrationTests { subscriptions: [.explicitSubscription(1)] ) ], streams: [StreamDescription(name: "stream", is_default: false)]))) - + // Subscriptions should be active now, but not marked as synced do { let status = try #require(await statusUpdates.next()) @@ -513,7 +529,7 @@ class InMemorySyncIntegrationTests { try #require(status.subscription.hasExplicitSubscription) } } - + try await channel.pushLine(.checkpointPartiallyComplete(lastOpId: "0", priority: BucketPriority(1))) do { let status = try #require(await statusUpdates.next()) @@ -521,7 +537,7 @@ class InMemorySyncIntegrationTests { try #require(status.forStream(stream: b)!.subscription.lastSyncedAt != nil) try await b.waitForFirstSync() } - + try await channel.pushLine(.checkpointComplete(lastOpId: "0")) try await a.waitForFirstSync() } @@ -530,12 +546,12 @@ class InMemorySyncIntegrationTests { let channel = AsyncThrowingChannel() let db = openDatabase(MockHttpClient { request in channel }) try await db.connect(connector: TestConnector(), options: ConnectOptions()) - + await waitForStatus(db.currentStatus) { $0.connected } var statusUpdates = db.currentStatus.asFlow().makeAsyncIterator() let _ = await statusUpdates.next() // Skip initial try await channel.pushLine(.fullCheckpoint(Checkpoint(last_op_id: "0", buckets: [], streams: [StreamDescription(name: "default_stream", is_default: true)]))) - + let status = try #require(await statusUpdates.next()) let stream = try #require(status.syncStreams?.first) try #require(stream.subscription.name == "default_stream") @@ -551,7 +567,7 @@ class InMemorySyncIntegrationTests { await lastRequest.withMutex { $0 = body } return AsyncThrowingChannel() }) - + try await db.connect(connector: TestConnector(), options: ConnectOptions()) await waitForStatus(db.currentStatus) { $0.connected } let request = try #require(await lastRequest.inner) @@ -579,17 +595,17 @@ class InMemorySyncIntegrationTests { } let _ = consume subscription } - + @Test func subscriptionsUpdateWhileOffline() async throws { let db = openDatabase(PlatformHttpClient.shared) var statusUpdates = db.currentStatus.asFlow().makeAsyncIterator() - + // Subscribing while offline should add the stream to subscriptions reported in the status. let subscription = try await db.syncStream(name: "a", params: nil).subscribe() let status = try #require(await statusUpdates.next()) let _ = try #require(status.forStream(stream: subscription)) } - + @Test func unsubscribingMultipleTimesHasNoEffect() async throws { let db = openDatabase(MockHttpClient { request in let body = try StreamingSyncClient.jsonDecoder.decode(JsonParam.self, from: try #require(request.httpBody)) @@ -607,21 +623,21 @@ class InMemorySyncIntegrationTests { return AsyncThrowingChannel() }) - + let a = try await db.syncStream(name: "a", params: nil).subscribe() let aAgain = try await db.syncStream(name: "a", params: nil).subscribe() try await a.unsubscribe() try await a.unsubscribe() - + // Pretend the streams are expired, they should still be requested because the // core extension extends the lifetime of streams currently referenced before connecting try await db.execute("UPDATE ps_stream_subscriptions SET expires_at = unixepoch() - 1000") try await db.connect(connector: TestConnector(), options: ConnectOptions()) await waitForStatus(db.currentStatus) { $0.connected } - + let _ = consume aAgain } - + @Test func unsubscribeAll() async throws { let didConnect = Signal() let db = openDatabase(MockHttpClient { request in @@ -671,7 +687,7 @@ let testCredentials = PowerSyncCredentials( private final class TestConnector: PowerSyncBackendConnectorProtocol { private let uploadDataCallback: @Sendable (_ database: any PowerSyncDatabaseProtocol) async throws -> () - + init( uploadDataCallback: @Sendable @escaping (_: any PowerSyncDatabaseProtocol) async throws -> Void = { db in let tx = try await db.getNextCrudTransaction() @@ -679,7 +695,7 @@ private final class TestConnector: PowerSyncBackendConnectorProtocol { }) { self.uploadDataCallback = uploadDataCallback } - + func fetchCredentials() async throws -> PowerSyncCredentials? { return testCredentials } @@ -701,14 +717,6 @@ private final class Signal: Sendable { } } -private final class Box: Sendable { - let inner: T - - init(inner: consuming T) { - self.inner = inner - } -} - func expectUserCount(_ db: PowerSyncDatabaseProtocol, _ amount: Int32) async throws { let users = try await db.getAll("SELECT name FROM users") { $0.getStringOptional(index: 0) } try #require(users.count == amount) @@ -718,7 +726,7 @@ func waitForStatus(_ status: SyncStatus, predicate: (borrowing SyncStatusData) - if predicate(status) { return } - + let _ = await status.asFlow().first(where: predicate) } From 642cdbb3deaacf57d0f628d645ad116e99e4c8c8 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 27 Apr 2026 10:26:40 +0200 Subject: [PATCH 26/40] Fix sync in demo app --- .../xcshareddata/swiftpm/Package.resolved | 13 +++++++++++-- .../Implementation/sync/StreamingSyncClient.swift | 2 +- Sources/PowerSync/Protocol/Schema/Index.swift | 13 ++++++++++--- Sources/PowerSync/Protocol/Schema/Table.swift | 5 ++++- Tests/PowerSyncTests/Schema/TableTests.swift | 3 ++- 5 files changed, 28 insertions(+), 8 deletions(-) diff --git a/Demos/PowerSyncExample/PowerSyncExample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Demos/PowerSyncExample/PowerSyncExample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index bc9c9ff..54d11b6 100644 --- a/Demos/PowerSyncExample/PowerSyncExample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/Demos/PowerSyncExample/PowerSyncExample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -33,8 +33,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/powersync-ja/powersync-sqlite-core-swift.git", "state" : { - "revision" : "9743740980cd488a11f50cffe62ed34a9739a135", - "version" : "0.4.10" + "revision" : "05c2af384558011f0915d757b6677f5dcbbc5c54", + "version" : "0.4.13" } }, { @@ -55,6 +55,15 @@ "version" : "1.3.0" } }, + { + "identity" : "swift-async-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-async-algorithms.git", + "state" : { + "revision" : "9d349bcc328ac3c31ce40e746b5882742a0d1272", + "version" : "1.1.3" + } + }, { "identity" : "swift-case-paths", "kind" : "remoteSourceControl", diff --git a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift index ef84e1d..a432708 100644 --- a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift +++ b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift @@ -440,5 +440,5 @@ struct WriteCheckpointResponse: Codable { } private func sleepForSeconds(seconds: TimeInterval) async throws { - try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_00)) + try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000)) } diff --git a/Sources/PowerSync/Protocol/Schema/Index.swift b/Sources/PowerSync/Protocol/Schema/Index.swift index 820af49..688ccf5 100644 --- a/Sources/PowerSync/Protocol/Schema/Index.swift +++ b/Sources/PowerSync/Protocol/Schema/Index.swift @@ -12,7 +12,7 @@ public protocol IndexProtocol: Sendable { var columns: [IndexedColumnProtocol] { get } } -public struct Index: IndexProtocol, Encodable { +public struct Index: IndexProtocol { public let name: String public let columns: [IndexedColumnProtocol] @@ -48,24 +48,31 @@ public struct Index: IndexProtocol, Encodable { return ascending(name: name, columns: [column]) } - public func encode(to encoder: any Encoder) throws { + internal func encode(table: borrowing Table, to: any UnkeyedEncodingContainer) throws { enum CodingKeys: CodingKey { case name case columns } - var container = encoder.container(keyedBy: CodingKeys.self) + var to = to + var container = to.nestedContainer(keyedBy: CodingKeys.self) try container.encode(name, forKey: .name) var columnsContainer = container.nestedUnkeyedContainer(forKey: .columns) for column in columns { enum IndexedColumnCodingKeys: CodingKey { case name case ascending + case type } var container = columnsContainer.nestedContainer(keyedBy: IndexedColumnCodingKeys.self) try container.encode(column.column, forKey: .name) try container.encode(column.ascending, forKey: .ascending) + guard let tableColumn = table.columns.first(where: { c in c.name == column.column }) else { + throw PowerSyncError.operationFailed(message: "Unserializable schema: Index \(self.name) references column \(column.column) which does not exist in \(table.name)") + } + + try container.encode(tableColumn.type, forKey: .type) } } } diff --git a/Sources/PowerSync/Protocol/Schema/Table.swift b/Sources/PowerSync/Protocol/Schema/Table.swift index 323d566..061ea52 100644 --- a/Sources/PowerSync/Protocol/Schema/Table.swift +++ b/Sources/PowerSync/Protocol/Schema/Table.swift @@ -190,7 +190,10 @@ public struct Table: TableProtocol, Encodable { try container.encode(name, forKey: .outer(.name)) try container.encodeIfPresent(viewNameOverride, forKey: .outer(.viewName)) try container.encode(columns, forKey: .outer(.columns)) - try container.encode(indexes, forKey: .outer(.indexes)) + let indexContainer = container.nestedUnkeyedContainer(forKey: .outer(.indexes)) + for index in indexes { + try index.encode(table: self, to: indexContainer) + } try options.serializeTo(container) } } diff --git a/Tests/PowerSyncTests/Schema/TableTests.swift b/Tests/PowerSyncTests/Schema/TableTests.swift index 53c91cf..c9c2dff 100644 --- a/Tests/PowerSyncTests/Schema/TableTests.swift +++ b/Tests/PowerSyncTests/Schema/TableTests.swift @@ -272,7 +272,8 @@ final class TableTests: XCTestCase { "columns" : [ { "ascending" : true, - "name" : "name" + "name" : "name", + "type" : "text" } ], "name" : "test_index" From 0f71c4af599758d13ab74aa0f1b09ed76b0589fd Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 27 Apr 2026 10:53:23 +0200 Subject: [PATCH 27/40] AI feedback --- .../Implementation/SyncStreams.swift | 1 - .../Implementation/sync/Status.swift | 9 ++++++--- .../sync/StreamingSyncClient.swift | 20 ++++++++++++------- .../Implementation/sync/SyncCoordinator.swift | 2 +- .../Kotlin/KotlinPowerSyncDatabaseImpl.swift | 5 ++++- .../Protocol/PowerSyncDatabaseProtocol.swift | 2 +- Sources/PowerSync/Utils/BroadcastStream.swift | 2 -- .../test-utils/MockHttpClient.swift | 1 - 8 files changed, 25 insertions(+), 17 deletions(-) diff --git a/Sources/PowerSync/Implementation/SyncStreams.swift b/Sources/PowerSync/Implementation/SyncStreams.swift index 264aa95..92d63c0 100644 --- a/Sources/PowerSync/Implementation/SyncStreams.swift +++ b/Sources/PowerSync/Implementation/SyncStreams.swift @@ -1,5 +1,4 @@ import Foundation -import Synchronization final class StreamTracker: Sendable { // For each active stream key, how many StreamSubscription instances are active in that key. diff --git a/Sources/PowerSync/Implementation/sync/Status.swift b/Sources/PowerSync/Implementation/sync/Status.swift index 1ed8fe1..85bea0b 100644 --- a/Sources/PowerSync/Implementation/sync/Status.swift +++ b/Sources/PowerSync/Implementation/sync/Status.swift @@ -1,5 +1,4 @@ import Foundation -import Synchronization /// The internal struct backing all sync status fields. /// @@ -84,7 +83,7 @@ fileprivate struct SyncStatusDataImpl: SyncStatusData { func statusForPriority(_ priority: BucketPriority) -> PriorityStatusEntry { for known in priorityStatusEntries { // Lower-priority buckets are synced after higher-priority buckets, and since priorityStatusEntries - // is sortedwe look for the first entry that doesn't have a higher priority. + // is sorted, we look for the first entry that doesn't have a higher priority. if known.priority <= priority { return known } @@ -95,7 +94,11 @@ fileprivate struct SyncStatusDataImpl: SyncStatusData { } func forStream(stream: any SyncStreamDescription) -> SyncStreamStatus? { - for found in syncStreams! { + guard let streams = syncStreams else { + return nil + } + + for found in streams { if found.subscription.name == stream.name && found.subscription.parameters == stream.parameters { return found } diff --git a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift index a432708..b949756 100644 --- a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift +++ b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift @@ -1,6 +1,5 @@ import AsyncAlgorithms import Foundation -import Synchronization fileprivate let tag = "StreamingSyncClient" @@ -158,7 +157,7 @@ The next upload iteration will be delayed. while (!Task.isCancelled) { do { // This async let ensures each iteration is a task scoped to this block. This allows us to spawn - // aditional tasks in run() that would get cancelled when the main iteration is complete. + // additional tasks in run() that would get cancelled when the main iteration is complete. async let iteration = ActiveSyncIteration(syncClient: self, signals: signals).run() result = try await iteration @@ -306,7 +305,10 @@ private struct ActiveSyncIteration: Sendable { private func powersyncControl(_ args: PowerSyncControlArguments) async throws -> [Instruction] { let rawInstructions = try await syncClient.db.writeTransaction { tx in try args.execute(tx) } - return try StreamingSyncClient.jsonDecoder.decode([Instruction].self, from: rawInstructions.data(using: .utf8)!) + guard let data = rawInstructions.data(using: .utf8) else { + throw PowerSyncError.operationFailed(message: "Could not encode raw instructions") + } + return try StreamingSyncClient.jsonDecoder.decode([Instruction].self, from: data) } private func execute(instr: consuming Instruction) async throws { @@ -335,10 +337,13 @@ private struct ActiveSyncIteration: Sendable { await syncClient.invalidateCredentials() } else { Task { - let _ = try await syncClient.connector.fetchCredentials(allowCached: false) - syncClient.db.logger.debug("Stopping because new credentials are available", tag: tag) - // Token has been refreshed, start another iteration - localEvents.dispatch(event: .didRefreshToken) + do { + let _ = try await syncClient.connector.fetchCredentials(allowCached: false) + syncClient.db.logger.debug("Stopping because new credentials are available", tag: tag) + localEvents.dispatch(event: .didRefreshToken) + } catch { + syncClient.db.logger.warning("Pre-fetching credentials that are about to expire has failed: \(error)", tag: tag) + } } } case .flushFileSystem: @@ -397,6 +402,7 @@ fileprivate enum ControlInvocationsFromStreamIterator: AsyncIteratorProtocol { self = .eof return .responseStreamEnd case .some(.text(contents: let contents)): + self = .isReceiving(iterator) return .textLine(line: contents) } case .eof: diff --git a/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift b/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift index e9b1f96..b6b3884 100644 --- a/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift +++ b/Sources/PowerSync/Implementation/sync/SyncCoordinator.swift @@ -19,7 +19,7 @@ actor SyncCoordinator { func disconnect() async { guard let task = activeSync else { - return // Not connecteed + return // Not connected } await self.finishSyncTask(task: task) diff --git a/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift b/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift index b6fe5cd..d26784d 100644 --- a/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift +++ b/Sources/PowerSync/Kotlin/KotlinPowerSyncDatabaseImpl.swift @@ -62,7 +62,10 @@ final class KotlinPowerSyncDatabaseImpl: PowerSyncDatabaseProtocol, private func resolveOfflineSyncStatus() async throws { let offlineSyncStatus = try await get("SELECT powersync_offline_sync_status()") { cursor in let raw = try cursor.getString(index: 0) - return try StreamingSyncClient.jsonDecoder.decode(CoreDownloadSyncStatus.self, from: raw.data(using: .utf8)!) + guard let data = raw.data(using: .utf8) else { + throw PowerSyncError.operationFailed(message: "Could not encode offline sync status") + } + return try StreamingSyncClient.jsonDecoder.decode(CoreDownloadSyncStatus.self, from: data) } syncStatus.mutateStatus { $0 = MutableSyncStatus(core: offlineSyncStatus) } diff --git a/Sources/PowerSync/Protocol/PowerSyncDatabaseProtocol.swift b/Sources/PowerSync/Protocol/PowerSyncDatabaseProtocol.swift index 6bda1be..eac729c 100644 --- a/Sources/PowerSync/Protocol/PowerSyncDatabaseProtocol.swift +++ b/Sources/PowerSync/Protocol/PowerSyncDatabaseProtocol.swift @@ -133,7 +133,7 @@ public struct ConnectOptions: Sendable { self.newClientImplementation = newClientImplementation self.clientConfiguration = clientConfiguration self.appMetadata = appMetadata - self.includeDefaultStreams = true + self.includeDefaultStreams = includeDefaultStreams } } diff --git a/Sources/PowerSync/Utils/BroadcastStream.swift b/Sources/PowerSync/Utils/BroadcastStream.swift index 58986e1..e10cf6c 100644 --- a/Sources/PowerSync/Utils/BroadcastStream.swift +++ b/Sources/PowerSync/Utils/BroadcastStream.swift @@ -1,5 +1,3 @@ -import Synchronization - /// Dispatches events to a number of listeners as an ``AsyncStream``. final class BroadcastStream: Sendable { private let listeners: Mutex>> = Mutex([]) diff --git a/Tests/PowerSyncTests/test-utils/MockHttpClient.swift b/Tests/PowerSyncTests/test-utils/MockHttpClient.swift index 1ca13e7..535a28d 100644 --- a/Tests/PowerSyncTests/test-utils/MockHttpClient.swift +++ b/Tests/PowerSyncTests/test-utils/MockHttpClient.swift @@ -2,7 +2,6 @@ import AsyncAlgorithms import Foundation @testable import PowerSync import Testing -import Synchronization final class MockHttpClient: HttpClient { private let _writeCheckpoint = PowerSync.Mutex(1000) From dfd3ef03890baa7544af33b4fe473143b2576d9c Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 27 Apr 2026 12:27:28 +0200 Subject: [PATCH 28/40] Add database groups --- .../Implementation/ActiveInstanceStore.swift | 65 +++++++++++++++++++ .../PowerSyncDatabaseImpl.swift | 19 +++--- .../Implementation/SyncStreams.swift | 6 +- .../sqlite3/NativeConnectionPool.swift | 1 - .../sync/StreamingSyncClient.swift | 4 +- Sources/PowerSync/PowerSyncDatabase.swift | 9 ++- .../Kotlin/ActiveInstanceStoreTests.swift | 37 +++++++++++ .../Kotlin/SqlCursorTests.swift | 3 +- Tests/PowerSyncTests/SyncTests.swift | 3 +- 9 files changed, 128 insertions(+), 19 deletions(-) create mode 100644 Sources/PowerSync/Implementation/ActiveInstanceStore.swift create mode 100644 Tests/PowerSyncTests/Kotlin/ActiveInstanceStoreTests.swift diff --git a/Sources/PowerSync/Implementation/ActiveInstanceStore.swift b/Sources/PowerSync/Implementation/ActiveInstanceStore.swift new file mode 100644 index 0000000..ab26782 --- /dev/null +++ b/Sources/PowerSync/Implementation/ActiveInstanceStore.swift @@ -0,0 +1,65 @@ +final class DatabaseGroupCollection: Sendable { + private let groups: Mutex<[ActiveDatabaseGroupData]> = Mutex([]) + + fileprivate func closeGroup(identifier: String) { + groups.withLock { $0.removeAll { group in group.identifier == identifier } } + } + + func referenceGroup(identifier: String, logger: LoggerProtocol) -> ActiveDatabaseGroup { + groups.withLock { activeDatabases in + let existingGroup = activeDatabases.first { $0.identifier == identifier } + let data: ActiveDatabaseGroupData + if let existingGroup { + logger.warning(""" +Multiple PowerSync instances for the same database have been detected. +This can cause unexpected results. +Please check your PowerSync client instantiation logic if this is not intentional. +""", tag: "DatabaseGroupCollection") + data = existingGroup + } else { + data = ActiveDatabaseGroupData(identifier: identifier) + activeDatabases.append(data) + } + + return ActiveDatabaseGroup(data: data, collection: self) + } + } + + static let shared = DatabaseGroupCollection() +} + +private final class ActiveDatabaseGroupData: Sendable { + let identifier: String + let syncCoordinator = SyncCoordinator() + + init(identifier: String) { + self.identifier = identifier + } +} + +/// A collection of PowerSync databases with the same path / identifier. +/// +/// We expect that each group will only ever have one database because we encourage users to write their databases as +/// singletons. We print a warning when two databasees are part of the same group. +/// Additionally, we want to avoid two databases in the same group having a sync stream open at the same time to avoid +/// duplicate resources being used. For this reason, each active database group has a single sync coordinator actor +/// responsible for initializing the sync process for all databases in the group. +final class ActiveDatabaseGroup: Sendable { + fileprivate let data: ActiveDatabaseGroupData + private weak let collection: DatabaseGroupCollection? + + fileprivate init(data: ActiveDatabaseGroupData, collection: DatabaseGroupCollection) { + self.data = data + self.collection = collection + } + + var syncCoordinator: SyncCoordinator { + data.syncCoordinator + } + + deinit { + if let collection { + collection.closeGroup(identifier: self.data.identifier) + } + } +} diff --git a/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift b/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift index d97effa..3186e32 100644 --- a/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift +++ b/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift @@ -3,7 +3,7 @@ import Foundation final class PowerSyncDatabaseImpl: PowerSyncDatabaseProtocol { let logger: any LoggerProtocol - let syncCoordinator = SyncCoordinator() + let group: ActiveDatabaseGroup let syncStatus = SwiftSyncStatus() private let dbFilename: String? private let httpClient: HttpClient @@ -13,6 +13,8 @@ final class PowerSyncDatabaseImpl: PowerSyncDatabaseProtocol { init( dbFilename: String? = nil, + identifier: String, + activeInstanceStore: DatabaseGroupCollection = .shared, logger: any LoggerProtocol, pool: any SQLiteConnectionPoolProtocol, httpClient: HttpClient, @@ -23,14 +25,15 @@ final class PowerSyncDatabaseImpl: PowerSyncDatabaseProtocol { self.schema = AsyncMutex(schema) self.httpClient = httpClient self.queries = ConnectionPoolQueries(pool: pool) + self.group = activeInstanceStore.referenceGroup(identifier: identifier, logger: logger) } - + var currentStatus: any SyncStatus { syncStatus } func resolveOfflineSyncStatusIfNotConnected() async throws { - try await syncCoordinator.guardNotConnected(inner: { + try await group.syncCoordinator.guardNotConnected(inner: { try await resolveOfflineSyncStatus() }, ifConnected: {}) } @@ -56,7 +59,7 @@ final class PowerSyncDatabaseImpl: PowerSyncDatabaseProtocol { func updateSchema(schema: any SchemaProtocol) async throws { try await initializer.ensureInitialized(db: self) - try await syncCoordinator.guardNotConnected( + try await group.syncCoordinator.guardNotConnected( inner: { let schema = Schema(other: schema) await self.schema.withMutex { $0 = schema } @@ -99,7 +102,7 @@ final class PowerSyncDatabaseImpl: PowerSyncDatabaseProtocol { } func disconnect() async throws { - await syncCoordinator.disconnect() + await group.syncCoordinator.disconnect() } func syncStream(name: String, params: JsonParam?) -> any SyncStream { @@ -109,7 +112,7 @@ final class PowerSyncDatabaseImpl: PowerSyncDatabaseProtocol { func close() async throws { try await initialize() try await initializer.close { - await syncCoordinator.disconnect() + await group.syncCoordinator.disconnect() try await queries.pool.close() } } @@ -124,12 +127,12 @@ final class PowerSyncDatabaseImpl: PowerSyncDatabaseProtocol { } func connect(connector: any PowerSyncBackendConnectorProtocol, options: ConnectOptions?) async throws { - await syncCoordinator.connect(db: self, connector: connector, options: options ?? ConnectOptions(), client: httpClient) + await group.syncCoordinator.connect(db: self, connector: connector, options: options ?? ConnectOptions(), client: httpClient) } func disconnectAndClear(clearLocal: Bool, soft: Bool) async throws { try await initialize() - try await syncCoordinator.disconnectAndThen { + try await group.syncCoordinator.disconnectAndThen { var flags = 0 if clearLocal { flags |= 1 diff --git a/Sources/PowerSync/Implementation/SyncStreams.swift b/Sources/PowerSync/Implementation/SyncStreams.swift index 13107ff..2897909 100644 --- a/Sources/PowerSync/Implementation/SyncStreams.swift +++ b/Sources/PowerSync/Implementation/SyncStreams.swift @@ -88,11 +88,11 @@ struct PendingSyncStream: SyncStream { } func subscribe(ttl: TimeInterval?, priority: BucketPriority?) async throws -> any SyncStreamSubscription { - return try await db.syncCoordinator.streams.subscribe(db: db, stream: self, ttl: ttl, priority: priority) + return try await db.group.syncCoordinator.streams.subscribe(db: db, stream: self, ttl: ttl, priority: priority) } func unsubscribeAll() async throws { - let tracker = db.syncCoordinator.streams + let tracker = db.group.syncCoordinator.streams let key = self.key tracker.removeStreamGroup(key: key) try await tracker.subscriptionsCommand(db: db, request: .unsubscribe(key)) @@ -125,7 +125,7 @@ final class SyncSubscriptionImplementation: SyncStreamSubscription { } deinit { - db.syncCoordinator.streams.decrementRefCount(key: key) + db.group.syncCoordinator.streams.decrementRefCount(key: key) } } diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift index f66d050..c92a177 100644 --- a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift +++ b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift @@ -24,7 +24,6 @@ final class NativeConnectionPool: Sendable { private func dispatchWrites(lease: NativeConnectionLease) throws { try lease.withIterator(sql: "SELECT powersync_update_hooks('get')", parameters: []) { rows in - var rows = rows let affectedTables = try rows.next { let decoder = JSONDecoder() return try decoder.decode(Set.self, from: try $0.getString(index: 0).data(using: .utf8)!) diff --git a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift index 6e2accb..9cff49f 100644 --- a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift +++ b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift @@ -245,7 +245,7 @@ private struct ActiveSyncIteration: Sendable { parameters: syncClient.options.params, schema: await syncClient.db.schema.inner, includeDefaults: syncClient.options.includeDefaultStreams, - activeStreams: syncClient.db.syncCoordinator.streams.currentStreams, + activeStreams: syncClient.db.group.syncCoordinator.streams.currentStreams, appMetadata: syncClient.options.appMetadata, ))) @@ -357,7 +357,7 @@ private struct ActiveSyncIteration: Sendable { } private func watchSyncStreams() async throws { - let changes = syncClient.db.syncCoordinator.streams.streamsChanged.subscribe() + let changes = syncClient.db.group.syncCoordinator.streams.streamsChanged.subscribe() for await change in changes { self.localEvents.dispatch(event: .updateSubscriptions(streams: change)) } diff --git a/Sources/PowerSync/PowerSyncDatabase.swift b/Sources/PowerSync/PowerSyncDatabase.swift index 5da4587..b849d59 100644 --- a/Sources/PowerSync/PowerSyncDatabase.swift +++ b/Sources/PowerSync/PowerSyncDatabase.swift @@ -16,14 +16,16 @@ public func PowerSyncDatabase( logger: (any LoggerProtocol) = DefaultLogger(), initialStatements: [String] = [] ) -> PowerSyncDatabaseProtocol { - let location = if dbFilename == ":memory:" { - DatabaseLocation.inMemory + let (location, group) = if dbFilename == ":memory:" { + (DatabaseLocation.inMemory, DatabaseGroupCollection()) } else { - DatabaseLocation.inDefaultDirectory(name: dbFilename) + (DatabaseLocation.inDefaultDirectory(name: dbFilename), .shared) } let pool = AsyncConnectionPool(location: location, initialStatements: initialStatements) return PowerSyncDatabaseImpl( dbFilename: dbFilename, + identifier: dbFilename, + activeInstanceStore: group, logger: logger, pool: pool, httpClient: PlatformHttpClient.shared, @@ -53,6 +55,7 @@ public func OpenedPowerSyncDatabase( logger: (any LoggerProtocol) = DefaultLogger() ) -> PowerSyncDatabaseProtocol { return PowerSyncDatabaseImpl( + identifier: identifier, logger: logger, pool: pool, httpClient: PlatformHttpClient.shared, diff --git a/Tests/PowerSyncTests/Kotlin/ActiveInstanceStoreTests.swift b/Tests/PowerSyncTests/Kotlin/ActiveInstanceStoreTests.swift new file mode 100644 index 0000000..9f7f408 --- /dev/null +++ b/Tests/PowerSyncTests/Kotlin/ActiveInstanceStoreTests.swift @@ -0,0 +1,37 @@ +@testable import PowerSync +import Testing + +@Suite +struct MultipleInstanceTest { + @Test func warnsAboutMultipleInstances() async throws { + let pool = AsyncConnectionPool(location: .inMemory) + let logWriter = TestLogWriterAdapter() + let logger = DefaultLogger(minSeverity: .warning, writers: [logWriter]) + let schema = Schema() + + let a = PowerSyncDatabaseImpl(identifier: "id", logger: logger, pool: pool, httpClient: PlatformHttpClient.shared, schema: schema) + try #require(logWriter.getLogs().isEmpty) + + let b = PowerSyncDatabaseImpl(identifier: "id", logger: logger, pool: pool, httpClient: PlatformHttpClient.shared, schema: schema) + let _ = try #require(logWriter.getLogs().first { $0.contains("Multiple PowerSync instances for the same database have been detected.") }) + + // Ensure databases are kept around until the end of the test (if a gets closed before, we would't see the warning). + let _ = consume a + let _ = consume b + } + + @Test func doesNotWarnForClosedInstances() async throws { + let pool = AsyncConnectionPool(location: .inMemory) + let logWriter = TestLogWriterAdapter() + let logger = DefaultLogger(minSeverity: .warning, writers: [logWriter]) + let schema = Schema() + + do { + let _ = PowerSyncDatabaseImpl(identifier: "id2", logger: logger, pool: pool, httpClient: PlatformHttpClient.shared, schema: schema) + } + + let b = PowerSyncDatabaseImpl(identifier: "id2", logger: logger, pool: pool, httpClient: PlatformHttpClient.shared, schema: schema) + try #require(logWriter.getLogs().isEmpty) + let _ = consume b + } +} diff --git a/Tests/PowerSyncTests/Kotlin/SqlCursorTests.swift b/Tests/PowerSyncTests/Kotlin/SqlCursorTests.swift index dbb156b..574db98 100644 --- a/Tests/PowerSyncTests/Kotlin/SqlCursorTests.swift +++ b/Tests/PowerSyncTests/Kotlin/SqlCursorTests.swift @@ -66,7 +66,8 @@ final class SqlCursorTests: XCTestCase { ]) database = PowerSyncDatabaseImpl( - dbFilename: ":memory:", + identifier: ":memory:", + activeInstanceStore: DatabaseGroupCollection(), logger: DefaultLogger(), pool: AsyncConnectionPool(location: .inMemory), httpClient: PlatformHttpClient.shared, diff --git a/Tests/PowerSyncTests/SyncTests.swift b/Tests/PowerSyncTests/SyncTests.swift index 57980f2..bdd9d52 100644 --- a/Tests/PowerSyncTests/SyncTests.swift +++ b/Tests/PowerSyncTests/SyncTests.swift @@ -673,7 +673,8 @@ let defaultSchema = Schema(tables: [ private func openDatabase(_ client: any HttpClient, schema: Schema = defaultSchema, logger: any LoggerProtocol = DefaultLogger()) -> PowerSyncDatabaseProtocol { return PowerSyncDatabaseImpl( - dbFilename: ":memory:", + identifier: ":memory:", + activeInstanceStore: DatabaseGroupCollection(), logger: logger, pool: AsyncConnectionPool(location: .inMemory), httpClient: client, From 1808f1c35b9ee0d167dd8d2a4a29eb2099ca6ae3 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 27 Apr 2026 15:45:44 +0200 Subject: [PATCH 29/40] Fix a bunch of AsyncSemaphore bugs --- Sources/PowerSync/Utils/AsyncMutex.swift | 190 ++++++++++++------ Sources/PowerSync/Utils/Mutex.swift | 2 +- .../Utils/AsyncMutexTests.swift | 53 +++-- 3 files changed, 171 insertions(+), 74 deletions(-) diff --git a/Sources/PowerSync/Utils/AsyncMutex.swift b/Sources/PowerSync/Utils/AsyncMutex.swift index cedad85..9804c48 100644 --- a/Sources/PowerSync/Utils/AsyncMutex.swift +++ b/Sources/PowerSync/Utils/AsyncMutex.swift @@ -1,7 +1,7 @@ import BasicContainers import DequeModule +import Foundation -/// A simple async mutex implemented through actors. /// An asynchronous mutex implemented as a simple actor. actor AsyncMutex { var inner: T @@ -20,15 +20,17 @@ actor AsyncMutex { /// Heavily inspired from https://github.com/powersync-ja/powersync-js/blob/main/packages/common/src/utils/mutex.ts. final class AsyncSemaphore: Sendable { let count: Int - private let state: Mutex> + fileprivate let state: Mutex> + /// Creates a semaphore by consuming a queue of items. init (_ values: consuming RigidDeque) { self.count = values.count state = Mutex(SemaphoreState( available: values )) } - + + /// Creates a semaphore from a single owned item. convenience init(singleElement: consuming T) { var queue = RigidDeque(capacity: 1) queue.append(singleElement) @@ -42,35 +44,19 @@ final class AsyncSemaphore: Sendable { } } } - - func acquire(count: Int) async throws -> SemaphoreGrant { + + /// Acquires a flexible amount of items from this semaphore. + func acquire(count: Int) async throws(CancellationError) -> SemaphoreGrant { precondition(count > 0 && count <= self.count) - try Task.checkCancellation() - - let waiter = Mutex(nil) - try await withTaskCancellationHandler(operation: { - try await withCheckedThrowingContinuation { continuation in - let node = state.withLock { state in - state.addWaiter(requestedItems: count, continuation: continuation) - } - - waiter.withLock { $0 = node } - } - }, onCancel: { - if let waiter = waiter.withLock({ $0 }) { - state.withLock { state in - state.abortWaiter(waiter: waiter) - } - } - }) - - let node = waiter.withLock { $0! } - let items: RigidArray? = node.consumeItems() - if let items, node.isFull { - return SemaphoreGrant(semaphore: self, items: items) - } else { + do { + try Task.checkCancellation() + } catch { + // As per checkCancellation() docs, the method exclusively throws cancellation errors. throw CancellationError() } + + let node = TypedWaitNode(semaphore: self) + return try await node.acquire(count) } } @@ -84,6 +70,9 @@ extension AsyncSemaphore where T: Copyable { } } +/// A grant to a resource in an ``AsyncSemaphore``. +/// +/// The grant is automatically returned when this struct goes out of scope. struct SemaphoreGrant: ~Copyable { private let semaphore: AsyncSemaphore var acquiredItems: RigidArray @@ -101,31 +90,31 @@ struct SemaphoreGrant: ~Copyable { private struct SemaphoreState: ~Copyable { // Available items that are not currently assigned to a waiter. var available: RigidDeque - var firstWaiter: SemaphoreWaitNode? - var lastWaiter: SemaphoreWaitNode? + // Wait nodes are guaranteed to outlive these references because we call deactiveWaiter before + // it gets freed. + unowned var firstWaiter: SemaphoreWaitNode? + unowned var lastWaiter: SemaphoreWaitNode? var size: Int { available.capacity } - + deinit { - // Clean up reference cycle in double-linked list. - var currentNode = firstWaiter - while let node = currentNode { - currentNode = node.next - node.next = nil - node.prev = nil - } + // This being called implies that the AsyncSemaphore is deallocated, which in turn implies + // that no pending waiters reference it. So the list of waiters must be empty. + assert(firstWaiter == nil) + assert(lastWaiter == nil) } private mutating func deactivateWaiter(waiter: SemaphoreWaitNode) { if !waiter.isActive { return } - - waiter.isActive = false + let prev = waiter.prev let next = waiter.next + waiter.prev = nil + waiter.next = nil if let prev { prev.next = next @@ -133,14 +122,15 @@ private struct SemaphoreState: ~Copyable { if let next { next.prev = prev } - + if waiter === firstWaiter { firstWaiter = next } if waiter === lastWaiter { lastWaiter = prev } - + + waiter.isActive = false waiter.continuation.resume(returning: ()) } @@ -156,13 +146,13 @@ private struct SemaphoreState: ~Copyable { available.append(item) } } - + mutating func returnItems(items: consuming RigidArray) { while !items.isEmpty { returnItem(item: items.removeLast()) } } - + mutating func abortWaiter(waiter: SemaphoreWaitNode) { let items: RigidArray? = waiter.consumeItems() deactivateWaiter(waiter: waiter) @@ -170,24 +160,25 @@ private struct SemaphoreState: ~Copyable { returnItems(items: items) } } - - mutating func addWaiter(requestedItems: Int, continuation: CheckedContinuation<(), any Error>) -> SemaphoreWaitNode { + + mutating func addWaiter(requestedItems: Int, continuation: CheckedContinuation<(), Never>) -> SemaphoreWaitNode { let node = SemaphoreWaitNode(requestedItems: requestedItems, continuation: continuation) if let lastWaiter { lastWaiter.next = node + node.prev = lastWaiter self.lastWaiter = node } else { // First waiter firstWaiter = node lastWaiter = node } - + // If there are items in the pool that haven't been assigned, we can pull them into this waiter. Note that this is // only the case if we're the first waiter (otherwise, items would have been assigned to an earlier waiter). while !available.isEmpty && !node.isFull { node.pushItem(item: available.removeFirst()) } - + if node.isFull { self.deactivateWaiter(waiter: node) } @@ -197,27 +188,36 @@ private struct SemaphoreState: ~Copyable { // This isn't actually sendable, but we don't use it concurrently: While waiting, it's only mutated with a lock on SemaphoreState. // Afterwards, it's sent to acquire() where it's only used in a single async context. +// +// This class is not generic: Making it generic (with `T: ~Copyable`) causes compiler errors because older runtimes can't compare +// objects with non-copyable type parameters by identity. private final class SemaphoreWaitNode: @unchecked Sendable { let requestedItems: Int var acquiredItems: Int - var itemsBuffer: UnsafeMutableRawPointer? // pointer to [T; requestedItems] - var continuation: CheckedContinuation<(), any Error> + // pointer to [T; requestedItems]. Note that the region from acquiredItems..requestItems is uninitialized + var itemsBuffer: UnsafeMutableRawPointer? + var continuation: CheckedContinuation<(), Never> var isActive = true - var prev: SemaphoreWaitNode? - var next: SemaphoreWaitNode? - init(requestedItems: Int, continuation: CheckedContinuation<(), any Error>) { + // Wait nodes are owned by a waiter. The only way for these to get dropped is by removing them + // from the linked list, which also unsets prev/next on adjacent nodes. + // This invariant is checked by also setting isActive = false when a wait node is disposed properly. + // In deinit, we crash if isActive is still set. + unowned var prev: SemaphoreWaitNode? + unowned var next: SemaphoreWaitNode? + + init(requestedItems: Int, continuation: CheckedContinuation<(), Never>) { self.requestedItems = requestedItems self.continuation = continuation self.acquiredItems = 0 } - + var isFull: Bool { acquiredItems == requestedItems } - + func pushItem(item: consuming T) { - precondition(!isFull) + precondition(!isFull && isActive) if let items = itemsBuffer { items.assumingMemoryBound(to: T.self).advanced(by: acquiredItems).initialize(to: item) @@ -244,8 +244,80 @@ private final class SemaphoreWaitNode: @unchecked Sendable { return nil } } - + deinit { - precondition(itemsBuffer == nil, "Wait node leaked items buffer") + assert(!isActive, "Wait node was dropped while active") + assert(itemsBuffer == nil, "Wait node leaked items") + assert(prev == nil && next == nil, "Wait node should be unlinked") + } +} + +private struct TypedWaitNode: Sendable, ~Copyable { + private enum TypedWaitNodeState: ~Copyable { + case hasWaiter(SemaphoreWaitNode) + case cancelled + } + + private let inner: Mutex = Mutex(nil) + private let semaphore: AsyncSemaphore + + init(semaphore: AsyncSemaphore) { + self.semaphore = semaphore + } + + /// Adds a wait node to the semaphore and waits for a grant or that node to be aborted. + private func acquireInternal(count: Int) async { + await withTaskCancellationHandler(operation: { + await withCheckedContinuation { continuation in + inner.withLock { state in + if state != nil { + continuation.resume() + return + } + + let waiter = semaphore.state.withLock { state in + state.addWaiter(requestedItems: count, continuation: continuation) + } + state = .hasWaiter(waiter) + } + } + }, onCancel: { + inner.withLock { state in + if case let .hasWaiter(waiter) = state { + semaphore.state.withLock { state in + state.abortWaiter(waiter: waiter) + } + } + state = .cancelled + } + }) + } + + consuming func acquire(_ count: Int) async throws(CancellationError) -> SemaphoreGrant { + await self.acquireInternal(count: count) + + var didComplete = false + let items = inner.withLock { state in + switch state! { + case .hasWaiter(let node): + assert(!node.isActive) + let items: RigidArray? = node.consumeItems() + didComplete = node.isFull + return items + case .cancelled: + return nil + } + } + + if let items { + if didComplete { + return SemaphoreGrant(semaphore: semaphore, items: items) + } + + // We were able to obtain some items before aborting the read. Return those now. + semaphore.returnItems(items: items) + } + + throw CancellationError() } } diff --git a/Sources/PowerSync/Utils/Mutex.swift b/Sources/PowerSync/Utils/Mutex.swift index d311317..80ff293 100644 --- a/Sources/PowerSync/Utils/Mutex.swift +++ b/Sources/PowerSync/Utils/Mutex.swift @@ -19,7 +19,7 @@ struct Mutex: @unchecked Sendable, ~Copyable { self.value.deallocate() } - func withLock(_ action: (_ value: inout T) throws -> R) rethrows -> R { + func withLock(_ action: (_ value: inout T) throws -> R) rethrows -> R { os_unfair_lock_lock(self.osLock) defer { os_unfair_lock_unlock(self.osLock) } diff --git a/Tests/PowerSyncTests/Utils/AsyncMutexTests.swift b/Tests/PowerSyncTests/Utils/AsyncMutexTests.swift index f72fc5f..649dec9 100644 --- a/Tests/PowerSyncTests/Utils/AsyncMutexTests.swift +++ b/Tests/PowerSyncTests/Utils/AsyncMutexTests.swift @@ -5,11 +5,11 @@ import Testing struct AsyncSemaphoreTests { @Test func dispatchesItemsInOrder() async throws { let semaphore = AsyncSemaphore(from: ["a", "b", "c"]) - + let grant1 = try await semaphore.acquire(count: 1) let grant2 = try await semaphore.acquire(count: 1) let grant3 = try await semaphore.acquire(count: 1) - + try #require(grant1.acquiredItems[0] == "a") try #require(grant2.acquiredItems[0] == "b") try #require(grant3.acquiredItems[0] == "c") @@ -17,7 +17,7 @@ struct AsyncSemaphoreTests { @Test @MainActor func returnsReleasedItemsToWaiters() async throws { let semaphore = AsyncSemaphore(from: ["x"]) - + let grant1 = try await semaphore.acquire(count: 1) var hasSecond = false @@ -37,27 +37,27 @@ struct AsyncSemaphoreTests { let semaphore = AsyncSemaphore(from: ["a", "b", "c"]) let grant1 = try await semaphore.acquire(count: 1) let grant2 = try await semaphore.acquire(count: 1) - + var hasAll = false let acquireAllTask = Task { let _ = try await semaphore.acquire(count: 3) hasAll = false } - + await Task.yield() try #require(!hasAll) - + let _ = consume grant1 await Task.yield() try #require(!hasAll) // Still waiting for item2 - + let _ = consume grant2 let _ = await acquireAllTask.result } @Test func canReturnMultiple() async throws { let semaphore = AsyncSemaphore(from: ["a", "b"]) - + let grantAll = try await semaphore.acquire(count: 2) let hasOther = Task { @@ -68,19 +68,19 @@ struct AsyncSemaphoreTests { try #require(anotherGrant.acquiredItems[0] == "a") return true } - + let _ = consume grantAll let _ = try await hasOther.value } @Test func canAbort() async throws { let semaphore = AsyncSemaphore(from: ["a"]) - + let grant1 = try await semaphore.acquire(count: 1) - let second = Task { await #expect(throws: CancellationError.self) { let _ = try await semaphore.acquire(count: 1) + print("has items") } } let third = Task { @@ -88,11 +88,36 @@ struct AsyncSemaphoreTests { try #require(grant.acquiredItems[0] == "a") return } - + + await Task.yield() + second.cancel() + let _ = await second.result + + let _ = consume grant1 + try (await third.result).get() + } + + @Test func canAbortPartial() async throws { + let semaphore = AsyncSemaphore(from: ["a", "b"]) + let grant1 = try await semaphore.acquire(count: 1) + let second = Task { + await #expect(throws: CancellationError.self) { + let _ = try await semaphore.acquire(count: 2) + } + } + let third = Task { + let grant = try await semaphore.acquire(count: 2) + try #require(grant.acquiredItems[0] == "b") + try #require(grant.acquiredItems[1] == "a") + return + } + await Task.yield() + // At this point, second obtained value b from the semaphore, but it's still waiting + // for the second node. Aborting it should b into the semaphore. second.cancel() - + let _ = await second.result let _ = consume grant1 - let _ = await (second.result, third.result) + try (await third.result).get() } } From b99fcd25f57b1e40f09caaac18b2032ad92ea942 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 27 Apr 2026 18:06:16 +0200 Subject: [PATCH 30/40] AI feedback --- CHANGELOG.md | 1 + .../Implementation/ActiveInstanceStore.swift | 2 +- .../Implementation/AsyncConnectionPool.swift | 17 +++-- .../PowerSyncDatabaseImpl.swift | 4 +- .../queries/ConnectionPoolQueries.swift | 5 +- .../Implementation/queries/LeaseContext.swift | 10 +-- .../sqlite3/NativeConnectionPool.swift | 42 +++++++---- .../sqlite3/NativeStatement.swift | 6 +- .../sync/StreamingSyncClient.swift | 4 -- Sources/PowerSync/PowerSyncDatabase.swift | 2 +- ...Sequence.swift => MergeItemSequence.swift} | 62 ++++++++-------- Sources/PowerSync/Utils/sleepForSeconds.swift | 9 +++ Sources/PowerSync/Utils/withSession.swift | 51 ------------- .../Kotlin/ActiveInstanceStoreTests.swift | 4 +- .../KotlinPowerSyncDatabaseImplTests.swift | 22 ++++++ .../Kotlin/SqlCursorTests.swift | 2 +- Tests/PowerSyncTests/SyncTests.swift | 2 +- .../Utils/MergeItemSequence.swift | 71 +++++++++++++++++++ 18 files changed, 188 insertions(+), 128 deletions(-) rename Sources/PowerSync/Utils/{ThrottledAsyncSequence.swift => MergeItemSequence.swift} (66%) create mode 100644 Sources/PowerSync/Utils/sleepForSeconds.swift delete mode 100644 Sources/PowerSync/Utils/withSession.swift create mode 100644 Tests/PowerSyncTests/Utils/MergeItemSequence.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 4244c41..93b2856 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * Add `opDataTyped` and `previousValuesTyped` to `CrudEntry`, providing typed values instead of strings. * Make `CrudBatch`, `CrudEntry` and `CrudTransaction` a concrete struct. Note that these can no longer be created in user code. +* Remove the internal `withSession` API. ## 1.13.1 diff --git a/Sources/PowerSync/Implementation/ActiveInstanceStore.swift b/Sources/PowerSync/Implementation/ActiveInstanceStore.swift index ab26782..5358f8f 100644 --- a/Sources/PowerSync/Implementation/ActiveInstanceStore.swift +++ b/Sources/PowerSync/Implementation/ActiveInstanceStore.swift @@ -40,7 +40,7 @@ private final class ActiveDatabaseGroupData: Sendable { /// A collection of PowerSync databases with the same path / identifier. /// /// We expect that each group will only ever have one database because we encourage users to write their databases as -/// singletons. We print a warning when two databasees are part of the same group. +/// singletons. We print a warning when two databases are part of the same group. /// Additionally, we want to avoid two databases in the same group having a sync stream open at the same time to avoid /// duplicate resources being used. For this reason, each active database group has a single sync coordinator actor /// responsible for initializing the sync process for all databases in the group. diff --git a/Sources/PowerSync/Implementation/AsyncConnectionPool.swift b/Sources/PowerSync/Implementation/AsyncConnectionPool.swift index 1b3fee7..387d88e 100644 --- a/Sources/PowerSync/Implementation/AsyncConnectionPool.swift +++ b/Sources/PowerSync/Implementation/AsyncConnectionPool.swift @@ -59,11 +59,13 @@ enum DatabaseLocation { final class AsyncConnectionPool: SQLiteConnectionPoolProtocol { private let location: DatabaseLocation private let initialStatements: [String] + private let logger: any LoggerProtocol private let tableUpdatesStream = BroadcastStream>() private let inner: AsyncSemaphore = AsyncSemaphore(singleElement: nil) - init(location: DatabaseLocation, initialStatements: [String] = []) { + init(location: DatabaseLocation, logger: any LoggerProtocol, initialStatements: [String] = []) { self.location = location + self.logger = logger self.initialStatements = initialStatements } @@ -84,17 +86,18 @@ final class AsyncConnectionPool: SQLiteConnectionPoolProtocol { for stmt in initialStatements { let _ = try context.execute(sql: stmt, parameters: []) } - + if isWriter { let _ = try context.execute(sql: "pragma journal_mode = WAL", parameters: []) } else { + // This is mainly an additional safety element, we also open read connections SQLITE_READONLY. let _ = try context.execute(sql: "pragma query_only = TRUE", parameters: []) } - + let _ = try context.execute(sql: "pragma journal_size_limit = \(6 * 1024 * 1024)", parameters: []) let _ = try context.execute(sql: "pragma busy_timeout = 30000", parameters: []) let _ = try context.execute(sql: "pragma cache_size = -\(50 * 1024)", parameters: []) - + if isWriter { // Older versions of the SDK used to set up an empty schema and raise the user version to 1. // Keep doing that for consistency. @@ -104,7 +107,7 @@ final class AsyncConnectionPool: SQLiteConnectionPoolProtocol { if let version, version < 1 { let _ = try context.execute(sql: "pragma user_version = 1", parameters: []) } - + let _ = try context.execute(sql: "select powersync_update_hooks('install')", parameters: []) } } @@ -125,7 +128,7 @@ final class AsyncConnectionPool: SQLiteConnectionPoolProtocol { try configureConnection(connection: writer, isWriter: true) if case .inMemory = location { - return NativeConnectionPool(singleConnection: writer, handleUpdates: handleUpdates) + return NativeConnectionPool(singleConnection: writer, logger: logger, handleUpdates: handleUpdates) } else { let numReaders = 4 var readers = RigidDeque(capacity: numReaders) @@ -135,7 +138,7 @@ final class AsyncConnectionPool: SQLiteConnectionPoolProtocol { readers.append(connection) } - return NativeConnectionPool(writer: writer, readers: readers, handleUpdates: handleUpdates) + return NativeConnectionPool(writer: writer, readers: readers, logger: logger, handleUpdates: handleUpdates) } } diff --git a/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift b/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift index 3186e32..45307a0 100644 --- a/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift +++ b/Sources/PowerSync/Implementation/PowerSyncDatabaseImpl.swift @@ -7,7 +7,7 @@ final class PowerSyncDatabaseImpl: PowerSyncDatabaseProtocol { let syncStatus = SwiftSyncStatus() private let dbFilename: String? private let httpClient: HttpClient - private let initializer = DatabaseInitizalizationActor() + private let initializer = DatabaseInitializationAction() fileprivate let queries: ConnectionPoolQueries let schema: AsyncMutex @@ -165,7 +165,7 @@ final class PowerSyncDatabaseImpl: PowerSyncDatabaseProtocol { static let maxOpId = Int64.max } -private actor DatabaseInitizalizationActor { +private actor DatabaseInitializationAction { private var isInitialized = false var powerSyncVersion: String? private var closed = false diff --git a/Sources/PowerSync/Implementation/queries/ConnectionPoolQueries.swift b/Sources/PowerSync/Implementation/queries/ConnectionPoolQueries.swift index d9ec650..6c507e0 100644 --- a/Sources/PowerSync/Implementation/queries/ConnectionPoolQueries.swift +++ b/Sources/PowerSync/Implementation/queries/ConnectionPoolQueries.swift @@ -40,9 +40,9 @@ struct ConnectionPoolQueries: Sendable, Queries { }.map { _ in () } // Allows emitting the first result even if there aren't changes let withInitial = AsyncAlgorithms.merge([()].async, updateNotifications) - let throttled = AsyncThrottleSequence(inner: withInitial, duration: options.throttle) + let merged = MergeItemSequence(inner: withInitial) - for try await _ in throttled { + for try await _ in merged { // Check if the outer task is cancelled try Task.checkCancellation() @@ -51,6 +51,7 @@ struct ConnectionPoolQueries: Sendable, Queries { parameters: options.parameters, mapper: options.mapper )) + try await sleepForSeconds(seconds: options.throttle) } continuation.finish() diff --git a/Sources/PowerSync/Implementation/queries/LeaseContext.swift b/Sources/PowerSync/Implementation/queries/LeaseContext.swift index 29f0958..7947943 100644 --- a/Sources/PowerSync/Implementation/queries/LeaseContext.swift +++ b/Sources/PowerSync/Implementation/queries/LeaseContext.swift @@ -7,7 +7,7 @@ final class ConnectionLeaseContext: ConnectionContext { } /// Maps any parameter array to typed SQLite values. - private func mapMarameters(_ parameters: [(any Sendable)?]?) throws -> [PowerSyncDataType?] { + private func mapParameters(_ parameters: [(any Sendable)?]?) throws -> [PowerSyncDataType?] { guard let parameters else { return [] } @@ -25,13 +25,13 @@ final class ConnectionLeaseContext: ConnectionContext { func execute(sql: String, parameters: [(any Sendable)?]?) throws -> Int64 { try lease.withLock { lease in - return try lease.execute(sql: sql, parameters: mapMarameters(parameters)) + return try lease.execute(sql: sql, parameters: mapParameters(parameters)) } } func getOptional(sql: String, parameters: [(any Sendable)?]?, mapper: @escaping @Sendable (any SqlCursor) throws -> RowType) throws -> RowType? { try lease.withLock { lease in - try lease.withIterator(sql: sql, parameters: mapMarameters(parameters)) { rows in + try lease.withIterator(sql: sql, parameters: mapParameters(parameters)) { rows in return try rows.next(callback: mapper) } } @@ -39,7 +39,7 @@ final class ConnectionLeaseContext: ConnectionContext { func getAll(sql: String, parameters: [(any Sendable)?]?, mapper: @escaping @Sendable (any SqlCursor) throws -> RowType) throws -> [RowType] { try lease.withLock { lease in - try lease.withIterator(sql: sql, parameters: mapMarameters(parameters)) { rows in + try lease.withIterator(sql: sql, parameters: mapParameters(parameters)) { rows in var result: [RowType] = [] while let row = try rows.next(callback: mapper) { result.append(row) @@ -51,7 +51,7 @@ final class ConnectionLeaseContext: ConnectionContext { func get(sql: String, parameters: [(any Sendable)?]?, mapper: @escaping @Sendable (any SqlCursor) throws -> RowType) throws -> RowType { try lease.withLock { lease in - try lease.withIterator(sql: sql, parameters: mapMarameters(parameters)) { rows in + try lease.withIterator(sql: sql, parameters: mapParameters(parameters)) { rows in guard let cursor = try rows.next(callback: mapper) else { throw PowerSyncError.operationFailed(message: "Expected \(sql) to return a row, but got an empty result set.") } diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift index c92a177..aad7062 100644 --- a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift +++ b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift @@ -9,29 +9,45 @@ final class NativeConnectionPool: Sendable { private let writer: AsyncSemaphore private let readers: AsyncSemaphore? private let handleUpdates: @Sendable (_: Set) -> () + private let logger: any LoggerProtocol - init(writer: consuming RawSqliteConnection, readers: consuming RigidDeque, handleUpdates: @escaping @Sendable (_: Set) -> ()) { + init( + writer: consuming RawSqliteConnection, + readers: consuming RigidDeque, + logger: any LoggerProtocol, + handleUpdates: @escaping @Sendable (_: Set) -> (), + ) { self.writer = AsyncSemaphore(singleElement: writer) self.readers = AsyncSemaphore(readers) self.handleUpdates = handleUpdates + self.logger = logger } - init(singleConnection: consuming RawSqliteConnection, handleUpdates: @escaping @Sendable (_: Set) -> ()) { + init( + singleConnection: consuming RawSqliteConnection, + logger: any LoggerProtocol, + handleUpdates: @escaping @Sendable (_: Set) -> (), + ) { self.writer = AsyncSemaphore(singleElement: singleConnection) self.readers = nil self.handleUpdates = handleUpdates + self.logger = logger } - private func dispatchWrites(lease: NativeConnectionLease) throws { - try lease.withIterator(sql: "SELECT powersync_update_hooks('get')", parameters: []) { rows in - let affectedTables = try rows.next { - let decoder = JSONDecoder() - return try decoder.decode(Set.self, from: try $0.getString(index: 0).data(using: .utf8)!) - } + private func dispatchWrites(lease: NativeConnectionLease) { + do { + try lease.withIterator(sql: "SELECT powersync_update_hooks('get')", parameters: []) { rows in + let affectedTables = try rows.next { + let decoder = JSONDecoder() + return try decoder.decode(Set.self, from: try $0.getString(index: 0).data(using: .utf8)!) + } - if let affectedTables, !affectedTables.isEmpty { - self.handleUpdates(affectedTables) + if let affectedTables, !affectedTables.isEmpty { + self.handleUpdates(affectedTables) + } } + } catch { + logger.warning("Could not read affected tables", tag: "NativeConnectionPool") } } @@ -46,14 +62,16 @@ final class NativeConnectionPool: Sendable { func write(onConnection: (NativeConnectionLease) async throws -> T) async throws -> T { let connection = try await writer.acquire(count: 1) let lease = connection.acquiredItems[0].asLease() + defer { dispatchWrites(lease: lease) } let result = try await onConnection(lease) - try dispatchWrites(lease: lease) return result } func withAllConnections(onConnection: (NativeConnectionLease, [NativeConnectionLease]) async throws -> T) async throws -> T{ let write = try await writer.acquire(count: 1) let writeLease = write.acquiredItems[0].asLease() + defer { dispatchWrites(lease: writeLease) } + let result: T if let readers { let acquiredReaders = try await readers.acquire(count: readers.count) @@ -67,8 +85,6 @@ final class NativeConnectionPool: Sendable { } else { result = try await onConnection(writeLease, []) } - - try dispatchWrites(lease: writeLease) return result } diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift b/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift index 36ad7f0..f1ce944 100644 --- a/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift +++ b/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift @@ -91,12 +91,8 @@ struct NativeSqliteStatement: ~Copyable { Int32(value.count), free, ) - - if rc != 0 { - free(buffer) - } } - + if rc != 0 { try throwDatabaseError(db: self.db, sql: self.sql) } diff --git a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift index 9cff49f..ca52d3c 100644 --- a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift +++ b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift @@ -444,7 +444,3 @@ struct WriteCheckpointData: Codable { struct WriteCheckpointResponse: Codable { let data: WriteCheckpointData } - -private func sleepForSeconds(seconds: TimeInterval) async throws { - try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000)) -} diff --git a/Sources/PowerSync/PowerSyncDatabase.swift b/Sources/PowerSync/PowerSyncDatabase.swift index b849d59..034b4e9 100644 --- a/Sources/PowerSync/PowerSyncDatabase.swift +++ b/Sources/PowerSync/PowerSyncDatabase.swift @@ -21,7 +21,7 @@ public func PowerSyncDatabase( } else { (DatabaseLocation.inDefaultDirectory(name: dbFilename), .shared) } - let pool = AsyncConnectionPool(location: location, initialStatements: initialStatements) + let pool = AsyncConnectionPool(location: location, logger: logger, initialStatements: initialStatements) return PowerSyncDatabaseImpl( dbFilename: dbFilename, identifier: dbFilename, diff --git a/Sources/PowerSync/Utils/ThrottledAsyncSequence.swift b/Sources/PowerSync/Utils/MergeItemSequence.swift similarity index 66% rename from Sources/PowerSync/Utils/ThrottledAsyncSequence.swift rename to Sources/PowerSync/Utils/MergeItemSequence.swift index 732e587..63e8e91 100644 --- a/Sources/PowerSync/Utils/ThrottledAsyncSequence.swift +++ b/Sources/PowerSync/Utils/MergeItemSequence.swift @@ -1,57 +1,53 @@ -import Foundation - -// Throttled async sequences that drop events emitted during a timeout. -// Inspired from https://github.com/apple/swift-async-algorithms/blob/main/Sources/AsyncAlgorithms/AsyncThrottleSequence.swift, -// but with changes to support older platforms. -struct AsyncThrottleSequence: AsyncSequence where Base.Element == () { +/// An ``AsyncSequence`` merging all items emitted between ``AsyncIteratorProtocol/next``. +struct MergeItemSequence: AsyncSequence where Base.Element == () { typealias AsyncIterator = IteratorImpl typealias Element = () private let inner: Base - private let duration: TimeInterval - - init(inner: Base, duration: TimeInterval) { + + init(inner: Base) { self.inner = inner - self.duration = duration } - + func makeAsyncIterator() -> IteratorImpl { - IteratorImpl(duration: duration, inner: inner) + IteratorImpl(inner: self.inner) } - + + private final class IteratorState: Sendable { + let inner = Mutex(MergeSequenceState.idle) + } + final class IteratorImpl: AsyncIteratorProtocol, Sendable { - fileprivate let duration: TimeInterval - private let state: LockedThrottleSequenceState + private let state: IteratorState let pollTask: Task<(), any Error> - init(duration: TimeInterval, inner: Base) { - self.duration = duration - let state = LockedThrottleSequenceState() + init(inner: Base) { + let state = IteratorState() self.pollTask = Task { - defer { state.state.withLock { $0.transitionToDone() } } - + defer { state.inner.withLock { $0.transitionToDone() } } + do { for try await event in inner { - state.state.withLock { $0.markHasEvent(event: .success(event)) } + state.inner.withLock { $0.markHasEvent(event: .success(event)) } } } catch { - state.state.withLock { $0.markHasEvent(event: .failure(error)) } + state.inner.withLock { $0.markHasEvent(event: .failure(error)) } } } - + self.state = state } - + func next() async throws -> ()? { try await withTaskCancellationHandler( operation: { try await withCheckedThrowingContinuation { continuation in - state.state.withLock { $0.registerListener(continuation) } + state.inner.withLock { $0.registerListener(continuation) } } }, onCancel: { pollTask.cancel() - state.state.withLock { + state.inner.withLock { if case .waitingForUpstream(let continuation) = $0 { continuation.resume(returning: nil) } @@ -60,14 +56,14 @@ struct AsyncThrottleSequence: AsyncSequence wher } ) } + + deinit { + self.pollTask.cancel() + } } } -private final class LockedThrottleSequenceState: Sendable { - let state = Mutex(ThrottleSequenceState.idle) -} - -private enum ThrottleSequenceState { +private enum MergeSequenceState { /// No one waiting on next(), no pending emit either. case idle /// We're waiting in next() for an upstream emission. @@ -89,7 +85,7 @@ private enum ThrottleSequenceState { continuation.resume(returning: nil) } } - + mutating func markHasEvent(event: Result<(), any Error>) { if case let .waitingForUpstream(continuation) = self { continuation.resume(with: event.map { _ in () }) @@ -98,7 +94,7 @@ private enum ThrottleSequenceState { self = .hasPendingEvent(event) } } - + mutating func transitionToDone() { if case let .waitingForUpstream(continuation) = self { continuation.resume(returning: nil) diff --git a/Sources/PowerSync/Utils/sleepForSeconds.swift b/Sources/PowerSync/Utils/sleepForSeconds.swift new file mode 100644 index 0000000..5b8d39c --- /dev/null +++ b/Sources/PowerSync/Utils/sleepForSeconds.swift @@ -0,0 +1,9 @@ +import Foundation + +func sleepForSeconds(seconds: TimeInterval) async throws { + if #available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) { + try await Task.sleep(for: .seconds(seconds)) + } else { + try await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000)) + } +} diff --git a/Sources/PowerSync/Utils/withSession.swift b/Sources/PowerSync/Utils/withSession.swift deleted file mode 100644 index c2c2467..0000000 --- a/Sources/PowerSync/Utils/withSession.swift +++ /dev/null @@ -1,51 +0,0 @@ -import Foundation - -public struct WithSessionResult: Sendable { - public let blockResult: Result - public let affectedTables: Set -} - -/// Executes an action within a SQLite database connection session and -/// returns a `WithSessionResult` containing the action result and the set of -/// tables that were affected during the session. -/// -/// The raw SQLite connection is only available in some niche scenarios. This helper is -/// intended for internal use. -/// -/// - The provided `action` is executed inside a database session. -/// - Any success or failure from the `action` is captured in -/// `WithSessionResult.blockResult`. -/// - The set of updated table names is returned in -/// `WithSessionResult.affectedTables`. -/// -/// Example usage: -/// ```swift -/// let result = try withSession(db: database) { -/// // perform database work and return a value -/// try someOperation() -/// } -/// -/// switch result.blockResult { -/// case .success(let value): -/// print("Operation succeeded with: \(value)") -/// case .failure(let error): -/// print("Operation failed: \(error)") -/// } -/// -/// print("Updated tables: \(result.affectedTables)") -/// ``` -/// -/// - Parameters: -/// - db: The raw SQLite connection pointer used to open the session. -/// - action: The operation to execute within the session. Its return value is -/// propagated into `WithSessionResult.blockResult` on success. -/// - Returns: A `WithSessionResult` containing the action's result and the set -/// of affected table names. -/// - Throws: Any error thrown while establishing the session or executing the -/// provided `action`. -public func withSession( - db: OpaquePointer, - action: @escaping () throws -> ReturnType -) throws -> WithSessionResult { - fatalError("todo") -} diff --git a/Tests/PowerSyncTests/Kotlin/ActiveInstanceStoreTests.swift b/Tests/PowerSyncTests/Kotlin/ActiveInstanceStoreTests.swift index 9f7f408..30aedc3 100644 --- a/Tests/PowerSyncTests/Kotlin/ActiveInstanceStoreTests.swift +++ b/Tests/PowerSyncTests/Kotlin/ActiveInstanceStoreTests.swift @@ -4,7 +4,7 @@ import Testing @Suite struct MultipleInstanceTest { @Test func warnsAboutMultipleInstances() async throws { - let pool = AsyncConnectionPool(location: .inMemory) + let pool = AsyncConnectionPool(location: .inMemory, logger: DefaultLogger()) let logWriter = TestLogWriterAdapter() let logger = DefaultLogger(minSeverity: .warning, writers: [logWriter]) let schema = Schema() @@ -21,7 +21,7 @@ struct MultipleInstanceTest { } @Test func doesNotWarnForClosedInstances() async throws { - let pool = AsyncConnectionPool(location: .inMemory) + let pool = AsyncConnectionPool(location: .inMemory, logger: DefaultLogger()) let logWriter = TestLogWriterAdapter() let logger = DefaultLogger(minSeverity: .warning, writers: [logWriter]) let schema = Schema() diff --git a/Tests/PowerSyncTests/Kotlin/KotlinPowerSyncDatabaseImplTests.swift b/Tests/PowerSyncTests/Kotlin/KotlinPowerSyncDatabaseImplTests.swift index c9b0a6c..6ee95f5 100644 --- a/Tests/PowerSyncTests/Kotlin/KotlinPowerSyncDatabaseImplTests.swift +++ b/Tests/PowerSyncTests/Kotlin/KotlinPowerSyncDatabaseImplTests.swift @@ -419,6 +419,28 @@ final class KotlinPowerSyncDatabaseImplTests: XCTestCase { XCTAssertEqual(result, 0) } + func testFailedWritesStillEmitUpdateNotifications() async throws { + var query = try database.watch("SELECT name FROM users") { try $0.getString(index: 0) }.makeAsyncIterator() + do { + let rows = try await query.next() + XCTAssertEqual(rows, []) + } + + do { + try await database.writeLock { ctx in + try ctx.execute(sql: "INSERT INTO users (id, name) VALUES (uuid(), ?)", parameters: ["test user"]) + throw PowerSyncError.operationFailed(message: "deliberate error from test") + } + } catch { + // Expected + } + + do { + let rows = try await query.next() + XCTAssertEqual(rows, ["test user"]) + } + } + func testReadTransaction() async throws { _ = try await database.execute( sql: "INSERT INTO users (id, name, email) VALUES (?, ?, ?)", diff --git a/Tests/PowerSyncTests/Kotlin/SqlCursorTests.swift b/Tests/PowerSyncTests/Kotlin/SqlCursorTests.swift index 574db98..b88dd50 100644 --- a/Tests/PowerSyncTests/Kotlin/SqlCursorTests.swift +++ b/Tests/PowerSyncTests/Kotlin/SqlCursorTests.swift @@ -69,7 +69,7 @@ final class SqlCursorTests: XCTestCase { identifier: ":memory:", activeInstanceStore: DatabaseGroupCollection(), logger: DefaultLogger(), - pool: AsyncConnectionPool(location: .inMemory), + pool: AsyncConnectionPool(location: .inMemory, logger: DefaultLogger()), httpClient: PlatformHttpClient.shared, schema: schema, ) diff --git a/Tests/PowerSyncTests/SyncTests.swift b/Tests/PowerSyncTests/SyncTests.swift index bdd9d52..36e0ff7 100644 --- a/Tests/PowerSyncTests/SyncTests.swift +++ b/Tests/PowerSyncTests/SyncTests.swift @@ -676,7 +676,7 @@ private func openDatabase(_ client: any HttpClient, schema: Schema = defaultSche identifier: ":memory:", activeInstanceStore: DatabaseGroupCollection(), logger: logger, - pool: AsyncConnectionPool(location: .inMemory), + pool: AsyncConnectionPool(location: .inMemory, logger: DefaultLogger()), httpClient: client, schema: schema, ) diff --git a/Tests/PowerSyncTests/Utils/MergeItemSequence.swift b/Tests/PowerSyncTests/Utils/MergeItemSequence.swift new file mode 100644 index 0000000..5ddb2bb --- /dev/null +++ b/Tests/PowerSyncTests/Utils/MergeItemSequence.swift @@ -0,0 +1,71 @@ +import AsyncAlgorithms +@testable import PowerSync +import Testing + +@Suite +struct MergeItemSequenceTest { + private let source = AsyncThrowingChannel<(), any Error>() + + private func generateMerged() -> MergeItemSequence> { + MergeItemSequence(inner: source) + } + + @Test func canReceiveItem() async throws { + let items = generateMerged().makeAsyncIterator() + async let didReceive = items.next() + await source.send(()) + try #require(await didReceive) + } + + @Test @MainActor func mergesItems() async throws { + let items = generateMerged().makeAsyncIterator() + await source.send(()) + await source.send(()) + await source.send(()) + + async let firstItem = items.next() + try #require(await firstItem) + + var hasSecondItem = false + let secondTask = Task { + try #require(await items.next()) + hasSecondItem = true + } + + try #require(!hasSecondItem) + await Task.yield() + try #require(!hasSecondItem) + + await source.send(()) + try await secondTask.value + } + + @Test func reportsErrors() async throws { + let items = generateMerged().makeAsyncIterator() + await #expect(throws: PowerSyncError.self) { + async let firstItem = items.next() + await source.fail(PowerSyncError.operationFailed(message: "error for test")) + + try await firstItem + } + } + + @Test func forwardsClose() async throws { + let items = generateMerged().makeAsyncIterator() + await source.send(()) + try #require(try await items.next()) + source.finish() + try #require(try await items.next() == nil) + try await items.pollTask.value + } + + @Test func closesOnDrop() async throws { + let task: Task + do { + let items = generateMerged().makeAsyncIterator() + task = items.pollTask + } + + try await task.value + } +} From 177433e131efa0865640198f8ab73243e2f5d42b Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 27 Apr 2026 18:10:18 +0200 Subject: [PATCH 31/40] Avoid race in "reportsErrors" test --- Tests/PowerSyncTests/Utils/MergeItemSequence.swift | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Tests/PowerSyncTests/Utils/MergeItemSequence.swift b/Tests/PowerSyncTests/Utils/MergeItemSequence.swift index 5ddb2bb..3c7f005 100644 --- a/Tests/PowerSyncTests/Utils/MergeItemSequence.swift +++ b/Tests/PowerSyncTests/Utils/MergeItemSequence.swift @@ -42,12 +42,8 @@ struct MergeItemSequenceTest { @Test func reportsErrors() async throws { let items = generateMerged().makeAsyncIterator() - await #expect(throws: PowerSyncError.self) { - async let firstItem = items.next() - await source.fail(PowerSyncError.operationFailed(message: "error for test")) - - try await firstItem - } + await source.fail(PowerSyncError.operationFailed(message: "error for test")) + await #expect(throws: PowerSyncError.self) { try await items.next() } } @Test func forwardsClose() async throws { @@ -58,7 +54,7 @@ struct MergeItemSequenceTest { try #require(try await items.next() == nil) try await items.pollTask.value } - + @Test func closesOnDrop() async throws { let task: Task do { From 81c7121031a7dabf748387d0bb2c233f55af93a2 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 27 Apr 2026 20:34:32 +0200 Subject: [PATCH 32/40] Properly use structured concurrency --- .../sync/StreamingSyncClient.swift | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift index b949756..fe541ad 100644 --- a/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift +++ b/Sources/PowerSync/Implementation/sync/StreamingSyncClient.swift @@ -156,11 +156,11 @@ The next upload iteration will be delayed. while (!Task.isCancelled) { do { - // This async let ensures each iteration is a task scoped to this block. This allows us to spawn - // additional tasks in run() that would get cancelled when the main iteration is complete. - async let iteration = ActiveSyncIteration(syncClient: self, signals: signals).run() - - result = try await iteration + try await withThrowingTaskGroup(of: Void.self) { group in + let iteration = ActiveSyncIteration(syncClient: self, signals: signals) + var group: ThrowingTaskGroup? = group + result = try await iteration.run(group: &group) + } } catch { result = SyncIterationResult() @@ -234,7 +234,7 @@ private struct ActiveSyncIteration: Sendable { self.signals = signals } - func run() async throws -> SyncIterationResult { + func run(group: inout ThrowingTaskGroup?) async throws -> SyncIterationResult { // Notify the core extension for changed Sync Stream subscriptions, as we might have to reconnect. async let _ = watchSyncStreams() // Notify the core extension for completed crud uploads, as we might want to retry applying a @@ -256,7 +256,7 @@ private struct ActiveSyncIteration: Sendable { let serviceEvents = try await syncClient.fetchSyncLines(request: request) controlArgs = AsyncAlgorithms.merge(serviceEvents, localEvents.subscribe()) } else { - try await self.execute(instr: instruction) + try await self.execute(instr: instruction, group: &group) } } @@ -273,7 +273,7 @@ private struct ActiveSyncIteration: Sendable { return SyncIterationResult(hideDisconnect: hideDisconnect) } - try await execute(instr: instr) + try await execute(instr: instr, group: &group) } if !hadSyncLine && arg.isSyncLine() { @@ -296,7 +296,9 @@ private struct ActiveSyncIteration: Sendable { return SyncIterationResult(hideDisconnect: hideDisconnect) } - try await execute(instr: instr) + // Don't pass the task group here, stop instructions shouldn't spawn further async work. + var group: ThrowingTaskGroup? = nil + try await execute(instr: instr, group: &group) } return SyncIterationResult() @@ -311,7 +313,7 @@ private struct ActiveSyncIteration: Sendable { return try StreamingSyncClient.jsonDecoder.decode([Instruction].self, from: data) } - private func execute(instr: consuming Instruction) async throws { + private func execute(instr: consuming Instruction, group: inout ThrowingTaskGroup?) async throws { switch (instr) { case .logLine(severity: let severity, line: let line): let logger = syncClient.db.logger @@ -336,7 +338,7 @@ private struct ActiveSyncIteration: Sendable { if didExpire { await syncClient.invalidateCredentials() } else { - Task { + group?.addTask { do { let _ = try await syncClient.connector.fetchCredentials(allowCached: false) syncClient.db.logger.debug("Stopping because new credentials are available", tag: tag) From c13ff323c815d5fd72f43e5977e94067af038905 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 28 Apr 2026 09:16:23 +0200 Subject: [PATCH 33/40] Make cursor extension methods public --- .../project.xcworkspace/xcshareddata/swiftpm/Package.resolved | 4 ++-- Sources/PowerSync/Protocol/db/SqlCursor.swift | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Demos/PowerSyncExample/PowerSyncExample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Demos/PowerSyncExample/PowerSyncExample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 54d11b6..ce2aaa3 100644 --- a/Demos/PowerSyncExample/PowerSyncExample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/Demos/PowerSyncExample/PowerSyncExample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -78,8 +78,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-collections", "state" : { - "revision" : "671108c96644956dddcd89dd59c203dcdb36cec7", - "version" : "1.1.4" + "revision" : "6675bc0ff86e61436e615df6fc5174e043e57924", + "version" : "1.4.1" } }, { diff --git a/Sources/PowerSync/Protocol/db/SqlCursor.swift b/Sources/PowerSync/Protocol/db/SqlCursor.swift index 52b0b21..42f8012 100644 --- a/Sources/PowerSync/Protocol/db/SqlCursor.swift +++ b/Sources/PowerSync/Protocol/db/SqlCursor.swift @@ -64,7 +64,7 @@ public protocol SqlCursor { var columnNames: [String: Int] { get } } -extension SqlCursor { +public extension SqlCursor { private func withResolvedIndex(name: String, read: (_ index: Int) throws(SqlCursorError) -> T) throws(SqlCursorError) -> T { if let index = self.columnNames[name] { do { From f73de9f6da3342fd317537b9293bb462f2659043 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 29 Apr 2026 09:52:04 +0200 Subject: [PATCH 34/40] Some additional cleanup --- .../Implementation/AsyncConnectionPool.swift | 100 ++++++++++-------- .../sqlite3/NativeConnectionPool.swift | 1 + .../sqlite3/NativeStatement.swift | 9 +- .../PowerSync/Utils/MergeItemSequence.swift | 7 +- ...esolvePowerSyncLoadableExtensionPath.swift | 2 +- 5 files changed, 70 insertions(+), 49 deletions(-) diff --git a/Sources/PowerSync/Implementation/AsyncConnectionPool.swift b/Sources/PowerSync/Implementation/AsyncConnectionPool.swift index 387d88e..42b5b13 100644 --- a/Sources/PowerSync/Implementation/AsyncConnectionPool.swift +++ b/Sources/PowerSync/Implementation/AsyncConnectionPool.swift @@ -61,18 +61,19 @@ final class AsyncConnectionPool: SQLiteConnectionPoolProtocol { private let initialStatements: [String] private let logger: any LoggerProtocol private let tableUpdatesStream = BroadcastStream>() - private let inner: AsyncSemaphore = AsyncSemaphore(singleElement: nil) + private let opener = PoolOpener() init(location: DatabaseLocation, logger: any LoggerProtocol, initialStatements: [String] = []) { self.location = location self.logger = logger self.initialStatements = initialStatements } - + var tableUpdates: AsyncStream> { tableUpdatesStream.subscribe() } - + + /// Asyncifies a synchronous unit of work on by running it on a suitable background thread. private func runBlocking(action: @escaping @Sendable () throws -> T, qos: DispatchQoS.QoSClass = .userInitiated) async throws -> T { return try await withCheckedThrowingContinuation { continuation in DispatchQueue.global(qos: qos).async { @@ -80,7 +81,7 @@ final class AsyncConnectionPool: SQLiteConnectionPoolProtocol { } } } - + private func configureConnection(connection: borrowing RawSqliteConnection, isWriter: Bool) throws { let context = connection.asLease() for stmt in initialStatements { @@ -111,40 +112,10 @@ final class AsyncConnectionPool: SQLiteConnectionPoolProtocol { let _ = try context.execute(sql: "select powersync_update_hooks('install')", parameters: []) } } - - private func obtainInner() async throws -> NativeConnectionPool { - var lease = try await inner.acquire(count: 1) - if let pool = lease.acquiredItems[0] { - return pool - } else { - try registerPowerSyncCoreExtension() - - @Sendable func handleUpdates(_ updates: Set) { - self.tableUpdatesStream.dispatch(event: updates) - } - - let pool = try await runBlocking { [self] in - let writer = try location.openConnection(writer: true) - try configureConnection(connection: writer, isWriter: true) - if case .inMemory = location { - return NativeConnectionPool(singleConnection: writer, logger: logger, handleUpdates: handleUpdates) - } else { - let numReaders = 4 - var readers = RigidDeque(capacity: numReaders) - while !readers.isFull { - let connection = try location.openConnection(writer: false) - try configureConnection(connection: connection, isWriter: false) - readers.append(connection) - } - - return NativeConnectionPool(writer: writer, readers: readers, logger: logger, handleUpdates: handleUpdates) - } - } - - lease.acquiredItems[0] = pool - return pool - } + /// Opens connections on a background thread to obtain the native connection pool. + private func obtainInner() async throws -> NativeConnectionPool { + try await opener.obtainPool(pool: self) } func read(onConnection: @escaping @Sendable (any SQLiteConnectionLease) throws -> T) async throws -> T { @@ -160,7 +131,7 @@ final class AsyncConnectionPool: SQLiteConnectionPoolProtocol { try await runBlocking { try onConnection(connection) } } } - + func withAllConnections(onConnection: @escaping @Sendable (any SQLiteConnectionLease, [any SQLiteConnectionLease]) throws -> T) async throws -> T { let pool = try await obtainInner() return try await pool.withAllConnections { writer, readers in @@ -169,10 +140,55 @@ final class AsyncConnectionPool: SQLiteConnectionPoolProtocol { } func close() async throws { - var lease = try await inner.acquire(count: 1) - if let pool = lease.acquiredItems[0] { - try await pool.close() - lease.acquiredItems[0] = nil + try await self.opener.close() + } + + private actor PoolOpener { + private var pool: NativeConnectionPool? = nil + private var isClosed = false + + func obtainPool(pool context: AsyncConnectionPool) async throws -> NativeConnectionPool { + if let pool { + return pool + } + + try registerPowerSyncCoreExtension() + let handleUpdates: @Sendable (_: Set) -> () = { [weak context] updates in + context?.tableUpdatesStream.dispatch(event: updates) + } + + let pool = try await context.runBlocking { + let writer = try context.location.openConnection(writer: true) + try context.configureConnection(connection: writer, isWriter: true) + + if case .inMemory = context.location { + return NativeConnectionPool(singleConnection: writer, logger: context.logger, handleUpdates: handleUpdates) + } else { + let numReaders = 4 + var readers = RigidDeque(capacity: numReaders) + while !readers.isFull { + let connection = try context.location.openConnection(writer: false) + try context.configureConnection(connection: connection, isWriter: false) + readers.append(connection) + } + + return NativeConnectionPool(writer: writer, readers: readers, logger: context.logger, handleUpdates: handleUpdates) + } + } + + self.pool = pool + return pool + } + + func close() async throws { + if isClosed { + return + } + + isClosed = true + if let pool { + try await pool.close() + } } } } diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift index aad7062..c30d570 100644 --- a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift +++ b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift @@ -6,6 +6,7 @@ import DequeModule /// /// This class does not configure or open connections (that is the responsibility of ``AsyncConnectionPool``). final class NativeConnectionPool: Sendable { + // This could be an async mutex, but AsyncSemaphore has better cancellation support. private let writer: AsyncSemaphore private let readers: AsyncSemaphore? private let handleUpdates: @Sendable (_: Set) -> () diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift b/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift index f1ce944..a79e2bf 100644 --- a/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift +++ b/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift @@ -35,13 +35,12 @@ struct NativeSqliteStatement: ~Copyable { var columnCount: Int { return Int(sqlite3_column_count(self.stmt)) } - - + var columnNames: [String : Int] { return resolvedColumnNames! } - borrowing func bindValues(_ parameters: [PowerSyncDataType?]) throws (PowerSyncError) { + borrowing func bindValues(_ parameters: [PowerSyncDataType?]) throws(PowerSyncError) { for (i, parameter) in parameters.enumerated() { let index = Int32(i + 1) @@ -53,7 +52,7 @@ struct NativeSqliteStatement: ~Copyable { } } - borrowing func bindValue(_ index: Int32, _ parameter: PowerSyncDataType?) throws (PowerSyncError) { + borrowing func bindValue(_ index: Int32, _ parameter: PowerSyncDataType?) throws(PowerSyncError) { let rc: Int32 switch parameter { @@ -98,7 +97,7 @@ struct NativeSqliteStatement: ~Copyable { } } - mutating func step() throws (PowerSyncError) -> Bool { + mutating func step() throws(PowerSyncError) -> Bool { let rc = sqlite3_step(self.stmt) if rc == SQLITE_DONE { return false diff --git a/Sources/PowerSync/Utils/MergeItemSequence.swift b/Sources/PowerSync/Utils/MergeItemSequence.swift index 63e8e91..257e931 100644 --- a/Sources/PowerSync/Utils/MergeItemSequence.swift +++ b/Sources/PowerSync/Utils/MergeItemSequence.swift @@ -1,4 +1,9 @@ -/// An ``AsyncSequence`` merging all items emitted between ``AsyncIteratorProtocol/next``. +/// An ``AsyncSequence`` merging all items emitted between calls to ``AsyncIteratorProtocol/next``. +/// +/// This is useful for sequences where we just want to know that an event has occurred, without needing +/// to know about the exact event. We use this internally to implement `watch()` queries with a throttle: +/// If any amount of events have occurred between throttled calls to `next()`, we want to dispatch a single +/// event. struct MergeItemSequence: AsyncSequence where Base.Element == () { typealias AsyncIterator = IteratorImpl typealias Element = () diff --git a/Sources/PowerSync/resolvePowerSyncLoadableExtensionPath.swift b/Sources/PowerSync/resolvePowerSyncLoadableExtensionPath.swift index ae46371..dfe51a2 100644 --- a/Sources/PowerSync/resolvePowerSyncLoadableExtensionPath.swift +++ b/Sources/PowerSync/resolvePowerSyncLoadableExtensionPath.swift @@ -5,7 +5,7 @@ /// function invokes `sqlite3_auto_extension` to load the core extension automatically. /// /// - Returns: `nil` -/// - Throws: An error if the extension could not be registered watchOS. +/// - Throws: An error if the extension could not be registered. public func resolvePowerSyncLoadableExtensionPath() throws(PowerSyncError) -> String? { try registerPowerSyncCoreExtension() return nil From 2321496ee88c33c95f96010b534c687d963a36da Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 29 Apr 2026 10:25:27 +0200 Subject: [PATCH 35/40] AI feedback, more tests --- CHANGELOG.md | 3 ++ .../Implementation/AsyncConnectionPool.swift | 2 +- .../sqlite3/NativeConnectionPool.swift | 4 +- .../sqlite3/NativeStatement.swift | 29 +++++++++----- .../Implementation/StatementTests.swift | 40 +++++++++++++++++++ 5 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 Tests/PowerSyncTests/Implementation/StatementTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 93b2856..0dedd49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ * Add `opDataTyped` and `previousValuesTyped` to `CrudEntry`, providing typed values instead of strings. * Make `CrudBatch`, `CrudEntry` and `CrudTransaction` a concrete struct. Note that these can no longer be created in user code. * Remove the internal `withSession` API. +* Remove internal dependency on the PowerSync Kotlin SDK. Going forward, the Swift SDK is implemented in Swift! +* Breaking (for internal `SQLiteConnectionPoolProtocol` implementers): Make callbacks generic. +* Breaking (for internal `SQLiteConnectionLease` implementers): Add methods to run statements. ## 1.13.1 diff --git a/Sources/PowerSync/Implementation/AsyncConnectionPool.swift b/Sources/PowerSync/Implementation/AsyncConnectionPool.swift index 42b5b13..a52767f 100644 --- a/Sources/PowerSync/Implementation/AsyncConnectionPool.swift +++ b/Sources/PowerSync/Implementation/AsyncConnectionPool.swift @@ -125,7 +125,7 @@ final class AsyncConnectionPool: SQLiteConnectionPoolProtocol { } } - func write(onConnection: @escaping @Sendable (any SQLiteConnectionLease) throws -> T) async throws -> T{ + func write(onConnection: @escaping @Sendable (any SQLiteConnectionLease) throws -> T) async throws -> T { let pool = try await obtainInner() return try await pool.write { connection in try await runBlocking { try onConnection(connection) } diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift index c30d570..74b09c2 100644 --- a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift +++ b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift @@ -68,7 +68,7 @@ final class NativeConnectionPool: Sendable { return result } - func withAllConnections(onConnection: (NativeConnectionLease, [NativeConnectionLease]) async throws -> T) async throws -> T{ + func withAllConnections(onConnection: (NativeConnectionLease, [NativeConnectionLease]) async throws -> T) async throws -> T { let write = try await writer.acquire(count: 1) let writeLease = write.acquiredItems[0].asLease() defer { dispatchWrites(lease: writeLease) } @@ -134,6 +134,8 @@ struct RawSqliteConnection: ~Copyable { } } +// We mark this as Sendable because it's only used in a mutex from `ConnectionLeaseContext`. +// We can't generally assume SQLite connections to be thread-safe. struct NativeConnectionLease: SQLiteConnectionLease, @unchecked Sendable { let pointer: OpaquePointer diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift b/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift index a79e2bf..cdf53b9 100644 --- a/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift +++ b/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift @@ -37,7 +37,10 @@ struct NativeSqliteStatement: ~Copyable { } var columnNames: [String : Int] { - return resolvedColumnNames! + guard let resolvedColumnNames else { + fatalError("columnNames is only available after step()") + } + return resolvedColumnNames } borrowing func bindValues(_ parameters: [PowerSyncDataType?]) throws(PowerSyncError) { @@ -79,17 +82,21 @@ struct NativeSqliteStatement: ~Copyable { case .double(let value): rc = sqlite3_bind_double(self.stmt, index, value) case .data(let value): - // Data object can be made up of multiple memory regions, so copy once. - let buffer = malloc(value.count)! - value.copyBytes(to: buffer.assumingMemoryBound(to: UInt8.self), count: value.count) + if value.count == 0 { + rc = sqlite3_bind_zeroblob(self.stmt, index, 0) + } else { + // Data object can be made up of multiple memory regions, so copy once. + let buffer = malloc(value.count)! + value.copyBytes(to: buffer.assumingMemoryBound(to: UInt8.self), count: value.count) - rc = sqlite3_bind_blob( - self.stmt, - index, - buffer, - Int32(value.count), - free, - ) + rc = sqlite3_bind_blob( + self.stmt, + index, + buffer, + Int32(value.count), + free, + ) + } } if rc != 0 { diff --git a/Tests/PowerSyncTests/Implementation/StatementTests.swift b/Tests/PowerSyncTests/Implementation/StatementTests.swift new file mode 100644 index 0000000..95e3293 --- /dev/null +++ b/Tests/PowerSyncTests/Implementation/StatementTests.swift @@ -0,0 +1,40 @@ +import Foundation +@testable import PowerSync +import Testing + +struct StatementTests { + @Test func bindValues() throws { + let connection = try DatabaseLocation.inMemory.openConnection(writer: true) + let lease = connection.asLease() + try lease.withIterator( + sql: "SELECT ?, ?, ?, ?, ?, ?, typeof(?), typeof(?)", + parameters: [ + nil, + .bool(false), + .int32(32), + .int64(64), + .double(3.14), + .string("hello"), + .data(Data()), + .data(Data([1, 2, 3])) + ], + callback: { iterator in + var hadRow = false + try iterator.next { cursor in + hadRow = true + + try #require(cursor.getStringOptional(index: 0) == nil) + try #require(cursor.getBoolean(index: 1) == false) + try #require(cursor.getInt(index: 2) == 32) + try #require(cursor.getInt(index: 3) == 64) + try #require(cursor.getDouble(index: 4) == 3.14) + try #require(cursor.getString(index: 5) == "hello") + try #require(cursor.getString(index: 6) == "blob") + try #require(cursor.getString(index: 7) == "blob") + } + + try #require(hadRow) + } + ) + } +} From 937ed8c9688737bba104fe5daee58de1122333f5 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 4 May 2026 09:38:12 +0200 Subject: [PATCH 36/40] Fix GRDB tests --- CHANGELOG.md | 4 +- .../GRDBDemo.xcodeproj/project.pbxproj | 4 +- .../xcshareddata/swiftpm/Package.resolved | 28 +++++- README.md | 9 -- .../Implementation/AsyncConnectionPool.swift | 6 +- .../Implementation/queries/LeaseContext.swift | 28 +++--- .../sqlite3/NativeConnectionPool.swift | 50 ++-------- .../sqlite3/NativeLeaseUtils.swift | 22 +++++ .../sqlite3/NativeStatement.swift | 14 +++ .../Protocol/SQLiteConnectionPool.swift | 9 -- .../Connections/GRDBConnectionLease.swift | 11 --- .../Connections/GRDBConnectionPool.swift | 21 ++++- .../PowerSyncGRDB/Connections/RowCursor.swift | 92 ------------------- .../Connections/collectWrites.swift | 30 ++++++ Tests/PowerSyncGRDBTests/BasicTest.swift | 19 ++++ .../Implementation/StatementTests.swift | 31 +++---- 16 files changed, 167 insertions(+), 211 deletions(-) create mode 100644 Sources/PowerSync/Implementation/sqlite3/NativeLeaseUtils.swift delete mode 100644 Sources/PowerSyncGRDB/Connections/RowCursor.swift create mode 100644 Sources/PowerSyncGRDB/Connections/collectWrites.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dedd49..97953a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,12 @@ ## 1.14.0 (unreleased) +* Remove internal dependency on the PowerSync Kotlin SDK. Going forward, the Swift SDK is implemented in Swift! + __Important__: While these changes are tested, they are a full rewrite of the internal connection pool logic. + Please also test queries in your app after upgrading. * Add `opDataTyped` and `previousValuesTyped` to `CrudEntry`, providing typed values instead of strings. * Make `CrudBatch`, `CrudEntry` and `CrudTransaction` a concrete struct. Note that these can no longer be created in user code. * Remove the internal `withSession` API. -* Remove internal dependency on the PowerSync Kotlin SDK. Going forward, the Swift SDK is implemented in Swift! * Breaking (for internal `SQLiteConnectionPoolProtocol` implementers): Make callbacks generic. * Breaking (for internal `SQLiteConnectionLease` implementers): Add methods to run statements. diff --git a/Demos/GRDBDemo/GRDBDemo.xcodeproj/project.pbxproj b/Demos/GRDBDemo/GRDBDemo.xcodeproj/project.pbxproj index 5f65cab..346c830 100644 --- a/Demos/GRDBDemo/GRDBDemo.xcodeproj/project.pbxproj +++ b/Demos/GRDBDemo/GRDBDemo.xcodeproj/project.pbxproj @@ -314,7 +314,7 @@ ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = ZGT7463CVJ; + DEVELOPMENT_TEAM = N2FNDTNV98; ENABLE_APP_SANDBOX = YES; ENABLE_HARDENED_RUNTIME = YES; ENABLE_INCOMING_NETWORK_CONNECTIONS = NO; @@ -368,7 +368,7 @@ ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = ZGT7463CVJ; + DEVELOPMENT_TEAM = N2FNDTNV98; ENABLE_APP_SANDBOX = YES; ENABLE_HARDENED_RUNTIME = YES; ENABLE_INCOMING_NETWORK_CONNECTIONS = NO; diff --git a/Demos/GRDBDemo/GRDBDemo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Demos/GRDBDemo/GRDBDemo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index e995dbb..6458b51 100644 --- a/Demos/GRDBDemo/GRDBDemo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/Demos/GRDBDemo/GRDBDemo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -1,13 +1,13 @@ { - "originHash" : "84d25347b5249e7ab78894935a203b13d9d55e5a01b7fe00745bdf028062df42", + "originHash" : "3de68e554d2ad75cf2c4dbdaf311828c8260cc6008b42e8738726e62912934ce", "pins" : [ { "identity" : "csqlite", "kind" : "remoteSourceControl", "location" : "https://github.com/powersync-ja/CSQLite.git", "state" : { - "revision" : "25f4a02fce2dcd588bad37ea6fc047c2bbe8ef5e", - "version" : "3.51.1" + "revision" : "9f10da4272e292c453db400342d9e16c15e59db5", + "version" : "3.51.2" } }, { @@ -33,8 +33,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/powersync-ja/powersync-sqlite-core-swift.git", "state" : { - "revision" : "9743740980cd488a11f50cffe62ed34a9739a135", - "version" : "0.4.10" + "revision" : "05c2af384558011f0915d757b6677f5dcbbc5c54", + "version" : "0.4.13" } }, { @@ -55,6 +55,15 @@ "version" : "1.4.0" } }, + { + "identity" : "swift-async-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-async-algorithms.git", + "state" : { + "revision" : "9d349bcc328ac3c31ce40e746b5882742a0d1272", + "version" : "1.1.3" + } + }, { "identity" : "swift-clocks", "kind" : "remoteSourceControl", @@ -64,6 +73,15 @@ "version" : "1.0.6" } }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "6675bc0ff86e61436e615df6fc5174e043e57924", + "version" : "1.4.1" + } + }, { "identity" : "swift-concurrency-extras", "kind" : "remoteSourceControl", diff --git a/README.md b/README.md index ad10fdf..35333ac 100644 --- a/README.md +++ b/README.md @@ -118,15 +118,6 @@ Feel free to use the `DatabasePool` for view logic and the `PowerSyncDatabase` f - Updating the PowerSync schema, with `updateSchema`, is not currently fully supported with GRDB connections. - This integration currently requires statically linking PowerSync and GRDB. - This implementation requires defining the PowerSync Schema and GRDB data types. This results in some duplication. -- This implementation uses the SQLite session API to track updates made by the PowerSync SDK. This could use more memory compared to the Standard PowerSync SQLite implementation. -- This implementation can cause warnings such as "Thread Performance Checker: Thread running at User-interactive quality-of-service class waiting on a lower QoS thread running at Default quality-of-service class. Investigate ways to avoid priority inversions". This will be addressed in a future release. - -## Underlying Kotlin Dependency - -The PowerSync Swift SDK makes use of the [PowerSync Kotlin SDK](https://github.com/powersync-ja/powersync-kotlin) and the API tool [SKIE](https://skie.touchlab.co/) under the hood to implement the Swift package. -However, this dependency is resolved internally and all public APIs are written entirely in Swift. - -For more details, see the [Swift SDK reference](https://docs.powersync.com/client-sdk-references/swift) and generated [API references](https://powersync-ja.github.io/powersync-swift/documentation/powersync/). ## Attachments diff --git a/Sources/PowerSync/Implementation/AsyncConnectionPool.swift b/Sources/PowerSync/Implementation/AsyncConnectionPool.swift index a52767f..5cdcf67 100644 --- a/Sources/PowerSync/Implementation/AsyncConnectionPool.swift +++ b/Sources/PowerSync/Implementation/AsyncConnectionPool.swift @@ -102,9 +102,9 @@ final class AsyncConnectionPool: SQLiteConnectionPoolProtocol { if isWriter { // Older versions of the SDK used to set up an empty schema and raise the user version to 1. // Keep doing that for consistency. - let version = try context.withIterator(sql: "pragma user_version", parameters: []) { rows in - try rows.next { try $0.getInt(index: 0) } - } + var stmt = try context.iterate(sql: "pragma user_version", parameters: []) + let version = try stmt.stepWithCursor { try $0.getInt(index: 0) } + let _ = consume stmt if let version, version < 1 { let _ = try context.execute(sql: "pragma user_version = 1", parameters: []) } diff --git a/Sources/PowerSync/Implementation/queries/LeaseContext.swift b/Sources/PowerSync/Implementation/queries/LeaseContext.swift index 7947943..4eabcb0 100644 --- a/Sources/PowerSync/Implementation/queries/LeaseContext.swift +++ b/Sources/PowerSync/Implementation/queries/LeaseContext.swift @@ -31,31 +31,31 @@ final class ConnectionLeaseContext: ConnectionContext { func getOptional(sql: String, parameters: [(any Sendable)?]?, mapper: @escaping @Sendable (any SqlCursor) throws -> RowType) throws -> RowType? { try lease.withLock { lease in - try lease.withIterator(sql: sql, parameters: mapParameters(parameters)) { rows in - return try rows.next(callback: mapper) - } + var stmt = try lease.iterate(sql: sql, parameters: mapParameters(parameters)) + return try stmt.stepWithCursor(callback: mapper) } } func getAll(sql: String, parameters: [(any Sendable)?]?, mapper: @escaping @Sendable (any SqlCursor) throws -> RowType) throws -> [RowType] { try lease.withLock { lease in - try lease.withIterator(sql: sql, parameters: mapParameters(parameters)) { rows in - var result: [RowType] = [] - while let row = try rows.next(callback: mapper) { - result.append(row) - } - return result + var stmt = try lease.iterate(sql: sql, parameters: mapParameters(parameters)) + var result: [RowType] = [] + + while let row = try stmt.stepWithCursor(callback: mapper) { + result.append(row) } + return result } } func get(sql: String, parameters: [(any Sendable)?]?, mapper: @escaping @Sendable (any SqlCursor) throws -> RowType) throws -> RowType { try lease.withLock { lease in - try lease.withIterator(sql: sql, parameters: mapParameters(parameters)) { rows in - guard let cursor = try rows.next(callback: mapper) else { - throw PowerSyncError.operationFailed(message: "Expected \(sql) to return a row, but got an empty result set.") - } - return cursor + var stmt = try lease.iterate(sql: sql, parameters: mapParameters(parameters)) + + if let row = try stmt.stepWithCursor(callback: mapper) { + return row + } else { + throw PowerSyncError.operationFailed(message: "Expected \(sql) to return a row, but got an empty result set.") } } } diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift index 74b09c2..3f76809 100644 --- a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift +++ b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift @@ -37,15 +37,14 @@ final class NativeConnectionPool: Sendable { private func dispatchWrites(lease: NativeConnectionLease) { do { - try lease.withIterator(sql: "SELECT powersync_update_hooks('get')", parameters: []) { rows in - let affectedTables = try rows.next { - let decoder = JSONDecoder() - return try decoder.decode(Set.self, from: try $0.getString(index: 0).data(using: .utf8)!) - } + var stmt = try lease.iterate(sql: "SELECT powersync_update_hooks('get')", parameters: []) + let affectedTables = try stmt.stepWithCursor { + let decoder = JSONDecoder() + return try decoder.decode(Set.self, from: try $0.getString(index: 0).data(using: .utf8)!) + } - if let affectedTables, !affectedTables.isEmpty { - self.handleUpdates(affectedTables) - } + if let affectedTables, !affectedTables.isEmpty { + self.handleUpdates(affectedTables) } } catch { logger.warning("Could not read affected tables", tag: "NativeConnectionPool") @@ -138,39 +137,4 @@ struct RawSqliteConnection: ~Copyable { // We can't generally assume SQLite connections to be thread-safe. struct NativeConnectionLease: SQLiteConnectionLease, @unchecked Sendable { let pointer: OpaquePointer - - func execute(sql: String, parameters: [PowerSyncDataType?]) throws -> Int64 { - do { - var stmt = try NativeSqliteStatement(db: pointer, sql: sql) - try stmt.bindValues(parameters) - while try stmt.step() { - // Iterate through the statement. - } - } - - return sqlite3_changes64(pointer) - } - - func withIterator(sql: String, parameters: [PowerSyncDataType?], callback: (SQLiteStatementIteratorProtocol) throws -> T) throws -> T { - var stmt = try NativeSqliteStatement(db: pointer, sql: sql) - try stmt.bindValues(parameters) - return try withUnsafeMutablePointer(to: &stmt) { ptr in - let iterator = NativeStatementIterator(stmt: ptr) - return try callback(iterator) - } - } -} - -private struct NativeStatementIterator: SQLiteStatementIteratorProtocol { - var stmt: UnsafeMutablePointer - - func next(callback: (any SqlCursor) throws -> T) throws -> T? { - if try stmt.pointee.step() { - let cursor = StatementCursor(stmt) - defer { cursor.invalidate() } - return try callback(cursor) - } else { - return nil - } - } } diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeLeaseUtils.swift b/Sources/PowerSync/Implementation/sqlite3/NativeLeaseUtils.swift new file mode 100644 index 0000000..dc000a4 --- /dev/null +++ b/Sources/PowerSync/Implementation/sqlite3/NativeLeaseUtils.swift @@ -0,0 +1,22 @@ +import CSQLite + +/// Implements functions to execute and iterate through SQL statements based on a SQLite connection pointer. +extension SQLiteConnectionLease { + func execute(sql: String, parameters: [PowerSyncDataType?]) throws -> Int64 { + do { + var stmt = try NativeSqliteStatement(db: pointer, sql: sql) + try stmt.bindValues(parameters) + while try stmt.step() { + // Iterate through the statement. + } + } + + return sqlite3_changes64(pointer) + } + + func iterate(sql: String, parameters: [PowerSyncDataType?]) throws -> NativeSqliteStatement { + let stmt = try NativeSqliteStatement(db: pointer, sql: sql) + try stmt.bindValues(parameters) + return stmt + } +} diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift b/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift index cdf53b9..db7007e 100644 --- a/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift +++ b/Sources/PowerSync/Implementation/sqlite3/NativeStatement.swift @@ -125,4 +125,18 @@ struct NativeSqliteStatement: ~Copyable { try throwDatabaseError(db: self.db, sql: self.sql) } } + + mutating func stepWithCursor(callback: (any SqlCursor) throws -> T) throws -> T? { + if try step() { + // Turn the static ~Copyable lifetime into a dynamic lifetime with explicit + // invalidation. + return try withUnsafePointer(to: self) { ptr in + let cursor = StatementCursor(ptr) + defer { cursor.invalidate() } + return try callback(cursor) + } + } else { + return nil + } + } } diff --git a/Sources/PowerSync/Protocol/SQLiteConnectionPool.swift b/Sources/PowerSync/Protocol/SQLiteConnectionPool.swift index 9b632d2..f50dd33 100644 --- a/Sources/PowerSync/Protocol/SQLiteConnectionPool.swift +++ b/Sources/PowerSync/Protocol/SQLiteConnectionPool.swift @@ -5,15 +5,6 @@ public protocol SQLiteConnectionLease { /// Pointer to the underlying SQLite connection. /// This pointer should not be used outside of the closure which provided the lease. var pointer: OpaquePointer { borrowing get } - - /// Executes an SQL statement and returns the amount of rows affected. - func execute(sql: String, parameters: [PowerSyncDataType?]) throws -> Int64 - - func withIterator(sql: String, parameters: [PowerSyncDataType?], callback: (_: SQLiteStatementIteratorProtocol) throws -> T) throws -> T -} - -public protocol SQLiteStatementIteratorProtocol { - func next(callback: (_ cursor: SqlCursor) throws -> T) throws -> T? } /// An implementation of a connection pool providing asynchronous access to a single writer and multiple readers. diff --git a/Sources/PowerSyncGRDB/Connections/GRDBConnectionLease.swift b/Sources/PowerSyncGRDB/Connections/GRDBConnectionLease.swift index a79614f..fee9f43 100644 --- a/Sources/PowerSyncGRDB/Connections/GRDBConnectionLease.swift +++ b/Sources/PowerSyncGRDB/Connections/GRDBConnectionLease.swift @@ -17,17 +17,6 @@ final class GRDBConnectionLease: SQLiteConnectionLease { self.pointer = connection self.database = database } - - func execute(sql: String, parameters: [PowerSync.PowerSyncDataType?]) throws -> Int64 { - try database.execute(sql: sql, arguments: StatementArguments(parameters)) - return Int64(database.changesCount) - } - - func withIterator(sql: String, parameters: [PowerSync.PowerSyncDataType?], callback: (any PowerSync.SQLiteStatementIteratorProtocol) throws -> T) throws -> T { - let statement = try database.makeStatement(sql: sql) - let rows = try Row.fetchCursor(statement, arguments: StatementArguments(parameters)) - return try callback(RowIterator(rows: rows)) - } } extension PowerSync.PowerSyncDataType: DatabaseValueConvertible { diff --git a/Sources/PowerSyncGRDB/Connections/GRDBConnectionPool.swift b/Sources/PowerSyncGRDB/Connections/GRDBConnectionPool.swift index 59df99e..f819ee7 100644 --- a/Sources/PowerSyncGRDB/Connections/GRDBConnectionPool.swift +++ b/Sources/PowerSyncGRDB/Connections/GRDBConnectionPool.swift @@ -51,11 +51,24 @@ actor GRDBConnectionPool: SQLiteConnectionPoolProtocol { onConnection: @Sendable @escaping (SQLiteConnectionLease) throws -> T ) async throws -> T { // Don't start an explicit transaction, we do this internally - return try await pool.writeWithoutTransaction { database in - return try onConnection( - GRDBConnectionLease(database: database) - ) + let (result, updates) = try await pool.writeWithoutTransaction { database in + try collectWrites(db: database.sqliteConnection!) { + try onConnection(GRDBConnectionLease(database: database)) + } + } + + if !updates.isEmpty { + tableUpdatesContinuation?.yield(updates) + + // Notify GRDB, this needs to be a write (transaction) + try await pool.write { database in + // Notify GRDB about these changes + for table in updates { + try database.notifyChanges(in: Table(table)) + } + } } + return result } func withAllConnections( diff --git a/Sources/PowerSyncGRDB/Connections/RowCursor.swift b/Sources/PowerSyncGRDB/Connections/RowCursor.swift deleted file mode 100644 index ad95d41..0000000 --- a/Sources/PowerSyncGRDB/Connections/RowCursor.swift +++ /dev/null @@ -1,92 +0,0 @@ -import GRDB -import PowerSync - -final class RowIterator: PowerSync.SQLiteStatementIteratorProtocol { - let rows: RowCursor - var columnNames: [String: Int]? = nil - - init(rows: RowCursor) { - self.rows = rows - } - - func next(callback: (any SqlCursor) throws -> T) throws -> T? { - guard let row = try rows.next() else { - return nil - } - - return try callback(RowSqlCursor(columnNames: resolveColumnNames(), row: row)) - } - - private func resolveColumnNames() -> [String: Int] { - if let columnNames { - return columnNames - } - - var names: [String: Int] = [:] - for (i, name) in rows.columnNames.enumerated() { - names[name] = i - } - columnNames = names - return names - } -} - -struct RowSqlCursor: PowerSync.SqlCursor { - let columnNames: [String: Int] - let row: Row - - private func checkNotNull(index: Int) throws(PowerSync.SqlCursorError) { - if row.hasNull(atIndex: index) { - throw .nullValueFound("\(index)") - } - } - - public func getBoolean(index: Int) throws(PowerSync.SqlCursorError) -> Bool { - try checkNotNull(index: index) - return row.self[index] - } - - public func getBooleanOptional(index: Int) -> Bool? { - return row.self[index] - } - - public func getDouble(index: Int) throws(PowerSync.SqlCursorError) -> Double { - try checkNotNull(index: index) - return row.self[index] - } - - public func getDoubleOptional(index: Int) -> Double? { - return row.self[index] - } - - public func getInt(index: Int) throws(PowerSync.SqlCursorError) -> Int { - try checkNotNull(index: index) - return row.self[index] - } - - public func getIntOptional(index: Int) -> Int? { - return row.self[index] - } - - public func getInt64(index: Int) throws(PowerSync.SqlCursorError) -> Int64 { - try self.checkNotNull(index: index) - return row.self[index] - } - - public func getInt64Optional(index: Int) -> Int64? { - return row.self[index] - } - - public func getString(index: Int) throws(PowerSync.SqlCursorError) -> String { - try self.checkNotNull(index: index) - return row.self[index] - } - - public func getStringOptional(index: Int) -> String? { - return row.self[index] - } - - public var columnCount: Int { - row.count - } -} diff --git a/Sources/PowerSyncGRDB/Connections/collectWrites.swift b/Sources/PowerSyncGRDB/Connections/collectWrites.swift new file mode 100644 index 0000000..800887f --- /dev/null +++ b/Sources/PowerSyncGRDB/Connections/collectWrites.swift @@ -0,0 +1,30 @@ +import CSQLite + +/// Collect writes that a callback makes on a database connection. +/// +/// We can't install commit / rollback hooks since GRDB keeps those installed, so this may also +/// return writes for transactions that have been rolled back. This may cause more queries than +/// intended to update again, which doesn't impact correctness. +func collectWrites(db: OpaquePointer, callback: () throws -> T) rethrows -> (T, Set) { + var notifications = Set() + // Install temporary update/commit/rollback hooks. GRDB should doesn't install hooks outside of + // statements, so this doesn't interfere with GRDB (but we have an assert to be sure). + let result = try withUnsafeMutablePointer(to: ¬ifications) { ptr in + let prevUpdates = sqlite3_update_hook(db, { context, type, dbName, tableName, rowId in + if let tableName, let context { + let table = String(cString: tableName) + context.assumingMemoryBound(to: Set.self).pointee.insert(table) + } + }, ptr) + assert(prevUpdates == nil, "Unexpected existing update hook") + + defer { + // Uninstall our hooks + sqlite3_update_hook(db, nil, nil) + } + + return try callback() + } + + return (result, notifications) +} diff --git a/Tests/PowerSyncGRDBTests/BasicTest.swift b/Tests/PowerSyncGRDBTests/BasicTest.swift index 1150b62..57f522e 100644 --- a/Tests/PowerSyncGRDBTests/BasicTest.swift +++ b/Tests/PowerSyncGRDBTests/BasicTest.swift @@ -352,6 +352,25 @@ final class GRDBTests: XCTestCase { watchTask.cancel() } + func testGRDBUpdatesFromIndirectPowerSyncFunction() async throws { + try await database.execute( + sql: "INSERT INTO users(id, name) VALUES(uuid(), ?)", + parameters: ["a user"] + ) + + var events = ValueObservation.tracking { + try User.order(User.Columns.name.asc).fetchAll($0) + }.values(in: pool).makeAsyncIterator() + let first = try await events.next() + XCTAssertEqual(first?.count, 1) + + // We want to assert that internal statements from the core extension still + // update GRDB value observations. + try await database.disconnectAndClear() + let second = try await events.next() + XCTAssertEqual(second?.count, 0) + } + func testShouldThrowErrorsFromPowerSync() async throws { do { try await database.execute( diff --git a/Tests/PowerSyncTests/Implementation/StatementTests.swift b/Tests/PowerSyncTests/Implementation/StatementTests.swift index 95e3293..77e9a7f 100644 --- a/Tests/PowerSyncTests/Implementation/StatementTests.swift +++ b/Tests/PowerSyncTests/Implementation/StatementTests.swift @@ -6,7 +6,7 @@ struct StatementTests { @Test func bindValues() throws { let connection = try DatabaseLocation.inMemory.openConnection(writer: true) let lease = connection.asLease() - try lease.withIterator( + var stmt = try lease.iterate( sql: "SELECT ?, ?, ?, ?, ?, ?, typeof(?), typeof(?)", parameters: [ nil, @@ -18,23 +18,18 @@ struct StatementTests { .data(Data()), .data(Data([1, 2, 3])) ], - callback: { iterator in - var hadRow = false - try iterator.next { cursor in - hadRow = true - - try #require(cursor.getStringOptional(index: 0) == nil) - try #require(cursor.getBoolean(index: 1) == false) - try #require(cursor.getInt(index: 2) == 32) - try #require(cursor.getInt(index: 3) == 64) - try #require(cursor.getDouble(index: 4) == 3.14) - try #require(cursor.getString(index: 5) == "hello") - try #require(cursor.getString(index: 6) == "blob") - try #require(cursor.getString(index: 7) == "blob") - } - - try #require(hadRow) - } ) + try #require(try stmt.step()) + try withUnsafePointer(to: stmt) { ptr in + let cursor = StatementCursor(ptr) + try #require(cursor.getStringOptional(index: 0) == nil) + try #require(cursor.getBoolean(index: 1) == false) + try #require(cursor.getInt(index: 2) == 32) + try #require(cursor.getInt(index: 3) == 64) + try #require(cursor.getDouble(index: 4) == 3.14) + try #require(cursor.getString(index: 5) == "hello") + try #require(cursor.getString(index: 6) == "blob") + try #require(cursor.getString(index: 7) == "blob") + } } } From ff32bec1631b6c36de4c5f845303f612a76efc4e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 4 May 2026 09:48:47 +0200 Subject: [PATCH 37/40] Remove unused GRDB data convertible code --- .../Connections/GRDBConnectionLease.swift | 25 +------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/Sources/PowerSyncGRDB/Connections/GRDBConnectionLease.swift b/Sources/PowerSyncGRDB/Connections/GRDBConnectionLease.swift index fee9f43..36b7be3 100644 --- a/Sources/PowerSyncGRDB/Connections/GRDBConnectionLease.swift +++ b/Sources/PowerSyncGRDB/Connections/GRDBConnectionLease.swift @@ -17,27 +17,4 @@ final class GRDBConnectionLease: SQLiteConnectionLease { self.pointer = connection self.database = database } -} - -extension PowerSync.PowerSyncDataType: DatabaseValueConvertible { - public var databaseValue: GRDB.DatabaseValue { - switch self { - case .bool(let value): - return value.databaseValue - case .string(let value): - return value.databaseValue - case .int64(let value): - return value.databaseValue - case .int32(let value): - return value.databaseValue - case .double(let value): - return value.databaseValue - case .data(let value): - return value.databaseValue - } - } - - public static func fromDatabaseValue(_ dbValue: GRDB.DatabaseValue) -> PowerSync.PowerSyncDataType? { - nil - } -} +} \ No newline at end of file From 0e4893d5e61a374ff8c69237fb36e25ce10220de Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 4 May 2026 10:13:16 +0200 Subject: [PATCH 38/40] Mark connection lease and pool protocols as internal --- Sources/PowerSync/Protocol/SQLiteConnectionPool.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Sources/PowerSync/Protocol/SQLiteConnectionPool.swift b/Sources/PowerSync/Protocol/SQLiteConnectionPool.swift index f50dd33..2a24b3b 100644 --- a/Sources/PowerSync/Protocol/SQLiteConnectionPool.swift +++ b/Sources/PowerSync/Protocol/SQLiteConnectionPool.swift @@ -1,6 +1,8 @@ import Foundation /// A lease representing a temporarily borrowed SQLite connection from the pool. +/// +/// This is an internal protocol and should not be implemented outside of the PowerSync SDK. public protocol SQLiteConnectionLease { /// Pointer to the underlying SQLite connection. /// This pointer should not be used outside of the closure which provided the lease. @@ -9,6 +11,8 @@ public protocol SQLiteConnectionLease { /// An implementation of a connection pool providing asynchronous access to a single writer and multiple readers. /// This is the underlying pool implementation on which the higher-level PowerSync Swift SDK is built on. +/// +/// This is an internal protocol and should not be implemented outside of the PowerSync SDK. public protocol SQLiteConnectionPoolProtocol: Sendable { var tableUpdates: AsyncStream> { get } From bae227e86ae3228b3313e555eb51b2fdf14f30c8 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 4 May 2026 11:18:28 +0200 Subject: [PATCH 39/40] Motivate update hooks with comment --- .../PowerSyncGRDB/Connections/GRDBConnectionPool.swift | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Sources/PowerSyncGRDB/Connections/GRDBConnectionPool.swift b/Sources/PowerSyncGRDB/Connections/GRDBConnectionPool.swift index f819ee7..ff9ae93 100644 --- a/Sources/PowerSyncGRDB/Connections/GRDBConnectionPool.swift +++ b/Sources/PowerSyncGRDB/Connections/GRDBConnectionPool.swift @@ -50,8 +50,16 @@ actor GRDBConnectionPool: SQLiteConnectionPoolProtocol { func write( onConnection: @Sendable @escaping (SQLiteConnectionLease) throws -> T ) async throws -> T { - // Don't start an explicit transaction, we do this internally + // Don't start an explicit transaction, we do this internally. let (result, updates) = try await pool.writeWithoutTransaction { database in + // This installs a temporary update hook, which breaks if GRDB had its own. Currently, we rely on + // GRDB only installing update hooks for statements that need it (see https://github.com/groue/GRDB.swift/blob/36e30a6f1ef10e4194f6af0cff90888526f0c115/GRDB/Core/TransactionObserver.swift#L266-L275), + // note that `statementObservations` is set in `statementWillExecute` and cleared after a statement + // has completed or failed. + // So since we have exclusive access to the write connection here, no GRDB-active statement can run and we + // can safely install our own hooks. + // In the future, we would like to use high-level GRDB APIs for this instead. However, we're blocked + // by https://github.com/groue/GRDB.swift/issues/1863 on that. try collectWrites(db: database.sqliteConnection!) { try onConnection(GRDBConnectionLease(database: database)) } From 99b9d3b2faaaa0ddbe84901c911d96d79f825e95 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 6 May 2026 10:08:17 +0200 Subject: [PATCH 40/40] Revert API changes --- .../Implementation/AsyncConnectionPool.swift | 6 +-- .../Implementation/queries/LeaseContext.swift | 28 ++++++------ .../sqlite3/NativeConnectionPool.swift | 15 ++++--- .../sqlite3/NativeLeaseUtils.swift | 22 ---------- .../Protocol/SQLiteConnectionPool.swift | 44 +++++++++++++++++++ .../Implementation/StatementTests.swift | 31 +++++++------ 6 files changed, 87 insertions(+), 59 deletions(-) delete mode 100644 Sources/PowerSync/Implementation/sqlite3/NativeLeaseUtils.swift diff --git a/Sources/PowerSync/Implementation/AsyncConnectionPool.swift b/Sources/PowerSync/Implementation/AsyncConnectionPool.swift index 5cdcf67..a52767f 100644 --- a/Sources/PowerSync/Implementation/AsyncConnectionPool.swift +++ b/Sources/PowerSync/Implementation/AsyncConnectionPool.swift @@ -102,9 +102,9 @@ final class AsyncConnectionPool: SQLiteConnectionPoolProtocol { if isWriter { // Older versions of the SDK used to set up an empty schema and raise the user version to 1. // Keep doing that for consistency. - var stmt = try context.iterate(sql: "pragma user_version", parameters: []) - let version = try stmt.stepWithCursor { try $0.getInt(index: 0) } - let _ = consume stmt + let version = try context.withIterator(sql: "pragma user_version", parameters: []) { rows in + try rows.next { try $0.getInt(index: 0) } + } if let version, version < 1 { let _ = try context.execute(sql: "pragma user_version = 1", parameters: []) } diff --git a/Sources/PowerSync/Implementation/queries/LeaseContext.swift b/Sources/PowerSync/Implementation/queries/LeaseContext.swift index 4eabcb0..7947943 100644 --- a/Sources/PowerSync/Implementation/queries/LeaseContext.swift +++ b/Sources/PowerSync/Implementation/queries/LeaseContext.swift @@ -31,31 +31,31 @@ final class ConnectionLeaseContext: ConnectionContext { func getOptional(sql: String, parameters: [(any Sendable)?]?, mapper: @escaping @Sendable (any SqlCursor) throws -> RowType) throws -> RowType? { try lease.withLock { lease in - var stmt = try lease.iterate(sql: sql, parameters: mapParameters(parameters)) - return try stmt.stepWithCursor(callback: mapper) + try lease.withIterator(sql: sql, parameters: mapParameters(parameters)) { rows in + return try rows.next(callback: mapper) + } } } func getAll(sql: String, parameters: [(any Sendable)?]?, mapper: @escaping @Sendable (any SqlCursor) throws -> RowType) throws -> [RowType] { try lease.withLock { lease in - var stmt = try lease.iterate(sql: sql, parameters: mapParameters(parameters)) - var result: [RowType] = [] - - while let row = try stmt.stepWithCursor(callback: mapper) { - result.append(row) + try lease.withIterator(sql: sql, parameters: mapParameters(parameters)) { rows in + var result: [RowType] = [] + while let row = try rows.next(callback: mapper) { + result.append(row) + } + return result } - return result } } func get(sql: String, parameters: [(any Sendable)?]?, mapper: @escaping @Sendable (any SqlCursor) throws -> RowType) throws -> RowType { try lease.withLock { lease in - var stmt = try lease.iterate(sql: sql, parameters: mapParameters(parameters)) - - if let row = try stmt.stepWithCursor(callback: mapper) { - return row - } else { - throw PowerSyncError.operationFailed(message: "Expected \(sql) to return a row, but got an empty result set.") + try lease.withIterator(sql: sql, parameters: mapParameters(parameters)) { rows in + guard let cursor = try rows.next(callback: mapper) else { + throw PowerSyncError.operationFailed(message: "Expected \(sql) to return a row, but got an empty result set.") + } + return cursor } } } diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift index 3f76809..f6defaf 100644 --- a/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift +++ b/Sources/PowerSync/Implementation/sqlite3/NativeConnectionPool.swift @@ -37,14 +37,15 @@ final class NativeConnectionPool: Sendable { private func dispatchWrites(lease: NativeConnectionLease) { do { - var stmt = try lease.iterate(sql: "SELECT powersync_update_hooks('get')", parameters: []) - let affectedTables = try stmt.stepWithCursor { - let decoder = JSONDecoder() - return try decoder.decode(Set.self, from: try $0.getString(index: 0).data(using: .utf8)!) - } + try lease.withIterator(sql: "SELECT powersync_update_hooks('get')", parameters: []) { rows in + let affectedTables = try rows.next { + let decoder = JSONDecoder() + return try decoder.decode(Set.self, from: try $0.getString(index: 0).data(using: .utf8)!) + } - if let affectedTables, !affectedTables.isEmpty { - self.handleUpdates(affectedTables) + if let affectedTables, !affectedTables.isEmpty { + self.handleUpdates(affectedTables) + } } } catch { logger.warning("Could not read affected tables", tag: "NativeConnectionPool") diff --git a/Sources/PowerSync/Implementation/sqlite3/NativeLeaseUtils.swift b/Sources/PowerSync/Implementation/sqlite3/NativeLeaseUtils.swift deleted file mode 100644 index dc000a4..0000000 --- a/Sources/PowerSync/Implementation/sqlite3/NativeLeaseUtils.swift +++ /dev/null @@ -1,22 +0,0 @@ -import CSQLite - -/// Implements functions to execute and iterate through SQL statements based on a SQLite connection pointer. -extension SQLiteConnectionLease { - func execute(sql: String, parameters: [PowerSyncDataType?]) throws -> Int64 { - do { - var stmt = try NativeSqliteStatement(db: pointer, sql: sql) - try stmt.bindValues(parameters) - while try stmt.step() { - // Iterate through the statement. - } - } - - return sqlite3_changes64(pointer) - } - - func iterate(sql: String, parameters: [PowerSyncDataType?]) throws -> NativeSqliteStatement { - let stmt = try NativeSqliteStatement(db: pointer, sql: sql) - try stmt.bindValues(parameters) - return stmt - } -} diff --git a/Sources/PowerSync/Protocol/SQLiteConnectionPool.swift b/Sources/PowerSync/Protocol/SQLiteConnectionPool.swift index 2a24b3b..8edb486 100644 --- a/Sources/PowerSync/Protocol/SQLiteConnectionPool.swift +++ b/Sources/PowerSync/Protocol/SQLiteConnectionPool.swift @@ -1,3 +1,4 @@ +import CSQLite import Foundation /// A lease representing a temporarily borrowed SQLite connection from the pool. @@ -7,6 +8,49 @@ public protocol SQLiteConnectionLease { /// Pointer to the underlying SQLite connection. /// This pointer should not be used outside of the closure which provided the lease. var pointer: OpaquePointer { borrowing get } + + /// Executes a SQL statement, returning the amount of affected rows. + func execute(sql: String, parameters: [PowerSyncDataType?]) throws -> Int64 + + /// Prepares a statement from the SQL text and parameters, then invokes callback with a cursor. + func withIterator(sql: String, parameters: [PowerSyncDataType?], callback: (SQLiteStatementIteratorProtocol) throws -> T) throws -> T +} + +extension SQLiteConnectionLease { + /// Default implementation of ``execute(sql:parameters:)`` based on raw sqlite3 APIs. + public func execute(sql: String, parameters: [PowerSyncDataType?]) throws -> Int64 { + do { + var stmt = try NativeSqliteStatement(db: pointer, sql: sql) + try stmt.bindValues(parameters) + while try stmt.step() { + // Iterate through the statement. + } + } + + return sqlite3_changes64(pointer) + } + + /// Default implementation of ``withIterator(sql:parameters:callback:)`` based on raw sqlite3 APIs. + public func withIterator(sql: String, parameters: [PowerSyncDataType?], callback: (SQLiteStatementIteratorProtocol) throws -> T) throws -> T { + var stmt = try NativeSqliteStatement(db: pointer, sql: sql) + try stmt.bindValues(parameters) + return try withUnsafeMutablePointer(to: &stmt) { ptr in + let iterator = NativeStatementIterator(stmt: ptr) + return try callback(iterator) + } + } +} + +private struct NativeStatementIterator: SQLiteStatementIteratorProtocol { + var stmt: UnsafeMutablePointer + + func next(callback: (any SqlCursor) throws -> T) throws -> T? { + return try stmt.pointee.stepWithCursor(callback: callback) + } +} + +public protocol SQLiteStatementIteratorProtocol { + func next(callback: (_ cursor: SqlCursor) throws -> T) throws -> T? } /// An implementation of a connection pool providing asynchronous access to a single writer and multiple readers. diff --git a/Tests/PowerSyncTests/Implementation/StatementTests.swift b/Tests/PowerSyncTests/Implementation/StatementTests.swift index 77e9a7f..95e3293 100644 --- a/Tests/PowerSyncTests/Implementation/StatementTests.swift +++ b/Tests/PowerSyncTests/Implementation/StatementTests.swift @@ -6,7 +6,7 @@ struct StatementTests { @Test func bindValues() throws { let connection = try DatabaseLocation.inMemory.openConnection(writer: true) let lease = connection.asLease() - var stmt = try lease.iterate( + try lease.withIterator( sql: "SELECT ?, ?, ?, ?, ?, ?, typeof(?), typeof(?)", parameters: [ nil, @@ -18,18 +18,23 @@ struct StatementTests { .data(Data()), .data(Data([1, 2, 3])) ], + callback: { iterator in + var hadRow = false + try iterator.next { cursor in + hadRow = true + + try #require(cursor.getStringOptional(index: 0) == nil) + try #require(cursor.getBoolean(index: 1) == false) + try #require(cursor.getInt(index: 2) == 32) + try #require(cursor.getInt(index: 3) == 64) + try #require(cursor.getDouble(index: 4) == 3.14) + try #require(cursor.getString(index: 5) == "hello") + try #require(cursor.getString(index: 6) == "blob") + try #require(cursor.getString(index: 7) == "blob") + } + + try #require(hadRow) + } ) - try #require(try stmt.step()) - try withUnsafePointer(to: stmt) { ptr in - let cursor = StatementCursor(ptr) - try #require(cursor.getStringOptional(index: 0) == nil) - try #require(cursor.getBoolean(index: 1) == false) - try #require(cursor.getInt(index: 2) == 32) - try #require(cursor.getInt(index: 3) == 64) - try #require(cursor.getDouble(index: 4) == 3.14) - try #require(cursor.getString(index: 5) == "hello") - try #require(cursor.getString(index: 6) == "blob") - try #require(cursor.getString(index: 7) == "blob") - } } }